From 5ea9b74efafd67e321126c050688aa0afedc6e9e Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Tue, 10 Nov 2026 11:54:46 +0000 Subject: [PATCH 1/4] Split services into workspaces and add the Node packager --- .github/workflows/ci.yml | 177 ++++++++- .github/workflows/cloudflare-events.yml | 23 ++ .gitignore | 5 +- .prettierignore | 3 +- AGENTS.md | 85 +---- README.md | 202 +++------- eslint.config.ts | 4 +- libraries/extension-packaging/package.json | 19 + .../extension-packaging/src/adapters/b2.ts | 21 +- .../extension-packaging/src/adapters/http.ts | 84 +++++ .../src/adapters}/sigv4.ts | 12 +- libraries/extension-packaging/src/checksum.ts | 6 + .../extension-packaging/src}/errors.ts | 0 .../extension-packaging/src}/images.ts | 0 libraries/extension-packaging/src/limits.ts | 8 + .../extension-packaging/src}/manifest.ts | 20 +- libraries/extension-packaging/src/process.ts | 67 ++++ .../extension-packaging/src/processor.ts | 54 +++ .../extension-packaging/src}/store.ts | 0 libraries/extension-packaging/src/urls.ts | 13 + .../extension-packaging/src/zip.ts | 29 +- .../extension-packaging/test/b2-store.test.ts | 8 +- .../extension-packaging/test/sigv4.test.ts | 2 +- .../extension-packaging/test/zip.test.ts | 8 +- libraries/extension-packaging/tsconfig.json | 4 + libraries/logging/package.json | 12 + {src/lib => libraries/logging/src}/logger.ts | 0 .../logging/test}/logger.test.ts | 2 +- libraries/logging/tsconfig.json | 4 + package-lock.json | 134 ++++--- package.json | 58 +-- services/api/AGENTS.md | 78 ++++ services/api/README.md | 239 ++++++++++++ .../api/drizzle.central-alerts.config.ts | 2 +- .../api/drizzle.extensions.config.ts | 2 +- services/api/package.json | 49 +++ services/api/scripts/prepare-deployment.ts | 115 ++++++ {src => services/api/src}/app/index.ts | 32 +- .../api/src}/lib/adapters/cloudflare/cache.ts | 0 .../lib/adapters/cloudflare/environment.ts | 0 .../api/src}/lib/adapters/cloudflare/index.ts | 9 + .../lib/adapters/cloudflare/package-assets.ts | 6 +- .../lib/adapters/cloudflare/packaging-jobs.ts | 54 ++- .../api/src}/lib/adapters/cloudflare/queue.ts | 0 .../api/src}/lib/adapters/node/cache.ts | 0 .../api/src}/lib/adapters/node/environment.ts | 0 .../api/src}/lib/adapters/node/index.ts | 0 .../api/src}/lib/auth/bearer-assertion.ts | 2 +- {src => services/api/src}/lib/auth/index.ts | 0 .../api/src}/lib/auth/interfaces.ts | 2 +- .../api/src}/lib/auth/m2m-bearer.ts | 2 +- .../api/src}/lib/auth/middleware.ts | 0 {src => services/api/src}/lib/cache.ts | 0 {src => services/api/src}/lib/context.ts | 25 +- {src => services/api/src}/lib/db.ts | 0 .../api/src}/lib/github-errors.ts | 0 {src => services/api/src}/lib/hash.ts | 0 {src => services/api/src}/lib/interfaces.ts | 2 + {src => services/api/src}/lib/json.ts | 0 {src => services/api/src}/lib/middleware.ts | 5 +- {src => services/api/src}/lib/pagination.ts | 0 {src => services/api/src}/lib/releases.ts | 0 .../src}/services/central-alerts/v1/README.md | 0 .../services/central-alerts/v1/database.ts | 0 .../migrations/0000_create_central_alerts.sql | 0 .../db/migrations/0001_seed_initial_alert.sql | 0 .../v1/db/migrations/meta/0000_snapshot.json | 0 .../v1/db/migrations/meta/0001_snapshot.json | 0 .../v1/db/migrations/meta/_journal.json | 0 .../services/central-alerts/v1/db/schema.ts | 0 .../src}/services/central-alerts/v1/index.ts | 5 +- .../services/central-alerts/v1/interfaces.ts | 0 .../api/src}/services/extensions/v1/README.md | 0 .../extensions/v1/cloudflare-rules.json | 0 .../src}/services/extensions/v1/database.ts | 0 .../src}/services/extensions/v1/db/schema.sql | 0 .../api/src}/services/extensions/v1/index.ts | 15 +- .../src}/services/extensions/v1/interfaces.ts | 0 .../src}/services/extensions/v1/list-cache.ts | 3 +- .../src}/services/extensions/v2/PACKAGING.md | 51 ++- .../api/src}/services/extensions/v2/README.md | 0 .../src}/services/extensions/v2/db/badges.ts | 0 .../src}/services/extensions/v2/db/batch.ts | 0 .../v2/db/claim-verification-budget.ts | 0 .../src}/services/extensions/v2/db/columns.ts | 0 .../src}/services/extensions/v2/db/cursor.ts | 0 .../extensions/v2/db/developer-approval.ts | 0 .../extensions/v2/db/developer-claims.ts | 0 .../extensions/v2/db/developer-profiles.ts | 0 .../extensions/v2/db/developer-transfers.ts | 0 .../src}/services/extensions/v2/db/errors.ts | 2 +- .../services/extensions/v2/db/extensions.ts | 0 .../extensions/v2/db/manifest-preflight.sql | 0 .../v2/db/migrations/0000_bootstrap_users.sql | 0 .../v2/db/migrations/0001_add_v2_tables.sql | 0 .../migrations/0002_add_author_approval.sql | 0 .../0003_add_author_profile_fields.sql | 0 .../0004_add_authors_owner_unique.sql | 0 .../db/migrations/0005_add_author_history.sql | 0 .../migrations/0006_add_author_transfers.sql | 0 .../db/migrations/0007_add_author_claims.sql | 0 .../0008_rename_authors_to_developers.sql | 0 .../0009_drop_developer_history_fk.sql | 0 .../db/migrations/0010_drop_developer_bio.sql | 0 .../0011_harden_review_workflows.sql | 0 ...d_developer_claims_github_verification.sql | 0 ...013_add_developers_github_verification.sql | 0 .../db/migrations/0014_drizzle_baseline.sql | 0 .../0015_add_developer_github_verified_at.sql | 0 ...0016_add_developer_github_url_verified.sql | 0 .../0017_add_developer_url_check_cooldown.sql | 0 .../0018_add_extension_catalogue_indexes.sql | 0 .../migrations/0019_add_user_deleted_at.sql | 0 .../0020_check_reserved_route_ids.sql | 0 .../0021_restructure_extensions_revisions.sql | 0 .../0022_add_extensions_delisted_at.sql | 0 .../v2/db/migrations/0023_fast_makkari.sql | 0 .../0024_claim_verification_budgets.sql | 0 .../0025_luxuriant_franklin_richards.sql | 0 .../v2/db/migrations/0026_resource_bounds.sql | 0 ...0027_bind_developer_profile_generation.sql | 0 .../migrations/0028_m2m_publisher_clients.sql | 0 .../v2/db/migrations/0029_align_manifest.sql | 0 .../db/migrations/0030_directory_identity.sql | 0 .../db/migrations/0031_directory_packages.sql | 0 .../v2/db/migrations/0032_package_jobs.sql | 0 .../v2/db/migrations/meta/0014_snapshot.json | 0 .../v2/db/migrations/meta/0015_snapshot.json | 0 .../v2/db/migrations/meta/0016_snapshot.json | 0 .../v2/db/migrations/meta/0017_snapshot.json | 0 .../v2/db/migrations/meta/0018_snapshot.json | 0 .../v2/db/migrations/meta/0019_snapshot.json | 0 .../v2/db/migrations/meta/0021_snapshot.json | 0 .../v2/db/migrations/meta/0022_snapshot.json | 0 .../v2/db/migrations/meta/0023_snapshot.json | 0 .../v2/db/migrations/meta/0024_snapshot.json | 0 .../v2/db/migrations/meta/0025_snapshot.json | 0 .../v2/db/migrations/meta/0026_snapshot.json | 0 .../v2/db/migrations/meta/0027_snapshot.json | 0 .../v2/db/migrations/meta/0028_snapshot.json | 0 .../v2/db/migrations/meta/0029_snapshot.json | 0 .../v2/db/migrations/meta/0030_snapshot.json | 0 .../v2/db/migrations/meta/0031_snapshot.json | 0 .../v2/db/migrations/meta/0032_snapshot.json | 0 .../v2/db/migrations/meta/_journal.json | 0 .../extensions/v2/db/package-builds.ts | 16 +- .../services/extensions/v2/db/packages.ts | 0 .../extensions/v2/db/packaging-jobs.ts | 16 +- .../extensions/v2/db/publisher-clients.ts | 0 .../extensions/v2/db/resource-inventory.sql | 0 .../extensions/v2/db/resource-maintenance.ts | 2 +- .../services/extensions/v2/db/revisions.ts | 0 .../src}/services/extensions/v2/db/schema.ts | 2 +- .../src}/services/extensions/v2/db/users.ts | 0 .../services/extensions/v2/email/config.ts | 0 .../services/extensions/v2/email/factory.ts | 0 .../services/extensions/v2/email/mxroute.ts | 0 .../services/extensions/v2/email/notify.ts | 2 +- .../extensions/v2/email/recipients.ts | 0 .../services/extensions/v2/email/resend.ts | 0 .../services/extensions/v2/email/templates.ts | 0 .../services/extensions/v2/email/types.ts | 0 .../services/extensions/v2/github/identity.ts | 0 .../extensions/v2/github/verification.ts | 2 +- .../api/src}/services/extensions/v2/index.ts | 0 .../src}/services/extensions/v2/middleware.ts | 14 +- .../services/extensions/v2/packages/build.ts | 104 +++--- .../services/extensions/v2/packages/jobs.ts | 24 +- .../extensions/v2/packages/manifest.ts | 0 .../extensions/v2/packages/prepare.ts | 37 +- .../extensions/v2/packages/repository.ts | 1 + .../services/extensions/v2/packages/store.ts | 18 +- .../services/extensions/v2/resource-limits.ts | 0 .../src}/services/extensions/v2/revalidate.ts | 31 +- .../services/extensions/v2/routes/account.ts | 2 +- .../src}/services/extensions/v2/routes/app.ts | 0 .../services/extensions/v2/routes/badges.ts | 0 .../v2/routes/developer-profiles.ts | 13 +- .../services/extensions/v2/routes/errors.ts | 0 .../src}/services/extensions/v2/routes/m2m.ts | 0 .../extensions/v2/routes/moderation.ts | 2 +- .../services/extensions/v2/routes/notify.ts | 5 +- .../extensions/v2/routes/owner-extensions.ts | 2 +- .../extensions/v2/routes/ownership.ts | 2 +- .../services/extensions/v2/routes/packages.ts | 46 +-- .../extensions/v2/routes/public-extensions.ts | 0 .../extensions/v2/routes/revisions.ts | 2 +- .../services/extensions/v2/schemas/badges.ts | 0 .../services/extensions/v2/schemas/common.ts | 12 - .../extensions/v2/schemas/developers.ts | 3 +- .../extensions/v2/schemas/extensions.ts | 10 +- .../services/extensions/v2/schemas/m2m.ts | 0 .../extensions/v2/schemas/manifest.ts | 26 ++ .../extensions/v2/schemas/ownership.ts | 0 .../extensions/v2/schemas/packages.ts | 6 +- .../extensions/v2/schemas/revisions.ts | 0 .../services/extensions/v2/schemas/users.ts | 0 .../api/src}/services/previews/v1/README.md | 0 .../api/src}/services/previews/v1/budget.ts | 0 .../api/src}/services/previews/v1/cache.ts | 5 +- .../services/previews/v1/github/artifacts.ts | 2 +- .../services/previews/v1/github/request.ts | 0 .../api/src}/services/previews/v1/index.ts | 0 .../api/src}/services/previews/v1/r2.ts | 0 .../api/src}/services/previews/v1/resolve.ts | 0 .../src}/services/previews/v1/routes/app.ts | 0 .../services/previews/v1/routes/commit.ts | 9 +- .../services/previews/v1/routes/errors.ts | 0 .../src}/services/previews/v1/routes/main.ts | 7 +- .../src}/services/previews/v1/routes/pr.ts | 26 +- .../services/previews/v1/routes/respond.ts | 0 .../services/previews/v1/schemas/previews.ts | 0 .../api/src}/services/stats/v1/README.md | 0 .../api/src}/services/stats/v1/dashboard.ts | 0 .../api/src}/services/stats/v1/index.ts | 4 +- .../api/src}/services/stats/v1/interfaces.ts | 0 .../api/src}/services/versions/v1/README.md | 0 .../api/src}/services/versions/v1/index.ts | 4 +- .../src}/services/versions/v1/interfaces.ts | 0 .../api/src}/services/versions/v1/r2.ts | 0 .../src}/services/versions/v1/responses.ts | 0 services/api/test/deployment.node.test.ts | 122 ++++++ {test => services/api/test}/env.d.ts | 0 .../api/test}/integration/app.test.ts | 0 .../api/test}/integration/scheduled.test.ts | 8 +- .../test}/integration/versions/index.test.ts | 2 +- .../lib/adapters/cloudflare/cache.test.ts | 0 .../adapters/cloudflare/environment.test.ts | 0 .../api/test}/lib/adapters/node/cache.test.ts | 0 .../lib/adapters/node/environment.test.ts | 0 .../api/test}/lib/adapters/node/index.test.ts | 0 .../api/test}/lib/auth/assertion-helper.ts | 0 .../test}/lib/auth/bearer-assertion.test.ts | 15 +- .../api/test}/lib/auth/m2m-bearer.test.ts | 4 +- .../api/test}/lib/auth/m2m-helper.ts | 0 {test => services/api/test}/lib/cache.test.ts | 0 .../api/test}/lib/context.test.ts | 17 +- .../api/test}/lib/github-errors.test.ts | 0 .../api/test}/lib/middleware.test.ts | 0 .../api/test}/lib/releases.test.ts | 0 {test => services/api/test}/mocks/README.md | 0 .../api/test}/mocks/github-releases.ts | 0 {test => services/api/test}/mocks/octokit.ts | 0 .../central-alerts/v1/database.test.ts | 0 .../services/central-alerts/v1/index.test.ts | 2 +- .../services/extensions/v1/index.test.ts | 2 +- .../services/extensions/v2/account.test.ts | 2 +- .../services/extensions/v2/badges.test.ts | 0 .../extensions/v2/claim-budget.test.ts | 0 .../services/extensions/v2/contract.test.ts | 0 .../services/extensions/v2/db-fixtures.ts | 0 .../services/extensions/v2/db-interceptor.ts | 0 .../extensions/v2/developer-approval.test.ts | 0 .../extensions/v2/developer-profiles.test.ts | 0 .../extensions/v2/directory-identity.test.ts | 0 .../services/extensions/v2/email.test.ts | 0 .../extensions/v2/extension-writes.test.ts | 0 .../test}/services/extensions/v2/harness.ts | 2 +- .../services/extensions/v2/index.test.ts | 0 .../services/extensions/v2/manifest.test.ts | 0 .../services/extensions/v2/migrations.test.ts | 0 .../extensions/v2/moderation-correct.test.ts | 0 .../extensions/v2/moderation-notify.test.ts | 0 .../v2/moderation-revalidate.test.ts | 26 +- .../v2/moderation-revocation.test.ts | 0 .../services/extensions/v2/moderation.test.ts | 0 .../services/extensions/v2/ownership.test.ts | 0 .../extensions/v2/package-build.node.test.ts | 18 +- .../services/extensions/v2/packages.test.ts | 335 +++++++---------- .../extensions/v2/packaging-jobs.node.test.ts | 34 +- .../extensions/v2/pending-edits.test.ts | 0 .../extensions/v2/public-extensions.test.ts | 0 .../services/extensions/v2/publisher.test.ts | 0 .../extensions/v2/resource-bounds.test.ts | 2 +- .../extensions/v2/revalidate.node.test.ts | 53 +++ .../test}/services/previews/v1/budget.test.ts | 2 +- .../test}/services/previews/v1/cache.test.ts | 0 .../test}/services/previews/v1/commit.test.ts | 2 +- .../test}/services/previews/v1/main.test.ts | 2 +- .../api/test}/services/previews/v1/pr.test.ts | 2 +- .../api/test}/services/stats/v1/index.test.ts | 2 +- .../test}/services/versions/v1/errors.test.ts | 2 +- .../test}/services/versions/v1/index.test.ts | 2 +- .../services/versions/v1/middleware.test.ts | 2 +- .../api/test}/utils/apply-migrations.ts | 0 .../api/test}/utils/isolate-edge-cache.ts | 0 .../api/test}/utils/mock-helpers.ts | 0 .../api/test}/utils/test-types.ts | 0 services/api/tsconfig.json | 11 + .../api/vitest.config.ts | 1 - .../api/vitest.node.config.ts | 2 +- .../api/worker-configuration.d.ts | 3 +- wrangler.jsonc => services/api/wrangler.jsonc | 32 +- services/extension-packager/README.md | 103 ++++++ services/extension-packager/netlify.toml | 10 + .../netlify/functions/extension-packager.ts | 4 + .../netlify/public/.gitkeep | 0 services/extension-packager/package.json | 22 ++ .../scripts/extension-packager-dev.ts | 33 ++ services/extension-packager/src/handler.ts | 69 ++++ services/extension-packager/src/images.ts | 49 +++ .../extension-packager/test/processor.test.ts | 348 ++++++++++++++++++ services/extension-packager/tsconfig.json | 9 + src/lib/adapters/cloudflare/package-images.ts | 62 ---- test/tsconfig.json | 8 - tsconfig.json | 9 +- 306 files changed, 2616 insertions(+), 1040 deletions(-) create mode 100644 .github/workflows/cloudflare-events.yml create mode 100644 libraries/extension-packaging/package.json rename src/lib/adapters/b2/package-store.ts => libraries/extension-packaging/src/adapters/b2.ts (93%) create mode 100644 libraries/extension-packaging/src/adapters/http.ts rename {src/lib/adapters/b2 => libraries/extension-packaging/src/adapters}/sigv4.ts (94%) create mode 100644 libraries/extension-packaging/src/checksum.ts rename {src/services/extensions/v2/packages => libraries/extension-packaging/src}/errors.ts (100%) rename {src/services/extensions/v2/packages => libraries/extension-packaging/src}/images.ts (100%) create mode 100644 libraries/extension-packaging/src/limits.ts rename {src/services/extensions/v2/schemas => libraries/extension-packaging/src}/manifest.ts (81%) create mode 100644 libraries/extension-packaging/src/process.ts create mode 100644 libraries/extension-packaging/src/processor.ts rename {src/services/extensions/v2/packages => libraries/extension-packaging/src}/store.ts (100%) create mode 100644 libraries/extension-packaging/src/urls.ts rename src/services/extensions/v2/package-builder.ts => libraries/extension-packaging/src/zip.ts (90%) rename test/services/extensions/v2/b2-store.node.test.ts => libraries/extension-packaging/test/b2-store.test.ts (97%) rename test/services/extensions/v2/sigv4.node.test.ts => libraries/extension-packaging/test/sigv4.test.ts (98%) rename test/services/extensions/v2/package-builder.test.ts => libraries/extension-packaging/test/zip.test.ts (98%) create mode 100644 libraries/extension-packaging/tsconfig.json create mode 100644 libraries/logging/package.json rename {src/lib => libraries/logging/src}/logger.ts (100%) rename {test/lib => libraries/logging/test}/logger.test.ts (98%) create mode 100644 libraries/logging/tsconfig.json create mode 100644 services/api/AGENTS.md create mode 100644 services/api/README.md rename drizzle.central-alerts.config.ts => services/api/drizzle.central-alerts.config.ts (83%) rename drizzle.extensions.config.ts => services/api/drizzle.extensions.config.ts (90%) create mode 100644 services/api/package.json create mode 100644 services/api/scripts/prepare-deployment.ts rename {src => services/api/src}/app/index.ts (76%) rename {src => services/api/src}/lib/adapters/cloudflare/cache.ts (100%) rename {src => services/api/src}/lib/adapters/cloudflare/environment.ts (100%) rename {src => services/api/src}/lib/adapters/cloudflare/index.ts (79%) rename {src => services/api/src}/lib/adapters/cloudflare/package-assets.ts (92%) rename src/lib/adapters/cloudflare/package-jobs.ts => services/api/src/lib/adapters/cloudflare/packaging-jobs.ts (53%) rename {src => services/api/src}/lib/adapters/cloudflare/queue.ts (100%) rename {src => services/api/src}/lib/adapters/node/cache.ts (100%) rename {src => services/api/src}/lib/adapters/node/environment.ts (100%) rename {src => services/api/src}/lib/adapters/node/index.ts (100%) rename {src => services/api/src}/lib/auth/bearer-assertion.ts (99%) rename {src => services/api/src}/lib/auth/index.ts (100%) rename {src => services/api/src}/lib/auth/interfaces.ts (95%) rename {src => services/api/src}/lib/auth/m2m-bearer.ts (98%) rename {src => services/api/src}/lib/auth/middleware.ts (100%) rename {src => services/api/src}/lib/cache.ts (100%) rename {src => services/api/src}/lib/context.ts (51%) rename {src => services/api/src}/lib/db.ts (100%) rename {src => services/api/src}/lib/github-errors.ts (100%) rename {src => services/api/src}/lib/hash.ts (100%) rename {src => services/api/src}/lib/interfaces.ts (84%) rename {src => services/api/src}/lib/json.ts (100%) rename {src => services/api/src}/lib/middleware.ts (79%) rename {src => services/api/src}/lib/pagination.ts (100%) rename {src => services/api/src}/lib/releases.ts (100%) rename {src => services/api/src}/services/central-alerts/v1/README.md (100%) rename {src => services/api/src}/services/central-alerts/v1/database.ts (100%) rename {src => services/api/src}/services/central-alerts/v1/db/migrations/0000_create_central_alerts.sql (100%) rename {src => services/api/src}/services/central-alerts/v1/db/migrations/0001_seed_initial_alert.sql (100%) rename {src => services/api/src}/services/central-alerts/v1/db/migrations/meta/0000_snapshot.json (100%) rename {src => services/api/src}/services/central-alerts/v1/db/migrations/meta/0001_snapshot.json (100%) rename {src => services/api/src}/services/central-alerts/v1/db/migrations/meta/_journal.json (100%) rename {src => services/api/src}/services/central-alerts/v1/db/schema.ts (100%) rename {src => services/api/src}/services/central-alerts/v1/index.ts (94%) rename {src => services/api/src}/services/central-alerts/v1/interfaces.ts (100%) rename {src => services/api/src}/services/extensions/v1/README.md (100%) rename {src => services/api/src}/services/extensions/v1/cloudflare-rules.json (100%) rename {src => services/api/src}/services/extensions/v1/database.ts (100%) rename {src => services/api/src}/services/extensions/v1/db/schema.sql (100%) rename {src => services/api/src}/services/extensions/v1/index.ts (92%) rename {src => services/api/src}/services/extensions/v1/interfaces.ts (100%) rename {src => services/api/src}/services/extensions/v1/list-cache.ts (96%) rename {src => services/api/src}/services/extensions/v2/PACKAGING.md (86%) rename {src => services/api/src}/services/extensions/v2/README.md (100%) rename {src => services/api/src}/services/extensions/v2/db/badges.ts (100%) rename {src => services/api/src}/services/extensions/v2/db/batch.ts (100%) rename {src => services/api/src}/services/extensions/v2/db/claim-verification-budget.ts (100%) rename {src => services/api/src}/services/extensions/v2/db/columns.ts (100%) rename {src => services/api/src}/services/extensions/v2/db/cursor.ts (100%) rename {src => services/api/src}/services/extensions/v2/db/developer-approval.ts (100%) rename {src => services/api/src}/services/extensions/v2/db/developer-claims.ts (100%) rename {src => services/api/src}/services/extensions/v2/db/developer-profiles.ts (100%) rename {src => services/api/src}/services/extensions/v2/db/developer-transfers.ts (100%) rename {src => services/api/src}/services/extensions/v2/db/errors.ts (99%) rename {src => services/api/src}/services/extensions/v2/db/extensions.ts (100%) rename {src => services/api/src}/services/extensions/v2/db/manifest-preflight.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0000_bootstrap_users.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0001_add_v2_tables.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0002_add_author_approval.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0003_add_author_profile_fields.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0004_add_authors_owner_unique.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0005_add_author_history.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0006_add_author_transfers.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0007_add_author_claims.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0008_rename_authors_to_developers.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0009_drop_developer_history_fk.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0010_drop_developer_bio.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0011_harden_review_workflows.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0012_add_developer_claims_github_verification.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0013_add_developers_github_verification.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0014_drizzle_baseline.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0015_add_developer_github_verified_at.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0016_add_developer_github_url_verified.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0017_add_developer_url_check_cooldown.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0018_add_extension_catalogue_indexes.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0019_add_user_deleted_at.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0020_check_reserved_route_ids.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0021_restructure_extensions_revisions.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0022_add_extensions_delisted_at.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0023_fast_makkari.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0024_claim_verification_budgets.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0025_luxuriant_franklin_richards.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0026_resource_bounds.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0027_bind_developer_profile_generation.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0028_m2m_publisher_clients.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0029_align_manifest.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0030_directory_identity.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0031_directory_packages.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/0032_package_jobs.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0014_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0015_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0016_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0017_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0018_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0019_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0021_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0022_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0023_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0024_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0025_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0026_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0027_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0028_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0029_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0030_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0031_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/0032_snapshot.json (100%) rename {src => services/api/src}/services/extensions/v2/db/migrations/meta/_journal.json (100%) rename {src => services/api/src}/services/extensions/v2/db/package-builds.ts (96%) rename {src => services/api/src}/services/extensions/v2/db/packages.ts (100%) rename src/services/extensions/v2/db/package-jobs.ts => services/api/src/services/extensions/v2/db/packaging-jobs.ts (91%) rename {src => services/api/src}/services/extensions/v2/db/publisher-clients.ts (100%) rename {src => services/api/src}/services/extensions/v2/db/resource-inventory.sql (100%) rename {src => services/api/src}/services/extensions/v2/db/resource-maintenance.ts (98%) rename {src => services/api/src}/services/extensions/v2/db/revisions.ts (100%) rename {src => services/api/src}/services/extensions/v2/db/schema.ts (99%) rename {src => services/api/src}/services/extensions/v2/db/users.ts (100%) rename {src => services/api/src}/services/extensions/v2/email/config.ts (100%) rename {src => services/api/src}/services/extensions/v2/email/factory.ts (100%) rename {src => services/api/src}/services/extensions/v2/email/mxroute.ts (100%) rename {src => services/api/src}/services/extensions/v2/email/notify.ts (99%) rename {src => services/api/src}/services/extensions/v2/email/recipients.ts (100%) rename {src => services/api/src}/services/extensions/v2/email/resend.ts (100%) rename {src => services/api/src}/services/extensions/v2/email/templates.ts (100%) rename {src => services/api/src}/services/extensions/v2/email/types.ts (100%) rename {src => services/api/src}/services/extensions/v2/github/identity.ts (100%) rename {src => services/api/src}/services/extensions/v2/github/verification.ts (99%) rename {src => services/api/src}/services/extensions/v2/index.ts (100%) rename {src => services/api/src}/services/extensions/v2/middleware.ts (97%) rename {src => services/api/src}/services/extensions/v2/packages/build.ts (67%) rename {src => services/api/src}/services/extensions/v2/packages/jobs.ts (80%) rename {src => services/api/src}/services/extensions/v2/packages/manifest.ts (100%) rename {src => services/api/src}/services/extensions/v2/packages/prepare.ts (59%) rename {src => services/api/src}/services/extensions/v2/packages/repository.ts (97%) rename src/lib/adapters/cloudflare/package-store.ts => services/api/src/services/extensions/v2/packages/store.ts (74%) rename {src => services/api/src}/services/extensions/v2/resource-limits.ts (100%) rename {src => services/api/src}/services/extensions/v2/revalidate.ts (73%) rename {src => services/api/src}/services/extensions/v2/routes/account.ts (99%) rename {src => services/api/src}/services/extensions/v2/routes/app.ts (100%) rename {src => services/api/src}/services/extensions/v2/routes/badges.ts (100%) rename {src => services/api/src}/services/extensions/v2/routes/developer-profiles.ts (97%) rename {src => services/api/src}/services/extensions/v2/routes/errors.ts (100%) rename {src => services/api/src}/services/extensions/v2/routes/m2m.ts (100%) rename {src => services/api/src}/services/extensions/v2/routes/moderation.ts (99%) rename {src => services/api/src}/services/extensions/v2/routes/notify.ts (91%) rename {src => services/api/src}/services/extensions/v2/routes/owner-extensions.ts (99%) rename {src => services/api/src}/services/extensions/v2/routes/ownership.ts (99%) rename {src => services/api/src}/services/extensions/v2/routes/packages.ts (91%) rename {src => services/api/src}/services/extensions/v2/routes/public-extensions.ts (100%) rename {src => services/api/src}/services/extensions/v2/routes/revisions.ts (97%) rename {src => services/api/src}/services/extensions/v2/schemas/badges.ts (100%) rename {src => services/api/src}/services/extensions/v2/schemas/common.ts (94%) rename {src => services/api/src}/services/extensions/v2/schemas/developers.ts (98%) rename {src => services/api/src}/services/extensions/v2/schemas/extensions.ts (97%) rename {src => services/api/src}/services/extensions/v2/schemas/m2m.ts (100%) create mode 100644 services/api/src/services/extensions/v2/schemas/manifest.ts rename {src => services/api/src}/services/extensions/v2/schemas/ownership.ts (100%) rename {src => services/api/src}/services/extensions/v2/schemas/packages.ts (90%) rename {src => services/api/src}/services/extensions/v2/schemas/revisions.ts (100%) rename {src => services/api/src}/services/extensions/v2/schemas/users.ts (100%) rename {src => services/api/src}/services/previews/v1/README.md (100%) rename {src => services/api/src}/services/previews/v1/budget.ts (100%) rename {src => services/api/src}/services/previews/v1/cache.ts (98%) rename {src => services/api/src}/services/previews/v1/github/artifacts.ts (99%) rename {src => services/api/src}/services/previews/v1/github/request.ts (100%) rename {src => services/api/src}/services/previews/v1/index.ts (100%) rename {src => services/api/src}/services/previews/v1/r2.ts (100%) rename {src => services/api/src}/services/previews/v1/resolve.ts (100%) rename {src => services/api/src}/services/previews/v1/routes/app.ts (100%) rename {src => services/api/src}/services/previews/v1/routes/commit.ts (96%) rename {src => services/api/src}/services/previews/v1/routes/errors.ts (100%) rename {src => services/api/src}/services/previews/v1/routes/main.ts (97%) rename {src => services/api/src}/services/previews/v1/routes/pr.ts (88%) rename {src => services/api/src}/services/previews/v1/routes/respond.ts (100%) rename {src => services/api/src}/services/previews/v1/schemas/previews.ts (100%) rename {src => services/api/src}/services/stats/v1/README.md (100%) rename {src => services/api/src}/services/stats/v1/dashboard.ts (100%) rename {src => services/api/src}/services/stats/v1/index.ts (98%) rename {src => services/api/src}/services/stats/v1/interfaces.ts (100%) rename {src => services/api/src}/services/versions/v1/README.md (100%) rename {src => services/api/src}/services/versions/v1/index.ts (99%) rename {src => services/api/src}/services/versions/v1/interfaces.ts (100%) rename {src => services/api/src}/services/versions/v1/r2.ts (100%) rename {src => services/api/src}/services/versions/v1/responses.ts (100%) create mode 100644 services/api/test/deployment.node.test.ts rename {test => services/api/test}/env.d.ts (100%) rename {test => services/api/test}/integration/app.test.ts (100%) rename {test => services/api/test}/integration/scheduled.test.ts (86%) rename {test => services/api/test}/integration/versions/index.test.ts (99%) rename {test => services/api/test}/lib/adapters/cloudflare/cache.test.ts (100%) rename {test => services/api/test}/lib/adapters/cloudflare/environment.test.ts (100%) rename {test => services/api/test}/lib/adapters/node/cache.test.ts (100%) rename {test => services/api/test}/lib/adapters/node/environment.test.ts (100%) rename {test => services/api/test}/lib/adapters/node/index.test.ts (100%) rename {test => services/api/test}/lib/auth/assertion-helper.ts (100%) rename {test => services/api/test}/lib/auth/bearer-assertion.test.ts (96%) rename {test => services/api/test}/lib/auth/m2m-bearer.test.ts (97%) rename {test => services/api/test}/lib/auth/m2m-helper.ts (100%) rename {test => services/api/test}/lib/cache.test.ts (100%) rename {test => services/api/test}/lib/context.test.ts (73%) rename {test => services/api/test}/lib/github-errors.test.ts (100%) rename {test => services/api/test}/lib/middleware.test.ts (100%) rename {test => services/api/test}/lib/releases.test.ts (100%) rename {test => services/api/test}/mocks/README.md (100%) rename {test => services/api/test}/mocks/github-releases.ts (100%) rename {test => services/api/test}/mocks/octokit.ts (100%) rename {test => services/api/test}/services/central-alerts/v1/database.test.ts (100%) rename {test => services/api/test}/services/central-alerts/v1/index.test.ts (99%) rename {test => services/api/test}/services/extensions/v1/index.test.ts (99%) rename {test => services/api/test}/services/extensions/v2/account.test.ts (99%) rename {test => services/api/test}/services/extensions/v2/badges.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/claim-budget.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/contract.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/db-fixtures.ts (100%) rename {test => services/api/test}/services/extensions/v2/db-interceptor.ts (100%) rename {test => services/api/test}/services/extensions/v2/developer-approval.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/developer-profiles.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/directory-identity.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/email.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/extension-writes.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/harness.ts (99%) rename {test => services/api/test}/services/extensions/v2/index.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/manifest.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/migrations.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/moderation-correct.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/moderation-notify.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/moderation-revalidate.test.ts (95%) rename {test => services/api/test}/services/extensions/v2/moderation-revocation.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/moderation.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/ownership.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/package-build.node.test.ts (90%) rename {test => services/api/test}/services/extensions/v2/packages.test.ts (86%) rename test/services/extensions/v2/package-jobs.node.test.ts => services/api/test/services/extensions/v2/packaging-jobs.node.test.ts (80%) rename {test => services/api/test}/services/extensions/v2/pending-edits.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/public-extensions.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/publisher.test.ts (100%) rename {test => services/api/test}/services/extensions/v2/resource-bounds.test.ts (99%) create mode 100644 services/api/test/services/extensions/v2/revalidate.node.test.ts rename {test => services/api/test}/services/previews/v1/budget.test.ts (99%) rename {test => services/api/test}/services/previews/v1/cache.test.ts (100%) rename {test => services/api/test}/services/previews/v1/commit.test.ts (99%) rename {test => services/api/test}/services/previews/v1/main.test.ts (99%) rename {test => services/api/test}/services/previews/v1/pr.test.ts (99%) rename {test => services/api/test}/services/stats/v1/index.test.ts (99%) rename {test => services/api/test}/services/versions/v1/errors.test.ts (99%) rename {test => services/api/test}/services/versions/v1/index.test.ts (99%) rename {test => services/api/test}/services/versions/v1/middleware.test.ts (98%) rename {test => services/api/test}/utils/apply-migrations.ts (100%) rename {test => services/api/test}/utils/isolate-edge-cache.ts (100%) rename {test => services/api/test}/utils/mock-helpers.ts (100%) rename {test => services/api/test}/utils/test-types.ts (100%) create mode 100644 services/api/tsconfig.json rename vitest.config.ts => services/api/vitest.config.ts (98%) rename vitest.node.config.ts => services/api/vitest.node.config.ts (89%) rename worker-configuration.d.ts => services/api/worker-configuration.d.ts (99%) rename wrangler.jsonc => services/api/wrangler.jsonc (79%) create mode 100644 services/extension-packager/README.md create mode 100644 services/extension-packager/netlify.toml create mode 100644 services/extension-packager/netlify/functions/extension-packager.ts create mode 100644 services/extension-packager/netlify/public/.gitkeep create mode 100644 services/extension-packager/package.json create mode 100644 services/extension-packager/scripts/extension-packager-dev.ts create mode 100644 services/extension-packager/src/handler.ts create mode 100644 services/extension-packager/src/images.ts create mode 100644 services/extension-packager/test/processor.test.ts create mode 100644 services/extension-packager/tsconfig.json delete mode 100644 src/lib/adapters/cloudflare/package-images.ts delete mode 100644 test/tsconfig.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62ec2ad5..6a517e10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: CI +name: CI/CD on: push: @@ -8,30 +8,185 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + jobs: - test: + validation: + name: Validation runs-on: ubuntu-latest - if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.base.repo.id != github.event.pull_request.head.repo.id) || github.event_name == 'push' }} + timeout-minutes: 15 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Checkout Code uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 - - name: Setup Node.js uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f with: node-version: "26.13.0" - cache: "npm" - + cache: npm - name: Install Dependencies + id: dependencies run: npm ci - - - name: Lint Code (ESLint) + - name: Check Formatting + if: ${{ !cancelled() && steps.dependencies.outcome == 'success' }} + run: npm run format:check + - name: Lint Code + if: ${{ !cancelled() && steps.dependencies.outcome == 'success' }} run: npm run lint - - name: Typecheck + if: ${{ !cancelled() && steps.dependencies.outcome == 'success' }} run: npm run typecheck - - name: Run Tests - run: npm run test:all + if: ${{ !cancelled() && steps.dependencies.outcome == 'success' }} + run: npm test + + packager-node24: + name: Packager on Node.js 24 + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout Code + uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f + with: + node-version: "24" + cache: npm + - name: Install Dependencies + run: npm ci + - name: Test Packager + run: npm test --workspace @fossbilling/extension-packager + + build: + name: Build API + runs-on: ubuntu-latest + timeout-minutes: 15 + outputs: + wrangler-version: ${{ steps.wrangler-version.outputs.version }} + steps: + - name: Checkout Code + uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f + with: + node-version: "26.13.0" + cache: npm + - name: Install Dependencies + run: npm ci + - name: Read Wrangler Version + id: wrangler-version + run: | + node <<'NODE' + const fs = require('node:fs'); + const { version } = require('./node_modules/wrangler/package.json'); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `version=${version}\n`); + NODE + - name: Build Deployment Artifact + run: npm run build:api + env: + API_PREVIEW_RESOURCES: ${{ vars.API_PREVIEW_RESOURCES }} + - name: Validate Deployment Artifact + run: npm exec -- wrangler deploy --dry-run --config services/api/dist/wrangler.json + - name: Upload Deployment Artifact + if: github.event_name == 'push' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: api-worker-build + overwrite: true + path: services/api/dist/ + if-no-files-found: error + retention-days: 1 + + migrations: + name: Apply API Migrations + needs: [validation, packager-node24, build] + if: github.event_name == 'push' && github.ref_type == 'branch' + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: + name: ${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }} + deployment: false + concurrency: + group: api-migrations-${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }} + cancel-in-progress: false + steps: + - name: Checkout Code + uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + with: + persist-credentials: false + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f + with: + node-version: "26.13.0" + cache: npm + - name: Install Dependencies + run: npm ci + - name: Download Deployment Artifact + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: api-worker-build + path: services/api/dist/ + - name: Apply Migrations + env: + DEPLOY_TARGET: ${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} + run: | + config=services/api/dist/wrangler.json + if [ "$DEPLOY_TARGET" = preview ]; then + config=services/api/dist/wrangler.preview-migrations.json + if [ ! -f "$config" ]; then + echo '::error::Configure the API_PREVIEW_RESOURCES repository variable before deploying Previews.' + exit 1 + fi + fi + for binding in DB_CENTRAL_ALERTS DB_EXTENSIONS; do + npm exec -- wrangler d1 migrations apply "$binding" --remote --config "$config" + done + + preview: + name: Cloudflare Preview + needs: [validation, packager-node24, build, migrations] + if: ${{ !cancelled() && github.event_name == 'push' && github.ref_type == 'branch' && github.ref != 'refs/heads/main' }} + permissions: + contents: read + actions: read + pull-requests: write + uses: FOSSBilling/.workflows/.github/workflows/cloudflare-workers.yml@384ecc11e315c77cf3d6abb947e859ce62b38cf5 # main + with: + operation: deploy + mode: preview + build-result: ${{ needs.build.result }} + checks-result: ${{ needs.validation.result == 'success' && needs.packager-node24.result == 'success' && needs.migrations.result == 'success' && 'success' || 'failure' }} + pr-comment: true + pr-base-branch: main + worker-name: api + environment: preview + artifact-name: api-worker-build + artifact-path: services/api/dist/ + config-path: services/api/dist/wrangler.json + wrangler-version: ${{ needs.build.outputs.wrangler-version }} + secrets: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} + + deploy: + name: Deploy Production + needs: [validation, packager-node24, build, migrations] + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: FOSSBilling/.workflows/.github/workflows/cloudflare-workers.yml@384ecc11e315c77cf3d6abb947e859ce62b38cf5 # main + with: + operation: deploy + mode: production + worker-name: api + environment: production + artifact-name: api-worker-build + artifact-path: services/api/dist/ + config-path: services/api/dist/wrangler.json + wrangler-version: ${{ needs.build.outputs.wrangler-version }} + secrets: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} diff --git a/.github/workflows/cloudflare-events.yml b/.github/workflows/cloudflare-events.yml new file mode 100644 index 00000000..340f2c9f --- /dev/null +++ b/.github/workflows/cloudflare-events.yml @@ -0,0 +1,23 @@ +name: Cloudflare Events + +on: + workflow_run: + workflows: ["CI/CD"] + types: [completed] + pull_request_target: + branches: [main] + types: [opened, reopened, synchronize] + +permissions: + contents: read + actions: read + pull-requests: write + +jobs: + comment: + uses: FOSSBilling/.workflows/.github/workflows/cloudflare-workers.yml@384ecc11e315c77cf3d6abb947e859ce62b38cf5 # main + with: + operation: sync-pr-comment + build-workflow: ci.yml + pr-base-branch: main + worker-name: api diff --git a/.gitignore b/.gitignore index 481f31b5..06026e2b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,12 @@ # prod -dist/ +**/dist/ # dev .vscode/* # deps node_modules/ +.netlify/ .wrangler # env @@ -20,7 +21,7 @@ logs/ npm-debug.log* # test coverage -coverage/ +**/coverage/ # misc .DS_Store diff --git a/.prettierignore b/.prettierignore index a50dd94a..1f011b51 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,2 +1 @@ -# Ignore workerd generated files -worker-configuration.d.ts \ No newline at end of file +**/worker-configuration.d.ts diff --git a/AGENTS.md b/AGENTS.md index 012befdf..cd1a27d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,74 +1,15 @@ # Repository Guidelines -## Project Structure & Module Organization - -- `src/app/index.ts` is the worker entrypoint and route wiring for Hono. -- Feature logic lives in `src/services/` and should stay runtime-agnostic. -- Platform interfaces and adapters are in `src/lib/` with Cloudflare and Node implementations under `src/lib/adapters/`. -- Tests mirror the source layout under `test/`, with shared helpers in `test/utils/` and mocks in `test/mocks/`. -- Runtime/config files include `wrangler.jsonc`, `worker-configuration.d.ts`, `tsconfig.json`, `eslint.config.ts`, and `prettier.config.ts`. - -### Service layout - -Small services are flat: `index.ts` + `interfaces.ts` plus narrow helpers (for example `database.ts`, `db/`, `r2.ts`). - -`extensions/v2` is the reference layout for anything larger, and new services should grow into it rather than inventing a third shape: - -- `index.ts` — app assembly only: middleware, route registration, OpenAPI document. Route registration order is load-bearing where static paths must beat parameter paths; those cases carry comments. -- `middleware.ts` — service-specific Hono middleware. -- `routes/` — one module per route group, each exporting `register*Routes(app)`. `routes/errors.ts` maps domain error codes to HTTP status; `routes/app.ts` holds the typed app alias. -- `db/` — `schema.ts`, `migrations/`, one `*Database` class per workflow, plus `errors.ts` (D1 constraint classification) and `batch.ts`. -- `schemas/` — zod/OpenAPI contract split by domain. There is deliberately **no barrel**: import from `schemas/` directly so a module's dependencies are visible. This is why `extensions/v2` has no `interfaces.ts`. -- `github/` — outbound GitHub calls, kept out of the persistence modules. -- `email/` (extensions v2) — provider abstraction and templates. Service-root helpers (`revalidate.ts`, `resource-limits.ts`) stay at the service root. `previews/v1` follows the same split (`routes/`, `github/`, `schemas/`, plus `budget.ts` / `cache.ts`). - -Route modules import `getExtensionsDb`/`getAuth`/`getPlatform` and middleware directly. There is no dependency-injection container; tests drive the real app through `app.request`. - -Each service documents its own contract and operational detail in its own `README.md` (`src/services///README.md`). Keep API behaviour there rather than here or in the root README: this file is for conventions that apply when modifying the code. - -## Build, Test, and Development Commands - -- `npm install`: install dependencies. -- `npm run dev`: start the local Cloudflare Workers dev server. -- `npm run cf-deploy`: apply remote D1 migrations, then deploy via Wrangler. -- `npm run cf-typegen`: regenerate `worker-configuration.d.ts`. -- `npm run db:migrate:all:local` / `db:migrate:all:remote`: apply all D1 migrations locally / remotely. -- `npm run test`: run the default Vitest suite (Workers pool). -- `npm run test:all`: run Workers pool plus Node adapter tests. -- `npm run test:node`: run Node adapter tests via `vitest.node.config.ts`. -- `npm run test:coverage`: run tests with coverage. -- `npm run typecheck`: TypeScript typecheck without emit. -- `npm run lint` / `npm run lint:fix`: lint the codebase (with optional fixes). -- `npm run format` / `npm run format:check`: format or verify formatting with Prettier. - -## Coding Style & Naming Conventions - -- TypeScript (ES modules), 2-space indentation, and Prettier formatting (`trailingComma: "none"`). -- ESLint uses `@typescript-eslint` with Prettier compatibility; keep lint clean before opening PRs. -- Use kebab-case for service folders (for example `central-alerts`), and name tests `*.test.ts`. - -## Testing Guidelines - -- Vitest is the primary test runner; integration tests live in `test/integration/`. -- Prefer colocating tests in the matching `test/` path (for example `src/services/versions/v1` -> `test/services/versions/v1`). -- Run `npm run test:all` before release-related changes to cover both worker and node adapters. - -## Commit & Pull Request Guidelines - -- Recent commits use short, imperative, sentence-case subjects (for example "Update API docs", "Refactor versions service"). -- No ticket prefixes are evident; keep messages focused and under ~72 chars. -- PRs should include a concise summary, testing notes (commands run), and call out config/binding changes in `wrangler.jsonc`. - -## Configuration & Secrets - -- Local secrets go in `.dev.vars` (for example `GITHUB_TOKEN="..."`). -- Bindings, cron, and vars are defined in `wrangler.jsonc` (D1, KV, R2, Durable Objects, rate limiters, service bindings); keep names aligned with `CloudflareBindings`. -- Use Wrangler secrets for production tokens instead of committing them. -- `ASSERTION_SIGNING_SECRET`: HMAC key the extensions v2 API uses to verify short-lived bearer assertions minted by the extensions site (`src/lib/auth/bearer-assertion.ts`). Not sent over the wire — only signs/verifies server-side in each Worker. Add `ASSERTION_SIGNING_SECRET="..."` to `.dev.vars` for local dev; set via `wrangler secret put ASSERTION_SIGNING_SECRET` in production, matching the value configured in the extensions site's Worker. -- `EXTENSIONS_REVALIDATE_SECRET`: bearer token the extensions v2 API sends to the extensions site's `POST /api/revalidate` (over the `EXTENSIONS_FRONTEND` service binding) to purge the site's CDN-cached extension pages after content mutations. Must match the value configured in the extensions site's Worker. Same local/prod setup as `ASSERTION_SIGNING_SECRET`. - -## Cache Revalidation - -Any endpoint that mutates publicly visible content (revision approve/reject, moderator-correct, delist/relist, developer approve, developer profile upsert via `PUT /developers/me`, profile deletion via `DELETE /developers/me`, claim approve/reject, extension withdraw) must call `revalidateExtensions(c)` from `src/services/extensions/v2/revalidate.ts` after a successful write. - -Skipping it does not break correctness — the site's `maxAge`/`stale-while-revalidate` windows bound staleness — but changes then take minutes instead of seconds to appear. Profile deletion keeps its purge even though the `developers` tag is currently a no-op. +This is an npm workspace repository. Deployable components live under `services/`; +reusable libraries live under `libraries/`. Service-specific instructions live in +`services/api/AGENTS.md`. + +- Install dependencies with `npm ci` from the repository root. Keep one root lockfile. +- Services must not import each other's source. Put code shared by actual callers in a private workspace package. +- Declare runtime dependencies in the workspace that uses them. Shared development tools are at the root. +- Preserve the API's existing feature structure inside `services/api/src/services/`. +- Keep provider entry points thin; domain and processing logic should remain portable. +- Local secrets and runtime state belong beside their service (`.dev.vars`, `.wrangler/`). Never commit credentials. +- Run `npm run format:check`, `npm run lint`, `npm run typecheck`, `npm test`, and `npm run build:api` before submitting changes. +- `npm run build:api` is a local dry run. `npm run cf-deploy` applies remote migrations and deploys production. +- Keep commits focused and use short imperative subjects. PRs should explain behaviour, validation and any deployment setup required. diff --git a/README.md b/README.md index 0f4186fb..700a391f 100644 --- a/README.md +++ b/README.md @@ -1,165 +1,77 @@ -# FOSSBilling API Worker +# FOSSBilling services -This is the API service that acts as the central hub for FOSSBilling instances. It serves version checks, system-wide alerts, the extensions directory, preview builds, and release stats. +This repository contains separately deployed backend services and the libraries they share. +It uses npm workspaces and one root `package-lock.json`. Install dependencies from this +repository root with `npm ci`; each workspace declares its own dependencies. -Everything is built on [Hono](https://hono.dev), making it lightweight and fast. While we currently deploy this to Cloudflare Workers, the code is designed to be platform-agnostic. +| Workspace | Purpose | Deployment | +| ---------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | ------------------------------------- | +| [`services/api`](services/api/README.md) | HTTP API, authentication, persistence, queues and publication | Cloudflare Workers | +| [`services/extension-packager`](services/extension-packager/README.md) | ZIP validation, manifest replacement and icon preparation | Netlify Function or another Node host | +| [`libraries/extension-packaging`](libraries/extension-packaging) | Shared manifest and processing contracts, ZIP implementation and storage/HTTP adapters | Bundled by its consumers | +| [`libraries/logging`](libraries/logging) | Structured logging and credential redaction used by both services | Bundled by its consumers | -## What it does - -The worker exposes five services: - -- **Versions** (`/versions/v1`) - The source of truth for FOSSBilling updates. It fetches release data from GitHub, caches it for performance, and helps instances decide if they need to update. - -- **Central Alerts** (`/central-alerts/v1`) - Allows the project to push critical notifications to all FOSSBilling installations—useful for security hotfixes or major announcements. - -- **Extensions** (`/extensions/v1`, `/extensions/v2`) - Owns the complete Extensions domain and its `DB_EXTENSIONS` schema. `v2` is authoritative. `v1` is deprecated; its badge URLs retire on **5 February 2027** ([migration and retirement policy](src/services/extensions/v1/README.md)). The separate Extensions site keeps OIDC/session state but accesses this domain through the - generated HTTPS API client; it must not bind or migrate `DB_EXTENSIONS`. - See [`src/services/extensions/v2/README.md`](src/services/extensions/v2/README.md). - -- **Previews** (`/previews/v1`) - Resolves FOSSBilling preview builds — the current main preview and per-PR/per-commit builds produced by FOSSBilling/FOSSBilling's GitHub Actions workflows. Read-only; GitHub Actions and R2 are the sources of truth, not this service. - See [`src/services/previews/v1/README.md`](src/services/previews/v1/README.md). - -- **Stats** (`/stats/v1`) - Aggregates release data through the versions service's shared releases cache into charts and JSON; a cold cache can trigger a GitHub fetch through that shared path. - See [`src/services/stats/v1/README.md`](src/services/stats/v1/README.md). - -## Architecture - -We've structured the app to separate the core logic from the specific runtime environment (Cloudflare, Node, etc.). - -- **Application Logic**: Found in `src/services/versions/v1`, `src/services/central-alerts/v1`, etc. These feature modules don't know they are running on Cloudflare. - Smaller services are a flat `index.ts` + `interfaces.ts`; `src/services/extensions/v2` is the reference layout for larger ones, splitting into `routes/`, `db/`, `schemas/`, and `github/`. - See `AGENTS.md` for what belongs in each. -- **Platform Layer**: Located in `src/lib`. This defines interfaces for things like Cache, Database, and Environment variables. -- **Adapters**: - - `src/lib/adapters/cloudflare`: Real implementations using KV and D1. - - `src/lib/adapters/node`: Reference implementations (useful for testing or alternative deployments). - -Cron and Durable Object code lives with its owning service and is wired in `src/app/index.ts`. - -## APIs - -Each service documents its own endpoints and behaviour: - -| Service | Base path | Docs | -| -------------- | ---------------------------------- | -------------------------------------------------------------------------------------- | -| Versions | `/versions/v1` | [`src/services/versions/v1/README.md`](src/services/versions/v1/README.md) | -| Central Alerts | `/central-alerts/v1` | [`src/services/central-alerts/v1/README.md`](src/services/central-alerts/v1/README.md) | -| Stats | `/stats/v1` | [`src/services/stats/v1/README.md`](src/services/stats/v1/README.md) | -| Extensions | `/extensions/v1`, `/extensions/v2` | [`src/services/extensions/v2/README.md`](src/services/extensions/v2/README.md) | -| Previews | `/previews/v1` | [`src/services/previews/v1/README.md`](src/services/previews/v1/README.md) | - -Extensions v2 and Previews v1 also publish a live OpenAPI document (`/extensions/v2/docs/openapi.json`, `/previews/v1/docs/openapi.json`) and a reference UI (`/extensions/v2/docs`, `/previews/v1/docs`). - -## Configuration - -If you're running this yourself, you'll need a few things set up. - -### Storage - -We use [Cloudflare D1](https://developers.cloudflare.com/d1/) and [KV](https://developers.cloudflare.com/kv/). - -- **D1 Database** (`DB_CENTRAL_ALERTS`): Stores the alert messages. -- **D1 Database** (`DB_EXTENSIONS`): Stores the complete Extensions domain. - Migrations are owned by extensions v2 and applied only from this repository — see [its README](src/services/extensions/v2/README.md#database) for the migration and adoption procedure. -- **KV Namespace** (`CACHE_KV`): Caches GitHub API responses so we don't hit rate limits. -- **KV Namespace** (`AUTH_KV`): Stores the `UPDATE_TOKEN` value for `/versions/v1/update`. -- **R2 Bucket** (`DOWNLOAD_BUCKET`): Shared by versions, previews, and stats for release/preview zips — see [`src/services/previews/v1/README.md`](src/services/previews/v1/README.md) and the comment in `wrangler.jsonc` for which bucket this points at and why. -- **Durable Object** (`PREVIEW_GITHUB_BUDGET`): Token bucket limiting previews GitHub calls. Policy lives in `src/services/previews/v1/budget.ts`. -- **Rate limiters** (`EXTENSION_WRITE_RATE_LIMITER`, `PROFILE_CREATION_RATE_LIMITER`): Edge pacing for extension writes and profile creation. Durable quotas are enforced in D1; see [extensions v2 README](src/services/extensions/v2/README.md#limits-and-budgets). -- **Service binding** (`EXTENSIONS_FRONTEND`): Private binding to the Extensions site worker, used to purge cached extension pages after mutations. -- **Cron** (`0 * * * *`): Hourly extension resource maintenance. See [extensions v2 README](src/services/extensions/v2/README.md#revisions-and-retention). - -### Environment Variables - -Production B2 connection details and MXroute server/username are managed in the -Worker dashboard under **Settings > Variables and Secrets** as runtime variables. -Credentials are stored as secrets. Build environment variables do not configure -the running Worker. `keep_vars: true` preserves dashboard-managed variables on -Wrangler deployments; keep their values out of the committed `vars` map and docs. -Public URLs and deployment controls remain in `wrangler.jsonc`. - -For local development, put operational values and credentials in the ignored -`.dev.vars` file. Document variable names and placeholders, not production values. -Package storage configuration is listed in [PACKAGING.md](src/services/extensions/v2/PACKAGING.md). - -- `GITHUB_TOKEN`: A GitHub Personal Access Token (classic) with public repo read access. -- `ASSERTION_SIGNING_SECRET`: Shared HMAC secret used to verify the short-lived bearer assertions minted by the Extensions site. Configure the same value in both Workers; it is never sent to clients. -- `ASSERTION_SIGNING_SECRET_PREVIOUS`: Optional previous HMAC secret accepted during a signing-key rotation. -- `EXTENSIONS_REVALIDATE_SECRET`: Bearer token the API sends to the Extensions site's `POST /api/revalidate` (over the `EXTENSIONS_FRONTEND` service binding) to purge the site's CDN-cached extension pages after content mutations. Configure the same value in both Workers; it is never sent to clients. -- `EXTENSIONS_V2_EMAIL_PROVIDER`: `mxroute` or `resend` to send moderator notification emails, or unset/`disabled` to skip sending. -- `EXTENSIONS_V2_EMAIL_FROM`: Sender address for notification emails (default `noreply@fossbilling.org`). -- `EXTENSIONS_V2_EMAIL_REPLY_TO`: Optional Reply-To address for a monitored inbox. Omit when unset (no Reply-To header is sent) — notification emails do not invite replies. -- `EXTENSIONS_V2_MXROUTE_SERVER`, `EXTENSIONS_V2_MXROUTE_USERNAME`: Dashboard-managed runtime variables identifying the mailbox for the MXroute SMTP API (`https://smtpapi.mxroute.com/`). Required when `EXTENSIONS_V2_EMAIL_PROVIDER=mxroute`. -- `EXTENSIONS_V2_MXROUTE_PASSWORD`: Mailbox password, stored as a secret. Required when `EXTENSIONS_V2_EMAIL_PROVIDER=mxroute`. -- `EXTENSIONS_V2_RESEND_API_KEY`: API key for Resend. Required when `EXTENSIONS_V2_EMAIL_PROVIDER=resend`. -- `EXTENSIONS_RETENTION_MODE`: `dry-run` (report only) or `compact` (prune old revision bodies). See [extensions v2 README](src/services/extensions/v2/README.md#revisions-and-retention). -- `M2M_ISSUER`, `M2M_JWKS_URL`, `M2M_AUDIENCE`: Issuer, JWKS endpoint, and expected audience for machine-to-machine access tokens minted by the central auth service (`FOSSBilling/auth`). Defaults live in `wrangler.jsonc`; preview environments need them pointed at the issuing preview. See [extensions v2 README](src/services/extensions/v2/README.md#machine-publishing). - -Only extensions v2 consumes these. For the assertion format and the rotation procedure, see [its README](src/services/extensions/v2/README.md#authentication). +Services do not import each other's source. Shared libraries are private npm +workspaces consumed directly from TypeScript source; there is no separate package +publication or library build step. Hosting configuration belongs to its service. ## Development -Get the dependencies installed: +Use Node.js 24 or later. The packager runs on Node.js 24; CI also tests the whole +repository on Node.js 26. Native dependencies are installed for the build host. -```bash -npm install +```sh +npm ci +npm run dev # API, using services/api/.dev.vars +npm run extension-packager:dev # Node packager, using services/extension-packager/.dev.vars ``` -### Local Setup - -1. Create a `.dev.vars` file for local runtime variables and secrets: - - ```env - GITHUB_TOKEN="your-token" - ASSERTION_SIGNING_SECRET="local-shared-secret" - EXTENSIONS_REVALIDATE_SECRET="local-revalidate-secret" - # Optional while rotating the shared assertion secret. - # ASSERTION_SIGNING_SECRET_PREVIOUS="previous-local-shared-secret" - ``` - -2. Apply migrations to the local D1 databases: - - ```bash - npm run db:migrate:all:local - ``` - -3. (Optional) Store an update token in KV for `/versions/v1/update`: +API migrations and other service-specific commands use npm's workspace option: - ```bash - npx wrangler kv:key put --binding AUTH_KV UPDATE_TOKEN "dev-secret" --local - ``` +```sh +npm run db:migrate:all:local --workspace @fossbilling/api +npm run test:workers --workspace @fossbilling/api +npm test --workspace @fossbilling/extension-packager +``` -4. Spin up the dev server: +See the service READMEs for local storage and authentication configuration. Runtime +secrets and local state stay beside their service and are ignored by Git. - ```bash - npm run dev - ``` +## Checks -You can now hit endpoints at `http://localhost:8787`. +```sh +npm run format:check +npm run lint +npm run typecheck +npm test +npm run build:api # Wrangler dry run; does not deploy or apply remote migrations +``` -### Testing +Each workspace owns its tests and type checking. The root test and type-check +commands run all workspaces. Keep the single root lockfile updated with `npm install` +from the repository root; do not create service-specific lockfiles. -We use Vitest. Unit tests run under the Workers pool; Node adapter tests use a separate config. +## Deployments -```bash -npm run test # workers pool -npm run test:all # workers + node adapters -``` +The services deploy independently from the same repository. Netlify configuration +and setup are documented in the [packager README](services/extension-packager/README.md). -```bash -npm run typecheck -npm run lint -``` +The API uses the same pinned [shared Cloudflare workflow](https://github.com/FOSSBilling/.workflows) +as `fossbilling.org`. [CI/CD](.github/workflows/ci.yml) validates the workspaces and +builds a self-contained Worker artifact, including deployment configuration and SQL. -### Deploy +- Pushes to `main` migrate production D1 and deploy the tested artifact. +- Other branch pushes migrate staging D1 and deploy Previews, with stable branch and + immutable commit URLs recorded on matching PRs. +- PR runs validate without deploying. The [Cloudflare Events workflow](.github/workflows/cloudflare-events.yml) + updates PR comments when a PR opens later or a run finishes, including failed attempts. -```bash -npm run cf-deploy -``` +See the [API deployment setup](services/api/README.md#deployment) for GitHub environments, +staging resource IDs, Preview secrets, migration behaviour and the Workers Builds cutover. +All production and Preview deployments require validation, the build and migrations +to succeed. Previews keep package processing disabled because Cloudflare Previews +cannot consume Queues. Production packaging also remains disabled pending Netlify +runtime verification and the controlled production package test. -This applies remote D1 migrations, then deploys with Wrangler. +`npm run cf-deploy` remains the explicit manual production command; it applies +remote migrations and deploys the API. It does not deploy the packager. diff --git a/eslint.config.ts b/eslint.config.ts index e4d754a2..a425eb45 100644 --- a/eslint.config.ts +++ b/eslint.config.ts @@ -26,8 +26,8 @@ export default defineConfig( } }, { - files: ["**/*.config.ts", "worker-configuration.d.ts"], + files: ["**/*.config.ts", "**/worker-configuration.d.ts"], extends: [tseslint.configs.disableTypeChecked] }, - globalIgnores(["coverage/", ".wrangler/"]) + globalIgnores(["**/coverage/", "**/.wrangler/", "**/dist/", "**/.netlify/"]) ); diff --git a/libraries/extension-packaging/package.json b/libraries/extension-packaging/package.json new file mode 100644 index 00000000..92834d8b --- /dev/null +++ b/libraries/extension-packaging/package.json @@ -0,0 +1,19 @@ +{ + "name": "@fossbilling/extension-packaging", + "private": true, + "version": "0.0.0", + "type": "module", + "license": "AGPL-3.0-only", + "exports": { + "./*": "./src/*.ts" + }, + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@zip.js/zip.js": "2.24.0", + "zod": "4.6.7", + "@fossbilling/logging": "*" + } +} diff --git a/src/lib/adapters/b2/package-store.ts b/libraries/extension-packaging/src/adapters/b2.ts similarity index 93% rename from src/lib/adapters/b2/package-store.ts rename to libraries/extension-packaging/src/adapters/b2.ts index eba8a97e..60ff135f 100644 --- a/src/lib/adapters/b2/package-store.ts +++ b/libraries/extension-packaging/src/adapters/b2.ts @@ -1,12 +1,7 @@ -import { - PACKAGE_LIMITS, - sha256 -} from "../../../services/extensions/v2/package-builder"; -import { PackageError } from "../../../services/extensions/v2/packages/errors"; -import { - boundedBytes, - type PackageStore -} from "../../../services/extensions/v2/packages/store"; +import { PACKAGE_LIMITS } from "../limits"; +import { sha256 } from "../checksum"; +import { PackageError } from "../errors"; +import { boundedBytes, type PackageStore } from "../store"; import { signS3Request } from "./sigv4"; // Includes generated manifest and ZIP headers as well as the source contents. @@ -14,7 +9,8 @@ const MAX_STORED_BYTES = PACKAGE_LIMITS.expandedBytes + 1024 * 1024; export function b2ObjectStore( getEnv: (name: string) => string | undefined, - kind: "packages" | "assets" + kind: "packages" | "assets", + signal?: AbortSignal ): PackageStore { const get = (suffix: string) => { const value = getEnv(`EXTENSION_${kind.toUpperCase()}_S3_${suffix}`); @@ -83,7 +79,10 @@ export function b2ObjectStore( redirect: "manual", // Read current origin state; public image caching belongs to the CDN. cache: "no-store", - signal: AbortSignal.timeout(15000) + signal: AbortSignal.any([ + ...(signal ? [signal] : []), + AbortSignal.timeout(15000) + ]) }, credentials ) diff --git a/libraries/extension-packaging/src/adapters/http.ts b/libraries/extension-packaging/src/adapters/http.ts new file mode 100644 index 00000000..da263296 --- /dev/null +++ b/libraries/extension-packaging/src/adapters/http.ts @@ -0,0 +1,84 @@ +import { logError } from "@fossbilling/logging"; +import { + ProcessingResultSchema, + type PackageProcessor, + type ProcessingInput +} from "../processor"; +import { PackageError } from "../errors"; +import { boundedBytes } from "../store"; + +export class HttpPackageProcessor implements PackageProcessor { + private readonly url: URL; + constructor( + url: string, + private readonly secret: string, + local = false + ) { + this.url = new URL(url); + if ( + !secret || + this.url.username || + this.url.password || + this.url.hash || + !( + this.url.protocol === "https:" || + (local && + this.url.protocol === "http:" && + ["localhost", "127.0.0.1", "[::1]"].includes(this.url.hostname)) + ) + ) + throw new PackageError( + "PACKAGES_DISABLED", + "Package processor is not configured correctly", + 503 + ); + } + async process(input: ProcessingInput) { + try { + const response = await fetch( + new Request(this.url, { + method: "POST", + headers: { + Authorization: `Bearer ${this.secret}`, + "Content-Type": "application/json" + }, + body: JSON.stringify(input), + redirect: "manual", + cache: "no-store", + signal: AbortSignal.timeout(50000) + }) + ); + const bytes = await boundedBytes(response.body, 16 * 1024); + const body = JSON.parse(new TextDecoder().decode(bytes)); + if (response.status === 422 && typeof body.error?.message === "string") + throw new PackageError( + body.error.code === "INVALID_ICON" + ? "INVALID_ICON" + : "INVALID_PACKAGE", + body.error.message.slice(0, 1000), + 422 + ); + if (!response.ok) throw new Error("Package processor request failed"); + const result = ProcessingResultSchema.parse(body.result); + const allowed: readonly string[] = input.icons ?? []; + if ( + result.icons.some((icon) => !allowed.includes(icon.id)) || + new Set(result.icons.map((icon) => icon.id)).size !== + result.icons.length + ) + throw new Error("Package processor returned unexpected output IDs"); + return result; + } catch (error) { + if (error instanceof PackageError && error.status === 422) throw error; + logError("extensions/packages", "Package processor request failed", { + objectId: input.archive, + error: error instanceof Error ? error.message : "Unknown error" + }); + throw new PackageError( + "PACKAGE_UNAVAILABLE", + "Package processor is temporarily unavailable. Please try again.", + 503 + ); + } + } +} diff --git a/src/lib/adapters/b2/sigv4.ts b/libraries/extension-packaging/src/adapters/sigv4.ts similarity index 94% rename from src/lib/adapters/b2/sigv4.ts rename to libraries/extension-packaging/src/adapters/sigv4.ts index d8d0925d..37bad7d6 100644 --- a/src/lib/adapters/b2/sigv4.ts +++ b/libraries/extension-packaging/src/adapters/sigv4.ts @@ -19,14 +19,16 @@ function toHex(bytes: ArrayBuffer): string { .join(""); } -async function sha256Hex(data: Uint8Array | string): Promise { +async function sha256Hex( + data: Uint8Array | string +): Promise { const bytes = typeof data === "string" ? new TextEncoder().encode(data) : data; return toHex(await crypto.subtle.digest("SHA-256", bytes)); } async function hmac( - key: Uint8Array | ArrayBuffer | string, + key: Uint8Array | ArrayBuffer | string, data: string ): Promise { const bytes = typeof key === "string" ? new TextEncoder().encode(key) : key; @@ -63,11 +65,11 @@ function canonicalQueryString(searchParams: URLSearchParams): string { } function bodyBytes( - body: BodyInit | null | undefined -): Uint8Array | string | null { + body: RequestInit["body"] +): Uint8Array | string | null { if (body == null) return null; if (typeof body === "string") return body; - if (body instanceof Uint8Array) return body; + if (body instanceof Uint8Array) return new Uint8Array(body); return null; } diff --git a/libraries/extension-packaging/src/checksum.ts b/libraries/extension-packaging/src/checksum.ts new file mode 100644 index 00000000..b550b9de --- /dev/null +++ b/libraries/extension-packaging/src/checksum.ts @@ -0,0 +1,6 @@ +export async function sha256(bytes: Uint8Array) { + const digest = await crypto.subtle.digest("SHA-256", bytes); + return Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0") + ).join(""); +} diff --git a/src/services/extensions/v2/packages/errors.ts b/libraries/extension-packaging/src/errors.ts similarity index 100% rename from src/services/extensions/v2/packages/errors.ts rename to libraries/extension-packaging/src/errors.ts diff --git a/src/services/extensions/v2/packages/images.ts b/libraries/extension-packaging/src/images.ts similarity index 100% rename from src/services/extensions/v2/packages/images.ts rename to libraries/extension-packaging/src/images.ts diff --git a/libraries/extension-packaging/src/limits.ts b/libraries/extension-packaging/src/limits.ts new file mode 100644 index 00000000..887260c6 --- /dev/null +++ b/libraries/extension-packaging/src/limits.ts @@ -0,0 +1,8 @@ +// Bounds for uploaded ZIPs and their decompressed contents. +export const PACKAGE_LIMITS = { + sourceBytes: 5 * 1024 * 1024, + expandedBytes: 16 * 1024 * 1024, + entries: 500, + pathLength: 1024, + pathDepth: 32 +} as const; diff --git a/src/services/extensions/v2/schemas/manifest.ts b/libraries/extension-packaging/src/manifest.ts similarity index 81% rename from src/services/extensions/v2/schemas/manifest.ts rename to libraries/extension-packaging/src/manifest.ts index e838dc6c..698dafe3 100644 --- a/src/services/extensions/v2/schemas/manifest.ts +++ b/libraries/extension-packaging/src/manifest.ts @@ -1,5 +1,5 @@ -import { z } from "@hono/zod-openapi"; -import { httpUrl } from "./common"; +import { z } from "zod"; +import { httpUrl } from "./urls"; // Package metadata follows FOSSBilling/modules manifest.schema.json (v1). // Icons use fixed filenames at the package root, outside the manifest. @@ -30,8 +30,7 @@ export const ManifestRequirementsSchema = z .record(z.string().regex(/^[a-z0-9_-]+$/), versionConstraint) .optional() }) - .strict() - .openapi("ManifestRequirements"); + .strict(); export const ManifestSchema = z .object({ @@ -61,17 +60,6 @@ export const ManifestSchema = z homepage_url: httpUrl().optional(), requires: ManifestRequirementsSchema }) - .strict() - .openapi("Manifest"); + .strict(); export type Manifest = z.infer; - -export function manifestMetadata( - content: Pick -) { - return { - manifest_version: content.manifest_version, - author: content.author, - requires: content.requires - }; -} diff --git a/libraries/extension-packaging/src/process.ts b/libraries/extension-packaging/src/process.ts new file mode 100644 index 00000000..11ac68d7 --- /dev/null +++ b/libraries/extension-packaging/src/process.ts @@ -0,0 +1,67 @@ +import { buildExtensionPackage } from "./zip"; +import { PACKAGE_LIMITS } from "./limits"; +import { sha256 } from "./checksum"; +import { PackageError } from "./errors"; +import type { PackageImageProcessor } from "./images"; +import { boundedBytes, type PackageStore } from "./store"; +import { + ProcessingResultSchema, + type ProcessingInput, + type ProcessingResult +} from "./processor"; + +export async function processPackage( + input: ProcessingInput, + store: PackageStore, + images: PackageImageProcessor, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted(); + const source = await boundedBytes( + await store.get(input.source.id, input.source.sha256), + PACKAGE_LIMITS.sourceBytes + ); + if (!source.length || (await sha256(source)) !== input.source.sha256) + throw new PackageError( + "SOURCE_UNAVAILABLE", + "Source ZIP is unavailable or failed checksum verification", + 503 + ); + let built; + try { + built = await buildExtensionPackage(source, input.manifest, signal); + } catch (error) { + signal?.throwIfAborted(); + throw new PackageError( + "INVALID_PACKAGE", + error instanceof Error + ? error.message + : "Source ZIP failed package validation", + 422 + ); + } + const variants = + input.icons && built.icon + ? await images.prepare(built.icon, built.iconName === "icon.svg") + : []; + signal?.throwIfAborted(); + await store.put(input.archive, built.bytes, built.packageSha256); + const icons: ProcessingResult["icons"] = []; + for (const [index, variant] of variants.entries()) { + signal?.throwIfAborted(); + const id = input.icons![index]; + const checksum = await sha256(variant.bytes); + await store.put(id, variant.bytes, checksum); + icons.push({ + id, + format: variant.format, + sha256: checksum, + bytes: variant.bytes.length + }); + } + signal?.throwIfAborted(); + return ProcessingResultSchema.parse({ + archive: { bytes: built.bytes.length, sha256: built.packageSha256 }, + icons + }); +} diff --git a/libraries/extension-packaging/src/processor.ts b/libraries/extension-packaging/src/processor.ts new file mode 100644 index 00000000..f2454625 --- /dev/null +++ b/libraries/extension-packaging/src/processor.ts @@ -0,0 +1,54 @@ +import { z } from "zod"; +import { ManifestSchema } from "./manifest"; +import { PACKAGE_LIMITS } from "./limits"; + +export const MAX_PACKAGE_OBJECT_BYTES = + PACKAGE_LIMITS.expandedBytes + 1024 * 1024; +const checksum = z.string().regex(/^[a-f0-9]{64}$/); +const output = z + .object({ + bytes: z.number().int().positive().max(MAX_PACKAGE_OBJECT_BYTES), + sha256: checksum + }) + .strict(); +export const ProcessingInputSchema = z + .object({ + manifest: ManifestSchema, + source: z.object({ id: z.uuid(), sha256: checksum }).strict(), + archive: z.uuid(), + icons: z.tuple([z.uuid(), z.uuid()]).optional() + }) + .strict() + .refine((value) => { + const keys = [value.source.id, value.archive, ...(value.icons ?? [])]; + return new Set(keys).size === keys.length; + }, "Processing object IDs must be distinct"); +export const ProcessingResultSchema = z + .object({ + archive: output, + icons: z + .array( + output + .extend({ + id: z.uuid(), + format: z.enum(["svg", "avif", "png"]) + }) + .strict() + ) + .max(2) + }) + .strict() + .refine((value) => { + const formats = value.icons + .map((icon) => icon.format) + .sort() + .join(","); + return formats === "" || formats === "svg" || formats === "avif,png"; + }, "Invalid prepared icon variants"); +export type ProcessingInput = z.infer; +export type ProcessingResult = z.infer; + +// Storage keys and manifests are provider-neutral; the runner never needs a database binding. +export interface PackageProcessor { + process(input: ProcessingInput): Promise; +} diff --git a/src/services/extensions/v2/packages/store.ts b/libraries/extension-packaging/src/store.ts similarity index 100% rename from src/services/extensions/v2/packages/store.ts rename to libraries/extension-packaging/src/store.ts diff --git a/libraries/extension-packaging/src/urls.ts b/libraries/extension-packaging/src/urls.ts new file mode 100644 index 00000000..a77762d1 --- /dev/null +++ b/libraries/extension-packaging/src/urls.ts @@ -0,0 +1,13 @@ +import { z } from "zod"; + +// Restricts to http(s) — z.string().url() alone accepts any scheme, +// including javascript:/data:, which is unsafe for fields a consumer may +// render as a link or image src. +export const httpUrl = () => + z + .string() + .max(2048) + .url() + .refine((value) => /^https?:\/\//i.test(value), { + message: "must use http or https" + }); diff --git a/src/services/extensions/v2/package-builder.ts b/libraries/extension-packaging/src/zip.ts similarity index 90% rename from src/services/extensions/v2/package-builder.ts rename to libraries/extension-packaging/src/zip.ts index 59aa497d..5c1a6e3e 100644 --- a/src/services/extensions/v2/package-builder.ts +++ b/libraries/extension-packaging/src/zip.ts @@ -1,19 +1,12 @@ +import { PACKAGE_LIMITS } from "./limits"; +import { sha256 } from "./checksum"; import { Uint8ArrayReader, Uint8ArrayWriter, ZipReader, ZipWriter } from "@zip.js/zip.js"; -import { ManifestSchema } from "./schemas/manifest"; - -// Bounds for uploaded ZIPs and their decompressed contents. -export const PACKAGE_LIMITS = { - sourceBytes: 5 * 1024 * 1024, - expandedBytes: 16 * 1024 * 1024, - entries: 500, - pathLength: 1024, - pathDepth: 32 -} as const; +import { ManifestSchema } from "./manifest"; const zipOptions = { useWebWorkers: false, @@ -23,13 +16,6 @@ const zipOptions = { checkOverlappingEntry: true }; -export async function sha256(bytes: Uint8Array) { - const digest = await crypto.subtle.digest("SHA-256", bytes); - return Array.from(new Uint8Array(digest), (byte) => - byte.toString(16).padStart(2, "0") - ).join(""); -} - function validatePath(filename: string, directory: boolean) { const path = directory ? filename.replace(/\/$/, "") : filename; if (path.length > PACKAGE_LIMITS.pathLength) @@ -58,13 +44,17 @@ function validatePath(filename: string, directory: boolean) { /** Validate source files and build a bounded archive with directory metadata. */ export async function buildExtensionPackage( source: Uint8Array, - manifestInput: unknown + manifestInput: unknown, + signal?: AbortSignal ) { const manifest = ManifestSchema.parse(manifestInput); if (source.byteLength > PACKAGE_LIMITS.sourceBytes) { throw new Error("Source ZIP exceeds size limit"); } - const reader = new ZipReader(new Uint8ArrayReader(source), zipOptions); + const reader = new ZipReader(new Uint8ArrayReader(source), { + ...zipOptions, + signal + }); const files: { path: string; bytes: Uint8Array; @@ -186,6 +176,7 @@ export async function buildExtensionPackage( files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); const writer = new ZipWriter(new Uint8ArrayWriter(), { ...zipOptions, + signal, level: 6, lastModDate: new Date("1980-01-01T00:00:00Z"), extendedTimestamp: false diff --git a/test/services/extensions/v2/b2-store.node.test.ts b/libraries/extension-packaging/test/b2-store.test.ts similarity index 97% rename from test/services/extensions/v2/b2-store.node.test.ts rename to libraries/extension-packaging/test/b2-store.test.ts index 3dafd7f7..7a5ab8e9 100644 --- a/test/services/extensions/v2/b2-store.node.test.ts +++ b/libraries/extension-packaging/test/b2-store.test.ts @@ -1,9 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { b2ObjectStore } from "../../../../src/lib/adapters/b2/package-store"; -import { - sha256, - PACKAGE_LIMITS -} from "../../../../src/services/extensions/v2/package-builder"; +import { b2ObjectStore } from "../src/adapters/b2"; +import { sha256 } from "../src/checksum"; +import { PACKAGE_LIMITS } from "../src/limits"; const objectId = "134cf413-b5ab-4f57-b2ad-520020af880d"; const config: Record = { diff --git a/test/services/extensions/v2/sigv4.node.test.ts b/libraries/extension-packaging/test/sigv4.test.ts similarity index 98% rename from test/services/extensions/v2/sigv4.node.test.ts rename to libraries/extension-packaging/test/sigv4.test.ts index 701e26de..8870367e 100644 --- a/test/services/extensions/v2/sigv4.node.test.ts +++ b/libraries/extension-packaging/test/sigv4.test.ts @@ -1,6 +1,6 @@ import { createHash, createHmac } from "node:crypto"; import { describe, expect, it } from "vitest"; -import { signS3Request } from "../../../../src/lib/adapters/b2/sigv4"; +import { signS3Request } from "../src/adapters/sigv4"; const credentials = { accessKeyId: "test-key", diff --git a/test/services/extensions/v2/package-builder.test.ts b/libraries/extension-packaging/test/zip.test.ts similarity index 98% rename from test/services/extensions/v2/package-builder.test.ts rename to libraries/extension-packaging/test/zip.test.ts index e3dc87a8..35eb402e 100644 --- a/test/services/extensions/v2/package-builder.test.ts +++ b/libraries/extension-packaging/test/zip.test.ts @@ -7,11 +7,9 @@ import { ZipReader, ZipWriter } from "@zip.js/zip.js"; -import { - buildExtensionPackage, - PACKAGE_LIMITS, - sha256 -} from "../../../../src/services/extensions/v2/package-builder"; +import { buildExtensionPackage } from "../src/zip"; +import { PACKAGE_LIMITS } from "../src/limits"; +import { sha256 } from "../src/checksum"; const manifest = { manifest_version: 1, diff --git a/libraries/extension-packaging/tsconfig.json b/libraries/extension-packaging/tsconfig.json new file mode 100644 index 00000000..e8b02ca4 --- /dev/null +++ b/libraries/extension-packaging/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/libraries/logging/package.json b/libraries/logging/package.json new file mode 100644 index 00000000..182d9480 --- /dev/null +++ b/libraries/logging/package.json @@ -0,0 +1,12 @@ +{ + "name": "@fossbilling/logging", + "private": true, + "version": "0.0.0", + "type": "module", + "license": "AGPL-3.0-only", + "exports": "./src/logger.ts", + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit" + } +} diff --git a/src/lib/logger.ts b/libraries/logging/src/logger.ts similarity index 100% rename from src/lib/logger.ts rename to libraries/logging/src/logger.ts diff --git a/test/lib/logger.test.ts b/libraries/logging/test/logger.test.ts similarity index 98% rename from test/lib/logger.test.ts rename to libraries/logging/test/logger.test.ts index 495e8117..6a714021 100644 --- a/test/lib/logger.test.ts +++ b/libraries/logging/test/logger.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; -import { logError, logWarn, logInfo } from "../../src/lib/logger"; +import { logError, logWarn, logInfo } from "../src/logger"; // console[level] receives one structured object ({ service, message, // context? }) - the shape Workers Logs serializes to filterable JSON. diff --git a/libraries/logging/tsconfig.json b/libraries/logging/tsconfig.json new file mode 100644 index 00000000..e8b02ca4 --- /dev/null +++ b/libraries/logging/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/package-lock.json b/package-lock.json index 0a47263b..8f563ec7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,45 +1,48 @@ { - "name": "api", + "name": "@fossbilling/services", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "api", + "name": "@fossbilling/services", "license": "AGPL-3.0-only", - "dependencies": { - "@hono/zod-openapi": "1.6.3", - "@octokit/graphql": "9.0.5", - "@octokit/request": "10.0.16", - "@scalar/hono-api-reference": "0.12.18", - "@zeropress/svg-hush": "0.1.1", - "@zip.js/zip.js": "2.24.0", - "badge-maker": "6.0.0", - "better-auth": "^1.7.14", - "drizzle-orm": "0.45.4", - "hono": "4.13.17", - "semver": "7.8.6", - "uuid": "14.0.3", - "zod": "4.6.7" - }, + "workspaces": [ + "services/*", + "libraries/*" + ], "devDependencies": { - "@cloudflare/vitest-plugin": "1.6.1", "@eslint/js": "10.0.1", "@types/node": "26.6.6", - "@types/semver": "7.8.0", "@typescript-eslint/eslint-plugin": "8.72.1", "@typescript-eslint/parser": "8.72.1", "@vitest/coverage-istanbul": "5.1.0", - "drizzle-kit": "0.31.11", "eslint": "10.14.0", "eslint-config-prettier": "10.1.8", "jiti": "2.7.0", "prettier": "3.9.10", "typescript": "6.0.3", "typescript-eslint": "8.72.1", - "vitest": "5.1.0", - "wrangler": "4.158.0" + "vitest": "5.1.0" + }, + "engines": { + "node": ">=24" } }, + "libraries/extension-packaging": { + "name": "@fossbilling/extension-packaging", + "version": "0.0.0", + "license": "AGPL-3.0-only", + "dependencies": { + "@fossbilling/logging": "*", + "@zip.js/zip.js": "2.24.0", + "zod": "4.6.7" + } + }, + "libraries/logging": { + "name": "@fossbilling/logging", + "version": "0.0.0", + "license": "AGPL-3.0-only" + }, "node_modules/@asteasolutions/zod-to-openapi": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/@asteasolutions/zod-to-openapi/-/zod-to-openapi-9.1.0.tgz", @@ -1454,6 +1457,22 @@ "node": "^20.19.0 || ^22.13.0 || >=24" } }, + "node_modules/@fossbilling/api": { + "resolved": "services/api", + "link": true + }, + "node_modules/@fossbilling/extension-packager": { + "resolved": "services/extension-packager", + "link": true + }, + "node_modules/@fossbilling/extension-packaging": { + "resolved": "libraries/extension-packaging", + "link": true + }, + "node_modules/@fossbilling/logging": { + "resolved": "libraries/logging", + "link": true + }, "node_modules/@hono/zod-openapi": { "version": "1.6.3", "resolved": "https://registry.npmjs.org/@hono/zod-openapi/-/zod-openapi-1.6.3.tgz", @@ -1552,7 +1571,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=18" @@ -1565,7 +1583,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -1588,7 +1605,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -1608,7 +1624,6 @@ "version": "0.35.5", "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.5.tgz", "integrity": "sha512-Y/z91nEZ4uIBX5X3nfTovjU9lHNKFYbL2lpHCLVNmXQK03VIZvXBBt0KxbPGp2SdGSF+2mQU4e+hQaWOt86iAw==", - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -1631,7 +1646,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1648,7 +1662,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1665,7 +1678,6 @@ "cpu": [ "arm" ], - "dev": true, "libc": [ "glibc" ], @@ -1685,7 +1697,6 @@ "cpu": [ "arm64" ], - "dev": true, "libc": [ "glibc" ], @@ -1705,7 +1716,6 @@ "cpu": [ "ppc64" ], - "dev": true, "libc": [ "glibc" ], @@ -1725,7 +1735,6 @@ "cpu": [ "riscv64" ], - "dev": true, "libc": [ "glibc" ], @@ -1745,7 +1754,6 @@ "cpu": [ "s390x" ], - "dev": true, "libc": [ "glibc" ], @@ -1765,7 +1773,6 @@ "cpu": [ "x64" ], - "dev": true, "libc": [ "glibc" ], @@ -1785,7 +1792,6 @@ "cpu": [ "arm64" ], - "dev": true, "libc": [ "musl" ], @@ -1805,7 +1811,6 @@ "cpu": [ "x64" ], - "dev": true, "libc": [ "musl" ], @@ -1825,7 +1830,6 @@ "cpu": [ "arm" ], - "dev": true, "libc": [ "glibc" ], @@ -1851,7 +1855,6 @@ "cpu": [ "arm64" ], - "dev": true, "libc": [ "glibc" ], @@ -1877,7 +1880,6 @@ "cpu": [ "ppc64" ], - "dev": true, "libc": [ "glibc" ], @@ -1903,7 +1905,6 @@ "cpu": [ "riscv64" ], - "dev": true, "libc": [ "glibc" ], @@ -1929,7 +1930,6 @@ "cpu": [ "s390x" ], - "dev": true, "libc": [ "glibc" ], @@ -1955,7 +1955,6 @@ "cpu": [ "x64" ], - "dev": true, "libc": [ "glibc" ], @@ -1981,7 +1980,6 @@ "cpu": [ "arm64" ], - "dev": true, "libc": [ "musl" ], @@ -2007,7 +2005,6 @@ "cpu": [ "x64" ], - "dev": true, "libc": [ "musl" ], @@ -2030,7 +2027,6 @@ "version": "0.35.5", "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.5.tgz", "integrity": "sha512-Ptsga1su4tQx+LLF1ECS9U6nz5kmrXKo6XVbtR48Ke3ZRxxgaWBu7IDtEe1quo8hiupwm6WFqxVlXaSf7IINGQ==", - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { @@ -2050,7 +2046,6 @@ "cpu": [ "wasm32" ], - "dev": true, "license": "Apache-2.0", "optional": true, "dependencies": { @@ -2070,7 +2065,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -2090,7 +2084,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -2110,7 +2103,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -3577,7 +3569,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=8" @@ -5529,7 +5520,6 @@ "version": "0.35.5", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.5.tgz", "integrity": "sha512-Ywn4OnzGukp7CDMrp08RQ50YKmuwG47brZgIVPTvBaaAfQlRlygrRqSrxdCiL9M+LlzLBiJ68IR1QqvzHyjC7g==", - "dev": true, "license": "Apache-2.0", "dependencies": { "@img/colour": "^1.1.0", @@ -6654,6 +6644,48 @@ "funding": { "url": "https://github.com/sponsors/colinhacks" } + }, + "services/api": { + "name": "@fossbilling/api", + "version": "0.0.0", + "license": "AGPL-3.0-only", + "dependencies": { + "@fossbilling/extension-packaging": "*", + "@fossbilling/logging": "*", + "@hono/zod-openapi": "1.6.3", + "@octokit/graphql": "9.0.5", + "@octokit/request": "10.0.16", + "@scalar/hono-api-reference": "0.12.18", + "badge-maker": "6.0.0", + "better-auth": "^1.7.14", + "drizzle-orm": "0.45.4", + "hono": "4.13.17", + "semver": "7.8.6", + "uuid": "14.0.3", + "zod": "4.6.7" + }, + "devDependencies": { + "@cloudflare/vitest-plugin": "1.6.1", + "@types/semver": "7.8.0", + "@zip.js/zip.js": "2.24.0", + "drizzle-kit": "0.31.11", + "wrangler": "4.158.0" + } + }, + "services/extension-packager": { + "name": "@fossbilling/extension-packager", + "version": "0.0.0", + "license": "AGPL-3.0-only", + "dependencies": { + "@fossbilling/extension-packaging": "*", + "@fossbilling/logging": "*", + "@zeropress/svg-hush": "0.1.1", + "sharp": "0.35.5" + }, + "devDependencies": { + "@zip.js/zip.js": "2.24.0", + "tsx": "4.23.15" + } } } } diff --git a/package.json b/package.json index 572de1d3..160a8b5a 100644 --- a/package.json +++ b/package.json @@ -1,61 +1,41 @@ { - "name": "api", + "name": "@fossbilling/services", + "private": true, "license": "AGPL-3.0-only", "type": "module", + "workspaces": [ + "services/*", + "libraries/*" + ], + "engines": { + "node": ">=24" + }, "scripts": { - "cf-typegen": "wrangler types --env-interface CloudflareBindings", - "cf-deploy": "npm run db:migrate:all:remote && wrangler deploy --minify", - "dev": "wrangler dev", + "dev": "npm run dev --workspace @fossbilling/api", + "extension-packager:dev": "npm run dev --workspace @fossbilling/extension-packager", + "cf-typegen": "npm run cf-typegen --workspace @fossbilling/api", + "cf-deploy": "npm run cf-deploy --workspace @fossbilling/api", + "build:api": "npm run build --workspace @fossbilling/api", + "test": "npm test --workspaces", + "typecheck": "tsc --noEmit && npm run typecheck --workspaces", + "lint": "eslint .", "format": "prettier --write .", "format:check": "prettier --check .", - "db:generate:extensions": "drizzle-kit generate --config drizzle.extensions.config.ts", - "db:generate:central-alerts": "drizzle-kit generate --config drizzle.central-alerts.config.ts", - "db:migrate:extensions-v2:local": "wrangler d1 migrations apply DB_EXTENSIONS --local", - "db:migrate:extensions-v2:remote": "wrangler d1 migrations apply DB_EXTENSIONS --remote", - "db:migrate:central-alerts:local": "wrangler d1 migrations apply DB_CENTRAL_ALERTS --local", - "db:migrate:central-alerts:remote": "wrangler d1 migrations apply DB_CENTRAL_ALERTS --remote", - "db:migrate:all:local": "npm run db:migrate:extensions-v2:local && npm run db:migrate:central-alerts:local", - "db:migrate:all:remote": "npm run db:migrate:extensions-v2:remote && npm run db:migrate:central-alerts:remote", - "lint": "eslint", - "lint:fix": "eslint --fix", - "test": "vitest run", - "test:node": "vitest run --config vitest.node.config.ts", - "test:all": "npm run test && npm run test:node", - "test:coverage": "vitest run --coverage", - "typecheck": "tsc --noEmit" - }, - "dependencies": { - "@hono/zod-openapi": "1.6.3", - "@octokit/graphql": "9.0.5", - "@octokit/request": "10.0.16", - "@scalar/hono-api-reference": "0.12.18", - "@zeropress/svg-hush": "0.1.1", - "@zip.js/zip.js": "2.24.0", - "badge-maker": "6.0.0", - "better-auth": "^1.7.14", - "drizzle-orm": "0.45.4", - "hono": "4.13.17", - "semver": "7.8.6", - "uuid": "14.0.3", - "zod": "4.6.7" + "cf-preview": "npm run cf-preview --workspace @fossbilling/api" }, "devDependencies": { - "@cloudflare/vitest-plugin": "1.6.1", "@eslint/js": "10.0.1", "@types/node": "26.6.6", - "@types/semver": "7.8.0", "@typescript-eslint/eslint-plugin": "8.72.1", "@typescript-eslint/parser": "8.72.1", "@vitest/coverage-istanbul": "5.1.0", - "drizzle-kit": "0.31.11", "eslint": "10.14.0", "eslint-config-prettier": "10.1.8", "jiti": "2.7.0", "prettier": "3.9.10", "typescript": "6.0.3", "typescript-eslint": "8.72.1", - "vitest": "5.1.0", - "wrangler": "4.158.0" + "vitest": "5.1.0" }, "overrides": { "@esbuild-kit/core-utils": { diff --git a/services/api/AGENTS.md b/services/api/AGENTS.md new file mode 100644 index 00000000..0f118e90 --- /dev/null +++ b/services/api/AGENTS.md @@ -0,0 +1,78 @@ +# Repository Guidelines + +## Project Structure & Module Organization + +- `src/app/index.ts` is the worker entrypoint and route wiring for Hono. +- Feature logic lives in `src/services/` and should stay runtime-agnostic. +- Platform interfaces and adapters are in `src/lib/` with Cloudflare and Node implementations under `src/lib/adapters/`. +- Tests mirror the source layout under `test/`, with shared helpers in `test/utils/` and mocks in `test/mocks/`. +- Service configuration includes `wrangler.jsonc`, `worker-configuration.d.ts`, `tsconfig.json`, and the Drizzle/Vitest configs. ESLint and Prettier configuration live at the repository root. + +### Service layout + +Small services are flat: `index.ts` + `interfaces.ts` plus narrow helpers (for example `database.ts`, `db/`, `r2.ts`). + +`extensions/v2` is the reference layout for anything larger, and new services should grow into it rather than inventing a third shape: + +- `index.ts` — app assembly only: middleware, route registration, OpenAPI document. Route registration order is load-bearing where static paths must beat parameter paths; those cases carry comments. +- `middleware.ts` — service-specific Hono middleware. +- `routes/` — one module per route group, each exporting `register*Routes(app)`. `routes/errors.ts` maps domain error codes to HTTP status; `routes/app.ts` holds the typed app alias. +- `db/` — `schema.ts`, `migrations/`, one `*Database` class per workflow, plus `errors.ts` (D1 constraint classification) and `batch.ts`. +- `schemas/` — zod/OpenAPI contract split by domain. There is deliberately **no barrel**: import from `schemas/` directly so a module's dependencies are visible. This is why `extensions/v2` has no `interfaces.ts`. +- `github/` — outbound GitHub calls, kept out of the persistence modules. +- `email/` (extensions v2) — provider abstraction and templates. Service-root helpers (`revalidate.ts`, `resource-limits.ts`) stay at the service root. `previews/v1` follows the same split (`routes/`, `github/`, `schemas/`, plus `budget.ts` / `cache.ts`). + +Use `getPlatform(c)` for cache access, service requests, rate-limit checks and background tasks. Keep binding conversion in platform adapters and inject background scheduling from the host entry point. + +Route modules import `getExtensionsDb`/`getAuth`/`getPlatform` and middleware directly. There is no dependency-injection container; tests drive the real app through `app.request`. + +Each service documents its own contract and operational detail in its own `README.md` (`src/services///README.md`). Keep API behaviour there rather than here or in the root README: this file is for conventions that apply when modifying the code. + +## Build, Test, and Development Commands + +- Run `npm ci` at the repository root to install workspace dependencies. +- `npm run dev`: start the local Cloudflare Workers dev server. +- `npm run cf-deploy`: apply remote D1 migrations, then deploy via Wrangler. +- `npm run cf-typegen`: regenerate `worker-configuration.d.ts`. +- `npm run build`: create a validated Worker artifact with deployment configuration and migrations. +- `npm run cf-preview`: deploy the prepared artifact as a Cloudflare Preview after staging migrations. +- `npm run db:migrate:all:local` / `db:migrate:all:remote`: apply all D1 migrations locally / remotely. +- `npm test`: run Workers and Node adapter tests for the API. +- `npm run test:workers`: run only the Workers pool. +- `npm run test:node`: run Node adapter tests via `vitest.node.config.ts`. +- `npm run test:coverage`: run tests with coverage. +- `npm run typecheck`: TypeScript typecheck without emit. +- Run `npm run lint` from the repository root. +- Run `npm run format` / `npm run format:check` from the repository root. + +## Coding Style & Naming Conventions + +- TypeScript (ES modules), 2-space indentation, and Prettier formatting (`trailingComma: "none"`). +- ESLint uses `@typescript-eslint` with Prettier compatibility; keep lint clean before opening PRs. +- Use kebab-case for service folders (for example `central-alerts`), and name tests `*.test.ts`. + +## Testing Guidelines + +- Vitest is the primary test runner; integration tests live in `test/integration/`. +- Prefer colocating tests in the matching `test/` path (for example `src/services/versions/v1` -> `test/services/versions/v1`). +- Run `npm test` at the repository root before release-related changes to cover all workspaces. + +## Commit & Pull Request Guidelines + +- Recent commits use short, imperative, sentence-case subjects (for example "Update API docs", "Refactor versions service"). +- No ticket prefixes are evident; keep messages focused and under ~72 chars. +- PRs should include a concise summary, testing notes (commands run), and call out config/binding changes in `wrangler.jsonc`. + +## Configuration & Secrets + +- Local secrets go in `.dev.vars` (for example `GITHUB_TOKEN="..."`). +- Bindings, cron, and vars are defined in `wrangler.jsonc` (D1, KV, R2, Durable Objects, rate limiters, service bindings); keep names aligned with `CloudflareBindings`. +- Use Wrangler secrets for production tokens instead of committing them. +- `ASSERTION_SIGNING_SECRET`: HMAC key the extensions v2 API uses to verify short-lived bearer assertions minted by the extensions site (`src/lib/auth/bearer-assertion.ts`). Not sent over the wire — only signs/verifies server-side in each Worker. Add `ASSERTION_SIGNING_SECRET="..."` to `.dev.vars` for local dev; set via `wrangler secret put ASSERTION_SIGNING_SECRET` in production, matching the value configured in the extensions site's Worker. +- `EXTENSIONS_REVALIDATE_SECRET`: bearer token the extensions v2 API sends to the extensions site's `POST /api/revalidate` (over the `EXTENSIONS_FRONTEND` service binding) to purge the site's CDN-cached extension pages after content mutations. Must match the value configured in the extensions site's Worker. Same local/prod setup as `ASSERTION_SIGNING_SECRET`. + +## Cache Revalidation + +Any endpoint that mutates publicly visible content (revision approve/reject, moderator-correct, delist/relist, developer approve, developer profile upsert via `PUT /developers/me`, profile deletion via `DELETE /developers/me`, claim approve/reject, extension withdraw) must call `revalidateExtensions(c)` from `src/services/extensions/v2/revalidate.ts` after a successful write. + +Skipping it does not break correctness — the site's `maxAge`/`stale-while-revalidate` windows bound staleness — but changes then take minutes instead of seconds to appear. Profile deletion keeps its purge even though the `developers` tag is currently a no-op. diff --git a/services/api/README.md b/services/api/README.md new file mode 100644 index 00000000..551bfbb1 --- /dev/null +++ b/services/api/README.md @@ -0,0 +1,239 @@ +# FOSSBilling API Worker + +This service is an npm workspace. Run `npm ci` from the [repository root](../../README.md). +The commands below run from `services/api` unless stated otherwise. + +This is the API service that acts as the central hub for FOSSBilling instances. It serves version checks, system-wide alerts, the extensions directory, preview builds, and release stats. + +The HTTP layer uses [Hono](https://hono.dev) and deploys to Cloudflare Workers. Shared packaging code and several platform capabilities use provider-neutral interfaces; database and specialised storage integrations still use Cloudflare bindings. + +## What it does + +The worker exposes five services: + +- **Versions** (`/versions/v1`) + The source of truth for FOSSBilling updates. It fetches release data from GitHub, caches it for performance, and helps instances decide if they need to update. + +- **Central Alerts** (`/central-alerts/v1`) + Allows the project to push critical notifications to all FOSSBilling installations—useful for security hotfixes or major announcements. + +- **Extensions** (`/extensions/v1`, `/extensions/v2`) + Owns the complete Extensions domain and its `DB_EXTENSIONS` schema. `v2` is authoritative. `v1` is deprecated; its badge URLs retire on **5 February 2027** ([migration and retirement policy](src/services/extensions/v1/README.md)). The separate Extensions site keeps OIDC/session state but accesses this domain through the + generated HTTPS API client; it must not bind or migrate `DB_EXTENSIONS`. + See [`src/services/extensions/v2/README.md`](src/services/extensions/v2/README.md). + +- **Previews** (`/previews/v1`) + Resolves FOSSBilling preview builds — the current main preview and per-PR/per-commit builds produced by FOSSBilling/FOSSBilling's GitHub Actions workflows. Read-only; GitHub Actions and R2 are the sources of truth, not this service. + See [`src/services/previews/v1/README.md`](src/services/previews/v1/README.md). + +- **Stats** (`/stats/v1`) + Aggregates release data through the versions service's shared releases cache into charts and JSON; a cold cache can trigger a GitHub fetch through that shared path. + See [`src/services/stats/v1/README.md`](src/services/stats/v1/README.md). + +## Architecture + +Feature modules live under `src/services/`. Smaller services are flat; larger ones, +such as `extensions/v2`, split into `routes/`, `db/`, `schemas/` and `github/`. + +`src/lib/context.ts` exposes caches, queues, environment values, service requests, +rate-limit checks and background-task scheduling. Cloudflare adapters translate +bindings into these capabilities. The Worker entry point supplies the request's +`waitUntil` callback; another host can supply its own task scheduling. Application +helpers use ordinary `Request`/`Response` values rather than Cloudflare `Fetcher`s. + +Database code uses Drizzle's D1 driver and some features access R2 or Durable Objects +directly. Moving the API to another host would still require replacing those +integrations. The Node adapters currently provide caches and environment access, +not a complete alternative API runtime. + +Package processing uses the shared library's storage, processor and image interfaces. +B2 storage configuration lives in `extensions/v2/packages/store.ts`; Cloudflare Queue +delivery and acknowledgement stay in `src/lib/adapters/cloudflare/packaging-jobs.ts`. +The separately hosted packager does not import API source or Cloudflare bindings. + +## APIs + +Each service documents its own endpoints and behaviour: + +| Service | Base path | Docs | +| -------------- | ---------------------------------- | -------------------------------------------------------------------------------------- | +| Versions | `/versions/v1` | [`src/services/versions/v1/README.md`](src/services/versions/v1/README.md) | +| Central Alerts | `/central-alerts/v1` | [`src/services/central-alerts/v1/README.md`](src/services/central-alerts/v1/README.md) | +| Stats | `/stats/v1` | [`src/services/stats/v1/README.md`](src/services/stats/v1/README.md) | +| Extensions | `/extensions/v1`, `/extensions/v2` | [`src/services/extensions/v2/README.md`](src/services/extensions/v2/README.md) | +| Previews | `/previews/v1` | [`src/services/previews/v1/README.md`](src/services/previews/v1/README.md) | + +Extensions v2 and Previews v1 also publish a live OpenAPI document (`/extensions/v2/docs/openapi.json`, `/previews/v1/docs/openapi.json`) and a reference UI (`/extensions/v2/docs`, `/previews/v1/docs`). + +## Configuration + +If you're running this yourself, you'll need a few things set up. + +### Storage + +We use [Cloudflare D1](https://developers.cloudflare.com/d1/) and [KV](https://developers.cloudflare.com/kv/). + +- **D1 Database** (`DB_CENTRAL_ALERTS`): Stores the alert messages. +- **D1 Database** (`DB_EXTENSIONS`): Stores the complete Extensions domain. + Migrations are owned by extensions v2 and applied only from this repository — see [its README](src/services/extensions/v2/README.md#database) for the migration and adoption procedure. +- **KV Namespace** (`CACHE_KV`): Caches GitHub API responses so we don't hit rate limits. +- **KV Namespace** (`AUTH_KV`): Stores the `UPDATE_TOKEN` value for `/versions/v1/update`. +- **R2 Bucket** (`DOWNLOAD_BUCKET`): Shared by versions, previews, and stats for release/preview zips — see [`src/services/previews/v1/README.md`](src/services/previews/v1/README.md) and the comment in `wrangler.jsonc` for which bucket this points at and why. +- **Durable Object** (`PREVIEW_GITHUB_BUDGET`): Token bucket limiting previews GitHub calls. Policy lives in `src/services/previews/v1/budget.ts`. +- **Rate limiters** (`EXTENSION_WRITE_RATE_LIMITER`, `PROFILE_CREATION_RATE_LIMITER`): Edge pacing for extension writes and profile creation. Durable quotas are enforced in D1; see [extensions v2 README](src/services/extensions/v2/README.md#limits-and-budgets). +- **Service binding** (`EXTENSIONS_FRONTEND`): Private binding to the Extensions site worker, used to purge cached extension pages after mutations. +- **Cron** (`0 * * * *`): Hourly extension resource maintenance. See [extensions v2 README](src/services/extensions/v2/README.md#revisions-and-retention). + +### Environment Variables + +Production B2 connection details and MXroute server/username are managed in the +Worker dashboard under **Settings > Variables and Secrets** as runtime variables. +Credentials are stored as secrets. Build environment variables do not configure +the running Worker. `keep_vars: true` preserves dashboard-managed variables on +Wrangler deployments; keep their values out of the committed `vars` map and docs. +Public URLs and deployment controls remain in `wrangler.jsonc`. + +For local development, put operational values and credentials in the ignored +`.dev.vars` file. Document variable names and placeholders, not production values. +Package storage configuration is listed in [PACKAGING.md](src/services/extensions/v2/PACKAGING.md). + +- `GITHUB_TOKEN`: A GitHub Personal Access Token (classic) with public repo read access. +- `ASSERTION_SIGNING_SECRET`: Shared HMAC secret used to verify the short-lived bearer assertions minted by the Extensions site. Configure the same value in both Workers; it is never sent to clients. +- `ASSERTION_SIGNING_SECRET_PREVIOUS`: Optional previous HMAC secret accepted during a signing-key rotation. +- `EXTENSIONS_REVALIDATE_SECRET`: Bearer token the API sends to the Extensions site's `POST /api/revalidate` (over the `EXTENSIONS_FRONTEND` service binding) to purge the site's CDN-cached extension pages after content mutations. Configure the same value in both Workers; it is never sent to clients. +- `EXTENSIONS_V2_EMAIL_PROVIDER`: `mxroute` or `resend` to send moderator notification emails, or unset/`disabled` to skip sending. +- `EXTENSIONS_V2_EMAIL_FROM`: Sender address for notification emails (default `noreply@fossbilling.org`). +- `EXTENSIONS_V2_EMAIL_REPLY_TO`: Optional Reply-To address for a monitored inbox. Omit when unset (no Reply-To header is sent) — notification emails do not invite replies. +- `EXTENSIONS_V2_MXROUTE_SERVER`, `EXTENSIONS_V2_MXROUTE_USERNAME`: Dashboard-managed runtime variables identifying the mailbox for the MXroute SMTP API (`https://smtpapi.mxroute.com/`). Required when `EXTENSIONS_V2_EMAIL_PROVIDER=mxroute`. +- `EXTENSIONS_V2_MXROUTE_PASSWORD`: Mailbox password, stored as a secret. Required when `EXTENSIONS_V2_EMAIL_PROVIDER=mxroute`. +- `EXTENSIONS_V2_RESEND_API_KEY`: API key for Resend. Required when `EXTENSIONS_V2_EMAIL_PROVIDER=resend`. +- `EXTENSIONS_RETENTION_MODE`: `dry-run` (report only) or `compact` (prune old revision bodies). See [extensions v2 README](src/services/extensions/v2/README.md#revisions-and-retention). +- `M2M_ISSUER`, `M2M_JWKS_URL`, `M2M_AUDIENCE`: Issuer, JWKS endpoint, and expected audience for machine-to-machine access tokens minted by the central auth service (`FOSSBilling/auth`). Defaults live in `wrangler.jsonc`; preview environments need them pointed at the issuing preview. See [extensions v2 README](src/services/extensions/v2/README.md#machine-publishing). + +Only extensions v2 consumes these. For the assertion format and the rotation procedure, see [its README](src/services/extensions/v2/README.md#authentication). + +## Development + +### Local Setup + +1. Create a `.dev.vars` file for local runtime variables and secrets: + + ```env + GITHUB_TOKEN="your-token" + ASSERTION_SIGNING_SECRET="local-shared-secret" + EXTENSIONS_REVALIDATE_SECRET="local-revalidate-secret" + # Optional while rotating the shared assertion secret. + # ASSERTION_SIGNING_SECRET_PREVIOUS="previous-local-shared-secret" + ``` + +2. Apply migrations to the local D1 databases: + + ```bash + npm run db:migrate:all:local + ``` + +3. (Optional) Store an update token in KV for `/versions/v1/update`: + + ```bash + npx wrangler kv:key put --binding AUTH_KV UPDATE_TOKEN "dev-secret" --local + ``` + +4. Spin up the dev server: + + ```bash + npm run dev + ``` + +You can now hit endpoints at `http://localhost:8787`. + +### Testing + +We use Vitest. Unit tests run under the Workers pool; Node adapter tests use a separate config. + +```bash +npm run test:workers # workers pool +npm test # workers + node adapters +``` + +```bash +npm run typecheck +npm run lint --prefix ../.. +``` + +### Deployment + +[CI/CD](../../.github/workflows/ci.yml) uses the pinned shared Cloudflare workflow +from [FOSSBilling/.workflows](https://github.com/FOSSBilling/.workflows), matching +`fossbilling.org`'s Preview URLs, deployment summaries and PR comment history. + +Validation and artifact builds run on branch pushes and PRs. A push can deploy only +after checks, the Node.js 24 packager test and D1 migrations succeed. `main` targets +production; other branches target Previews. PR runs, including forks, never receive +deployment credentials or apply remote migrations. + +The artifact contains the compiled Worker, `wrangler.json`, and SQL migrations. +Deployment does not rebuild the Worker. Production migrations use that artifact's +production bindings. Preview migrations use its generated +`wrangler.preview-migrations.json`, derived from the same bindings used by the Preview. +Migration jobs serialize access to the shared staging databases. + +#### Setup before switching deployment + +1. Configure GitHub `production` and `preview` environments, each with its own + `CLOUDFLARE_API_TOKEN` secret. Set `CLOUDFLARE_ACCOUNT_ID` as a repository variable + shared by both deployment targets. Restrict the production environment to `main`. + Deployment tokens need Workers deployment and + D1 migration permissions; Preview credentials and resources must be non-production. +2. Provision staging D1 databases, KV namespaces, a download bucket and a separately + deployed staging Extensions frontend. Set the repository variable `API_PREVIEW_RESOURCES` + to their identifiers, using the following JSON shape: + + ```json + { + "DB_EXTENSIONS": "", + "DB_CENTRAL_ALERTS": "", + "AUTH_KV": "", + "CACHE_KV": "", + "DOWNLOAD_BUCKET": "", + "EXTENSIONS_FRONTEND": "" + } + ``` + + The build refuses production resource IDs and duplicated IDs. These are operational + identifiers, not credentials. The variable is optional for local production bundle + checks, but required for Preview migrations and deployment. Previews share these + staging resources; they do not automatically receive separate D1/KV/R2 data. + +3. Configure Preview runtime secrets in the API Worker's **Previews Base** settings. + Use a separate assertion-signing secret, revalidation secret and read-only GitHub + token. Match the revalidation secret with the staging frontend. Set M2M issuer, + JWKS and audience variables there when using a separate test issuer. Operational + production variables and secrets remain dashboard-managed with `keep_vars: true`. +4. Configure required checks for the new validation/build job names, then disable the + API's Cloudflare Workers Builds Git integration to avoid duplicate deployments. + +The workflow compiles the branch's Preview settings into the artifact. Each Preview +receives isolated Durable Object state; the data bindings above are shared staging +resources. A frontend service binding calls that Worker's normal deployment, so use a +staging Worker name rather than the production Extensions worker. + +Cloudflare Previews cannot consume Queues or receive Cron Triggers. Their configuration +therefore disables packaging and notification email, and omits Queue bindings. Full +packaging/consumer testing needs a separately deployed staging Worker and Queue, or the +controlled production test. See [Cloudflare's Preview resource behaviour](https://developers.cloudflare.com/workers/previews/resources/). + +The [Cloudflare Events workflow](../../.github/workflows/cloudflare-events.yml) runs the +trusted shared comment handler on workflow completion and PR events. It reads deployment +metadata without checking out or executing PR code. Renovate tracks the pinned shared +workflow references in both files. + +#### Local deployment commands + +From the repository root, `npm run build:api` validates the bundle and prepares the +artifact without remote writes. To prepare a Preview artifact, supply +`API_PREVIEW_RESOURCES` in the environment when building. Apply its staging migrations +before running `npm run cf-preview`; CI performs those steps automatically. + +`npm run cf-deploy` remains a deliberate manual production command. It applies remote +migrations and deploys through Wrangler. Neither command deploys the separate +[extension packager](../extension-packager/README.md). Production packaging remains disabled. diff --git a/drizzle.central-alerts.config.ts b/services/api/drizzle.central-alerts.config.ts similarity index 83% rename from drizzle.central-alerts.config.ts rename to services/api/drizzle.central-alerts.config.ts index c27a05b9..0afe4f17 100644 --- a/drizzle.central-alerts.config.ts +++ b/services/api/drizzle.central-alerts.config.ts @@ -4,6 +4,6 @@ import { defineConfig } from "drizzle-kit"; // migrations_dir instead of using drizzle-kit's own D1 driver to apply. export default defineConfig({ dialect: "sqlite", - schema: "./src/services/central-alerts/v1/db/schema.ts", + schema: "./src/services/central-alerts/v1/db/schema", out: "./src/services/central-alerts/v1/db/migrations" }); diff --git a/drizzle.extensions.config.ts b/services/api/drizzle.extensions.config.ts similarity index 90% rename from drizzle.extensions.config.ts rename to services/api/drizzle.extensions.config.ts index a7078d22..02e1c473 100644 --- a/drizzle.extensions.config.ts +++ b/services/api/drizzle.extensions.config.ts @@ -7,6 +7,6 @@ import { defineConfig } from "drizzle-kit"; // Cloudflare credentials are needed just to generate a migration. export default defineConfig({ dialect: "sqlite", - schema: "./src/services/extensions/v2/db/schema.ts", + schema: "./src/services/extensions/v2/db/schema", out: "./src/services/extensions/v2/db/migrations" }); diff --git a/services/api/package.json b/services/api/package.json new file mode 100644 index 00000000..a50a79e0 --- /dev/null +++ b/services/api/package.json @@ -0,0 +1,49 @@ +{ + "name": "@fossbilling/api", + "private": true, + "version": "0.0.0", + "type": "module", + "license": "AGPL-3.0-only", + "scripts": { + "cf-typegen": "wrangler types --env-interface CloudflareBindings", + "cf-deploy": "npm run db:migrate:all:remote && wrangler deploy --minify", + "dev": "wrangler dev", + "db:generate:extensions": "drizzle-kit generate --config drizzle.extensions.config.ts", + "db:generate:central-alerts": "drizzle-kit generate --config drizzle.central-alerts.config.ts", + "db:migrate:extensions-v2:local": "wrangler d1 migrations apply DB_EXTENSIONS --local", + "db:migrate:extensions-v2:remote": "wrangler d1 migrations apply DB_EXTENSIONS --remote", + "db:migrate:central-alerts:local": "wrangler d1 migrations apply DB_CENTRAL_ALERTS --local", + "db:migrate:central-alerts:remote": "wrangler d1 migrations apply DB_CENTRAL_ALERTS --remote", + "db:migrate:all:local": "npm run db:migrate:extensions-v2:local && npm run db:migrate:central-alerts:local", + "db:migrate:all:remote": "npm run db:migrate:extensions-v2:remote && npm run db:migrate:central-alerts:remote", + "test": "npm run test:workers && npm run test:node", + "test:node": "vitest run --config vitest.node.config.ts", + "test:coverage": "vitest run --coverage", + "typecheck": "tsc --noEmit", + "test:workers": "vitest run", + "build": "wrangler deploy --dry-run --outdir dist --minify && node scripts/prepare-deployment.ts", + "cf-preview": "wrangler preview --config dist/wrangler.json" + }, + "dependencies": { + "@hono/zod-openapi": "1.6.3", + "@octokit/graphql": "9.0.5", + "@octokit/request": "10.0.16", + "@scalar/hono-api-reference": "0.12.18", + "badge-maker": "6.0.0", + "better-auth": "^1.7.14", + "drizzle-orm": "0.45.4", + "hono": "4.13.17", + "semver": "7.8.6", + "uuid": "14.0.3", + "zod": "4.6.7", + "@fossbilling/extension-packaging": "*", + "@fossbilling/logging": "*" + }, + "devDependencies": { + "@cloudflare/vitest-plugin": "1.6.1", + "@types/semver": "7.8.0", + "drizzle-kit": "0.31.11", + "wrangler": "4.158.0", + "@zip.js/zip.js": "2.24.0" + } +} diff --git a/services/api/scripts/prepare-deployment.ts b/services/api/scripts/prepare-deployment.ts new file mode 100644 index 00000000..da9851ff --- /dev/null +++ b/services/api/scripts/prepare-deployment.ts @@ -0,0 +1,115 @@ +import { + copyFileSync, + mkdirSync, + readdirSync, + rmSync, + writeFileSync +} from "node:fs"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { experimental_readRawConfig, type Unstable_RawConfig } from "wrangler"; +import { z } from "zod"; + +const PreviewResourcesSchema = z.strictObject({ + DB_EXTENSIONS: z.uuid().toLowerCase(), + DB_CENTRAL_ALERTS: z.uuid().toLowerCase(), + AUTH_KV: z.string().regex(/^[a-f0-9]{32}$/), + CACHE_KV: z.string().regex(/^[a-f0-9]{32}$/), + DOWNLOAD_BUCKET: z.string().min(1), + EXTENSIONS_FRONTEND: z.string().min(1) +}); + +// Keep the bundle, runtime bindings and migration targets in one deployable artifact. +export function prepareDeployment(root: string, previewResources?: string) { + const config: Unstable_RawConfig = experimental_readRawConfig({ + config: join(root, "wrangler.jsonc") + }).rawConfig; + const output = join(root, "dist"); + const resources = previewResources + ? PreviewResourcesSchema.parse(JSON.parse(previewResources)) + : undefined; + if (resources) { + const production = new Set([ + ...(config.d1_databases ?? []).map((value) => value.database_id), + ...(config.kv_namespaces ?? []).map((value) => value.id), + ...(config.r2_buckets ?? []).map((value) => value.bucket_name), + ...(config.services ?? []).map((value) => value.service) + ]); + const values = Object.values(resources); + if ( + values.some((value) => production.has(value)) || + resources.DB_EXTENSIONS === resources.DB_CENTRAL_ALERTS || + resources.AUTH_KV === resources.CACHE_KV + ) + throw new Error( + "API_PREVIEW_RESOURCES must use distinct non-production resources" + ); + } + mkdirSync(output, { recursive: true }); + rmSync(join(output, "migrations"), { recursive: true, force: true }); + rmSync(join(output, "wrangler.preview-migrations.json"), { force: true }); + config.d1_databases = (config.d1_databases ?? []).map((database) => { + if (!database.migrations_dir) + throw new Error(`Missing migrations_dir for ${database.binding}`); + const migrations = `migrations/${database.binding}`; + mkdirSync(join(output, migrations), { recursive: true }); + for (const filename of readdirSync( + join(root, database.migrations_dir) + ).filter((value) => value.endsWith(".sql"))) + copyFileSync( + join(root, database.migrations_dir, filename), + join(output, migrations, filename) + ); + return { ...database, migrations_dir: migrations }; + }); + if (resources) { + const databaseIds: Record = { + DB_EXTENSIONS: resources.DB_EXTENSIONS, + DB_CENTRAL_ALERTS: resources.DB_CENTRAL_ALERTS + }; + const stagingDatabases = (config.d1_databases ?? []).map((database) => { + const id = databaseIds[database.binding]; + if (!id) + throw new Error( + `No Preview database configured for ${database.binding}` + ); + return { ...database, database_id: id, database_name: undefined }; + }); + config.previews = { + ...config.previews, + d1_databases: stagingDatabases, + kv_namespaces: [ + { binding: "AUTH_KV", id: resources.AUTH_KV }, + { binding: "CACHE_KV", id: resources.CACHE_KV } + ], + r2_buckets: [ + { binding: "DOWNLOAD_BUCKET", bucket_name: resources.DOWNLOAD_BUCKET } + ], + services: [ + { + binding: "EXTENSIONS_FRONTEND", + service: resources.EXTENSIONS_FRONTEND + } + ] + }; + writeFileSync( + join(output, "wrangler.preview-migrations.json"), + JSON.stringify({ d1_databases: stagingDatabases }, null, 2) + "\n" + ); + } + Reflect.deleteProperty(config, "$schema"); + config.main = "index.js"; + config.no_bundle = true; + writeFileSync( + join(output, "wrangler.json"), + JSON.stringify(config, null, 2) + "\n" + ); +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + prepareDeployment(process.cwd(), process.env.API_PREVIEW_RESOURCES); + console.log("Prepared API deployment configuration and migrations."); +} diff --git a/src/app/index.ts b/services/api/src/app/index.ts similarity index 76% rename from src/app/index.ts rename to services/api/src/app/index.ts index 0f775643..9df2d587 100644 --- a/src/app/index.ts +++ b/services/api/src/app/index.ts @@ -1,7 +1,7 @@ import { - consumePackageJobs, - recoverPackageJobs -} from "../lib/adapters/cloudflare/package-jobs"; + consumePackagingJobs, + recoverPackagingJobs +} from "../lib/adapters/cloudflare/packaging-jobs"; import { getExtensionsDb } from "../lib/db"; import { maintainExtensionResources, @@ -9,21 +9,23 @@ import { } from "../services/extensions/v2/db/resource-maintenance"; import { Hono } from "hono"; import { HTTPException } from "hono/http-exception"; -import centralAlertsV1 from "../services/central-alerts/v1"; -import extensionsV1 from "../services/extensions/v1"; -import extensionsV2 from "../services/extensions/v2"; -import previewsV1 from "../services/previews/v1"; -import versionsV1 from "../services/versions/v1"; -import statsV1 from "../services/stats/v1"; +import centralAlertsV1 from "../services/central-alerts/v1/index"; +import extensionsV1 from "../services/extensions/v1/index"; +import extensionsV2 from "../services/extensions/v2/index"; +import previewsV1 from "../services/previews/v1/index"; +import versionsV1 from "../services/versions/v1/index"; +import statsV1 from "../services/stats/v1/index"; import { platformMiddleware } from "../lib/middleware"; -import { createCloudflareBindings } from "../lib/adapters/cloudflare"; -import { logError } from "../lib/logger"; +import { createCloudflareBindings } from "../lib/adapters/cloudflare/index"; +import { logError } from "@fossbilling/logging"; const app = new Hono<{ Bindings: CloudflareBindings }>(); app.use("*", async (c, next) => { const bindings = createCloudflareBindings(c.env); - const middleware = platformMiddleware(bindings); + const middleware = platformMiddleware(bindings, (promise) => + c.executionCtx.waitUntil(promise) + ); return middleware(c, next); }); @@ -79,7 +81,7 @@ app.all("/*", (c) => { }); export default { - queue: consumePackageJobs, + queue: consumePackagingJobs, fetch: app.fetch, request: app.request.bind(app), scheduled: async ( @@ -87,9 +89,9 @@ export default { env: CloudflareBindings ): Promise => { try { - await recoverPackageJobs(env); + await recoverPackagingJobs(env); } catch (error) { - logError("extensions/packages", "Package job recovery failed", { + logError("extensions/packages", "Packaging job recovery failed", { error: error instanceof Error ? error.message : String(error) }); } diff --git a/src/lib/adapters/cloudflare/cache.ts b/services/api/src/lib/adapters/cloudflare/cache.ts similarity index 100% rename from src/lib/adapters/cloudflare/cache.ts rename to services/api/src/lib/adapters/cloudflare/cache.ts diff --git a/src/lib/adapters/cloudflare/environment.ts b/services/api/src/lib/adapters/cloudflare/environment.ts similarity index 100% rename from src/lib/adapters/cloudflare/environment.ts rename to services/api/src/lib/adapters/cloudflare/environment.ts diff --git a/src/lib/adapters/cloudflare/index.ts b/services/api/src/lib/adapters/cloudflare/index.ts similarity index 79% rename from src/lib/adapters/cloudflare/index.ts rename to services/api/src/lib/adapters/cloudflare/index.ts index 89f80c7b..cdac7578 100644 --- a/src/lib/adapters/cloudflare/index.ts +++ b/services/api/src/lib/adapters/cloudflare/index.ts @@ -26,6 +26,15 @@ export function createCloudflareBindings( ) } : {}, + services: { + EXTENSIONS_FRONTEND: (request) => env.EXTENSIONS_FRONTEND.fetch(request) + }, + rateLimiters: { + EXTENSION_WRITE_RATE_LIMITER: async (key) => + (await env.EXTENSION_WRITE_RATE_LIMITER.limit({ key })).success, + PROFILE_CREATION_RATE_LIMITER: async (key) => + (await env.PROFILE_CREATION_RATE_LIMITER.limit({ key })).success + }, caches: { CACHE_KV: new CloudflareKVAdapter(env.CACHE_KV), AUTH_KV: new CloudflareKVAdapter(env.AUTH_KV) diff --git a/src/lib/adapters/cloudflare/package-assets.ts b/services/api/src/lib/adapters/cloudflare/package-assets.ts similarity index 92% rename from src/lib/adapters/cloudflare/package-assets.ts rename to services/api/src/lib/adapters/cloudflare/package-assets.ts index 82b2e283..0acb9494 100644 --- a/src/lib/adapters/cloudflare/package-assets.ts +++ b/services/api/src/lib/adapters/cloudflare/package-assets.ts @@ -1,9 +1,9 @@ import { PackageBuildDatabase } from "../../../services/extensions/v2/db/package-builds"; -import { logError } from "../../logger"; +import { logError } from "@fossbilling/logging"; import { revalidateExtensions } from "../../../services/extensions/v2/revalidate"; import type { Context } from "hono"; import { getPlatform } from "../../middleware"; -import { PackageError } from "../../../services/extensions/v2/packages/errors"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; // Publishing changes the listing URL only; icons are already stored in B2. async function updatePublishedIconUrl( @@ -48,7 +48,7 @@ export function schedulePublishedIconUpdate( extensionId: string ) { if (!getPlatform(c).getEnv("EXTENSION_ASSETS_S3_ENDPOINT")) return; - c.executionCtx.waitUntil( + getPlatform(c).waitUntil( updatePublishedIconUrl(c, extensionId) .then((changed) => { if (changed) revalidateExtensions(c); diff --git a/src/lib/adapters/cloudflare/package-jobs.ts b/services/api/src/lib/adapters/cloudflare/packaging-jobs.ts similarity index 53% rename from src/lib/adapters/cloudflare/package-jobs.ts rename to services/api/src/lib/adapters/cloudflare/packaging-jobs.ts index 8e50757e..74a2291d 100644 --- a/src/lib/adapters/cloudflare/package-jobs.ts +++ b/services/api/src/lib/adapters/cloudflare/packaging-jobs.ts @@ -1,25 +1,25 @@ -import { logError } from "../../logger"; +import { logError } from "@fossbilling/logging"; import { createCloudflareBindings } from "./index"; -import { PackageJobsDatabase } from "../../../services/extensions/v2/db/package-jobs"; +import { PackagingJobsDatabase } from "../../../services/extensions/v2/db/packaging-jobs"; import { PackageBuildDatabase } from "../../../services/extensions/v2/db/package-builds"; import { prepareQueuedPackage, type PackagePreparation } from "../../../services/extensions/v2/packages/prepare"; import { - processPackageJob, - dispatchPackageJob + processPackagingJob, + dispatchPackagingJob } from "../../../services/extensions/v2/packages/jobs"; -import { PackageError } from "../../../services/extensions/v2/packages/errors"; -import { configuredPackageStore } from "./package-store"; -import { CloudflarePackageImages } from "./package-images"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; +import { configuredPackageStore } from "../../../services/extensions/v2/packages/store"; +import { HttpPackageProcessor } from "@fossbilling/extension-packaging/adapters/http"; export function packagePreparation( env: CloudflareBindings ): PackagePreparation { const bindings = createCloudflareBindings(env); const getEnv = (name: string) => bindings.environment.get(name); - const store = configuredPackageStore(env, getEnv); + const store = configuredPackageStore(getEnv); const baseUrl = getEnv("EXTENSION_PACKAGES_BASE_URL"); if (!store || !baseUrl) throw new PackageError( @@ -27,19 +27,31 @@ export function packagePreparation( "Package storage or download URL is not configured", 503 ); + const processorUrl = getEnv("EXTENSION_PACKAGER_URL"); + const processorSecret = getEnv("EXTENSION_PACKAGER_SECRET"); + if (!processorUrl || !processorSecret) + throw new PackageError( + "PACKAGES_DISABLED", + "Package processor is not configured", + 503 + ); return { repository: new PackageBuildDatabase(env.DB_EXTENSIONS), store, - images: new CloudflarePackageImages(env.PACKAGE_IMAGES), + processor: new HttpPackageProcessor( + processorUrl, + processorSecret, + getEnv("ENVIRONMENT") !== "production" + ), baseUrl }; } -export async function consumePackageJobs( +export async function consumePackagingJobs( batch: MessageBatch<{ jobId: string }>, env: CloudflareBindings ) { - const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); + const jobs = new PackagingJobsDatabase(env.DB_EXTENSIONS); for (const message of batch.messages) { if ( !message.body || @@ -50,19 +62,25 @@ export async function consumePackageJobs( continue; } try { - const result = await processPackageJob(jobs, message.body.jobId, (job) => - prepareQueuedPackage(packagePreparation(env), job) + const result = await processPackagingJob( + jobs, + message.body.jobId, + (job) => prepareQueuedPackage(packagePreparation(env), job) ); if (result === "busy") message.retry({ delaySeconds: 1200 }); else if (result === "retry") message.retry({ delaySeconds: 60 }); else message.ack(); - } catch { + } catch (error) { + logError("extensions/packages", "Package consumer failed", { + jobId: message.body.jobId, + error: error instanceof Error ? error.message : String(error) + }); message.retry({ delaySeconds: 60 }); } } } -export async function recoverPackageJobs(env: CloudflareBindings) { +export async function recoverPackagingJobs(env: CloudflareBindings) { const mode = createCloudflareBindings(env).environment.get( "EXTENSION_PACKAGES_MODE" ); @@ -71,14 +89,14 @@ export async function recoverPackageJobs(env: CloudflareBindings) { !["local", "enabled"].includes(mode ?? "") ) return; - const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); + const jobs = new PackagingJobsDatabase(env.DB_EXTENSIONS); await jobs.expire(); const queue = createCloudflareBindings(env).queues!.EXTENSION_PACKAGE_QUEUE; for (const id of await jobs.pending()) { try { - await dispatchPackageJob(jobs, queue, id); + await dispatchPackagingJob(jobs, queue, id); } catch (error) { - logError("extensions/packages", "Package job dispatch failed", { + logError("extensions/packages", "Packaging job dispatch failed", { jobId: id, error: error instanceof Error ? error.message : String(error) }); diff --git a/src/lib/adapters/cloudflare/queue.ts b/services/api/src/lib/adapters/cloudflare/queue.ts similarity index 100% rename from src/lib/adapters/cloudflare/queue.ts rename to services/api/src/lib/adapters/cloudflare/queue.ts diff --git a/src/lib/adapters/node/cache.ts b/services/api/src/lib/adapters/node/cache.ts similarity index 100% rename from src/lib/adapters/node/cache.ts rename to services/api/src/lib/adapters/node/cache.ts diff --git a/src/lib/adapters/node/environment.ts b/services/api/src/lib/adapters/node/environment.ts similarity index 100% rename from src/lib/adapters/node/environment.ts rename to services/api/src/lib/adapters/node/environment.ts diff --git a/src/lib/adapters/node/index.ts b/services/api/src/lib/adapters/node/index.ts similarity index 100% rename from src/lib/adapters/node/index.ts rename to services/api/src/lib/adapters/node/index.ts diff --git a/src/lib/auth/bearer-assertion.ts b/services/api/src/lib/auth/bearer-assertion.ts similarity index 99% rename from src/lib/auth/bearer-assertion.ts rename to services/api/src/lib/auth/bearer-assertion.ts index df38571b..377b7856 100644 --- a/src/lib/auth/bearer-assertion.ts +++ b/services/api/src/lib/auth/bearer-assertion.ts @@ -1,5 +1,5 @@ import { verify as verifyJwt } from "hono/jwt"; -import { logWarn } from "../logger"; +import { logWarn } from "@fossbilling/logging"; import { AuthPrincipal, TokenVerifier } from "./interfaces"; const CLOCK_SKEW_SECONDS = 5; diff --git a/src/lib/auth/index.ts b/services/api/src/lib/auth/index.ts similarity index 100% rename from src/lib/auth/index.ts rename to services/api/src/lib/auth/index.ts diff --git a/src/lib/auth/interfaces.ts b/services/api/src/lib/auth/interfaces.ts similarity index 95% rename from src/lib/auth/interfaces.ts rename to services/api/src/lib/auth/interfaces.ts index 9865b87b..863a81da 100644 --- a/src/lib/auth/interfaces.ts +++ b/services/api/src/lib/auth/interfaces.ts @@ -21,6 +21,6 @@ export type AuthPrincipal = export interface TokenVerifier { verify( token: string, - platform: PlatformContext + platform: Pick ): Promise; } diff --git a/src/lib/auth/m2m-bearer.ts b/services/api/src/lib/auth/m2m-bearer.ts similarity index 98% rename from src/lib/auth/m2m-bearer.ts rename to services/api/src/lib/auth/m2m-bearer.ts index a3316b8b..472181ed 100644 --- a/src/lib/auth/m2m-bearer.ts +++ b/services/api/src/lib/auth/m2m-bearer.ts @@ -1,6 +1,6 @@ import { verifyBearerToken } from "better-auth/oauth2"; import type { AuthPrincipal, TokenVerifier } from "./interfaces"; -import { logWarn } from "../logger"; +import { logWarn } from "@fossbilling/logging"; // Machine-to-machine bearer tokens minted by the central auth service's // client_credentials grant (FOSSBilling/auth). Tokens carrying a `resource` diff --git a/src/lib/auth/middleware.ts b/services/api/src/lib/auth/middleware.ts similarity index 100% rename from src/lib/auth/middleware.ts rename to services/api/src/lib/auth/middleware.ts diff --git a/src/lib/cache.ts b/services/api/src/lib/cache.ts similarity index 100% rename from src/lib/cache.ts rename to services/api/src/lib/cache.ts diff --git a/src/lib/context.ts b/services/api/src/lib/context.ts similarity index 51% rename from src/lib/context.ts rename to services/api/src/lib/context.ts index 5210bb67..29f20d3e 100644 --- a/src/lib/context.ts +++ b/services/api/src/lib/context.ts @@ -1,16 +1,17 @@ -import { Context } from "hono"; import { ICache, IMessageQueue, IPlatformBindings } from "./interfaces"; export interface PlatformContext { getQueue(name: string): IMessageQueue; getCache(name: string): ICache; getEnv(key: string): string | undefined; - readonly raw: Context; + fetchService(name: string, request: Request): Promise; + checkRateLimit(name: string, key: string): Promise; + waitUntil(promise: Promise): void; } export function createPlatformContext( - c: Context, - bindings: IPlatformBindings + bindings: IPlatformBindings, + waitUntil: (promise: Promise) => void ): PlatformContext { return { getQueue(name: string): IMessageQueue { @@ -30,8 +31,18 @@ export function createPlatformContext( return bindings.environment.get(key); }, - get raw(): Context { - return c; - } + async fetchService(name: string, request: Request): Promise { + const service = bindings.services?.[name]; + if (!service) throw new Error(`Service binding '${name}' not found`); + return service(request); + }, + + async checkRateLimit(name: string, key: string): Promise { + const limiter = bindings.rateLimiters?.[name]; + if (!limiter) throw new Error(`Rate limiter binding '${name}' not found`); + return limiter(key); + }, + + waitUntil }; } diff --git a/src/lib/db.ts b/services/api/src/lib/db.ts similarity index 100% rename from src/lib/db.ts rename to services/api/src/lib/db.ts diff --git a/src/lib/github-errors.ts b/services/api/src/lib/github-errors.ts similarity index 100% rename from src/lib/github-errors.ts rename to services/api/src/lib/github-errors.ts diff --git a/src/lib/hash.ts b/services/api/src/lib/hash.ts similarity index 100% rename from src/lib/hash.ts rename to services/api/src/lib/hash.ts diff --git a/src/lib/interfaces.ts b/services/api/src/lib/interfaces.ts similarity index 84% rename from src/lib/interfaces.ts rename to services/api/src/lib/interfaces.ts index a4523baa..3ce40e8a 100644 --- a/src/lib/interfaces.ts +++ b/services/api/src/lib/interfaces.ts @@ -27,6 +27,8 @@ export interface IMessageQueue { export interface IPlatformBindings { queues?: Record; + services?: Record Promise>; + rateLimiters?: Record Promise>; caches: Record; environment: IEnvironment; } diff --git a/src/lib/json.ts b/services/api/src/lib/json.ts similarity index 100% rename from src/lib/json.ts rename to services/api/src/lib/json.ts diff --git a/src/lib/middleware.ts b/services/api/src/lib/middleware.ts similarity index 79% rename from src/lib/middleware.ts rename to services/api/src/lib/middleware.ts index 404eb5ca..46b1e304 100644 --- a/src/lib/middleware.ts +++ b/services/api/src/lib/middleware.ts @@ -9,10 +9,11 @@ declare module "hono" { } export function platformMiddleware( - bindings: IPlatformBindings + bindings: IPlatformBindings, + waitUntil: (promise: Promise) => void ): MiddlewareHandler { return async (c, next) => { - c.set("platform", createPlatformContext(c, bindings)); + c.set("platform", createPlatformContext(bindings, waitUntil)); await next(); }; } diff --git a/src/lib/pagination.ts b/services/api/src/lib/pagination.ts similarity index 100% rename from src/lib/pagination.ts rename to services/api/src/lib/pagination.ts diff --git a/src/lib/releases.ts b/services/api/src/lib/releases.ts similarity index 100% rename from src/lib/releases.ts rename to services/api/src/lib/releases.ts diff --git a/src/services/central-alerts/v1/README.md b/services/api/src/services/central-alerts/v1/README.md similarity index 100% rename from src/services/central-alerts/v1/README.md rename to services/api/src/services/central-alerts/v1/README.md diff --git a/src/services/central-alerts/v1/database.ts b/services/api/src/services/central-alerts/v1/database.ts similarity index 100% rename from src/services/central-alerts/v1/database.ts rename to services/api/src/services/central-alerts/v1/database.ts diff --git a/src/services/central-alerts/v1/db/migrations/0000_create_central_alerts.sql b/services/api/src/services/central-alerts/v1/db/migrations/0000_create_central_alerts.sql similarity index 100% rename from src/services/central-alerts/v1/db/migrations/0000_create_central_alerts.sql rename to services/api/src/services/central-alerts/v1/db/migrations/0000_create_central_alerts.sql diff --git a/src/services/central-alerts/v1/db/migrations/0001_seed_initial_alert.sql b/services/api/src/services/central-alerts/v1/db/migrations/0001_seed_initial_alert.sql similarity index 100% rename from src/services/central-alerts/v1/db/migrations/0001_seed_initial_alert.sql rename to services/api/src/services/central-alerts/v1/db/migrations/0001_seed_initial_alert.sql diff --git a/src/services/central-alerts/v1/db/migrations/meta/0000_snapshot.json b/services/api/src/services/central-alerts/v1/db/migrations/meta/0000_snapshot.json similarity index 100% rename from src/services/central-alerts/v1/db/migrations/meta/0000_snapshot.json rename to services/api/src/services/central-alerts/v1/db/migrations/meta/0000_snapshot.json diff --git a/src/services/central-alerts/v1/db/migrations/meta/0001_snapshot.json b/services/api/src/services/central-alerts/v1/db/migrations/meta/0001_snapshot.json similarity index 100% rename from src/services/central-alerts/v1/db/migrations/meta/0001_snapshot.json rename to services/api/src/services/central-alerts/v1/db/migrations/meta/0001_snapshot.json diff --git a/src/services/central-alerts/v1/db/migrations/meta/_journal.json b/services/api/src/services/central-alerts/v1/db/migrations/meta/_journal.json similarity index 100% rename from src/services/central-alerts/v1/db/migrations/meta/_journal.json rename to services/api/src/services/central-alerts/v1/db/migrations/meta/_journal.json diff --git a/src/services/central-alerts/v1/db/schema.ts b/services/api/src/services/central-alerts/v1/db/schema.ts similarity index 100% rename from src/services/central-alerts/v1/db/schema.ts rename to services/api/src/services/central-alerts/v1/db/schema.ts diff --git a/src/services/central-alerts/v1/index.ts b/services/api/src/services/central-alerts/v1/index.ts similarity index 94% rename from src/services/central-alerts/v1/index.ts rename to services/api/src/services/central-alerts/v1/index.ts index 60a16a00..9bba90ac 100644 --- a/src/services/central-alerts/v1/index.ts +++ b/services/api/src/services/central-alerts/v1/index.ts @@ -1,9 +1,10 @@ +import { getPlatform } from "../../../lib/middleware"; import { Hono } from "hono"; import { cors } from "hono/cors"; import { trimTrailingSlash } from "hono/trailing-slash"; import { CentralAlertsDatabase } from "./database"; import { getCentralAlertsDb } from "../../../lib/db"; -import { logError } from "../../../lib/logger"; +import { logError } from "@fossbilling/logging"; import { parseLegacyPagination } from "../../../lib/pagination"; const centralAlertsV1 = new Hono<{ Bindings: CloudflareBindings }>(); @@ -86,7 +87,7 @@ centralAlertsV1.get("/list", async (c) => { error: null }); if (edgeCache) { - c.executionCtx.waitUntil(edgeCache.put(cacheUrl.href, response.clone())); + getPlatform(c).waitUntil(edgeCache.put(cacheUrl.href, response.clone())); } return response; }); diff --git a/src/services/central-alerts/v1/interfaces.ts b/services/api/src/services/central-alerts/v1/interfaces.ts similarity index 100% rename from src/services/central-alerts/v1/interfaces.ts rename to services/api/src/services/central-alerts/v1/interfaces.ts diff --git a/src/services/extensions/v1/README.md b/services/api/src/services/extensions/v1/README.md similarity index 100% rename from src/services/extensions/v1/README.md rename to services/api/src/services/extensions/v1/README.md diff --git a/src/services/extensions/v1/cloudflare-rules.json b/services/api/src/services/extensions/v1/cloudflare-rules.json similarity index 100% rename from src/services/extensions/v1/cloudflare-rules.json rename to services/api/src/services/extensions/v1/cloudflare-rules.json diff --git a/src/services/extensions/v1/database.ts b/services/api/src/services/extensions/v1/database.ts similarity index 100% rename from src/services/extensions/v1/database.ts rename to services/api/src/services/extensions/v1/database.ts diff --git a/src/services/extensions/v1/db/schema.sql b/services/api/src/services/extensions/v1/db/schema.sql similarity index 100% rename from src/services/extensions/v1/db/schema.sql rename to services/api/src/services/extensions/v1/db/schema.sql diff --git a/src/services/extensions/v1/index.ts b/services/api/src/services/extensions/v1/index.ts similarity index 92% rename from src/services/extensions/v1/index.ts rename to services/api/src/services/extensions/v1/index.ts index 74d254aa..8a0f38ea 100644 --- a/src/services/extensions/v1/index.ts +++ b/services/api/src/services/extensions/v1/index.ts @@ -1,3 +1,4 @@ +import { getPlatform } from "../../../lib/middleware"; import { Hono, type Context } from "hono"; import { cors } from "hono/cors"; import { trimTrailingSlash } from "hono/trailing-slash"; @@ -32,6 +33,7 @@ extensionsV1.use("/*", trimTrailingSlash()); // nondeterministic row order. extensionsV1.get("/list", async (c) => { + const platform = getPlatform(c); const db = new ExtensionsDatabase(getExtensionsDb(c.env.DB_EXTENSIONS)); const type = c.req.query("type"); @@ -56,7 +58,7 @@ extensionsV1.get("/list", async (c) => { async (): Promise => { let cached: string | null = null; try { - cached = await c.env.CACHE_KV.get(cacheKey); + cached = await platform.getCache("CACHE_KV").get(cacheKey); } catch { // Fall through to the D1 build. } @@ -77,10 +79,13 @@ extensionsV1.get("/list", async (c) => { } : {}) }); - c.executionCtx.waitUntil( - c.env.CACHE_KV.put(cacheKey, serialized, { - expirationTtl: LIST_CACHE_TTL_SECONDS - }).catch(() => {}) + platform.waitUntil( + getPlatform(c) + .getCache("CACHE_KV") + .put(cacheKey, serialized, { + expirationTtl: LIST_CACHE_TTL_SECONDS + }) + .catch(() => {}) ); return serialized; } diff --git a/src/services/extensions/v1/interfaces.ts b/services/api/src/services/extensions/v1/interfaces.ts similarity index 100% rename from src/services/extensions/v1/interfaces.ts rename to services/api/src/services/extensions/v1/interfaces.ts diff --git a/src/services/extensions/v1/list-cache.ts b/services/api/src/services/extensions/v1/list-cache.ts similarity index 96% rename from src/services/extensions/v1/list-cache.ts rename to services/api/src/services/extensions/v1/list-cache.ts index 6e276f8b..af734243 100644 --- a/src/services/extensions/v1/list-cache.ts +++ b/services/api/src/services/extensions/v1/list-cache.ts @@ -1,3 +1,4 @@ +import type { ICache } from "../../../lib/interfaces"; // Cache-key surface for the v1 list body cache (see index.ts). Kept separate // so extensions/v2's revalidateExtensions can invalidate it after extension directory // mutations without importing the v1 app module (which would drag the whole @@ -27,7 +28,7 @@ export function listCacheKey( export const LIST_CACHE_TTL_SECONDS = 60; export function invalidateListCache( - kv: KVNamespace, + kv: ICache, waitUntil: (promise: Promise) => void ): void { // Mirrors listCacheKey's encoding: the "all" sentinel for the unfiltered diff --git a/src/services/extensions/v2/PACKAGING.md b/services/api/src/services/extensions/v2/PACKAGING.md similarity index 86% rename from src/services/extensions/v2/PACKAGING.md rename to services/api/src/services/extensions/v2/PACKAGING.md index 7cabbd67..384febe0 100644 --- a/src/services/extensions/v2/PACKAGING.md +++ b/services/api/src/services/extensions/v2/PACKAGING.md @@ -14,11 +14,12 @@ Run the API through the normal `npm run dev` command. Use ignored `.dev.vars` for local credentials, set `ENVIRONMENT=local` and `EXTENSION_PACKAGES_MODE=local`, and set `EXTENSION_PACKAGES_BASE_URL` to the local API's `/extensions/v2` URL. Apply local migrations with `npm run db:migrate:all:local`. The configured Queue -and Images bindings run locally under Wrangler; B2 requests use the configured -buckets and development keys, so use a separate key and package object prefix. +runs locally under Wrangler; start the Node packager as described below. B2 +requests use the configured buckets and development keys, so use a separate key +and package object prefix. -`npm run test:all` covers the package lifecycle through the real API routes and -D1 repository, with mocked S3 responses and emulated image processing. Node tests +From the repository root, `npm test` covers the package lifecycle through the real API routes and +D1 repository, with mocked S3 responses and prepared image fixtures. Node tests also exercise package preparation and B2 storage without a Cloudflare runtime. ## API and review @@ -32,7 +33,7 @@ The OpenAPI reference is available at `/extensions/v2/docs`. `extensions:push` and an active mapped developer binding. Moderators may rebuild published stored source for a correction, but cannot upload new files for another developer. -- `GET /extensions/v2/package-jobs/{jobId}` returns `queued`, `processing`, `ready` +- `GET /extensions/v2/packaging-jobs/{jobId}` returns `queued`, `processing`, `ready` or `failed`. Ready jobs include the artifact; failed jobs include an actionable error. Reads require the current owning developer/publisher or a moderator, and are never publicly cached. Identical requests reuse queued/completed work. @@ -63,12 +64,15 @@ processing. The staged upload becomes the immutable source object after validati so it is not uploaded to package storage a second time. The generic `IMessageQueue` producer interface lives in `src/lib/interfaces.ts`; -`CloudflareQueueAdapter` implements it. `processPackageJob` depends on a -`PackageJobRepository` and a preparation callback, not Cloudflare MessageBatch, -acknowledgements or retry APIs. Package preparation accepts a `PackageRepository`, `PackageStore` and -`PackageImageProcessor`. SQL and guarded writes live in `db/package-builds.ts`; -Cloudflare bindings, queue delivery and image processing live under -`src/lib/adapters/cloudflare/`, and the B2 store lives under `src/lib/adapters/b2/`. +`CloudflareQueueAdapter` implements it. `processPackagingJob` depends on a +`PackagingJobRepository` and a preparation callback, not Cloudflare MessageBatch, +acknowledgements or retry APIs. Package preparation accepts a `PackageRepository`, +`PackageStore` and `PackageProcessor`. SQL and guarded writes live in `db/package-builds.ts`; +Cloudflare bindings and queue delivery live under `src/lib/adapters/cloudflare/`. +The `@fossbilling/extension-packaging` workspace defines the shared contracts and adapters. +A `PackageProcessor` handles ZIP validation, manifest replacement and image preparation. +The API calls the shared HTTP adapter; the Netlify entry point runs the +same Node handler used locally. The B2 store lives under `libraries/extension-packaging/src/adapters/b2.ts`. The package workflow contains no Cloudflare binding types or SDK calls. A different runtime supplies these interfaces and its own queue consumer; the rest of the API still has existing platform integrations outside this package workflow. @@ -128,10 +132,11 @@ UUIDv7 draft identity after package validation, before the first icon is stored. The first review submission reuses that draft identity. SVGs are sanitized with Cloudflare svg-hush through the pinned `@zeropress/svg-hush` -Workers adapter and served directly. SVG inputs are bounded by the ZIP expansion -limit; there is no separate SVG size cap. Raster inputs use the local `PACKAGE_IMAGES` binding to produce +Node adapter and served directly. SVG inputs are bounded by the ZIP expansion +limit and the sanitizer's built-in input/output limits. Raster inputs use Sharp in the Node packager to produce AVIF and PNG variants within 256 × 256 pixels, preserving proportions and -transparency without enlarging small images. Requests prefer SVG when present, +transparency without enlarging small images. Raster inputs are limited to 16 megapixels. +Requests prefer SVG when present, otherwise AVIF when explicitly accepted, with PNG fallback. They read stored images directly and never reopen a ZIP. @@ -259,8 +264,8 @@ Before deployment (including an automatic deployment after merge): the private package bucket. Supply separate `EXTENSION_ASSETS_S3_ACCESS_KEY_ID` and `EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY` for the public asset bucket. Development keys in `.dev.vars` are not production secrets. -3. Confirm the Images binding is enabled for the account, the scoped CDN security - headers remain configured, and B2 caps/alerts are in place. +3. Configure the packager URL and shared secret described below, confirm the + scoped CDN security headers remain configured, and keep B2 caps/alerts in place. 4. Apply the package and job tables through the normal D1 deployment workflow. 5. After queue-consumer capacity and moderation requirements are agreed, explicitly set `EXTENSION_PACKAGES_MODE=enabled` in the reviewed production configuration. @@ -270,7 +275,7 @@ Before deployment (including an automatic deployment after merge): Production requires HTTPS download URLs with the `/extensions/v2` base path, complete credentials for both B2 buckets, an explicit non-`local` ZIP object prefix, -a public asset URL and the Images binding. An incomplete configuration fails with 503; +a public asset URL and an authenticated extension packager. An incomplete configuration fails with 503; it cannot fall back to local ZIP storage. Private package objects use the configured prefix. Public image objects use the UUID-based layout described above. @@ -280,5 +285,15 @@ it is not an upload-only pause switch. Prepared CDN images remain accessible ind Before enabling package mode, measure admission and queue-consumer invocations separately: CPU and wall duration, memory failures, queue delay, retries, and -storage/image-service latency. Include near-limit ZIPs and SVG/raster icons. +storage and packager latency. Include near-limit ZIPs and SVG/raster icons. Local functional tests do not establish production capacity or billing. + +## Extension packager + +The API dispatches ZIP and image work to a separately deployed Node service through +an authenticated HTTP adapter. Requests and responses contain metadata and private +B2 object keys, not ZIP bytes. The API retains ownership, leases and publication. + +See the [packager setup and deployment guide](../../../../../extension-packager/README.md) for Netlify configuration, +local development, credentials, deadlines and runtime verification. The shared +contracts and portable ZIP implementation live in `libraries/extension-packaging`. diff --git a/src/services/extensions/v2/README.md b/services/api/src/services/extensions/v2/README.md similarity index 100% rename from src/services/extensions/v2/README.md rename to services/api/src/services/extensions/v2/README.md diff --git a/src/services/extensions/v2/db/badges.ts b/services/api/src/services/extensions/v2/db/badges.ts similarity index 100% rename from src/services/extensions/v2/db/badges.ts rename to services/api/src/services/extensions/v2/db/badges.ts diff --git a/src/services/extensions/v2/db/batch.ts b/services/api/src/services/extensions/v2/db/batch.ts similarity index 100% rename from src/services/extensions/v2/db/batch.ts rename to services/api/src/services/extensions/v2/db/batch.ts diff --git a/src/services/extensions/v2/db/claim-verification-budget.ts b/services/api/src/services/extensions/v2/db/claim-verification-budget.ts similarity index 100% rename from src/services/extensions/v2/db/claim-verification-budget.ts rename to services/api/src/services/extensions/v2/db/claim-verification-budget.ts diff --git a/src/services/extensions/v2/db/columns.ts b/services/api/src/services/extensions/v2/db/columns.ts similarity index 100% rename from src/services/extensions/v2/db/columns.ts rename to services/api/src/services/extensions/v2/db/columns.ts diff --git a/src/services/extensions/v2/db/cursor.ts b/services/api/src/services/extensions/v2/db/cursor.ts similarity index 100% rename from src/services/extensions/v2/db/cursor.ts rename to services/api/src/services/extensions/v2/db/cursor.ts diff --git a/src/services/extensions/v2/db/developer-approval.ts b/services/api/src/services/extensions/v2/db/developer-approval.ts similarity index 100% rename from src/services/extensions/v2/db/developer-approval.ts rename to services/api/src/services/extensions/v2/db/developer-approval.ts diff --git a/src/services/extensions/v2/db/developer-claims.ts b/services/api/src/services/extensions/v2/db/developer-claims.ts similarity index 100% rename from src/services/extensions/v2/db/developer-claims.ts rename to services/api/src/services/extensions/v2/db/developer-claims.ts diff --git a/src/services/extensions/v2/db/developer-profiles.ts b/services/api/src/services/extensions/v2/db/developer-profiles.ts similarity index 100% rename from src/services/extensions/v2/db/developer-profiles.ts rename to services/api/src/services/extensions/v2/db/developer-profiles.ts diff --git a/src/services/extensions/v2/db/developer-transfers.ts b/services/api/src/services/extensions/v2/db/developer-transfers.ts similarity index 100% rename from src/services/extensions/v2/db/developer-transfers.ts rename to services/api/src/services/extensions/v2/db/developer-transfers.ts diff --git a/src/services/extensions/v2/db/errors.ts b/services/api/src/services/extensions/v2/db/errors.ts similarity index 99% rename from src/services/extensions/v2/db/errors.ts rename to services/api/src/services/extensions/v2/db/errors.ts index 7ad32e3a..02a30bfc 100644 --- a/src/services/extensions/v2/db/errors.ts +++ b/services/api/src/services/extensions/v2/db/errors.ts @@ -2,7 +2,7 @@ import { DatabaseError } from "../../../../lib/interfaces"; import { ExtensionsDb } from "../../../../lib/db"; import { UsersDatabase } from "./users"; import { DatabaseResult } from "../../../../lib/interfaces"; -import { logInfo, logError } from "../../../../lib/logger"; +import { logInfo, logError } from "@fossbilling/logging"; // Drizzle wraps driver errors; constraint classifiers inspect the cause chain. export function errorMessageChain(error: unknown): string { diff --git a/src/services/extensions/v2/db/extensions.ts b/services/api/src/services/extensions/v2/db/extensions.ts similarity index 100% rename from src/services/extensions/v2/db/extensions.ts rename to services/api/src/services/extensions/v2/db/extensions.ts diff --git a/src/services/extensions/v2/db/manifest-preflight.sql b/services/api/src/services/extensions/v2/db/manifest-preflight.sql similarity index 100% rename from src/services/extensions/v2/db/manifest-preflight.sql rename to services/api/src/services/extensions/v2/db/manifest-preflight.sql diff --git a/src/services/extensions/v2/db/migrations/0000_bootstrap_users.sql b/services/api/src/services/extensions/v2/db/migrations/0000_bootstrap_users.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0000_bootstrap_users.sql rename to services/api/src/services/extensions/v2/db/migrations/0000_bootstrap_users.sql diff --git a/src/services/extensions/v2/db/migrations/0001_add_v2_tables.sql b/services/api/src/services/extensions/v2/db/migrations/0001_add_v2_tables.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0001_add_v2_tables.sql rename to services/api/src/services/extensions/v2/db/migrations/0001_add_v2_tables.sql diff --git a/src/services/extensions/v2/db/migrations/0002_add_author_approval.sql b/services/api/src/services/extensions/v2/db/migrations/0002_add_author_approval.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0002_add_author_approval.sql rename to services/api/src/services/extensions/v2/db/migrations/0002_add_author_approval.sql diff --git a/src/services/extensions/v2/db/migrations/0003_add_author_profile_fields.sql b/services/api/src/services/extensions/v2/db/migrations/0003_add_author_profile_fields.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0003_add_author_profile_fields.sql rename to services/api/src/services/extensions/v2/db/migrations/0003_add_author_profile_fields.sql diff --git a/src/services/extensions/v2/db/migrations/0004_add_authors_owner_unique.sql b/services/api/src/services/extensions/v2/db/migrations/0004_add_authors_owner_unique.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0004_add_authors_owner_unique.sql rename to services/api/src/services/extensions/v2/db/migrations/0004_add_authors_owner_unique.sql diff --git a/src/services/extensions/v2/db/migrations/0005_add_author_history.sql b/services/api/src/services/extensions/v2/db/migrations/0005_add_author_history.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0005_add_author_history.sql rename to services/api/src/services/extensions/v2/db/migrations/0005_add_author_history.sql diff --git a/src/services/extensions/v2/db/migrations/0006_add_author_transfers.sql b/services/api/src/services/extensions/v2/db/migrations/0006_add_author_transfers.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0006_add_author_transfers.sql rename to services/api/src/services/extensions/v2/db/migrations/0006_add_author_transfers.sql diff --git a/src/services/extensions/v2/db/migrations/0007_add_author_claims.sql b/services/api/src/services/extensions/v2/db/migrations/0007_add_author_claims.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0007_add_author_claims.sql rename to services/api/src/services/extensions/v2/db/migrations/0007_add_author_claims.sql diff --git a/src/services/extensions/v2/db/migrations/0008_rename_authors_to_developers.sql b/services/api/src/services/extensions/v2/db/migrations/0008_rename_authors_to_developers.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0008_rename_authors_to_developers.sql rename to services/api/src/services/extensions/v2/db/migrations/0008_rename_authors_to_developers.sql diff --git a/src/services/extensions/v2/db/migrations/0009_drop_developer_history_fk.sql b/services/api/src/services/extensions/v2/db/migrations/0009_drop_developer_history_fk.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0009_drop_developer_history_fk.sql rename to services/api/src/services/extensions/v2/db/migrations/0009_drop_developer_history_fk.sql diff --git a/src/services/extensions/v2/db/migrations/0010_drop_developer_bio.sql b/services/api/src/services/extensions/v2/db/migrations/0010_drop_developer_bio.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0010_drop_developer_bio.sql rename to services/api/src/services/extensions/v2/db/migrations/0010_drop_developer_bio.sql diff --git a/src/services/extensions/v2/db/migrations/0011_harden_review_workflows.sql b/services/api/src/services/extensions/v2/db/migrations/0011_harden_review_workflows.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0011_harden_review_workflows.sql rename to services/api/src/services/extensions/v2/db/migrations/0011_harden_review_workflows.sql diff --git a/src/services/extensions/v2/db/migrations/0012_add_developer_claims_github_verification.sql b/services/api/src/services/extensions/v2/db/migrations/0012_add_developer_claims_github_verification.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0012_add_developer_claims_github_verification.sql rename to services/api/src/services/extensions/v2/db/migrations/0012_add_developer_claims_github_verification.sql diff --git a/src/services/extensions/v2/db/migrations/0013_add_developers_github_verification.sql b/services/api/src/services/extensions/v2/db/migrations/0013_add_developers_github_verification.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0013_add_developers_github_verification.sql rename to services/api/src/services/extensions/v2/db/migrations/0013_add_developers_github_verification.sql diff --git a/src/services/extensions/v2/db/migrations/0014_drizzle_baseline.sql b/services/api/src/services/extensions/v2/db/migrations/0014_drizzle_baseline.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0014_drizzle_baseline.sql rename to services/api/src/services/extensions/v2/db/migrations/0014_drizzle_baseline.sql diff --git a/src/services/extensions/v2/db/migrations/0015_add_developer_github_verified_at.sql b/services/api/src/services/extensions/v2/db/migrations/0015_add_developer_github_verified_at.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0015_add_developer_github_verified_at.sql rename to services/api/src/services/extensions/v2/db/migrations/0015_add_developer_github_verified_at.sql diff --git a/src/services/extensions/v2/db/migrations/0016_add_developer_github_url_verified.sql b/services/api/src/services/extensions/v2/db/migrations/0016_add_developer_github_url_verified.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0016_add_developer_github_url_verified.sql rename to services/api/src/services/extensions/v2/db/migrations/0016_add_developer_github_url_verified.sql diff --git a/src/services/extensions/v2/db/migrations/0017_add_developer_url_check_cooldown.sql b/services/api/src/services/extensions/v2/db/migrations/0017_add_developer_url_check_cooldown.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0017_add_developer_url_check_cooldown.sql rename to services/api/src/services/extensions/v2/db/migrations/0017_add_developer_url_check_cooldown.sql diff --git a/src/services/extensions/v2/db/migrations/0018_add_extension_catalogue_indexes.sql b/services/api/src/services/extensions/v2/db/migrations/0018_add_extension_catalogue_indexes.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0018_add_extension_catalogue_indexes.sql rename to services/api/src/services/extensions/v2/db/migrations/0018_add_extension_catalogue_indexes.sql diff --git a/src/services/extensions/v2/db/migrations/0019_add_user_deleted_at.sql b/services/api/src/services/extensions/v2/db/migrations/0019_add_user_deleted_at.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0019_add_user_deleted_at.sql rename to services/api/src/services/extensions/v2/db/migrations/0019_add_user_deleted_at.sql diff --git a/src/services/extensions/v2/db/migrations/0020_check_reserved_route_ids.sql b/services/api/src/services/extensions/v2/db/migrations/0020_check_reserved_route_ids.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0020_check_reserved_route_ids.sql rename to services/api/src/services/extensions/v2/db/migrations/0020_check_reserved_route_ids.sql diff --git a/src/services/extensions/v2/db/migrations/0021_restructure_extensions_revisions.sql b/services/api/src/services/extensions/v2/db/migrations/0021_restructure_extensions_revisions.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0021_restructure_extensions_revisions.sql rename to services/api/src/services/extensions/v2/db/migrations/0021_restructure_extensions_revisions.sql diff --git a/src/services/extensions/v2/db/migrations/0022_add_extensions_delisted_at.sql b/services/api/src/services/extensions/v2/db/migrations/0022_add_extensions_delisted_at.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0022_add_extensions_delisted_at.sql rename to services/api/src/services/extensions/v2/db/migrations/0022_add_extensions_delisted_at.sql diff --git a/src/services/extensions/v2/db/migrations/0023_fast_makkari.sql b/services/api/src/services/extensions/v2/db/migrations/0023_fast_makkari.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0023_fast_makkari.sql rename to services/api/src/services/extensions/v2/db/migrations/0023_fast_makkari.sql diff --git a/src/services/extensions/v2/db/migrations/0024_claim_verification_budgets.sql b/services/api/src/services/extensions/v2/db/migrations/0024_claim_verification_budgets.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0024_claim_verification_budgets.sql rename to services/api/src/services/extensions/v2/db/migrations/0024_claim_verification_budgets.sql diff --git a/src/services/extensions/v2/db/migrations/0025_luxuriant_franklin_richards.sql b/services/api/src/services/extensions/v2/db/migrations/0025_luxuriant_franklin_richards.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0025_luxuriant_franklin_richards.sql rename to services/api/src/services/extensions/v2/db/migrations/0025_luxuriant_franklin_richards.sql diff --git a/src/services/extensions/v2/db/migrations/0026_resource_bounds.sql b/services/api/src/services/extensions/v2/db/migrations/0026_resource_bounds.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0026_resource_bounds.sql rename to services/api/src/services/extensions/v2/db/migrations/0026_resource_bounds.sql diff --git a/src/services/extensions/v2/db/migrations/0027_bind_developer_profile_generation.sql b/services/api/src/services/extensions/v2/db/migrations/0027_bind_developer_profile_generation.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0027_bind_developer_profile_generation.sql rename to services/api/src/services/extensions/v2/db/migrations/0027_bind_developer_profile_generation.sql diff --git a/src/services/extensions/v2/db/migrations/0028_m2m_publisher_clients.sql b/services/api/src/services/extensions/v2/db/migrations/0028_m2m_publisher_clients.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0028_m2m_publisher_clients.sql rename to services/api/src/services/extensions/v2/db/migrations/0028_m2m_publisher_clients.sql diff --git a/src/services/extensions/v2/db/migrations/0029_align_manifest.sql b/services/api/src/services/extensions/v2/db/migrations/0029_align_manifest.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0029_align_manifest.sql rename to services/api/src/services/extensions/v2/db/migrations/0029_align_manifest.sql diff --git a/src/services/extensions/v2/db/migrations/0030_directory_identity.sql b/services/api/src/services/extensions/v2/db/migrations/0030_directory_identity.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0030_directory_identity.sql rename to services/api/src/services/extensions/v2/db/migrations/0030_directory_identity.sql diff --git a/src/services/extensions/v2/db/migrations/0031_directory_packages.sql b/services/api/src/services/extensions/v2/db/migrations/0031_directory_packages.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0031_directory_packages.sql rename to services/api/src/services/extensions/v2/db/migrations/0031_directory_packages.sql diff --git a/src/services/extensions/v2/db/migrations/0032_package_jobs.sql b/services/api/src/services/extensions/v2/db/migrations/0032_package_jobs.sql similarity index 100% rename from src/services/extensions/v2/db/migrations/0032_package_jobs.sql rename to services/api/src/services/extensions/v2/db/migrations/0032_package_jobs.sql diff --git a/src/services/extensions/v2/db/migrations/meta/0014_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0014_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0014_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0014_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0015_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0015_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0015_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0015_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0016_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0016_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0016_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0016_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0017_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0017_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0017_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0017_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0018_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0018_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0018_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0018_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0019_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0019_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0019_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0019_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0021_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0021_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0021_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0021_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0022_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0022_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0022_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0022_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0023_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0023_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0023_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0023_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0024_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0024_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0024_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0024_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0025_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0025_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0025_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0025_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0026_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0026_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0026_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0026_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0027_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0027_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0027_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0027_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0028_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0028_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0028_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0028_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0029_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0029_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0029_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0029_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0030_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0030_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0030_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0030_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0031_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0031_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/0032_snapshot.json b/services/api/src/services/extensions/v2/db/migrations/meta/0032_snapshot.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/0032_snapshot.json rename to services/api/src/services/extensions/v2/db/migrations/meta/0032_snapshot.json diff --git a/src/services/extensions/v2/db/migrations/meta/_journal.json b/services/api/src/services/extensions/v2/db/migrations/meta/_journal.json similarity index 100% rename from src/services/extensions/v2/db/migrations/meta/_journal.json rename to services/api/src/services/extensions/v2/db/migrations/meta/_journal.json diff --git a/src/services/extensions/v2/db/package-builds.ts b/services/api/src/services/extensions/v2/db/package-builds.ts similarity index 96% rename from src/services/extensions/v2/db/package-builds.ts rename to services/api/src/services/extensions/v2/db/package-builds.ts index 6881017b..97b58ce8 100644 --- a/src/services/extensions/v2/db/package-builds.ts +++ b/services/api/src/services/extensions/v2/db/package-builds.ts @@ -10,7 +10,7 @@ import type { ArtifactRow, IconObject } from "../packages/repository"; -import { PackageError } from "../packages/errors"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; export const MAX_PACKAGE_STORAGE_BYTES = 512 * 1024 * 1024; export const MAX_PACKAGE_OBJECTS = 256; @@ -112,6 +112,20 @@ export class PackageBuildDatabase implements PackageRepository { 409 ); } + async setObjectSize(id: string, bytes: number) { + const result = await this.db + .prepare( + "UPDATE package_objects SET bytes=? WHERE id=? AND deleting=0 AND bytes>=?" + ) + .bind(bytes, id, bytes) + .run(); + if (!result.meta.changes) + throw new PackageError( + "PACKAGE_UNAVAILABLE", + "Prepared package exceeds its storage reservation", + 503 + ); + } async removeObject(id: string) { await this.db .prepare("DELETE FROM package_objects WHERE id=?") diff --git a/src/services/extensions/v2/db/packages.ts b/services/api/src/services/extensions/v2/db/packages.ts similarity index 100% rename from src/services/extensions/v2/db/packages.ts rename to services/api/src/services/extensions/v2/db/packages.ts diff --git a/src/services/extensions/v2/db/package-jobs.ts b/services/api/src/services/extensions/v2/db/packaging-jobs.ts similarity index 91% rename from src/services/extensions/v2/db/package-jobs.ts rename to services/api/src/services/extensions/v2/db/packaging-jobs.ts index 0a943ac9..08978092 100644 --- a/src/services/extensions/v2/db/package-jobs.ts +++ b/services/api/src/services/extensions/v2/db/packaging-jobs.ts @@ -1,5 +1,5 @@ -import type { PackageJob, PackageJobRepository } from "../packages/jobs"; -import { PackageError } from "../packages/errors"; +import type { PackagingJob, PackagingJobRepository } from "../packages/jobs"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; interface JobRow { id: string; @@ -9,7 +9,7 @@ interface JobRow { source_id: string | null; upload_object_id: string | null; upload_sha256: string | null; - status: PackageJob["status"]; + status: PackagingJob["status"]; attempts: number; artifact_id: string | null; error_code: string | null; @@ -18,7 +18,7 @@ interface JobRow { created_at: string; updated_at: string; } -const job = (row: JobRow): PackageJob => ({ +const job = (row: JobRow): PackagingJob => ({ id: row.id, fingerprint: row.fingerprint, owner: JSON.parse(row.owner), @@ -40,7 +40,7 @@ const job = (row: JobRow): PackageJob => ({ updatedAt: row.updated_at }); -export class PackageJobsDatabase implements PackageJobRepository { +export class PackagingJobsDatabase implements PackagingJobRepository { constructor(private readonly db: D1Database) {} async get(id: string) { @@ -61,7 +61,7 @@ export class PackageJobsDatabase implements PackageJobRepository { } async create( input: Pick< - PackageJob, + PackagingJob, | "fingerprint" | "owner" | "manifest" @@ -93,7 +93,7 @@ export class PackageJobsDatabase implements PackageJobRepository { if (!result) throw new PackageError( "PACKAGE_ADMISSION", - "Package job allowance is exhausted; try again later", + "Packaging job allowance is exhausted; try again later", 429 ); return result; @@ -119,7 +119,7 @@ export class PackageJobsDatabase implements PackageJobRepository { async fail( id: string, lease: string, - error: NonNullable, + error: NonNullable, retry: boolean ) { await this.db diff --git a/src/services/extensions/v2/db/publisher-clients.ts b/services/api/src/services/extensions/v2/db/publisher-clients.ts similarity index 100% rename from src/services/extensions/v2/db/publisher-clients.ts rename to services/api/src/services/extensions/v2/db/publisher-clients.ts diff --git a/src/services/extensions/v2/db/resource-inventory.sql b/services/api/src/services/extensions/v2/db/resource-inventory.sql similarity index 100% rename from src/services/extensions/v2/db/resource-inventory.sql rename to services/api/src/services/extensions/v2/db/resource-inventory.sql diff --git a/src/services/extensions/v2/db/resource-maintenance.ts b/services/api/src/services/extensions/v2/db/resource-maintenance.ts similarity index 98% rename from src/services/extensions/v2/db/resource-maintenance.ts rename to services/api/src/services/extensions/v2/db/resource-maintenance.ts index a85e0116..c1116e09 100644 --- a/src/services/extensions/v2/db/resource-maintenance.ts +++ b/services/api/src/services/extensions/v2/db/resource-maintenance.ts @@ -1,6 +1,6 @@ import { sql } from "drizzle-orm"; import { ExtensionsDb } from "../../../../lib/db"; -import { logInfo, logWarn } from "../../../../lib/logger"; +import { logInfo, logWarn } from "@fossbilling/logging"; import { sha256Hex } from "../../../../lib/hash"; import { MAX_CONTENT_BYTES, diff --git a/src/services/extensions/v2/db/revisions.ts b/services/api/src/services/extensions/v2/db/revisions.ts similarity index 100% rename from src/services/extensions/v2/db/revisions.ts rename to services/api/src/services/extensions/v2/db/revisions.ts diff --git a/src/services/extensions/v2/db/schema.ts b/services/api/src/services/extensions/v2/db/schema.ts similarity index 99% rename from src/services/extensions/v2/db/schema.ts rename to services/api/src/services/extensions/v2/db/schema.ts index 53630359..e5e0c3d5 100644 --- a/src/services/extensions/v2/db/schema.ts +++ b/services/api/src/services/extensions/v2/db/schema.ts @@ -592,7 +592,7 @@ export const packageIcons = sqliteTable("package_icons", { svgSha256: text("svg_sha256") }); -export const packageJobs = sqliteTable( +export const packagingJobs = sqliteTable( "package_jobs", { id: text("id").primaryKey(), diff --git a/src/services/extensions/v2/db/users.ts b/services/api/src/services/extensions/v2/db/users.ts similarity index 100% rename from src/services/extensions/v2/db/users.ts rename to services/api/src/services/extensions/v2/db/users.ts diff --git a/src/services/extensions/v2/email/config.ts b/services/api/src/services/extensions/v2/email/config.ts similarity index 100% rename from src/services/extensions/v2/email/config.ts rename to services/api/src/services/extensions/v2/email/config.ts diff --git a/src/services/extensions/v2/email/factory.ts b/services/api/src/services/extensions/v2/email/factory.ts similarity index 100% rename from src/services/extensions/v2/email/factory.ts rename to services/api/src/services/extensions/v2/email/factory.ts diff --git a/src/services/extensions/v2/email/mxroute.ts b/services/api/src/services/extensions/v2/email/mxroute.ts similarity index 100% rename from src/services/extensions/v2/email/mxroute.ts rename to services/api/src/services/extensions/v2/email/mxroute.ts diff --git a/src/services/extensions/v2/email/notify.ts b/services/api/src/services/extensions/v2/email/notify.ts similarity index 99% rename from src/services/extensions/v2/email/notify.ts rename to services/api/src/services/extensions/v2/email/notify.ts index 3ee434c1..169ab4a2 100644 --- a/src/services/extensions/v2/email/notify.ts +++ b/services/api/src/services/extensions/v2/email/notify.ts @@ -1,5 +1,5 @@ import type { ExtensionsDb } from "../../../../lib/db"; -import { logError } from "../../../../lib/logger"; +import { logError } from "@fossbilling/logging"; import { createEmailSender, DisabledSender } from "./factory"; import { resolveClaimantEmail, diff --git a/src/services/extensions/v2/email/recipients.ts b/services/api/src/services/extensions/v2/email/recipients.ts similarity index 100% rename from src/services/extensions/v2/email/recipients.ts rename to services/api/src/services/extensions/v2/email/recipients.ts diff --git a/src/services/extensions/v2/email/resend.ts b/services/api/src/services/extensions/v2/email/resend.ts similarity index 100% rename from src/services/extensions/v2/email/resend.ts rename to services/api/src/services/extensions/v2/email/resend.ts diff --git a/src/services/extensions/v2/email/templates.ts b/services/api/src/services/extensions/v2/email/templates.ts similarity index 100% rename from src/services/extensions/v2/email/templates.ts rename to services/api/src/services/extensions/v2/email/templates.ts diff --git a/src/services/extensions/v2/email/types.ts b/services/api/src/services/extensions/v2/email/types.ts similarity index 100% rename from src/services/extensions/v2/email/types.ts rename to services/api/src/services/extensions/v2/email/types.ts diff --git a/src/services/extensions/v2/github/identity.ts b/services/api/src/services/extensions/v2/github/identity.ts similarity index 100% rename from src/services/extensions/v2/github/identity.ts rename to services/api/src/services/extensions/v2/github/identity.ts diff --git a/src/services/extensions/v2/github/verification.ts b/services/api/src/services/extensions/v2/github/verification.ts similarity index 99% rename from src/services/extensions/v2/github/verification.ts rename to services/api/src/services/extensions/v2/github/verification.ts index d33b6da5..5f49fd29 100644 --- a/src/services/extensions/v2/github/verification.ts +++ b/services/api/src/services/extensions/v2/github/verification.ts @@ -5,7 +5,7 @@ import { RateLimitError, NotFoundError } from "../../../../lib/github-errors"; -import { logWarn } from "../../../../lib/logger"; +import { logWarn } from "@fossbilling/logging"; import { Developer } from "../schemas/developers"; import { GithubIdentity } from "../db/users"; diff --git a/src/services/extensions/v2/index.ts b/services/api/src/services/extensions/v2/index.ts similarity index 100% rename from src/services/extensions/v2/index.ts rename to services/api/src/services/extensions/v2/index.ts diff --git a/src/services/extensions/v2/middleware.ts b/services/api/src/services/extensions/v2/middleware.ts similarity index 97% rename from src/services/extensions/v2/middleware.ts rename to services/api/src/services/extensions/v2/middleware.ts index b7d7e6b3..c05d662b 100644 --- a/src/services/extensions/v2/middleware.ts +++ b/services/api/src/services/extensions/v2/middleware.ts @@ -1,12 +1,13 @@ -import { logInfo, logError } from "../../../lib/logger"; +import { getPlatform } from "../../../lib/middleware"; +import { logInfo, logError } from "@fossbilling/logging"; import { MAX_RAW_BODY_BYTES } from "./resource-limits"; import { type Context, type MiddlewareHandler } from "hono"; import { bearerAssertionVerifier, identitySyncAssertionVerifier } from "../../../lib/auth/bearer-assertion"; -import { getAuth, requireAuth } from "../../../lib/auth"; -import type { AuthPrincipal } from "../../../lib/auth"; +import { getAuth, requireAuth } from "../../../lib/auth/index"; +import type { AuthPrincipal } from "../../../lib/auth/index"; import { getExtensionsDb } from "../../../lib/db"; import { sha256Hex } from "../../../lib/hash"; import { UsersDatabase } from "./db/users"; @@ -355,9 +356,10 @@ function paceContentAttempts(identity: "ip" | "account"): MiddlewareHandler { const bucket = identity === "ip" ? "ip" : auth?.scope === "m2m" ? "client" : "account"; try { - const { success } = await c.env.EXTENSION_WRITE_RATE_LIMITER.limit({ - key: `${bucket}:${subject}` - }); + const success = await getPlatform(c).checkRateLimit( + "EXTENSION_WRITE_RATE_LIMITER", + `${bucket}:${subject}` + ); if (!success) { c.header("Retry-After", "60"); logInfo("extensions-v2", "Resource admission rejected", { diff --git a/src/services/extensions/v2/packages/build.ts b/services/api/src/services/extensions/v2/packages/build.ts similarity index 67% rename from src/services/extensions/v2/packages/build.ts rename to services/api/src/services/extensions/v2/packages/build.ts index 371aef56..c2faaf53 100644 --- a/src/services/extensions/v2/packages/build.ts +++ b/services/api/src/services/extensions/v2/packages/build.ts @@ -1,14 +1,17 @@ +import { sha256 } from "@fossbilling/extension-packaging/checksum"; import { - buildExtensionPackage, - PACKAGE_LIMITS, - sha256 -} from "../package-builder"; + MAX_PACKAGE_OBJECT_BYTES, + type PackageProcessor +} from "@fossbilling/extension-packaging/processor"; import type { Manifest } from "../schemas/manifest"; import { canonicalJson } from "./manifest"; import type { PackageRepository, PackageOwner, IconObject } from "./repository"; -import type { PackageImageProcessor, IconVariant } from "./images"; -import { PackageError } from "./errors"; -import { boundedBytes, type PackageStore } from "./store"; +import type { IconVariant } from "@fossbilling/extension-packaging/images"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; +import { + boundedBytes, + type PackageStore +} from "@fossbilling/extension-packaging/store"; export async function reserveObject( repository: PackageRepository, @@ -41,10 +44,10 @@ export async function buildStoredPackage(input: { store: PackageStore; owner: PackageOwner; manifest: Manifest; - upload?: { bytes: Uint8Array; objectId: string }; + upload?: { sha256: string; objectId: string }; sourceId?: string; baseUrl: string; - images: PackageImageProcessor; + processor: PackageProcessor; }) { const { repository, store, owner, manifest } = input; let source = await repository.findSource( @@ -58,8 +61,7 @@ export async function buildStoredPackage(input: { "Source not found for this extension version", 404 ); - let sourceBytes = input.upload?.bytes; - if (sourceBytes && source && (await sha256(sourceBytes)) !== source.sha256) + if (input.upload && source && input.upload.sha256 !== source.sha256) throw new PackageError( "SOURCE_CHANGED", "This version already has a different source ZIP; use a new extension version", @@ -73,42 +75,50 @@ export async function buildStoredPackage(input: { const existing = await repository.findArtifact(source.id, manifestHash); if (existing) return existing; } - if (!sourceBytes) { - if (!source) - throw new PackageError("NOT_FOUND", "Source ZIP not found", 404); - sourceBytes = await boundedBytes( - await store.get(source.object_id), - PACKAGE_LIMITS.sourceBytes + const upload = + input.upload ?? + (source ? { objectId: source.object_id, sha256: source.sha256 } : null); + if (!upload) throw new PackageError("NOT_FOUND", "Source ZIP not found", 404); + const prepared = source && (await repository.hasIcon(source.id)); + const objectId = crypto.randomUUID(); + const iconIds: [string, string] | undefined = prepared + ? undefined + : [crypto.randomUUID(), crypto.randomUUID()]; + // Reserve worst-case output space before an external runner can write to B2. + // Unused/failed reservations remain discoverable by the normal cleanup job. + for (const id of [objectId, ...(iconIds ?? [])]) + await repository.reserveObject( + owner, + manifest.id, + id, + MAX_PACKAGE_OBJECT_BYTES ); - if ((await sha256(sourceBytes)) !== source.sha256) + const built = await input.processor.process({ + manifest, + source: { id: upload.objectId, sha256: upload.sha256 }, + archive: objectId, + icons: iconIds + }); + await repository.setObjectSize(objectId, built.archive.bytes); + const variants: IconVariant[] = []; + for (const icon of built.icons) { + const bytes = await boundedBytes( + await store.get(icon.id, icon.sha256), + MAX_PACKAGE_OBJECT_BYTES + ); + if (bytes.length !== icon.bytes || (await sha256(bytes)) !== icon.sha256) throw new PackageError( - "SOURCE_UNAVAILABLE", - "Stored source failed checksum verification", + "PACKAGE_UNAVAILABLE", + "Prepared icon failed checksum verification", 503 ); + variants.push({ format: icon.format, bytes }); } - let built; - try { - built = await buildExtensionPackage(sourceBytes, manifest); - } catch { - throw new PackageError( - "INVALID_PACKAGE", - "Source ZIP failed package validation", - 422 - ); - } - const prepared = source && (await repository.hasIcon(source.id)); - const variants = - prepared || !built.icon - ? [] - : await input.images.prepare(built.icon, built.iconName === "icon.svg"); if (!source) { - // Without an upload, the stored-source lookup above must have succeeded. - const objectId = input.upload!.objectId; source = await repository.linkSource(owner, manifest, { id: crypto.randomUUID(), - sha256: built.sourceSha256, - object_id: objectId + sha256: upload.sha256, + object_id: upload.objectId }); if (!source) throw new PackageError( @@ -116,7 +126,7 @@ export async function buildStoredPackage(input: { "Ownership changed while building", 403 ); - if (source.sha256 !== built.sourceSha256) + if (source.sha256 !== upload.sha256) throw new PackageError( "SOURCE_CHANGED", "Another upload reserved this version with different source files", @@ -134,21 +144,13 @@ export async function buildStoredPackage(input: { sourceId: source.id, variants }); - const objectId = await reserveObject( - repository, - store, - owner, - manifest.id, - built.bytes, - built.packageSha256 - ); const artifactId = crypto.randomUUID(); const downloadUrl = `${input.baseUrl}/packages/${artifactId}/download`; const artifact = await repository.linkArtifact(owner, manifest, { id: artifactId, sourceId: source.id, manifestHash, - sha256: built.packageSha256, + sha256: built.archive.sha256, objectId, downloadUrl }); @@ -162,6 +164,10 @@ export async function buildStoredPackage(input: { await store.delete(objectId); await repository.removeObject(objectId); } + for (const id of iconIds ?? []) { + await store.delete(id); + await repository.removeObject(id); + } return artifact; } diff --git a/src/services/extensions/v2/packages/jobs.ts b/services/api/src/services/extensions/v2/packages/jobs.ts similarity index 80% rename from src/services/extensions/v2/packages/jobs.ts rename to services/api/src/services/extensions/v2/packages/jobs.ts index 0af883d0..816504d5 100644 --- a/src/services/extensions/v2/packages/jobs.ts +++ b/services/api/src/services/extensions/v2/packages/jobs.ts @@ -1,10 +1,10 @@ -import { logError } from "../../../../lib/logger"; +import { logError } from "@fossbilling/logging"; import type { IMessageQueue } from "../../../../lib/interfaces"; import type { Manifest } from "../schemas/manifest"; import type { PackageOwner } from "./repository"; -import { PackageError } from "./errors"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; -export interface PackageJob { +export interface PackagingJob { id: string; fingerprint: string; owner: PackageOwner; @@ -20,22 +20,22 @@ export interface PackageJob { updatedAt: string; } -export interface PackageJobRepository { - get(id: string): Promise; - claim(id: string, lease: string): Promise; +export interface PackagingJobRepository { + get(id: string): Promise; + claim(id: string, lease: string): Promise; complete(id: string, lease: string, artifactId: string): Promise; fail( id: string, lease: string, - error: NonNullable, + error: NonNullable, retry: boolean ): Promise; pending(): Promise; markEnqueued(id: string): Promise; } -export async function dispatchPackageJob( - jobs: PackageJobRepository, +export async function dispatchPackagingJob( + jobs: PackagingJobRepository, queue: IMessageQueue, id: string ) { @@ -44,10 +44,10 @@ export async function dispatchPackageJob( } // Delivery-specific acknowledgement/retry APIs stay in the platform adapter. -export async function processPackageJob( - jobs: PackageJobRepository, +export async function processPackagingJob( + jobs: PackagingJobRepository, id: string, - prepare: (job: PackageJob) => Promise, + prepare: (job: PackagingJob) => Promise, lease = crypto.randomUUID() ): Promise<"ack" | "retry" | "busy"> { const existing = await jobs.get(id); diff --git a/src/services/extensions/v2/packages/manifest.ts b/services/api/src/services/extensions/v2/packages/manifest.ts similarity index 100% rename from src/services/extensions/v2/packages/manifest.ts rename to services/api/src/services/extensions/v2/packages/manifest.ts diff --git a/src/services/extensions/v2/packages/prepare.ts b/services/api/src/services/extensions/v2/packages/prepare.ts similarity index 59% rename from src/services/extensions/v2/packages/prepare.ts rename to services/api/src/services/extensions/v2/packages/prepare.ts index 35061266..c924f8e3 100644 --- a/src/services/extensions/v2/packages/prepare.ts +++ b/services/api/src/services/extensions/v2/packages/prepare.ts @@ -1,22 +1,20 @@ import type { PackageRepository } from "./repository"; -import type { PackageImageProcessor } from "./images"; -import type { PackageStore } from "./store"; -import type { PackageJob } from "./jobs"; -import { PackageError } from "./errors"; -import { boundedBytes } from "./store"; +import type { PackageProcessor } from "@fossbilling/extension-packaging/processor"; +import type { PackageStore } from "@fossbilling/extension-packaging/store"; +import type { PackagingJob } from "./jobs"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; import { buildStoredPackage } from "./build"; -import { PACKAGE_LIMITS } from "../package-builder"; export interface PackagePreparation { repository: PackageRepository; store: PackageStore; - images: PackageImageProcessor; + processor: PackageProcessor; baseUrl: string; } export async function prepareQueuedPackage( services: PackagePreparation, - job: PackageJob + job: PackagingJob ): Promise { const { repository, store } = services; const previous = job.owner; @@ -41,23 +39,10 @@ export async function prepareQueuedPackage( "Package ownership changed while queued", 403 ); - let upload: { bytes: Uint8Array; objectId: string } | undefined; - if (job.uploadObjectId) { - const body = await store.get( - job.uploadObjectId, - job.uploadSha256 ?? undefined - ); - if (!body) - throw new PackageError( - "SOURCE_UNAVAILABLE", - "Queued source ZIP is unavailable", - 503 - ); - upload = { - bytes: await boundedBytes(body, PACKAGE_LIMITS.sourceBytes), - objectId: job.uploadObjectId - }; - } + const upload = + job.uploadObjectId && job.uploadSha256 + ? { objectId: job.uploadObjectId, sha256: job.uploadSha256 } + : undefined; const artifact = await buildStoredPackage({ repository, store, @@ -66,7 +51,7 @@ export async function prepareQueuedPackage( upload, sourceId: job.sourceId ?? undefined, baseUrl: services.baseUrl.replace(/\/$/, ""), - images: services.images + processor: services.processor }); return artifact.id; } diff --git a/src/services/extensions/v2/packages/repository.ts b/services/api/src/services/extensions/v2/packages/repository.ts similarity index 97% rename from src/services/extensions/v2/packages/repository.ts rename to services/api/src/services/extensions/v2/packages/repository.ts index a43e097c..b3d25488 100644 --- a/src/services/extensions/v2/packages/repository.ts +++ b/services/api/src/services/extensions/v2/packages/repository.ts @@ -65,6 +65,7 @@ export interface PackageRepository { id: string, bytes: number ): Promise; + setObjectSize(id: string, bytes: number): Promise; removeObject(id: string): Promise; claimCleanup(): Promise; finishCleanup(id: string): Promise; diff --git a/src/lib/adapters/cloudflare/package-store.ts b/services/api/src/services/extensions/v2/packages/store.ts similarity index 74% rename from src/lib/adapters/cloudflare/package-store.ts rename to services/api/src/services/extensions/v2/packages/store.ts index cfc073a8..71ccd8da 100644 --- a/src/lib/adapters/cloudflare/package-store.ts +++ b/services/api/src/services/extensions/v2/packages/store.ts @@ -1,17 +1,14 @@ -import { PackageError } from "../../../services/extensions/v2/packages/errors"; -import type { PackageStore } from "../../../services/extensions/v2/packages/store"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; +import type { PackageStore } from "@fossbilling/extension-packaging/store"; import type { Context } from "hono"; -import { b2ObjectStore } from "../b2/package-store"; -import { getPlatform } from "../../middleware"; +import { b2ObjectStore } from "@fossbilling/extension-packaging/adapters/b2"; +import { getPlatform } from "../../../../lib/middleware"; -export function packageStore( - c: Context<{ Bindings: CloudflareBindings }> -): PackageStore | null { - return configuredPackageStore(c.env, (name) => getPlatform(c).getEnv(name)); +export function packageStore(c: Context): PackageStore | null { + return configuredPackageStore((name) => getPlatform(c).getEnv(name)); } export function configuredPackageStore( - env: CloudflareBindings, getEnv: (name: string) => string | undefined ): PackageStore | null { const mode = getEnv("EXTENSION_PACKAGES_MODE"); @@ -40,8 +37,7 @@ export function configuredPackageStore( !getEnv("EXTENSION_PACKAGES_S3_PREFIX") || getEnv("EXTENSION_PACKAGES_S3_PREFIX") === "local" || !getEnv("EXTENSION_ASSETS_S3_ENDPOINT") || - !getEnv("EXTENSION_ASSETS_PUBLIC_URL") || - !env.PACKAGE_IMAGES + !getEnv("EXTENSION_ASSETS_PUBLIC_URL") ) { throw new PackageError( "PACKAGES_DISABLED", diff --git a/src/services/extensions/v2/resource-limits.ts b/services/api/src/services/extensions/v2/resource-limits.ts similarity index 100% rename from src/services/extensions/v2/resource-limits.ts rename to services/api/src/services/extensions/v2/resource-limits.ts diff --git a/src/services/extensions/v2/revalidate.ts b/services/api/src/services/extensions/v2/revalidate.ts similarity index 73% rename from src/services/extensions/v2/revalidate.ts rename to services/api/src/services/extensions/v2/revalidate.ts index ffa6cacd..6e1a5b43 100644 --- a/src/services/extensions/v2/revalidate.ts +++ b/services/api/src/services/extensions/v2/revalidate.ts @@ -19,26 +19,29 @@ import { invalidateListCache } from "../v1/list-cache"; const REVALIDATE_TAGS = ["extensions", "developers"]; const REVALIDATE_URL = "https://extensions.fossbilling.org/api/revalidate"; -export function revalidateExtensions( - c: Context<{ Bindings: CloudflareBindings }> -): void { - invalidateListCache(c.env.CACHE_KV, (p) => c.executionCtx.waitUntil(p)); +export function revalidateExtensions(c: Context): void { + const platform = getPlatform(c); + invalidateListCache(platform.getCache("CACHE_KV"), platform.waitUntil); - const secret = getPlatform(c).getEnv("EXTENSIONS_REVALIDATE_SECRET"); + const secret = platform.getEnv("EXTENSIONS_REVALIDATE_SECRET"); if (!secret) { // Unconfigured (e.g. local dev without the secret): skipping is safe — // the site's cache windows bound staleness regardless. return; } - const purge = c.env.EXTENSIONS_FRONTEND.fetch(REVALIDATE_URL, { - method: "POST", - headers: { - authorization: `Bearer ${secret}`, - "content-type": "application/json" - }, - body: JSON.stringify({ tags: REVALIDATE_TAGS }) - }) + const purge = platform + .fetchService( + "EXTENSIONS_FRONTEND", + new Request(REVALIDATE_URL, { + method: "POST", + headers: { + authorization: `Bearer ${secret}`, + "content-type": "application/json" + }, + body: JSON.stringify({ tags: REVALIDATE_TAGS }) + }) + ) .then(async (response) => { if (!response.ok) { console.error( @@ -50,5 +53,5 @@ export function revalidateExtensions( console.error("[revalidate] purge request failed:", error); }); - c.executionCtx.waitUntil(purge); + platform.waitUntil(purge); } diff --git a/src/services/extensions/v2/routes/account.ts b/services/api/src/services/extensions/v2/routes/account.ts similarity index 99% rename from src/services/extensions/v2/routes/account.ts rename to services/api/src/services/extensions/v2/routes/account.ts index cc30b416..7a41dc4b 100644 --- a/src/services/extensions/v2/routes/account.ts +++ b/services/api/src/services/extensions/v2/routes/account.ts @@ -3,7 +3,7 @@ import { requireIdentitySync, requireUser } from "../middleware"; -import { getAuth } from "../../../../lib/auth"; +import { getAuth } from "../../../../lib/auth/index"; import { getExtensionsDb } from "../../../../lib/db"; import { createRoute, z } from "@hono/zod-openapi"; import { errorBody, statusFromErrorCode } from "./errors"; diff --git a/src/services/extensions/v2/routes/app.ts b/services/api/src/services/extensions/v2/routes/app.ts similarity index 100% rename from src/services/extensions/v2/routes/app.ts rename to services/api/src/services/extensions/v2/routes/app.ts diff --git a/src/services/extensions/v2/routes/badges.ts b/services/api/src/services/extensions/v2/routes/badges.ts similarity index 100% rename from src/services/extensions/v2/routes/badges.ts rename to services/api/src/services/extensions/v2/routes/badges.ts diff --git a/src/services/extensions/v2/routes/developer-profiles.ts b/services/api/src/services/extensions/v2/routes/developer-profiles.ts similarity index 97% rename from src/services/extensions/v2/routes/developer-profiles.ts rename to services/api/src/services/extensions/v2/routes/developer-profiles.ts index 4a76dbad..216d3386 100644 --- a/src/services/extensions/v2/routes/developer-profiles.ts +++ b/services/api/src/services/extensions/v2/routes/developer-profiles.ts @@ -6,7 +6,7 @@ import { } from "../middleware"; import { getExtensionsDb } from "../../../../lib/db"; import { getPlatform } from "../../../../lib/middleware"; -import { getAuth } from "../../../../lib/auth"; +import { getAuth } from "../../../../lib/auth/index"; import { createRoute, z } from "@hono/zod-openapi"; import { errorBody, @@ -195,16 +195,11 @@ export function registerDeveloperProfileRoutes(app: ExtensionsV2App): void { auth.userId, body, platform.getEnv("GITHUB_TOKEN"), - // Cloudflare enforces the configured 3-per-60s account allowance across - // isolates. Keep this as a callback so upsertOwn can run its cheap + // Keep this as a callback so upsertOwn can run its cheap // existing-profile/id checks first; updates and known-taken ids must not // spend creation allowance. - async () => - ( - await c.env.PROFILE_CREATION_RATE_LIMITER.limit({ - key: auth.userId - }) - ).success + () => + platform.checkRateLimit("PROFILE_CREATION_RATE_LIMITER", auth.userId) ); if (error || !data) { const status = diff --git a/src/services/extensions/v2/routes/errors.ts b/services/api/src/services/extensions/v2/routes/errors.ts similarity index 100% rename from src/services/extensions/v2/routes/errors.ts rename to services/api/src/services/extensions/v2/routes/errors.ts diff --git a/src/services/extensions/v2/routes/m2m.ts b/services/api/src/services/extensions/v2/routes/m2m.ts similarity index 100% rename from src/services/extensions/v2/routes/m2m.ts rename to services/api/src/services/extensions/v2/routes/m2m.ts diff --git a/src/services/extensions/v2/routes/moderation.ts b/services/api/src/services/extensions/v2/routes/moderation.ts similarity index 99% rename from src/services/extensions/v2/routes/moderation.ts rename to services/api/src/services/extensions/v2/routes/moderation.ts index ebdde3a3..6de0a685 100644 --- a/src/services/extensions/v2/routes/moderation.ts +++ b/services/api/src/services/extensions/v2/routes/moderation.ts @@ -6,7 +6,7 @@ import { getPublisher } from "../middleware"; import { getExtensionsDb } from "../../../../lib/db"; -import { getAuth } from "../../../../lib/auth"; +import { getAuth } from "../../../../lib/auth/index"; import { createRoute, z } from "@hono/zod-openapi"; import { errorBody, diff --git a/src/services/extensions/v2/routes/notify.ts b/services/api/src/services/extensions/v2/routes/notify.ts similarity index 91% rename from src/services/extensions/v2/routes/notify.ts rename to services/api/src/services/extensions/v2/routes/notify.ts index f5c4706f..61602f4a 100644 --- a/src/services/extensions/v2/routes/notify.ts +++ b/services/api/src/services/extensions/v2/routes/notify.ts @@ -26,7 +26,6 @@ export async function notifyAuthor( ): Promise { revalidateExtensions(c); if (!notifyRequested(query)) return false; - return sendModerationNotification(getPlatform(c), extDb, input, (p) => - c.executionCtx.waitUntil(p) - ); + const platform = getPlatform(c); + return sendModerationNotification(platform, extDb, input, platform.waitUntil); } diff --git a/src/services/extensions/v2/routes/owner-extensions.ts b/services/api/src/services/extensions/v2/routes/owner-extensions.ts similarity index 99% rename from src/services/extensions/v2/routes/owner-extensions.ts rename to services/api/src/services/extensions/v2/routes/owner-extensions.ts index dfd786cb..845c6dcf 100644 --- a/src/services/extensions/v2/routes/owner-extensions.ts +++ b/services/api/src/services/extensions/v2/routes/owner-extensions.ts @@ -16,7 +16,7 @@ import { getPublisher } from "../middleware"; import { getExtensionsDb } from "../../../../lib/db"; -import { getAuth } from "../../../../lib/auth"; +import { getAuth } from "../../../../lib/auth/index"; import { createRoute, z } from "@hono/zod-openapi"; import { ActiveAccountRequiredResponse, diff --git a/src/services/extensions/v2/routes/ownership.ts b/services/api/src/services/extensions/v2/routes/ownership.ts similarity index 99% rename from src/services/extensions/v2/routes/ownership.ts rename to services/api/src/services/extensions/v2/routes/ownership.ts index 587386b7..e3f8b8c4 100644 --- a/src/services/extensions/v2/routes/ownership.ts +++ b/services/api/src/services/extensions/v2/routes/ownership.ts @@ -1,7 +1,7 @@ import { requireActiveAuth, requireModerator } from "../middleware"; import { getExtensionsDb } from "../../../../lib/db"; import { getPlatform } from "../../../../lib/middleware"; -import { getAuth } from "../../../../lib/auth"; +import { getAuth } from "../../../../lib/auth/index"; import { createRoute, z } from "@hono/zod-openapi"; import { errorBody, diff --git a/src/services/extensions/v2/routes/packages.ts b/services/api/src/services/extensions/v2/routes/packages.ts similarity index 91% rename from src/services/extensions/v2/routes/packages.ts rename to services/api/src/services/extensions/v2/routes/packages.ts index 1cdacdd9..8f403dad 100644 --- a/src/services/extensions/v2/routes/packages.ts +++ b/services/api/src/services/extensions/v2/routes/packages.ts @@ -1,9 +1,9 @@ -import { PackageJobsDatabase } from "../db/package-jobs"; -import { dispatchPackageJob, type PackageJob } from "../packages/jobs"; -import { sha256 } from "../package-builder"; +import { PackagingJobsDatabase } from "../db/packaging-jobs"; +import { dispatchPackagingJob, type PackagingJob } from "../packages/jobs"; +import { sha256 } from "@fossbilling/extension-packaging/checksum"; import { createRoute, z } from "@hono/zod-openapi"; import type { Context, MiddlewareHandler } from "hono"; -import { getAuth } from "../../../../lib/auth"; +import { getAuth } from "../../../../lib/auth/index"; import { getPlatform } from "../../../../lib/middleware"; import { canonicalJson } from "../packages/manifest"; import type { ArtifactRow } from "../packages/repository"; @@ -18,14 +18,14 @@ import { paceContentIp } from "../middleware"; import { reserveObject, cleanupPackageObjects } from "../packages/build"; -import { PackageError } from "../packages/errors"; -import { boundedBytes } from "../packages/store"; -import { packageStore } from "../../../../lib/adapters/cloudflare/package-store"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; +import { boundedBytes } from "@fossbilling/extension-packaging/store"; +import { packageStore } from "../packages/store"; import { ManifestSchema } from "../schemas/manifest"; import { PackageArtifactSchema, - PackageJobSchema, - PackageJobParamSchema, + PackagingJobSchema, + PackagingJobParamSchema, PackageBuildSchema, PackageParamSchema } from "../schemas/packages"; @@ -62,7 +62,7 @@ function artifactResponse(row: ArtifactRow) { } }; } -async function jobResponse(db: D1Database, job: PackageJob) { +async function jobResponse(db: D1Database, job: PackagingJob) { const artifact = job.artifactId ? await new PackageBuildDatabase(db).getArtifact(job.artifactId) : null; @@ -111,7 +111,7 @@ export function registerPackageRoutes(app: ExtensionsV2App) { }; app.use("/packages", availability); app.use("/packages/*", availability); - app.use("/package-jobs/*", availability); + app.use("/packaging-jobs/*", availability); app.on("POST", "/packages", paceContentIp()); const boundUpload: MiddlewareHandler = async (c, next) => { try { @@ -148,7 +148,7 @@ export function registerPackageRoutes(app: ExtensionsV2App) { "Private processing job; identical requests reuse queued or completed work", content: { "application/json": { - schema: z.object({ result: PackageJobSchema }) + schema: z.object({ result: PackagingJobSchema }) } } }, @@ -208,7 +208,7 @@ export function registerPackageRoutes(app: ExtensionsV2App) { "Package download URL is not configured", 503 ); - const jobs = new PackageJobsDatabase(db); + const jobs = new PackagingJobsDatabase(db); const source = form.source ? new Uint8Array(await form.source.arrayBuffer()) : undefined; @@ -246,14 +246,14 @@ export function registerPackageRoutes(app: ExtensionsV2App) { }); } if (job.status === "queued") - await dispatchPackageJob( + await dispatchPackagingJob( jobs, getPlatform(c).getQueue("EXTENSION_PACKAGE_QUEUE"), job.id ); c.header( "Location", - `${baseUrl.replace(/\/$/, "")}/package-jobs/${job.id}` + `${baseUrl.replace(/\/$/, "")}/packaging-jobs/${job.id}` ); c.header("Retry-After", "1"); return c.json(await jobResponse(db, job), 202); @@ -266,18 +266,18 @@ export function registerPackageRoutes(app: ExtensionsV2App) { app.openapi( createRoute({ method: "get", - path: "/package-jobs/{jobId}", + path: "/packaging-jobs/{jobId}", tags: ["Packages"], - summary: "Read an owning developer's private package-processing job", + summary: "Read an owning developer's private packaging job", security: [{ Bearer: [] }], middleware: [requireContentWrite()] as const, - request: { params: PackageJobParamSchema }, + request: { params: PackagingJobParamSchema }, responses: { 200: { description: "Package processing state and completed artifact", content: { "application/json": { - schema: z.object({ result: PackageJobSchema }) + schema: z.object({ result: PackagingJobSchema }) } } }, @@ -288,11 +288,11 @@ export function registerPackageRoutes(app: ExtensionsV2App) { try { const db = c.env.DB_EXTENSIONS; const repository = new PackageBuildDatabase(db); - const job = await new PackageJobsDatabase(db).get( + const job = await new PackagingJobsDatabase(db).get( c.req.valid("param").jobId ); if (!job) - throw new PackageError("NOT_FOUND", "Package job not found", 404); + throw new PackageError("NOT_FOUND", "Packaging job not found", 404); const userId = getAuth(c).userId; const moderator = await repository.isModerator(userId); if (!moderator) { @@ -308,7 +308,7 @@ export function registerPackageRoutes(app: ExtensionsV2App) { owner.ownership_epoch !== job.owner.ownership_epoch || (publisher && publisher.clientId !== job.owner.publisher_client_id) ) - throw new PackageError("NOT_FOUND", "Package job not found", 404); + throw new PackageError("NOT_FOUND", "Packaging job not found", 404); } return c.json(await jobResponse(db, job), 200); } catch (error) { @@ -341,7 +341,7 @@ export function registerPackageRoutes(app: ExtensionsV2App) { }), async (c) => { try { - await new PackageJobsDatabase(c.env.DB_EXTENSIONS).expire(); + await new PackagingJobsDatabase(c.env.DB_EXTENSIONS).expire(); return c.json( { result: { diff --git a/src/services/extensions/v2/routes/public-extensions.ts b/services/api/src/services/extensions/v2/routes/public-extensions.ts similarity index 100% rename from src/services/extensions/v2/routes/public-extensions.ts rename to services/api/src/services/extensions/v2/routes/public-extensions.ts diff --git a/src/services/extensions/v2/routes/revisions.ts b/services/api/src/services/extensions/v2/routes/revisions.ts similarity index 97% rename from src/services/extensions/v2/routes/revisions.ts rename to services/api/src/services/extensions/v2/routes/revisions.ts index 2222d613..2f86d7a8 100644 --- a/src/services/extensions/v2/routes/revisions.ts +++ b/services/api/src/services/extensions/v2/routes/revisions.ts @@ -1,4 +1,4 @@ -import { getAuth } from "../../../../lib/auth"; +import { getAuth } from "../../../../lib/auth/index"; import { getExtensionsDb } from "../../../../lib/db"; import { createRoute, z } from "@hono/zod-openapi"; import { errorBody, listErrorStatus, listPayload } from "./errors"; diff --git a/src/services/extensions/v2/schemas/badges.ts b/services/api/src/services/extensions/v2/schemas/badges.ts similarity index 100% rename from src/services/extensions/v2/schemas/badges.ts rename to services/api/src/services/extensions/v2/schemas/badges.ts diff --git a/src/services/extensions/v2/schemas/common.ts b/services/api/src/services/extensions/v2/schemas/common.ts similarity index 94% rename from src/services/extensions/v2/schemas/common.ts rename to services/api/src/services/extensions/v2/schemas/common.ts index c709a03d..6ba3ea40 100644 --- a/src/services/extensions/v2/schemas/common.ts +++ b/services/api/src/services/extensions/v2/schemas/common.ts @@ -8,18 +8,6 @@ export const lowercaseId = (label: string) => message: `${label} id must be a lowercase alphanumeric slug` }); -// Restricts to http(s) — z.string().url() alone accepts any scheme, -// including javascript:/data:, which is unsafe for fields a consumer may -// render as a link or image src. -export const httpUrl = () => - z - .string() - .max(2048) - .url() - .refine((value) => /^https?:\/\//i.test(value), { - message: "must use http or https" - }); - export const ErrorResponseSchema = z .object({ error: z.object({ diff --git a/src/services/extensions/v2/schemas/developers.ts b/services/api/src/services/extensions/v2/schemas/developers.ts similarity index 98% rename from src/services/extensions/v2/schemas/developers.ts rename to services/api/src/services/extensions/v2/schemas/developers.ts index bd4af04a..13ac32b1 100644 --- a/src/services/extensions/v2/schemas/developers.ts +++ b/services/api/src/services/extensions/v2/schemas/developers.ts @@ -1,5 +1,6 @@ import { z } from "@hono/zod-openapi"; -import { httpUrl, lowercaseId, CursorPaginationQuerySchema } from "./common"; +import { httpUrl } from "@fossbilling/extension-packaging/urls"; +import { lowercaseId, CursorPaginationQuerySchema } from "./common"; // GET /developers/unapproved has been merged into GET /developers?scope=. // "unapproved" is therefore no longer a shadowed static route, but stays diff --git a/src/services/extensions/v2/schemas/extensions.ts b/services/api/src/services/extensions/v2/schemas/extensions.ts similarity index 97% rename from src/services/extensions/v2/schemas/extensions.ts rename to services/api/src/services/extensions/v2/schemas/extensions.ts index be505f97..ddb6ea18 100644 --- a/src/services/extensions/v2/schemas/extensions.ts +++ b/services/api/src/services/extensions/v2/schemas/extensions.ts @@ -1,11 +1,9 @@ +import { MANIFEST_EXTENSION_TYPES } from "@fossbilling/extension-packaging/manifest"; import { MAX_CONTENT_BYTES } from "../resource-limits"; import { z } from "@hono/zod-openapi"; -import { - ManifestSchema, - MANIFEST_EXTENSION_TYPES, - ManifestRequirementsSchema -} from "./manifest"; -import { cursorPaginationFields, httpUrl, PaginationSchema } from "./common"; +import { ManifestSchema, ManifestRequirementsSchema } from "./manifest"; +import { cursorPaginationFields, PaginationSchema } from "./common"; +import { httpUrl } from "@fossbilling/extension-packaging/urls"; import { PublicDeveloperSchema } from "./developers"; export const EXTENSION_TYPES = MANIFEST_EXTENSION_TYPES; diff --git a/src/services/extensions/v2/schemas/m2m.ts b/services/api/src/services/extensions/v2/schemas/m2m.ts similarity index 100% rename from src/services/extensions/v2/schemas/m2m.ts rename to services/api/src/services/extensions/v2/schemas/m2m.ts diff --git a/services/api/src/services/extensions/v2/schemas/manifest.ts b/services/api/src/services/extensions/v2/schemas/manifest.ts new file mode 100644 index 00000000..aa35b320 --- /dev/null +++ b/services/api/src/services/extensions/v2/schemas/manifest.ts @@ -0,0 +1,26 @@ +import { z } from "@hono/zod-openapi"; +import { + ManifestSchema as PackageManifestSchema, + ManifestRequirementsSchema as PackageRequirementsSchema +} from "@fossbilling/extension-packaging/manifest"; + +export const ManifestRequirementsSchema = z + .strictObject(PackageRequirementsSchema.shape) + .openapi("ManifestRequirements"); +export const ManifestSchema = z + .strictObject({ + ...PackageManifestSchema.shape, + requires: ManifestRequirementsSchema + }) + .openapi("Manifest"); +export type Manifest = z.infer; + +export function manifestMetadata( + content: Pick +) { + return { + manifest_version: content.manifest_version, + author: content.author, + requires: content.requires + }; +} diff --git a/src/services/extensions/v2/schemas/ownership.ts b/services/api/src/services/extensions/v2/schemas/ownership.ts similarity index 100% rename from src/services/extensions/v2/schemas/ownership.ts rename to services/api/src/services/extensions/v2/schemas/ownership.ts diff --git a/src/services/extensions/v2/schemas/packages.ts b/services/api/src/services/extensions/v2/schemas/packages.ts similarity index 90% rename from src/services/extensions/v2/schemas/packages.ts rename to services/api/src/services/extensions/v2/schemas/packages.ts index 64f90d40..c76c3172 100644 --- a/src/services/extensions/v2/schemas/packages.ts +++ b/services/api/src/services/extensions/v2/schemas/packages.ts @@ -32,8 +32,8 @@ export const PackageArtifactSchema = z }) .openapi("PackageArtifact"); -export const PackageJobParamSchema = z.object({ jobId: z.uuid() }); -export const PackageJobSchema = z +export const PackagingJobParamSchema = z.object({ jobId: z.uuid() }); +export const PackagingJobSchema = z .object({ id: z.uuid(), extension_identifier: z.string(), @@ -49,4 +49,4 @@ export const PackageJobSchema = z created_at: z.string(), updated_at: z.string() }) - .openapi("PackageJob"); + .openapi("PackagingJob"); diff --git a/src/services/extensions/v2/schemas/revisions.ts b/services/api/src/services/extensions/v2/schemas/revisions.ts similarity index 100% rename from src/services/extensions/v2/schemas/revisions.ts rename to services/api/src/services/extensions/v2/schemas/revisions.ts diff --git a/src/services/extensions/v2/schemas/users.ts b/services/api/src/services/extensions/v2/schemas/users.ts similarity index 100% rename from src/services/extensions/v2/schemas/users.ts rename to services/api/src/services/extensions/v2/schemas/users.ts diff --git a/src/services/previews/v1/README.md b/services/api/src/services/previews/v1/README.md similarity index 100% rename from src/services/previews/v1/README.md rename to services/api/src/services/previews/v1/README.md diff --git a/src/services/previews/v1/budget.ts b/services/api/src/services/previews/v1/budget.ts similarity index 100% rename from src/services/previews/v1/budget.ts rename to services/api/src/services/previews/v1/budget.ts diff --git a/src/services/previews/v1/cache.ts b/services/api/src/services/previews/v1/cache.ts similarity index 98% rename from src/services/previews/v1/cache.ts rename to services/api/src/services/previews/v1/cache.ts index a340434a..561c09f5 100644 --- a/src/services/previews/v1/cache.ts +++ b/services/api/src/services/previews/v1/cache.ts @@ -1,3 +1,4 @@ +import type { ICache } from "../../../lib/interfaces"; import { GithubLookupResult } from "./github/artifacts"; import { singleFlight } from "../../../lib/cache"; @@ -33,7 +34,7 @@ export type CachedRead = { status: "hit"; value: T } | { status: "negative" } | { status: "miss" }; export async function readCachedValue( - kv: KVNamespace, + kv: ICache, key: string ): Promise> { const cached = await kv.get(key); @@ -59,7 +60,7 @@ export async function readCachedValue( // waitUntil, when provided, receives the KV writes so they don't sit on // the response path. export async function cachedLookup( - kv: KVNamespace, + kv: ICache, key: string, resolve: () => Promise>, ttlSeconds: number | ((data: T) => number) = DEFAULT_CACHE_TTL_SECONDS, diff --git a/src/services/previews/v1/github/artifacts.ts b/services/api/src/services/previews/v1/github/artifacts.ts similarity index 99% rename from src/services/previews/v1/github/artifacts.ts rename to services/api/src/services/previews/v1/github/artifacts.ts index 27dfd6c2..24818d5d 100644 --- a/src/services/previews/v1/github/artifacts.ts +++ b/services/api/src/services/previews/v1/github/artifacts.ts @@ -4,7 +4,7 @@ import { GitHubError, NotFoundError } from "../../../../lib/github-errors"; -import { logWarn } from "../../../../lib/logger"; +import { logWarn } from "@fossbilling/logging"; const REPO_OWNER = "FOSSBilling"; const REPO_NAME = "FOSSBilling"; diff --git a/src/services/previews/v1/github/request.ts b/services/api/src/services/previews/v1/github/request.ts similarity index 100% rename from src/services/previews/v1/github/request.ts rename to services/api/src/services/previews/v1/github/request.ts diff --git a/src/services/previews/v1/index.ts b/services/api/src/services/previews/v1/index.ts similarity index 100% rename from src/services/previews/v1/index.ts rename to services/api/src/services/previews/v1/index.ts diff --git a/src/services/previews/v1/r2.ts b/services/api/src/services/previews/v1/r2.ts similarity index 100% rename from src/services/previews/v1/r2.ts rename to services/api/src/services/previews/v1/r2.ts diff --git a/src/services/previews/v1/resolve.ts b/services/api/src/services/previews/v1/resolve.ts similarity index 100% rename from src/services/previews/v1/resolve.ts rename to services/api/src/services/previews/v1/resolve.ts diff --git a/src/services/previews/v1/routes/app.ts b/services/api/src/services/previews/v1/routes/app.ts similarity index 100% rename from src/services/previews/v1/routes/app.ts rename to services/api/src/services/previews/v1/routes/app.ts diff --git a/src/services/previews/v1/routes/commit.ts b/services/api/src/services/previews/v1/routes/commit.ts similarity index 96% rename from src/services/previews/v1/routes/commit.ts rename to services/api/src/services/previews/v1/routes/commit.ts index f549b076..2172e126 100644 --- a/src/services/previews/v1/routes/commit.ts +++ b/services/api/src/services/previews/v1/routes/commit.ts @@ -1,3 +1,4 @@ +import { getPlatform } from "../../../../lib/middleware"; import { previewGitHub } from "../github/request"; import { createRoute } from "@hono/zod-openapi"; import { @@ -92,11 +93,11 @@ export function registerCommitRoutes(app: PreviewsV1App): void { resolveArtifactPreview(github, sha, null) ) : await cachedLookup( - c.env.CACHE_KV, + getPlatform(c).getCache("CACHE_KV"), cacheKeyForSha(sha), () => resolveArtifactPreview(github, sha, null), ttlForArtifact, - (p) => c.executionCtx.waitUntil(p) + getPlatform(c).waitUntil ); return respondWithLookup( @@ -137,11 +138,11 @@ export function registerCommitRoutes(app: PreviewsV1App): void { resolveArtifactPreview(github, sha, null) ) : await cachedLookup( - c.env.CACHE_KV, + getPlatform(c).getCache("CACHE_KV"), cacheKeyForSha(sha), () => resolveArtifactPreview(github, sha, null), ttlForArtifact, - (p) => c.executionCtx.waitUntil(p) + getPlatform(c).waitUntil ); return respondWithDownloadRedirect( c, diff --git a/src/services/previews/v1/routes/errors.ts b/services/api/src/services/previews/v1/routes/errors.ts similarity index 100% rename from src/services/previews/v1/routes/errors.ts rename to services/api/src/services/previews/v1/routes/errors.ts diff --git a/src/services/previews/v1/routes/main.ts b/services/api/src/services/previews/v1/routes/main.ts similarity index 97% rename from src/services/previews/v1/routes/main.ts rename to services/api/src/services/previews/v1/routes/main.ts index e0b23d3d..b216de60 100644 --- a/src/services/previews/v1/routes/main.ts +++ b/services/api/src/services/previews/v1/routes/main.ts @@ -1,3 +1,4 @@ +import { getPlatform } from "../../../../lib/middleware"; import { PreviewGitHub, previewGitHub } from "../github/request"; import { createRoute } from "@hono/zod-openapi"; import { Context } from "hono"; @@ -92,7 +93,7 @@ async function resolveMainPreview( c: Context<{ Bindings: CloudflareBindings }> ): Promise { const result = await cachedLookup( - c.env.CACHE_KV, + getPlatform(c).getCache("CACHE_KV"), MAIN_CACHE_KEY, async () => { const object = await resolveMainObject(c); @@ -114,7 +115,7 @@ async function resolveMainPreview( }; }, MAIN_CACHE_TTL_SECONDS, - (p) => c.executionCtx.waitUntil(p) + getPlatform(c).waitUntil ); return result.status === "found" ? result.data : null; } @@ -124,7 +125,7 @@ async function resolveMainDownloadUrl( c: Context<{ Bindings: CloudflareBindings }> ): Promise { const read = await readCachedValue( - c.env.CACHE_KV, + getPlatform(c).getCache("CACHE_KV"), MAIN_CACHE_KEY ); if (read.status === "negative") return null; // authoritative negative diff --git a/src/services/previews/v1/routes/pr.ts b/services/api/src/services/previews/v1/routes/pr.ts similarity index 88% rename from src/services/previews/v1/routes/pr.ts rename to services/api/src/services/previews/v1/routes/pr.ts index f105d7dd..ece9eb26 100644 --- a/src/services/previews/v1/routes/pr.ts +++ b/services/api/src/services/previews/v1/routes/pr.ts @@ -1,3 +1,5 @@ +import type { ICache } from "../../../../lib/interfaces"; +import { getPlatform } from "../../../../lib/middleware"; import { PreviewGitHub, previewGitHub } from "../github/request"; import { createRoute } from "@hono/zod-openapi"; import { @@ -22,7 +24,7 @@ import { PreviewsV1App } from "./app"; async function resolvePrPreview( github: PreviewGitHub, prNumber: number, - kv: KVNamespace, + kv: ICache, waitUntil?: (promise: Promise) => void ): Promise { const head = await resolvePullRequestHeadSha(github, prNumber); @@ -72,14 +74,17 @@ export function registerPrRoutes(app: PreviewsV1App): void { const github = previewGitHub(c); const result = await cachedLookup( - c.env.CACHE_KV, + getPlatform(c).getCache("CACHE_KV"), `preview:pr:${number}`, () => - resolvePrPreview(github, number, c.env.CACHE_KV, (p) => - c.executionCtx.waitUntil(p) + resolvePrPreview( + github, + number, + getPlatform(c).getCache("CACHE_KV"), + getPlatform(c).waitUntil ), DEFAULT_CACHE_TTL_SECONDS, - (p) => c.executionCtx.waitUntil(p) + getPlatform(c).waitUntil ); return respondWithLookup(c, result, notFoundMessage(number), false); @@ -110,14 +115,17 @@ export function registerPrRoutes(app: PreviewsV1App): void { // GitHub API calls (PR->SHA, SHA->artifact, then the redirect) instead // of the 1 that's actually unavoidable. const artifact = await cachedLookup( - c.env.CACHE_KV, + getPlatform(c).getCache("CACHE_KV"), `preview:pr:${number}`, () => - resolvePrPreview(github, number, c.env.CACHE_KV, (p) => - c.executionCtx.waitUntil(p) + resolvePrPreview( + github, + number, + getPlatform(c).getCache("CACHE_KV"), + getPlatform(c).waitUntil ), DEFAULT_CACHE_TTL_SECONDS, - (p) => c.executionCtx.waitUntil(p) + getPlatform(c).waitUntil ); return respondWithDownloadRedirect( c, diff --git a/src/services/previews/v1/routes/respond.ts b/services/api/src/services/previews/v1/routes/respond.ts similarity index 100% rename from src/services/previews/v1/routes/respond.ts rename to services/api/src/services/previews/v1/routes/respond.ts diff --git a/src/services/previews/v1/schemas/previews.ts b/services/api/src/services/previews/v1/schemas/previews.ts similarity index 100% rename from src/services/previews/v1/schemas/previews.ts rename to services/api/src/services/previews/v1/schemas/previews.ts diff --git a/src/services/stats/v1/README.md b/services/api/src/services/stats/v1/README.md similarity index 100% rename from src/services/stats/v1/README.md rename to services/api/src/services/stats/v1/README.md diff --git a/src/services/stats/v1/dashboard.ts b/services/api/src/services/stats/v1/dashboard.ts similarity index 100% rename from src/services/stats/v1/dashboard.ts rename to services/api/src/services/stats/v1/dashboard.ts diff --git a/src/services/stats/v1/index.ts b/services/api/src/services/stats/v1/index.ts similarity index 98% rename from src/services/stats/v1/index.ts rename to services/api/src/services/stats/v1/index.ts index 266bf5e8..2becccfb 100644 --- a/src/services/stats/v1/index.ts +++ b/services/api/src/services/stats/v1/index.ts @@ -16,7 +16,7 @@ import { STATS_DASHBOARD_HTML } from "./dashboard"; import { getPlatform } from "../../../lib/middleware"; import { ICache } from "../../../lib/interfaces"; import { publicCacheKey, publicResponseCache } from "../../../lib/cache"; -import { logError, logInfo } from "../../../lib/logger"; +import { logError, logInfo } from "@fossbilling/logging"; import { GitHubError } from "../../../lib/github-errors"; type StatsEnv = { Bindings: CloudflareBindings }; @@ -225,7 +225,7 @@ registerCachedRoute("/data", async (c) => { platform.getCache("CACHE_KV"), platform.getEnv("GITHUB_TOKEN") || "", c.env.DOWNLOAD_BUCKET, - (p) => c.executionCtx.waitUntil(p) + platform.waitUntil ); if (result.error && result.stats.releaseSizes.length === 0) { diff --git a/src/services/stats/v1/interfaces.ts b/services/api/src/services/stats/v1/interfaces.ts similarity index 100% rename from src/services/stats/v1/interfaces.ts rename to services/api/src/services/stats/v1/interfaces.ts diff --git a/src/services/versions/v1/README.md b/services/api/src/services/versions/v1/README.md similarity index 100% rename from src/services/versions/v1/README.md rename to services/api/src/services/versions/v1/README.md diff --git a/src/services/versions/v1/index.ts b/services/api/src/services/versions/v1/index.ts similarity index 99% rename from src/services/versions/v1/index.ts rename to services/api/src/services/versions/v1/index.ts index 740721c4..e0d16abe 100644 --- a/src/services/versions/v1/index.ts +++ b/services/api/src/services/versions/v1/index.ts @@ -28,7 +28,7 @@ import { publicResponseCache, singleFlight } from "../../../lib/cache"; -import { logError, logWarn, logInfo } from "../../../lib/logger"; +import { logError, logWarn, logInfo } from "@fossbilling/logging"; import { GitHubError, AuthError, @@ -176,7 +176,7 @@ async function loadReleases( platform.getEnv("GITHUB_TOKEN") || "", c.env.DOWNLOAD_BUCKET, updateCache, - (p) => c.executionCtx.waitUntil(p) + platform.waitUntil ); } diff --git a/src/services/versions/v1/interfaces.ts b/services/api/src/services/versions/v1/interfaces.ts similarity index 100% rename from src/services/versions/v1/interfaces.ts rename to services/api/src/services/versions/v1/interfaces.ts diff --git a/src/services/versions/v1/r2.ts b/services/api/src/services/versions/v1/r2.ts similarity index 100% rename from src/services/versions/v1/r2.ts rename to services/api/src/services/versions/v1/r2.ts diff --git a/src/services/versions/v1/responses.ts b/services/api/src/services/versions/v1/responses.ts similarity index 100% rename from src/services/versions/v1/responses.ts rename to services/api/src/services/versions/v1/responses.ts diff --git a/services/api/test/deployment.node.test.ts b/services/api/test/deployment.node.test.ts new file mode 100644 index 00000000..7492cb9e --- /dev/null +++ b/services/api/test/deployment.node.test.ts @@ -0,0 +1,122 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + copyFileSync, + existsSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath, URL } from "node:url"; +import { experimental_readRawConfig, type Unstable_RawConfig } from "wrangler"; +import { prepareDeployment } from "../scripts/prepare-deployment"; + +const service = fileURLToPath(new URL("../", import.meta.url)); +const resources = { + DB_EXTENSIONS: "11111111-1111-4111-8111-111111111111", + DB_CENTRAL_ALERTS: "22222222-2222-4222-8222-222222222222", + AUTH_KV: "a".repeat(32), + CACHE_KV: "b".repeat(32), + DOWNLOAD_BUCKET: "api-staging", + EXTENSIONS_FRONTEND: "api-staging" +}; +const folders: string[] = []; +afterEach(() => { + for (const folder of folders.splice(0)) + rmSync(folder, { recursive: true, force: true }); +}); +function fixture() { + const folder = mkdtempSync(join(tmpdir(), "api-deployment-")); + folders.push(folder); + copyFileSync(join(service, "wrangler.jsonc"), join(folder, "wrangler.jsonc")); + symlinkSync(join(service, "src"), join(folder, "src"), "dir"); + return folder; +} +const read = (folder: string, file: string) => + JSON.parse(readFileSync(join(folder, "dist", file), "utf8")); + +describe("API deployment artifact", () => { + it("keeps production and Preview targets separate and packages the exact SQL used by both", () => { + const folder = fixture(); + const source: Unstable_RawConfig = experimental_readRawConfig({ + config: join(folder, "wrangler.jsonc") + }).rawConfig; + prepareDeployment(folder, JSON.stringify(resources)); + const artifact = read(folder, "wrangler.json"); + expect(artifact.main).toBe("index.js"); + expect(artifact.no_bundle).toBe(true); + expect( + artifact.d1_databases.map((db: { database_id: string }) => db.database_id) + ).toEqual(source.d1_databases!.map((db) => db.database_id)); + expect( + artifact.previews.d1_databases.map( + (db: { binding: string; database_id: string }) => [ + db.binding, + db.database_id + ] + ) + ).toEqual( + source.d1_databases!.map((db) => [ + db.binding, + resources[db.binding as "DB_EXTENSIONS" | "DB_CENTRAL_ALERTS"] + ]) + ); + expect( + read(folder, "wrangler.preview-migrations.json").d1_databases + ).toEqual(artifact.previews.d1_databases); + expect(artifact.previews.kv_namespaces).toEqual([ + { binding: "AUTH_KV", id: resources.AUTH_KV }, + { binding: "CACHE_KV", id: resources.CACHE_KV } + ]); + expect(artifact.previews.r2_buckets).toEqual([ + { binding: "DOWNLOAD_BUCKET", bucket_name: resources.DOWNLOAD_BUCKET } + ]); + expect(artifact.previews.services).toEqual([ + { binding: "EXTENSIONS_FRONTEND", service: resources.EXTENSIONS_FRONTEND } + ]); + expect(artifact.previews.vars.EXTENSION_PACKAGES_MODE).toBe("disabled"); + for (const database of source.d1_databases!) { + const packaged = artifact.d1_databases.find( + (db: { binding: string }) => db.binding === database.binding + ); + const original = join(service, database.migrations_dir!); + const copied = join(folder, "dist", packaged.migrations_dir); + const sql = readdirSync(original) + .filter((name) => name.endsWith(".sql")) + .sort(); + expect(readdirSync(copied).sort()).toEqual(sql); + for (const name of sql) + expect(readFileSync(join(copied, name))).toEqual( + readFileSync(join(original, name)) + ); + } + // Rebuilding without Preview resources cannot reuse a stale migration target. + prepareDeployment(folder); + expect( + existsSync(join(folder, "dist/wrangler.preview-migrations.json")) + ).toBe(false); + expect(read(folder, "wrangler.json").previews.d1_databases).toBeUndefined(); + }); + it("rejects production resource reuse and duplicate staging databases before writing an artifact", () => { + const folder = fixture(); + const source: Unstable_RawConfig = experimental_readRawConfig({ + config: join(folder, "wrangler.jsonc") + }).rawConfig; + for (const DB_EXTENSIONS of [ + source.d1_databases![0].database_id, + source.d1_databases![0].database_id!.toUpperCase(), + resources.DB_CENTRAL_ALERTS + ]) { + expect(() => + prepareDeployment( + folder, + JSON.stringify({ ...resources, DB_EXTENSIONS }) + ) + ).toThrow("distinct non-production resources"); + expect(existsSync(join(folder, "dist/wrangler.json"))).toBe(false); + } + }); +}); diff --git a/test/env.d.ts b/services/api/test/env.d.ts similarity index 100% rename from test/env.d.ts rename to services/api/test/env.d.ts diff --git a/test/integration/app.test.ts b/services/api/test/integration/app.test.ts similarity index 100% rename from test/integration/app.test.ts rename to services/api/test/integration/app.test.ts diff --git a/test/integration/scheduled.test.ts b/services/api/test/integration/scheduled.test.ts similarity index 86% rename from test/integration/scheduled.test.ts rename to services/api/test/integration/scheduled.test.ts index 0a17d647..39b336ce 100644 --- a/test/integration/scheduled.test.ts +++ b/services/api/test/integration/scheduled.test.ts @@ -5,15 +5,15 @@ const mocks = vi.hoisted(() => ({ maintain: vi.fn(), report: vi.fn() })); -vi.mock("../../src/lib/adapters/cloudflare/package-jobs", () => ({ - consumePackageJobs: vi.fn(), - recoverPackageJobs: mocks.recover +vi.mock("../../src/lib/adapters/cloudflare/packaging-jobs", () => ({ + consumePackagingJobs: vi.fn(), + recoverPackagingJobs: mocks.recover })); vi.mock("../../src/services/extensions/v2/db/resource-maintenance", () => ({ maintainExtensionResources: mocks.maintain, reportExtensionResources: mocks.report })); -import app from "../../src/app"; +import app from "../../src/app/index"; afterEach(() => vi.restoreAllMocks()); describe("scheduled maintenance", () => { it("continues resource maintenance when package recovery fails", async () => { diff --git a/test/integration/versions/index.test.ts b/services/api/test/integration/versions/index.test.ts similarity index 99% rename from test/integration/versions/index.test.ts rename to services/api/test/integration/versions/index.test.ts index e705565f..4c79e760 100644 --- a/test/integration/versions/index.test.ts +++ b/services/api/test/integration/versions/index.test.ts @@ -5,7 +5,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../src/app"; +import app from "../../../src/app/index"; const BYPASS_CACHE = { authorization: "test-bypass-cache" } as const; import { diff --git a/test/lib/adapters/cloudflare/cache.test.ts b/services/api/test/lib/adapters/cloudflare/cache.test.ts similarity index 100% rename from test/lib/adapters/cloudflare/cache.test.ts rename to services/api/test/lib/adapters/cloudflare/cache.test.ts diff --git a/test/lib/adapters/cloudflare/environment.test.ts b/services/api/test/lib/adapters/cloudflare/environment.test.ts similarity index 100% rename from test/lib/adapters/cloudflare/environment.test.ts rename to services/api/test/lib/adapters/cloudflare/environment.test.ts diff --git a/test/lib/adapters/node/cache.test.ts b/services/api/test/lib/adapters/node/cache.test.ts similarity index 100% rename from test/lib/adapters/node/cache.test.ts rename to services/api/test/lib/adapters/node/cache.test.ts diff --git a/test/lib/adapters/node/environment.test.ts b/services/api/test/lib/adapters/node/environment.test.ts similarity index 100% rename from test/lib/adapters/node/environment.test.ts rename to services/api/test/lib/adapters/node/environment.test.ts diff --git a/test/lib/adapters/node/index.test.ts b/services/api/test/lib/adapters/node/index.test.ts similarity index 100% rename from test/lib/adapters/node/index.test.ts rename to services/api/test/lib/adapters/node/index.test.ts diff --git a/test/lib/auth/assertion-helper.ts b/services/api/test/lib/auth/assertion-helper.ts similarity index 100% rename from test/lib/auth/assertion-helper.ts rename to services/api/test/lib/auth/assertion-helper.ts diff --git a/test/lib/auth/bearer-assertion.test.ts b/services/api/test/lib/auth/bearer-assertion.test.ts similarity index 96% rename from test/lib/auth/bearer-assertion.test.ts rename to services/api/test/lib/auth/bearer-assertion.test.ts index 44ef7c4d..fc70ee66 100644 --- a/test/lib/auth/bearer-assertion.test.ts +++ b/services/api/test/lib/auth/bearer-assertion.test.ts @@ -3,11 +3,11 @@ import { bearerAssertionVerifier, identitySyncAssertionVerifier } from "../../../src/lib/auth/bearer-assertion"; -import { logWarn } from "../../../src/lib/logger"; +import { logWarn } from "@fossbilling/logging"; import { PlatformContext } from "../../../src/lib/context"; import { base64UrlEncodeString, signAssertion } from "./assertion-helper"; -vi.mock("../../../src/lib/logger", () => ({ +vi.mock("@fossbilling/logging", () => ({ logError: vi.fn(), logWarn: vi.fn(), logInfo: vi.fn() @@ -18,20 +18,13 @@ const SECRET = "test-secret"; function platformWithSecret( secret: string | undefined, previousSecret?: string -): PlatformContext { +): Pick { return { - getQueue: () => { - throw new Error("No queue in auth test"); - }, - getCache: () => { - throw new Error("not implemented"); - }, getEnv: (key: string) => { if (key === "ASSERTION_SIGNING_SECRET") return secret; if (key === "ASSERTION_SIGNING_SECRET_PREVIOUS") return previousSecret; return undefined; - }, - raw: undefined as unknown as PlatformContext["raw"] + } }; } diff --git a/test/lib/auth/m2m-bearer.test.ts b/services/api/test/lib/auth/m2m-bearer.test.ts similarity index 97% rename from test/lib/auth/m2m-bearer.test.ts rename to services/api/test/lib/auth/m2m-bearer.test.ts index 762a023c..3c6071d7 100644 --- a/test/lib/auth/m2m-bearer.test.ts +++ b/services/api/test/lib/auth/m2m-bearer.test.ts @@ -16,11 +16,11 @@ const platform = { M2M_JWKS_URL: "https://auth.example.test/.well-known/jwks.json", M2M_AUDIENCE: "https://api.fossbilling.net" })[key] -} as unknown as PlatformContext; +} satisfies Pick; const emptyPlatform = { getEnv: () => undefined -} as unknown as PlatformContext; +} satisfies Pick; const verifier = m2mBearerVerifier; diff --git a/test/lib/auth/m2m-helper.ts b/services/api/test/lib/auth/m2m-helper.ts similarity index 100% rename from test/lib/auth/m2m-helper.ts rename to services/api/test/lib/auth/m2m-helper.ts diff --git a/test/lib/cache.test.ts b/services/api/test/lib/cache.test.ts similarity index 100% rename from test/lib/cache.test.ts rename to services/api/test/lib/cache.test.ts diff --git a/test/lib/context.test.ts b/services/api/test/lib/context.test.ts similarity index 73% rename from test/lib/context.test.ts rename to services/api/test/lib/context.test.ts index 86d305c9..0f17ee1d 100644 --- a/test/lib/context.test.ts +++ b/services/api/test/lib/context.test.ts @@ -1,5 +1,4 @@ import { describe, it, expect } from "vitest"; -import { Context } from "hono"; import { createPlatformContext } from "../../src/lib/context"; import { IPlatformBindings } from "../../src/lib/interfaces"; @@ -22,30 +21,22 @@ describe("createPlatformContext", () => { } }; - const mockHonoContext = { - get: () => ({}), - set: () => {}, - req: {}, - res: {} - } as unknown as Context; - - it("exposes the binding accessors and the raw context", () => { - const context = createPlatformContext(mockHonoContext, mockBindings); + it("exposes the binding accessors", () => { + const context = createPlatformContext(mockBindings, () => {}); expect(context.getCache("testCache")).toBe(mockBindings.caches.testCache); expect(context.getEnv("TEST_VAR")).toBe("test-value"); expect(context.getQueue("testQueue")).toBe(mockBindings.queues!.testQueue); - expect(context.raw).toBe(mockHonoContext); }); it("throws when a queue binding is missing", () => { - const context = createPlatformContext(mockHonoContext, mockBindings); + const context = createPlatformContext(mockBindings, () => {}); expect(() => context.getQueue("missing")).toThrow( "Queue binding 'missing' not found" ); }); it("throws when cache binding not found", () => { - const context = createPlatformContext(mockHonoContext, mockBindings); + const context = createPlatformContext(mockBindings, () => {}); expect(() => context.getCache("nonexistent")).toThrow( "Cache binding 'nonexistent' not found" diff --git a/test/lib/github-errors.test.ts b/services/api/test/lib/github-errors.test.ts similarity index 100% rename from test/lib/github-errors.test.ts rename to services/api/test/lib/github-errors.test.ts diff --git a/test/lib/middleware.test.ts b/services/api/test/lib/middleware.test.ts similarity index 100% rename from test/lib/middleware.test.ts rename to services/api/test/lib/middleware.test.ts diff --git a/test/lib/releases.test.ts b/services/api/test/lib/releases.test.ts similarity index 100% rename from test/lib/releases.test.ts rename to services/api/test/lib/releases.test.ts diff --git a/test/mocks/README.md b/services/api/test/mocks/README.md similarity index 100% rename from test/mocks/README.md rename to services/api/test/mocks/README.md diff --git a/test/mocks/github-releases.ts b/services/api/test/mocks/github-releases.ts similarity index 100% rename from test/mocks/github-releases.ts rename to services/api/test/mocks/github-releases.ts diff --git a/test/mocks/octokit.ts b/services/api/test/mocks/octokit.ts similarity index 100% rename from test/mocks/octokit.ts rename to services/api/test/mocks/octokit.ts diff --git a/test/services/central-alerts/v1/database.test.ts b/services/api/test/services/central-alerts/v1/database.test.ts similarity index 100% rename from test/services/central-alerts/v1/database.test.ts rename to services/api/test/services/central-alerts/v1/database.test.ts diff --git a/test/services/central-alerts/v1/index.test.ts b/services/api/test/services/central-alerts/v1/index.test.ts similarity index 99% rename from test/services/central-alerts/v1/index.test.ts rename to services/api/test/services/central-alerts/v1/index.test.ts index b4b17903..3ede01c8 100644 --- a/test/services/central-alerts/v1/index.test.ts +++ b/services/api/test/services/central-alerts/v1/index.test.ts @@ -4,7 +4,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import type { CentralAlertsResponse } from "../../../utils/test-types"; import { applyTestMigrations } from "../../../utils/apply-migrations"; diff --git a/test/services/extensions/v1/index.test.ts b/services/api/test/services/extensions/v1/index.test.ts similarity index 99% rename from test/services/extensions/v1/index.test.ts rename to services/api/test/services/extensions/v1/index.test.ts index 41f6adbc..92a8e618 100644 --- a/test/services/extensions/v1/index.test.ts +++ b/services/api/test/services/extensions/v1/index.test.ts @@ -6,7 +6,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { getExtensionsDb } from "../../../../src/lib/db"; import { developers, diff --git a/test/services/extensions/v2/account.test.ts b/services/api/test/services/extensions/v2/account.test.ts similarity index 99% rename from test/services/extensions/v2/account.test.ts rename to services/api/test/services/extensions/v2/account.test.ts index 749359c8..ebb494f0 100644 --- a/test/services/extensions/v2/account.test.ts +++ b/services/api/test/services/extensions/v2/account.test.ts @@ -1,5 +1,5 @@ import { UUID_V7 } from "./db-fixtures"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { env } from "cloudflare:workers"; import { createExecutionContext, diff --git a/test/services/extensions/v2/badges.test.ts b/services/api/test/services/extensions/v2/badges.test.ts similarity index 100% rename from test/services/extensions/v2/badges.test.ts rename to services/api/test/services/extensions/v2/badges.test.ts diff --git a/test/services/extensions/v2/claim-budget.test.ts b/services/api/test/services/extensions/v2/claim-budget.test.ts similarity index 100% rename from test/services/extensions/v2/claim-budget.test.ts rename to services/api/test/services/extensions/v2/claim-budget.test.ts diff --git a/test/services/extensions/v2/contract.test.ts b/services/api/test/services/extensions/v2/contract.test.ts similarity index 100% rename from test/services/extensions/v2/contract.test.ts rename to services/api/test/services/extensions/v2/contract.test.ts diff --git a/test/services/extensions/v2/db-fixtures.ts b/services/api/test/services/extensions/v2/db-fixtures.ts similarity index 100% rename from test/services/extensions/v2/db-fixtures.ts rename to services/api/test/services/extensions/v2/db-fixtures.ts diff --git a/test/services/extensions/v2/db-interceptor.ts b/services/api/test/services/extensions/v2/db-interceptor.ts similarity index 100% rename from test/services/extensions/v2/db-interceptor.ts rename to services/api/test/services/extensions/v2/db-interceptor.ts diff --git a/test/services/extensions/v2/developer-approval.test.ts b/services/api/test/services/extensions/v2/developer-approval.test.ts similarity index 100% rename from test/services/extensions/v2/developer-approval.test.ts rename to services/api/test/services/extensions/v2/developer-approval.test.ts diff --git a/test/services/extensions/v2/developer-profiles.test.ts b/services/api/test/services/extensions/v2/developer-profiles.test.ts similarity index 100% rename from test/services/extensions/v2/developer-profiles.test.ts rename to services/api/test/services/extensions/v2/developer-profiles.test.ts diff --git a/test/services/extensions/v2/directory-identity.test.ts b/services/api/test/services/extensions/v2/directory-identity.test.ts similarity index 100% rename from test/services/extensions/v2/directory-identity.test.ts rename to services/api/test/services/extensions/v2/directory-identity.test.ts diff --git a/test/services/extensions/v2/email.test.ts b/services/api/test/services/extensions/v2/email.test.ts similarity index 100% rename from test/services/extensions/v2/email.test.ts rename to services/api/test/services/extensions/v2/email.test.ts diff --git a/test/services/extensions/v2/extension-writes.test.ts b/services/api/test/services/extensions/v2/extension-writes.test.ts similarity index 100% rename from test/services/extensions/v2/extension-writes.test.ts rename to services/api/test/services/extensions/v2/extension-writes.test.ts diff --git a/test/services/extensions/v2/harness.ts b/services/api/test/services/extensions/v2/harness.ts similarity index 99% rename from test/services/extensions/v2/harness.ts rename to services/api/test/services/extensions/v2/harness.ts index c7e77ff6..99e85ca2 100644 --- a/test/services/extensions/v2/harness.ts +++ b/services/api/test/services/extensions/v2/harness.ts @@ -5,7 +5,7 @@ import { } from "cloudflare:test"; import { env } from "cloudflare:workers"; import { request as ghRequest } from "@octokit/request"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { signAssertion } from "../../../lib/auth/assertion-helper"; import { MockGitHubRequest } from "../../../utils/test-types"; import { applyTestMigrations } from "../../../utils/apply-migrations"; diff --git a/test/services/extensions/v2/index.test.ts b/services/api/test/services/extensions/v2/index.test.ts similarity index 100% rename from test/services/extensions/v2/index.test.ts rename to services/api/test/services/extensions/v2/index.test.ts diff --git a/test/services/extensions/v2/manifest.test.ts b/services/api/test/services/extensions/v2/manifest.test.ts similarity index 100% rename from test/services/extensions/v2/manifest.test.ts rename to services/api/test/services/extensions/v2/manifest.test.ts diff --git a/test/services/extensions/v2/migrations.test.ts b/services/api/test/services/extensions/v2/migrations.test.ts similarity index 100% rename from test/services/extensions/v2/migrations.test.ts rename to services/api/test/services/extensions/v2/migrations.test.ts diff --git a/test/services/extensions/v2/moderation-correct.test.ts b/services/api/test/services/extensions/v2/moderation-correct.test.ts similarity index 100% rename from test/services/extensions/v2/moderation-correct.test.ts rename to services/api/test/services/extensions/v2/moderation-correct.test.ts diff --git a/test/services/extensions/v2/moderation-notify.test.ts b/services/api/test/services/extensions/v2/moderation-notify.test.ts similarity index 100% rename from test/services/extensions/v2/moderation-notify.test.ts rename to services/api/test/services/extensions/v2/moderation-notify.test.ts diff --git a/test/services/extensions/v2/moderation-revalidate.test.ts b/services/api/test/services/extensions/v2/moderation-revalidate.test.ts similarity index 95% rename from test/services/extensions/v2/moderation-revalidate.test.ts rename to services/api/test/services/extensions/v2/moderation-revalidate.test.ts index f2b3fb7f..05ae419a 100644 --- a/test/services/extensions/v2/moderation-revalidate.test.ts +++ b/services/api/test/services/extensions/v2/moderation-revalidate.test.ts @@ -46,9 +46,9 @@ function stubFrontend(status = 200): FetcherStub { return fetcher; } -function fetchCalls(fetcher: FetcherStub): Array<[unknown, RequestInit]> { +function fetchCalls(fetcher: FetcherStub): Array<[Request]> { return (fetcher.fetch as ReturnType).mock.calls as Array< - [unknown, RequestInit] + [Request] >; } @@ -136,16 +136,16 @@ describe("CDN cache revalidation on extension mutations", () => { expect(res.status).toBe(200); expect(fetchCalls(fetcher)).toHaveLength(1); - const [url, init] = fetchCalls(fetcher)[0]; - expect(String(url)).toBe( + const [request] = fetchCalls(fetcher)[0]; + expect(request.url).toBe( "https://extensions.fossbilling.org/api/revalidate" ); - expect(init.method).toBe("POST"); - expect(init.headers).toMatchObject({ + expect(request.method).toBe("POST"); + expect(Object.fromEntries(request.headers)).toMatchObject({ authorization: `Bearer ${SECRET}`, "content-type": "application/json" }); - expect(JSON.parse(String(init.body))).toEqual({ + expect(await request.json()).toEqual({ tags: ["extensions", "developers"] }); }); @@ -163,12 +163,12 @@ describe("CDN cache revalidation on extension mutations", () => { expect(res.status).toBe(200); expect(fetchCalls(fetcher)).toHaveLength(1); - const [url, init] = fetchCalls(fetcher)[0]; - expect(String(url)).toBe( + const [request] = fetchCalls(fetcher)[0]; + expect(request.url).toBe( "https://extensions.fossbilling.org/api/revalidate" ); - expect(init.method).toBe("POST"); - expect(JSON.parse(String(init.body))).toEqual({ + expect(request.method).toBe("POST"); + expect(await request.json()).toEqual({ tags: ["extensions", "developers"] }); }); @@ -338,10 +338,10 @@ describe("CDN cache revalidation on extension mutations", () => { }); }); - it("still delists when the purge request throws", async () => { + it("still delists when the frontend binding throws synchronously", async () => { await seedModAndExtension(); const fetcher: FetcherStub = { - fetch: vi.fn(async () => { + fetch: vi.fn(() => { throw new Error("binding unavailable"); }) }; diff --git a/test/services/extensions/v2/moderation-revocation.test.ts b/services/api/test/services/extensions/v2/moderation-revocation.test.ts similarity index 100% rename from test/services/extensions/v2/moderation-revocation.test.ts rename to services/api/test/services/extensions/v2/moderation-revocation.test.ts diff --git a/test/services/extensions/v2/moderation.test.ts b/services/api/test/services/extensions/v2/moderation.test.ts similarity index 100% rename from test/services/extensions/v2/moderation.test.ts rename to services/api/test/services/extensions/v2/moderation.test.ts diff --git a/test/services/extensions/v2/ownership.test.ts b/services/api/test/services/extensions/v2/ownership.test.ts similarity index 100% rename from test/services/extensions/v2/ownership.test.ts rename to services/api/test/services/extensions/v2/ownership.test.ts diff --git a/test/services/extensions/v2/package-build.node.test.ts b/services/api/test/services/extensions/v2/package-build.node.test.ts similarity index 90% rename from test/services/extensions/v2/package-build.node.test.ts rename to services/api/test/services/extensions/v2/package-build.node.test.ts index 97806995..06746310 100644 --- a/test/services/extensions/v2/package-build.node.test.ts +++ b/services/api/test/services/extensions/v2/package-build.node.test.ts @@ -7,6 +7,8 @@ import { ZipReader, ZipWriter } from "@zip.js/zip.js"; +import { type ProcessingInput } from "@fossbilling/extension-packaging/processor"; +import { processPackage } from "@fossbilling/extension-packaging/process"; import { prepareQueuedPackage } from "../../../../src/services/extensions/v2/packages/prepare"; import type { PackageRepository, @@ -14,9 +16,9 @@ import type { SourceRow, IconObject } from "../../../../src/services/extensions/v2/packages/repository"; -import type { PackageJob } from "../../../../src/services/extensions/v2/packages/jobs"; -import type { PackageStore } from "../../../../src/services/extensions/v2/packages/store"; -import { sha256 } from "../../../../src/services/extensions/v2/package-builder"; +import type { PackagingJob } from "../../../../src/services/extensions/v2/packages/jobs"; +import type { PackageStore } from "@fossbilling/extension-packaging/store"; +import { sha256 } from "@fossbilling/extension-packaging/checksum"; const zipOptions = { useWebWorkers: false, useCompressionStream: true }; describe("package preparation outside Cloudflare", () => { @@ -51,7 +53,8 @@ describe("package preparation outside Cloudflare", () => { findArtifact: async (_sourceId, hash) => artifacts.get(hash) ?? null, hasIcon: async () => icons !== null, reserveObject: async () => {}, - removeObject: unexpected, + setObjectSize: async () => {}, + removeObject: async () => {}, claimCleanup: unexpected, finishCleanup: unexpected, linkSource: async (_owner, _manifest, input) => (source ??= input), @@ -116,7 +119,7 @@ describe("package preparation outside Cloudflare", () => { const upload = await writer.close(); const uploadId = crypto.randomUUID(); objects.set(uploadId, upload); - const job: PackageJob = { + const job: PackagingJob = { id: crypto.randomUUID(), fingerprint: "node", owner, @@ -134,7 +137,10 @@ describe("package preparation outside Cloudflare", () => { const services = { repository, store, - images, + processor: { + process: (input: ProcessingInput) => + processPackage(input, store, images) + }, baseUrl: "https://directory.example.test/extensions/v2" }; const id = await prepareQueuedPackage(services, job); diff --git a/test/services/extensions/v2/packages.test.ts b/services/api/test/services/extensions/v2/packages.test.ts similarity index 86% rename from test/services/extensions/v2/packages.test.ts rename to services/api/test/services/extensions/v2/packages.test.ts index 8db2efeb..b6aa2c20 100644 --- a/test/services/extensions/v2/packages.test.ts +++ b/services/api/test/services/extensions/v2/packages.test.ts @@ -1,9 +1,12 @@ -import { PackageJobsDatabase } from "../../../../src/services/extensions/v2/db/package-jobs"; -import { processPackageJob } from "../../../../src/services/extensions/v2/packages/jobs"; +import { processPackage } from "@fossbilling/extension-packaging/process"; +import { ProcessingInputSchema } from "@fossbilling/extension-packaging/processor"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; +import { PackagingJobsDatabase } from "../../../../src/services/extensions/v2/db/packaging-jobs"; +import { processPackagingJob } from "../../../../src/services/extensions/v2/packages/jobs"; import { - recoverPackageJobs, + recoverPackagingJobs, packagePreparation -} from "../../../../src/lib/adapters/cloudflare/package-jobs"; +} from "../../../../src/lib/adapters/cloudflare/packaging-jobs"; import { prepareQueuedPackage } from "../../../../src/services/extensions/v2/packages/prepare"; import { beforeEach, afterEach, describe, expect, it, vi } from "vitest"; import { env } from "cloudflare:workers"; @@ -21,7 +24,7 @@ import { ZipReader, TextWriter } from "@zip.js/zip.js"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { signAccessToken, activateM2mJwks, @@ -31,7 +34,7 @@ import { PackageArtifactSchema } from "../../../../src/services/extensions/v2/sc import { ManifestSchema } from "../../../../src/services/extensions/v2/schemas/manifest"; import { ExtensionContentSchema } from "../../../../src/services/extensions/v2/schemas/extensions"; import { contentManifest } from "../../../../src/services/extensions/v2/packages/manifest"; -import { sha256 } from "../../../../src/services/extensions/v2/package-builder"; +import { sha256 } from "@fossbilling/extension-packaging/checksum"; import { setupExtensionsV2Tests, authHeaders, @@ -57,9 +60,25 @@ const manifest = () => ); const packageKey = (id: string) => `/test-packages/local/packages/${id}`; let b2: ReturnType; +const preparedSvg = + ''; +const png = Uint8Array.from( + atob( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADUlEQVQImWNQTvnfAAAEOwIHOVGdNAAAAABJRU5ErkJggg==" + ), + (c) => c.charCodeAt(0) +); +const avif = Uint8Array.from( + atob( + "AAAAHGZ0eXBhdmlmAAAAAG1pZjFhdmlmbWlhZgAAAXBtZXRhAAAAAAAAACFoZGxyAAAAAAAAAABwaWN0AAAAAAAAAAAAAAAAAAAAADRpbG9jAAAAAERAAAIAAQAAAAABlAABAAAAAAAAACMAAgAAAAABtwABAAAAAAAAABAAAAA4aWluZgAAAAAAAgAAABVpbmZlAgAAAAABAABhdjAxAAAAABVpbmZlAgAAAAACAABhdjAxAAAAAA5waXRtAAAAAAABAAAAr2lwcnAAAACKaXBjbwAAAAxhdjFDgSACAAAAABRpc3BlAAAAAAAAAAEAAAABAAAAEHBpeGkAAAAAAwgICAAAAAxhdjFDgQAcAAAAAA5waXhpAAAAAAEIAAAAOGF1eEMAAAAAdXJuOm1wZWc6bXBlZ0I6Y2ljcDpzeXN0ZW1zOmF1eGlsaWFyeTphbHBoYQAAAAAdaXBtYQAAAAAAAAACAAEDgQIDAAIEhAIFhgAAABppcmVmAAAAAAAAAA5hdXhsAAIAAQABAAAAO21kYXQSAAoHOAAGkBDQaTIWGUJjBMAANAAAkEDJHGFRR0szJUz6VhIACgQYAAYVMgYYAAABQA8=" + ), + (c) => c.charCodeAt(0) +); function mockB2() { Object.assign(env, { + EXTENSION_PACKAGER_URL: "https://processor.example.test/process", + EXTENSION_PACKAGER_SECRET: "test-processor-secret", EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.example.test", EXTENSION_PACKAGES_S3_REGION: "test-region", EXTENSION_PACKAGES_S3_BUCKET: "test-packages", @@ -73,6 +92,23 @@ function mockB2() { EXTENSION_ASSETS_S3_SECRET_ACCESS_KEY: "test-secret", EXTENSION_ASSETS_PUBLIC_URL: "https://extensions-assets.fossbilling.net" }); + // The remote runner's conversion is tested with Sharp in the Node suite. + // Here it supplies prepared fixtures so these tests cover API publication. + const images = { + prepare: vi.fn(async (_icon: Uint8Array, isSvg: boolean) => + isSvg + ? [ + { + format: "svg" as const, + bytes: new Uint8Array(new TextEncoder().encode(preparedSvg)) + } + ] + : [ + { format: "avif" as const, bytes: avif }, + { format: "png" as const, bytes: png } + ] + ) + }; const calls: Request[] = []; const objects = new Map< string, @@ -84,6 +120,28 @@ function mockB2() { >(); const fetch = vi.fn(async (request: Request) => { const url = new URL(request.url); + if (url.hostname === "processor.example.test") { + const input = ProcessingInputSchema.parse(await request.json()); + const { b2ObjectStore } = + await import("@fossbilling/extension-packaging/adapters/b2"); + const store = b2ObjectStore((name) => Reflect.get(env, name), "packages"); + try { + return Response.json({ + result: await processPackage(input, store, images) + }); + } catch (error) { + const e = error as { code?: string; message: string; status?: number }; + return Response.json( + { + error: { + code: e.code, + message: e.message + } + }, + { status: e.status ?? 503 } + ); + } + } calls.push(request.clone()); const key = url.pathname; if (request.method === "PUT") { @@ -112,7 +170,7 @@ function mockB2() { }); }); vi.stubGlobal("fetch", fetch); - return { calls, objects, fetch }; + return { calls, objects, fetch, images }; } beforeEach(async () => { @@ -150,10 +208,20 @@ afterEach(() => { "PREFIX" ]) Reflect.deleteProperty(env, `EXTENSION_PACKAGES_S3_${suffix}`); + Reflect.deleteProperty(env, "EXTENSION_PACKAGER_URL"); + Reflect.deleteProperty(env, "EXTENSION_PACKAGER_SECRET"); env.EXTENSION_PACKAGES_MODE = "disabled"; // Assert outside the adapter's catch, including calls with injected failures. for (const request of requests) { const url = new URL(request.url); + if (url.hostname === "processor.example.test") { + expect(request.method).toBe("POST"); + expect(request.headers.get("Authorization")).toBe( + "Bearer test-processor-secret" + ); + expect(request.cache).toBe("no-store"); + continue; + } expect(url.hostname).toBe("s3.example.test"); expect(request.headers.get("Authorization")).toMatch(/^AWS4-HMAC-SHA256 /); expect(["PUT", "GET", "HEAD", "DELETE"]).toContain(request.method); @@ -198,14 +266,14 @@ async function upload( async function finishJob(response: Response): Promise { if (response.status !== 202) return response; const { result } = (await response.json()) as { result: { id: string } }; - const jobs = new PackageJobsDatabase(env.DB_EXTENSIONS); + const jobs = new PackagingJobsDatabase(env.DB_EXTENSIONS); let job = (await jobs.get(result.id))!; for ( let attempt = 0; attempt < 100 && job.status !== "ready" && job.status !== "failed"; attempt++ ) { - await processPackageJob(jobs, result.id, (job) => + await processPackagingJob(jobs, result.id, (job) => prepareQueuedPackage(packagePreparation(env), job) ); job = (await jobs.get(result.id))!; @@ -291,7 +359,9 @@ describe("publisher package builds", () => { vi.stubGlobal("fetch", (input: RequestInfo | URL, init?: RequestInit) => { const request = input instanceof Request ? input : new Request(input, init); - return new URL(request.url).hostname === "s3.example.test" + return ["s3.example.test", "processor.example.test"].includes( + new URL(request.url).hostname + ) ? b2.fetch(request) : jwksFetch(input, init); }); @@ -333,7 +403,7 @@ describe("publisher package builds", () => { const headers = { Authorization: `Bearer ${await signAccessToken({ clientId: "package-publisher", scopes: ["extensions:push"] })}` }; - const pending = await get(`${root}/package-jobs/${job.id}`, headers); + const pending = await get(`${root}/packaging-jobs/${job.id}`, headers); expect(pending.status).toBe(200); expect(await pending.json()).toMatchObject({ result: { status: "queued", artifact: null } @@ -341,7 +411,7 @@ describe("publisher package builds", () => { expect( (await finishJob(Response.json({ result: job }, { status: 202 }))).status ).toBe(200); - const ready = await get(`${root}/package-jobs/${job.id}`, headers); + const ready = await get(`${root}/packaging-jobs/${job.id}`, headers); expect(ready.status).toBe(200); expect(await ready.json()).toMatchObject({ result: { @@ -354,9 +424,9 @@ describe("publisher package builds", () => { "UPDATE m2m_publisher_clients SET revoked_at=CURRENT_TIMESTAMP WHERE client_id='package-publisher'" ) .run(); - expect((await get(`${root}/package-jobs/${job.id}`, headers)).status).toBe( - 403 - ); + expect( + (await get(`${root}/packaging-jobs/${job.id}`, headers)).status + ).toBe(403); }); it("builds under the mapped developer while retaining owner privacy", async () => { const response = await machineBuild(); @@ -482,11 +552,13 @@ describe("queued package preparation", () => { expect( ((await repeat.json()) as { result: { id: string } }).result.id ).toBe(job.id); - expect((await get(`${root}/package-jobs/${job.id}`, {})).status).toBe(401); + expect((await get(`${root}/packaging-jobs/${job.id}`, {})).status).toBe( + 401 + ); expect( ( await get( - `${root}/package-jobs/${job.id}`, + `${root}/packaging-jobs/${job.id}`, await authHeaders("stranger") ) ).status @@ -506,7 +578,7 @@ describe("queued package preparation", () => { retryMessages: [] }); const status = await get( - `${root}/package-jobs/${job.id}`, + `${root}/packaging-jobs/${job.id}`, await authHeaders("owner") ); expect( @@ -545,13 +617,13 @@ describe("queued package preparation", () => { expect( (await upload(manifest(), undefined, undefined, "owner", false)).status ).toBe(503); - const jobs = new PackageJobsDatabase(db); + const jobs = new PackagingJobsDatabase(db); const ids = await jobs.pending(); expect(ids).toHaveLength(1); - const { dispatchPackageJob } = + const { dispatchPackagingJob } = await import("../../../../src/services/extensions/v2/packages/jobs"); const send = vi.fn().mockResolvedValue(undefined); - await dispatchPackageJob(jobs, { send }, ids[0]); + await dispatchPackagingJob(jobs, { send }, ids[0]); expect(send).toHaveBeenCalledWith({ jobId: ids[0] }); expect(await jobs.pending()).toEqual([]); }); @@ -569,9 +641,9 @@ describe("queued package preparation", () => { vi.mocked(env.EXTENSION_PACKAGE_QUEUE.send) .mockClear() .mockRejectedValueOnce(new Error("queue unavailable")); - await recoverPackageJobs(env); + await recoverPackagingJobs(env); expect(env.EXTENSION_PACKAGE_QUEUE.send).toHaveBeenCalledTimes(2); - expect(await new PackageJobsDatabase(db).pending()).toHaveLength(1); + expect(await new PackagingJobsDatabase(db).pending()).toHaveLength(1); expect(logged).toHaveBeenCalledWith( expect.objectContaining({ context: expect.objectContaining({ error: "queue unavailable" }) @@ -588,24 +660,24 @@ describe("queued package preparation", () => { false ); const { result } = (await response.json()) as { result: { id: string } }; - const jobs = new PackageJobsDatabase(db); + const jobs = new PackagingJobsDatabase(db); const job = (await jobs.get(result.id))!; const key = packageKey(job.uploadObjectId!); const staged = b2.objects.get(key)!; b2.objects.delete(key); expect( - await processPackageJob(jobs, job.id, (job) => + await processPackagingJob(jobs, job.id, (job) => prepareQueuedPackage(packagePreparation(env), job) ) ).toBe("retry"); expect(await jobs.get(job.id)).toMatchObject({ status: "queued", attempts: 1, - error: { code: "SOURCE_UNAVAILABLE", status: 503 } + error: { code: "PACKAGE_UNAVAILABLE", status: 503 } }); b2.objects.set(key, staged); expect( - await processPackageJob(jobs, job.id, (job) => + await processPackagingJob(jobs, job.id, (job) => prepareQueuedPackage(packagePreparation(env), job) ) ).toBe("ack"); @@ -626,7 +698,7 @@ describe("queued package preparation", () => { const { result: job } = (await response.json()) as { result: { id: string }; }; - const jobs = new PackageJobsDatabase(db); + const jobs = new PackagingJobsDatabase(db); expect(await jobs.claim(job.id, "first")).not.toBeNull(); expect(await jobs.claim(job.id, "second")).toBeNull(); await db @@ -662,15 +734,15 @@ describe("queued package preparation", () => { const { result: job } = (await response.json()) as { result: { id: string }; }; - const jobs = new PackageJobsDatabase(db); + const jobs = new PackagingJobsDatabase(db); const prepare = vi.fn( ( - job: import("../../../../src/services/extensions/v2/packages/jobs").PackageJob + job: import("../../../../src/services/extensions/v2/packages/jobs").PackagingJob ) => prepareQueuedPackage(packagePreparation(env), job) ); await Promise.all([ - processPackageJob(jobs, job.id, prepare), - processPackageJob(jobs, job.id, prepare) + processPackagingJob(jobs, job.id, prepare), + processPackagingJob(jobs, job.id, prepare) ]); expect(prepare).toHaveBeenCalledTimes(1); expect((await jobs.get(job.id))!.status).toBe("ready"); @@ -696,14 +768,14 @@ describe("stored directory packages", () => { ); expect(revised.source_id).toBe(first.source_id); const uploads = requests.filter((request) => request.method === "PUT"); - expect(uploads).toHaveLength(4); // source, two manifests and one shared icon + expect(uploads).toHaveLength(5); // source, two manifests, staged icon and shared public icon expect( uploads.filter((request) => new URL(request.url).pathname.startsWith( "/test-packages/local/packages/" ) ) - ).toHaveLength(3); + ).toHaveLength(4); expect( uploads.filter((request) => new URL(request.url).pathname.startsWith("/test-assets/extensions/") @@ -789,8 +861,11 @@ describe("stored directory packages", () => { const preview = await get(path, await authHeaders("owner")); expect(preview.status).toBe(200); expect(preview.headers.get("Content-Type")).toBe("image/svg+xml"); - expect(preview.headers.get("Content-Security-Policy")).toContain("sandbox"); - expect(await preview.text()).toContain(" { .run(); expect((await get(path, {})).status).toBe(404); }); - it("sanitizes SVG scripts, events, foreign content, and stylesheets while preserving vector artwork", async () => { - const original = ` - -
HTML
 - - - - `; - const writer = new ZipWriter(new Uint8ArrayWriter(), options); - await writer.add("icon.svg", new TextReader(original)); - const artifact = await built(manifest(), await writer.close()); - const response = await get( - `${root}/packages/${artifact.id}/icon`, - await authHeaders("owner") - ); - const svg = await response.text(); - expect(response.headers.get("Content-Type")).toBe("image/svg+xml"); - expect(response.headers.get("Content-Security-Policy")).toBe( - "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:" - ); - expect(response.headers.get("X-Content-Type-Options")).toBe("nosniff"); - expect(svg).not.toMatch( - /script|onload|foreignObject|untrusted\.example|xml-stylesheet/ - ); - expect(svg).toContain("linearGradient"); - expect(svg).toContain("url(#paint)"); - expect(svg).toContain('viewBox="0 0 24 24"'); - const row = await db - .prepare( - "SELECT svg_object_id,avif_object_id,png_object_id FROM package_icons WHERE source_id=?" - ) - .bind(artifact.source_id) - .first<{ - svg_object_id: string; - avif_object_id: null; - png_object_id: null; - }>(); - expect(row?.svg_object_id).toMatch( - /^extensions\/[a-f0-9-]{36}\/icons\/[a-f0-9-]{36}\.svg$/ - ); - expect(row?.avif_object_id).toBeNull(); - expect(row?.png_object_id).toBeNull(); - const download = await get( - `${root}/packages/${artifact.id}/download`, - await authHeaders("owner") - ); - const zip = new ZipReader( - new Uint8ArrayReader(new Uint8Array(await download.arrayBuffer())), - options - ); - try { - const icon = (await zip.getEntries()).find( - (entry) => entry.filename === "icon.svg" - )!; - if (icon.directory) throw new Error("Expected icon file"); - expect(await icon.getData(new TextWriter())).toBe(original); - } finally { - await zip.close(); - } - }); - it.each([ - "', - '" - ])("rejects malformed, non-SVG, and oversized SVG files %#", async (svg) => { - const writer = new ZipWriter(new Uint8ArrayWriter(), options); - await writer.add("icon.svg", new TextReader(svg)); - expect((await upload(manifest(), await writer.close())).status).toBe(422); - expect( - await db.prepare("SELECT count(*) AS count FROM package_sources").first() - ).toEqual({ count: 0 }); - }); - it("sanitizes SVG content even with a raster filename", async () => { - const writer = new ZipWriter(new Uint8ArrayWriter(), options); - await writer.add( - "icon.png", - new TextReader( - '' - ) - ); - const artifact = await built(manifest(), await writer.close()); - const response = await get( - `${root}/packages/${artifact.id}/icon`, - await authHeaders("owner") - ); - expect(response.headers.get("Content-Type")).toBe("image/svg+xml"); - expect(await response.text()).not.toContain("onload"); + it("serves prepared AVIF or PNG according to the visitor's Accept header", async () => { + const artifact = await built(manifest(), await iconSource("png")); + const path = `${root}/packages/${artifact.id}/icon`; + const headers = await authHeaders("owner"); + const response = await get(path, headers); + expect(response.headers.get("Content-Type")).toBe("image/png"); + expect(new Uint8Array(await response.arrayBuffer())).toEqual(png); + const preferred = await get(path, { + ...headers, + Accept: "image/avif,image/png" + }); + expect(preferred.headers.get("Content-Type")).toBe("image/avif"); + expect(new Uint8Array(await preferred.arrayBuffer())).toEqual(avif); + const excluded = await get(path, { + ...headers, + Accept: "image/avif;q=0,image/png" + }); + expect(excluded.headers.get("Content-Type")).toBe("image/png"); }); - it.each(["png", "webp", "avif"] as const)( - "converts icon.%s to bounded AVIF and PNG images", - async (format) => { - const svg = - ''; - const image = await env - .PACKAGE_IMAGES!.input(new Blob([svg]).stream()) - .output({ format: `image/${format}` }); - const original = new Uint8Array(await image.response().arrayBuffer()); - const writer = new ZipWriter(new Uint8ArrayWriter(), options); - await writer.add(`icon.${format}`, new Uint8ArrayReader(original)); - const artifact = await built(manifest(), await writer.close()); - const response = await get( - `${root}/packages/${artifact.id}/icon`, - await authHeaders("owner") - ); - const avif = await get(`${root}/packages/${artifact.id}/icon`, { - ...(await authHeaders("owner")), - Accept: "image/avif,image/png" - }); - expect(avif.headers.get("Content-Type")).toBe("image/avif"); - expect( - new TextDecoder().decode((await avif.arrayBuffer()).slice(4, 12)) - ).toBe("ftypavif"); - expect( - ( - await get(`${root}/packages/${artifact.id}/icon`, { - ...(await authHeaders("owner")), - Accept: "image/avif;q=0,image/png" - }) - ).headers.get("Content-Type") - ).toBe("image/png"); - const info = await env.PACKAGE_IMAGES!.info(response.body!); - expect(info).toMatchObject({ - format: "image/png", - width: 256, - height: 128 - }); - const download = await get( - `${root}/packages/${artifact.id}/download`, - await authHeaders("owner") - ); - const zip = new ZipReader( - new Uint8ArrayReader(new Uint8Array(await download.arrayBuffer())), - options - ); - try { - const entry = (await zip.getEntries()).find( - (entry) => entry.filename === `icon.${format}` - )!; - if (entry.directory) throw new Error("Expected icon file"); - expect(await entry.getData(new Uint8ArrayWriter())).toEqual(original); - } finally { - await zip.close(); - } - } - ); it("lets moderators rebuild published source for a correction without uploading new files", async () => { const artifact = await built(); @@ -1017,7 +964,14 @@ describe("stored directory packages", () => { ).toBe(200); }); - it("rejects invalid icons without reserving the source version", async () => { + it("propagates runner icon rejection without reserving the source version", async () => { + b2.images.prepare.mockRejectedValueOnce( + new PackageError( + "INVALID_ICON", + "Package icon could not be processed", + 422 + ) + ); const writer = new ZipWriter(new Uint8ArrayWriter(), options); await writer.add("icon.png", new TextReader("not an image")); const response = await upload(manifest(), await writer.close()); @@ -1034,18 +988,7 @@ describe("stored directory packages", () => { async function iconSource(format: "svg" | "png" = "svg") { const svg = ''; - const bytes = - format === "svg" - ? new TextEncoder().encode(svg) - : new Uint8Array( - await ( - await env - .PACKAGE_IMAGES!.input(new Blob([svg]).stream()) - .output({ format: "image/png" }) - ) - .response() - .arrayBuffer() - ); + const bytes = format === "svg" ? new TextEncoder().encode(svg) : png; const writer = new ZipWriter(new Uint8ArrayWriter(), options); await writer.add( `icon.${format}`, @@ -1624,7 +1567,7 @@ describe("stored directory packages", () => { ); expect(response.status).toBe(202); expect(response.headers.get("Location")).toMatch( - /^https:\/\/api\.fossbilling\.net\/extensions\/v2\/package-jobs\// + /^https:\/\/api\.fossbilling\.net\/extensions\/v2\/packaging-jobs\// ); expect(requests.length).toBeGreaterThan(0); expect( diff --git a/test/services/extensions/v2/package-jobs.node.test.ts b/services/api/test/services/extensions/v2/packaging-jobs.node.test.ts similarity index 80% rename from test/services/extensions/v2/package-jobs.node.test.ts rename to services/api/test/services/extensions/v2/packaging-jobs.node.test.ts index 818706a3..b53055fd 100644 --- a/test/services/extensions/v2/package-jobs.node.test.ts +++ b/services/api/test/services/extensions/v2/packaging-jobs.node.test.ts @@ -1,14 +1,14 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { - dispatchPackageJob, - processPackageJob, - type PackageJob, - type PackageJobRepository + dispatchPackagingJob, + processPackagingJob, + type PackagingJob, + type PackagingJobRepository } from "../../../../src/services/extensions/v2/packages/jobs"; -import { PackageError } from "../../../../src/services/extensions/v2/packages/errors"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; function repository() { - const job: PackageJob = { + const job: PackagingJob = { id: crypto.randomUUID(), fingerprint: "test", owner: { @@ -39,7 +39,7 @@ function repository() { updatedAt: "now" }; let activeLease: string | null = null; - const jobs: PackageJobRepository = { + const jobs: PackagingJobRepository = { get: async () => ({ ...job }), claim: async (_id, lease) => { if (job.status !== "queued") return null; @@ -70,11 +70,11 @@ function repository() { afterEach(() => vi.restoreAllMocks()); -describe("provider-independent package job processing", () => { +describe("provider-independent packaging job processing", () => { it("publishes only an identifier through the queue interface", async () => { const { jobs, job } = repository(); const send = vi.fn().mockResolvedValue(undefined); - await dispatchPackageJob(jobs, { send }, job.id); + await dispatchPackagingJob(jobs, { send }, job.id); expect(send).toHaveBeenCalledWith({ jobId: job.id }); expect(jobs.markEnqueued).toHaveBeenCalledWith(job.id); }); @@ -82,7 +82,7 @@ describe("provider-independent package job processing", () => { const { jobs, job } = repository(); job.status = "processing"; const prepare = vi.fn(); - expect(await processPackageJob(jobs, job.id, prepare)).toBe("busy"); + expect(await processPackagingJob(jobs, job.id, prepare)).toBe("busy"); expect(job.status).toBe("processing"); expect(job.attempts).toBe(0); expect(prepare).not.toHaveBeenCalled(); @@ -91,7 +91,7 @@ describe("provider-independent package job processing", () => { const { jobs, job } = repository(); job.attempts = 3; const prepare = vi.fn(); - expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); + expect(await processPackagingJob(jobs, job.id, prepare)).toBe("ack"); expect(job.attempts).toBe(4); expect(job.status).toBe("failed"); expect(job.error).toMatchObject({ @@ -103,8 +103,8 @@ describe("provider-independent package job processing", () => { it("does not run completed jobs twice", async () => { const { jobs, job } = repository(); const prepare = vi.fn().mockResolvedValue("artifact"); - expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); - expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); + expect(await processPackagingJob(jobs, job.id, prepare)).toBe("ack"); + expect(await processPackagingJob(jobs, job.id, prepare)).toBe("ack"); expect(prepare).toHaveBeenCalledTimes(1); expect(job.artifactId).toBe("artifact"); }); @@ -112,9 +112,9 @@ describe("provider-independent package job processing", () => { const { jobs, job } = repository(); const logged = vi.spyOn(console, "error").mockImplementation(() => {}); const prepare = vi.fn().mockRejectedValue(new Error("storage unavailable")); - expect(await processPackageJob(jobs, job.id, prepare)).toBe("retry"); - expect(await processPackageJob(jobs, job.id, prepare)).toBe("retry"); - expect(await processPackageJob(jobs, job.id, prepare)).toBe("ack"); + expect(await processPackagingJob(jobs, job.id, prepare)).toBe("retry"); + expect(await processPackagingJob(jobs, job.id, prepare)).toBe("retry"); + expect(await processPackagingJob(jobs, job.id, prepare)).toBe("ack"); expect(job.status).toBe("failed"); expect(job.error?.message).not.toContain("storage unavailable"); expect(logged).toHaveBeenCalledWith( @@ -130,7 +130,7 @@ describe("provider-independent package job processing", () => { for (const status of [422, 403] as const) { const { jobs, job } = repository(); expect( - await processPackageJob(jobs, job.id, async () => { + await processPackagingJob(jobs, job.id, async () => { throw new PackageError("REJECTED", "Rejected", status); }) ).toBe("ack"); diff --git a/test/services/extensions/v2/pending-edits.test.ts b/services/api/test/services/extensions/v2/pending-edits.test.ts similarity index 100% rename from test/services/extensions/v2/pending-edits.test.ts rename to services/api/test/services/extensions/v2/pending-edits.test.ts diff --git a/test/services/extensions/v2/public-extensions.test.ts b/services/api/test/services/extensions/v2/public-extensions.test.ts similarity index 100% rename from test/services/extensions/v2/public-extensions.test.ts rename to services/api/test/services/extensions/v2/public-extensions.test.ts diff --git a/test/services/extensions/v2/publisher.test.ts b/services/api/test/services/extensions/v2/publisher.test.ts similarity index 100% rename from test/services/extensions/v2/publisher.test.ts rename to services/api/test/services/extensions/v2/publisher.test.ts diff --git a/test/services/extensions/v2/resource-bounds.test.ts b/services/api/test/services/extensions/v2/resource-bounds.test.ts similarity index 99% rename from test/services/extensions/v2/resource-bounds.test.ts rename to services/api/test/services/extensions/v2/resource-bounds.test.ts index 35d91192..0a27a633 100644 --- a/test/services/extensions/v2/resource-bounds.test.ts +++ b/services/api/test/services/extensions/v2/resource-bounds.test.ts @@ -5,7 +5,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { getExtensionsDb } from "../../../../src/lib/db"; import { maintainExtensionResources, diff --git a/services/api/test/services/extensions/v2/revalidate.node.test.ts b/services/api/test/services/extensions/v2/revalidate.node.test.ts new file mode 100644 index 00000000..edc85b06 --- /dev/null +++ b/services/api/test/services/extensions/v2/revalidate.node.test.ts @@ -0,0 +1,53 @@ +import { expect, it } from "vitest"; +import { Hono } from "hono"; +import { createMemoryCache } from "../../../../src/lib/adapters/node/cache"; +import { platformMiddleware } from "../../../../src/lib/middleware"; +import { listCacheKey } from "../../../../src/services/extensions/v1/list-cache"; +import { revalidateExtensions } from "../../../../src/services/extensions/v2/revalidate"; + +it("revalidates through the platform capabilities without a Cloudflare context", async () => { + const cache = createMemoryCache(); + const tasks: Promise[] = []; + const requests: Request[] = []; + const app = new Hono(); + app.use( + "*", + platformMiddleware( + { + caches: { CACHE_KV: cache }, + environment: { + get: (name) => + name === "EXTENSIONS_REVALIDATE_SECRET" ? "local-secret" : undefined + }, + services: { + EXTENSIONS_FRONTEND: async (request) => { + requests.push(request); + return new Response(null, { status: 204 }); + } + } + }, + (task) => tasks.push(task) + ) + ); + app.post("/revalidate", (c) => { + revalidateExtensions(c); + return c.body(null, 204); + }); + + try { + const key = listCacheKey("module", undefined); + await cache.put(key, "cached extension list"); + await cache.put("unrelated", "keep"); + expect((await app.request("/revalidate", { method: "POST" })).status).toBe( + 204 + ); + expect(tasks.length).toBeGreaterThan(0); + await Promise.all(tasks); + expect(await cache.get(key)).toBeNull(); + expect(await cache.get("unrelated")).toBe("keep"); + expect(requests).toHaveLength(1); + expect(requests[0]).toBeInstanceOf(Request); + } finally { + cache.close(); + } +}); diff --git a/test/services/previews/v1/budget.test.ts b/services/api/test/services/previews/v1/budget.test.ts similarity index 99% rename from test/services/previews/v1/budget.test.ts rename to services/api/test/services/previews/v1/budget.test.ts index 48d0cfbd..02654372 100644 --- a/test/services/previews/v1/budget.test.ts +++ b/services/api/test/services/previews/v1/budget.test.ts @@ -5,7 +5,7 @@ import { runInDurableObject, waitOnExecutionContext } from "cloudflare:test"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { Hono } from "hono"; import { previewGitHub, diff --git a/test/services/previews/v1/cache.test.ts b/services/api/test/services/previews/v1/cache.test.ts similarity index 100% rename from test/services/previews/v1/cache.test.ts rename to services/api/test/services/previews/v1/cache.test.ts diff --git a/test/services/previews/v1/commit.test.ts b/services/api/test/services/previews/v1/commit.test.ts similarity index 99% rename from test/services/previews/v1/commit.test.ts rename to services/api/test/services/previews/v1/commit.test.ts index 467545a3..cb00c077 100644 --- a/test/services/previews/v1/commit.test.ts +++ b/services/api/test/services/previews/v1/commit.test.ts @@ -5,7 +5,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { MockGitHubRequest } from "../../../utils/test-types"; import { suppressConsole } from "../../../utils/mock-helpers"; diff --git a/test/services/previews/v1/main.test.ts b/services/api/test/services/previews/v1/main.test.ts similarity index 99% rename from test/services/previews/v1/main.test.ts rename to services/api/test/services/previews/v1/main.test.ts index 0ad538d6..0c3e605a 100644 --- a/test/services/previews/v1/main.test.ts +++ b/services/api/test/services/previews/v1/main.test.ts @@ -5,7 +5,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { MockGitHubRequest } from "../../../utils/test-types"; import { suppressConsole } from "../../../utils/mock-helpers"; diff --git a/test/services/previews/v1/pr.test.ts b/services/api/test/services/previews/v1/pr.test.ts similarity index 99% rename from test/services/previews/v1/pr.test.ts rename to services/api/test/services/previews/v1/pr.test.ts index c077cda7..dc4baefa 100644 --- a/test/services/previews/v1/pr.test.ts +++ b/services/api/test/services/previews/v1/pr.test.ts @@ -5,7 +5,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { MockGitHubRequest } from "../../../utils/test-types"; import { suppressConsole } from "../../../utils/mock-helpers"; diff --git a/test/services/stats/v1/index.test.ts b/services/api/test/services/stats/v1/index.test.ts similarity index 99% rename from test/services/stats/v1/index.test.ts rename to services/api/test/services/stats/v1/index.test.ts index d2b399e4..2bf89d6a 100644 --- a/test/services/stats/v1/index.test.ts +++ b/services/api/test/services/stats/v1/index.test.ts @@ -5,7 +5,7 @@ import { createExecutionContext, waitOnExecutionContext } from "cloudflare:test"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { mockGitHubReleases, mockComposerJson, diff --git a/test/services/versions/v1/errors.test.ts b/services/api/test/services/versions/v1/errors.test.ts similarity index 99% rename from test/services/versions/v1/errors.test.ts rename to services/api/test/services/versions/v1/errors.test.ts index 80b47887..c58db157 100644 --- a/test/services/versions/v1/errors.test.ts +++ b/services/api/test/services/versions/v1/errors.test.ts @@ -5,7 +5,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { mockGitHubReleases, mockComposerJson diff --git a/test/services/versions/v1/index.test.ts b/services/api/test/services/versions/v1/index.test.ts similarity index 99% rename from test/services/versions/v1/index.test.ts rename to services/api/test/services/versions/v1/index.test.ts index 5eb02bb4..f166eef3 100644 --- a/test/services/versions/v1/index.test.ts +++ b/services/api/test/services/versions/v1/index.test.ts @@ -6,7 +6,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { mockGitHubReleases, diff --git a/test/services/versions/v1/middleware.test.ts b/services/api/test/services/versions/v1/middleware.test.ts similarity index 98% rename from test/services/versions/v1/middleware.test.ts rename to services/api/test/services/versions/v1/middleware.test.ts index fbf981f6..633b5170 100644 --- a/test/services/versions/v1/middleware.test.ts +++ b/services/api/test/services/versions/v1/middleware.test.ts @@ -5,7 +5,7 @@ import { waitOnExecutionContext } from "cloudflare:test"; import { env } from "cloudflare:workers"; -import app from "../../../../src/app"; +import app from "../../../../src/app/index"; import { mockGitHubReleases, mockComposerJson diff --git a/test/utils/apply-migrations.ts b/services/api/test/utils/apply-migrations.ts similarity index 100% rename from test/utils/apply-migrations.ts rename to services/api/test/utils/apply-migrations.ts diff --git a/test/utils/isolate-edge-cache.ts b/services/api/test/utils/isolate-edge-cache.ts similarity index 100% rename from test/utils/isolate-edge-cache.ts rename to services/api/test/utils/isolate-edge-cache.ts diff --git a/test/utils/mock-helpers.ts b/services/api/test/utils/mock-helpers.ts similarity index 100% rename from test/utils/mock-helpers.ts rename to services/api/test/utils/mock-helpers.ts diff --git a/test/utils/test-types.ts b/services/api/test/utils/test-types.ts similarity index 100% rename from test/utils/test-types.ts rename to services/api/test/utils/test-types.ts diff --git a/services/api/tsconfig.json b/services/api/tsconfig.json new file mode 100644 index 00000000..b4672532 --- /dev/null +++ b/services/api/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*.ts", "test/**/*.ts", "*.ts", "scripts/**/*.ts"], + "compilerOptions": { + "types": [ + "./worker-configuration.d.ts", + "@cloudflare/vitest-plugin", + "node" + ] + } +} diff --git a/vitest.config.ts b/services/api/vitest.config.ts similarity index 98% rename from vitest.config.ts rename to services/api/vitest.config.ts index 0fd362d1..b2beaf96 100644 --- a/vitest.config.ts +++ b/services/api/vitest.config.ts @@ -18,7 +18,6 @@ export default defineConfig({ cloudflareTest({ wrangler: { configPath: "./wrangler.jsonc" }, miniflare: { - images: { binding: "PACKAGE_IMAGES" }, // Add test environment variables bindings: { GITHUB_TOKEN: "test-github-token", diff --git a/vitest.node.config.ts b/services/api/vitest.node.config.ts similarity index 89% rename from vitest.node.config.ts rename to services/api/vitest.node.config.ts index 11e93c0b..22bdcf76 100644 --- a/vitest.node.config.ts +++ b/services/api/vitest.node.config.ts @@ -6,7 +6,7 @@ export default defineConfig({ include: [ "test/lib/adapters/node/**/*.test.ts", "test/services/extensions/v2/migrations.test.ts", - "test/services/extensions/v2/*.node.test.ts" + "test/**/*.node.test.ts" ], testTimeout: 10000, diff --git a/worker-configuration.d.ts b/services/api/worker-configuration.d.ts similarity index 99% rename from worker-configuration.d.ts rename to services/api/worker-configuration.d.ts index 5e25d541..ea247648 100644 --- a/worker-configuration.d.ts +++ b/services/api/worker-configuration.d.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: e610663ea57a18e0636f4e14dece74ae) +// Generated by Wrangler by running `wrangler types --env-interface=CloudflareBindings` (hash: d1888a571fcbd45ad7ca22833c048474) // Runtime types generated with workerd@1.20261104.1 2026-10-01 interface __BaseEnv_CloudflareBindings { AUTH_KV: KVNamespace; @@ -10,7 +10,6 @@ interface __BaseEnv_CloudflareBindings { EXTENSION_PACKAGE_QUEUE: Queue; EXTENSION_WRITE_RATE_LIMITER: RateLimit; PROFILE_CREATION_RATE_LIMITER: RateLimit; - PACKAGE_IMAGES: ImagesBinding; ENVIRONMENT: "production"; EXTENSION_PACKAGES_MODE: "disabled"; EXTENSION_PACKAGES_BASE_URL: "https://api.fossbilling.net/extensions/v2"; diff --git a/wrangler.jsonc b/services/api/wrangler.jsonc similarity index 79% rename from wrangler.jsonc rename to services/api/wrangler.jsonc index 9ea8ef5c..465eab43 100644 --- a/wrangler.jsonc +++ b/services/api/wrangler.jsonc @@ -1,5 +1,5 @@ { - "$schema": "node_modules/wrangler/config-schema.json", + "$schema": "../../node_modules/wrangler/config-schema.json", "name": "api", "main": "src/app/index.ts", "compatibility_date": "2026-10-01", @@ -24,7 +24,7 @@ "keep_vars": true, "vars": { "ENVIRONMENT": "production", - // Enable only after the queue, Images binding and B2 secrets are configured. + // Enable only after the queue, HTTP processor and B2 secrets are configured. "EXTENSION_PACKAGES_MODE": "disabled", "EXTENSION_PACKAGES_BASE_URL": "https://api.fossbilling.net/extensions/v2", "EXTENSION_ASSETS_PUBLIC_URL": "https://extensions-assets.fossbilling.net", @@ -77,7 +77,6 @@ } ] }, - "images": { "binding": "PACKAGE_IMAGES", "remote": false }, "r2_buckets": [ { "binding": "DOWNLOAD_BUCKET", @@ -119,5 +118,30 @@ "period": 60 } } - ] + ], + "previews": { + "vars": { + "ENVIRONMENT": "preview", + "EXTENSION_PACKAGES_MODE": "disabled", + "EXTENSIONS_V2_EMAIL_PROVIDER": "disabled", + "EXTENSIONS_RETENTION_MODE": "dry-run" + }, + "durable_objects": { + "bindings": [ + { "name": "PREVIEW_GITHUB_BUDGET", "class_name": "PreviewGitHubBudget" } + ] + }, + "ratelimits": [ + { + "name": "EXTENSION_WRITE_RATE_LIMITER", + "namespace_id": "2002", + "simple": { "limit": 60, "period": 60 } + }, + { + "name": "PROFILE_CREATION_RATE_LIMITER", + "namespace_id": "2001", + "simple": { "limit": 3, "period": 60 } + } + ] + } } diff --git a/services/extension-packager/README.md b/services/extension-packager/README.md new file mode 100644 index 00000000..305d2492 --- /dev/null +++ b/services/extension-packager/README.md @@ -0,0 +1,103 @@ +# Extension packager + +A separately deployed Node service for ZIP validation, manifest replacement and icon +preparation. It imports the shared `@fossbilling/extension-packaging` contracts and +implementation; it has no dependency on the API service source, database or Cloudflare +bindings. Netlify's entry point in `netlify/functions/extension-packager.ts` delegates +to the same standard Request/Response handler used locally in `src/handler.ts`. + +## Local development + +Install dependencies with `npm ci` from the repository root. Put development settings +in the ignored `services/extension-packager/.dev.vars` file. It needs: + +```dotenv +EXTENSION_PACKAGER_SECRET=local-shared-secret +EXTENSION_PACKAGES_S3_ENDPOINT=https://s3.example.invalid +EXTENSION_PACKAGES_S3_REGION=development-region +EXTENSION_PACKAGES_S3_BUCKET=development-bucket +EXTENSION_PACKAGES_S3_PREFIX=local +EXTENSION_PACKAGES_S3_ACCESS_KEY_ID=development-key +EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY=development-secret +``` + +Use the actual development bucket settings locally, matching the API's private bucket +and object prefix. Configure `services/api/.dev.vars` with +`EXTENSION_PACKAGER_URL=http://127.0.0.1:8790/process` and the same shared secret. +The public asset bucket credentials remain configured only on the API. + +From the repository root: + +```sh +npm run extension-packager:dev +npm test --workspace @fossbilling/extension-packager +``` + +The server listens on `127.0.0.1:8790`; `EXTENSION_PACKAGER_PORT` can override the port. +Tests use simulated S3 responses and do not touch live B2 buckets. + +## Netlify deployment + +Create a dedicated Netlify project connected to this repository: + +- Base directory: leave unset to install from the repository root and its lockfile. +- Package directory: `services/extension-packager`. +- Build command: leave blank; Netlify bundles the function automatically. +- Configuration: `services/extension-packager/netlify.toml`. +- Publish directory: `services/extension-packager/netlify/public` (empty placeholder). +- Functions directory: `services/extension-packager/netlify/functions`. + +Paths in `netlify.toml` are relative to the repository-root base directory, following +[Netlify's monorepo configuration](https://docs.netlify.com/build/configure-builds/monorepos/). +The config selects Node.js 24 and keeps the native Sharp and sanitizer dependencies +external so Netlify includes their binaries and WASM files. Verify these files and +Node.js 24 in the deployed artifact/runtime before enabling processing. + +Set the seven variables listed above through Netlify's dashboard, CLI or API for the +function runtime. Use production credentials and a separate randomly generated shared +secret for production. Keep production values in the production deploy context. +Do not provide public asset bucket credentials, D1 credentials or API administration +tokens. Netlify runtime variables do not come from `netlify.toml`; redeploy after changing them. + +After deployment, set the API's dashboard-managed `EXTENSION_PACKAGER_URL` to +`https://.netlify.app/.netlify/functions/extension-packager`, and set its +`EXTENSION_PACKAGER_SECRET` to the same secret. The HTTP adapter requires +HTTPS in production and allows loopback HTTP for local development. + +Deploying this project only deploys the packager and its imported code. It does not +start or deploy the API. A Git-connected project can redeploy its configured branch +independently of the API's GitHub Actions deployment. + +## Processing and recovery + +The API retains D1 ownership checks, job leases, immutable source/artifact records, +and publication. ZIP validation, manifest replacement and image preparation run +outside Workers Free. The API has no inline-processing fallback. + +The API reserves bounded private B2 output keys before invoking `PackageProcessor`. +The runner reads the source and writes the resulting ZIP and temporary prepared +icons directly to the private package bucket. Requests and responses contain only +metadata, not ZIP bytes, avoiding function request/response size limits. +The API then creates/reuses the permanent extension draft, copies the prepared +icons to their public UUID paths, records the artifact, and deletes temporary icons. +No ZIP is reopened to serve an icon. The runner needs no D1 credentials or public +asset bucket key. + +Output reservations use the maximum object size until processing succeeds; the +archive reservation then shrinks to its actual size. Unused icon reservations are +removed after successful publication to the asset bucket. Failed or interrupted +outputs remain tracked for normal cleanup, including a runner whose response was +lost after it wrote to B2. + +This is a regular synchronous function. The existing API Queue provides background +delivery and waits for the completed response before acknowledging the job. Do not +use Netlify background-function mode, which returns 202 before processing completes. +The packager deadline is 45 seconds, the HTTP client timeout is 50 seconds, and the +Netlify synchronous limit is 60 seconds. Invalid ZIPs and icons return 422; transport +failures, timeouts and unavailable storage return retryable 503 errors. Existing API +attempt limits and lease recovery apply. + +Production package processing remains disabled in `services/api/wrangler.jsonc`. +Before enablement, verify the Netlify deployment, measure runtime and credit usage, +and repeat the controlled production package test, including near-limit ZIPs, +SVG/raster icons, stored-source rebuilds, approval and cleanup. diff --git a/services/extension-packager/netlify.toml b/services/extension-packager/netlify.toml new file mode 100644 index 00000000..e33daf7f --- /dev/null +++ b/services/extension-packager/netlify.toml @@ -0,0 +1,10 @@ +[build] + publish = "services/extension-packager/netlify/public" + +[build.environment] + NODE_VERSION = "24" + +[functions] + directory = "services/extension-packager/netlify/functions" + node_bundler = "esbuild" + external_node_modules = ["sharp", "@zeropress/svg-hush"] diff --git a/services/extension-packager/netlify/functions/extension-packager.ts b/services/extension-packager/netlify/functions/extension-packager.ts new file mode 100644 index 00000000..b1c4af11 --- /dev/null +++ b/services/extension-packager/netlify/functions/extension-packager.ts @@ -0,0 +1,4 @@ +import { handlePackageProcessing } from "../../src/handler"; + +export default (request: Request) => + handlePackageProcessing(request, (name) => process.env[name]); diff --git a/services/extension-packager/netlify/public/.gitkeep b/services/extension-packager/netlify/public/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/services/extension-packager/package.json b/services/extension-packager/package.json new file mode 100644 index 00000000..00ba1210 --- /dev/null +++ b/services/extension-packager/package.json @@ -0,0 +1,22 @@ +{ + "name": "@fossbilling/extension-packager", + "private": true, + "version": "0.0.0", + "type": "module", + "license": "AGPL-3.0-only", + "scripts": { + "dev": "tsx --env-file=.dev.vars scripts/extension-packager-dev.ts", + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@fossbilling/extension-packaging": "*", + "@fossbilling/logging": "*", + "@zeropress/svg-hush": "0.1.1", + "sharp": "0.35.5" + }, + "devDependencies": { + "tsx": "4.23.15", + "@zip.js/zip.js": "2.24.0" + } +} diff --git a/services/extension-packager/scripts/extension-packager-dev.ts b/services/extension-packager/scripts/extension-packager-dev.ts new file mode 100644 index 00000000..fe558806 --- /dev/null +++ b/services/extension-packager/scripts/extension-packager-dev.ts @@ -0,0 +1,33 @@ +import { createServer } from "node:http"; +import { Readable } from "node:stream"; +import { handlePackageProcessing } from "../src/handler"; + +const port = Number(process.env.EXTENSION_PACKAGER_PORT ?? 8790); +createServer(async (incoming, outgoing) => { + const controller = new AbortController(); + outgoing.on("close", () => { + if (!outgoing.writableFinished) controller.abort(); + }); + try { + const init = { + method: incoming.method, + headers: incoming.headers as Record, + body: ["GET", "HEAD"].includes(incoming.method ?? "GET") + ? undefined + : Readable.toWeb(incoming), + signal: controller.signal, + duplex: "half" as const + }; + const response = await handlePackageProcessing( + new Request(`http://localhost:${port}${incoming.url}`, init), + (name) => process.env[name] + ); + outgoing.writeHead(response.status, Object.fromEntries(response.headers)); + outgoing.end(new Uint8Array(await response.arrayBuffer())); + } catch { + outgoing.writeHead(500); + outgoing.end(); + } +}).listen(port, "127.0.0.1", () => + console.log(`Extension packager listening on http://127.0.0.1:${port}`) +); diff --git a/services/extension-packager/src/handler.ts b/services/extension-packager/src/handler.ts new file mode 100644 index 00000000..0c178f10 --- /dev/null +++ b/services/extension-packager/src/handler.ts @@ -0,0 +1,69 @@ +import { logError } from "@fossbilling/logging"; +import { timingSafeEqual } from "node:crypto"; +import { b2ObjectStore } from "@fossbilling/extension-packaging/adapters/b2"; +import { NodePackageImages } from "./images"; +import { ProcessingInputSchema } from "@fossbilling/extension-packaging/processor"; +import { processPackage } from "@fossbilling/extension-packaging/process"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; +import { boundedBytes } from "@fossbilling/extension-packaging/store"; + +// Standard Request/Response keeps the handler usable on Netlify or another Node host. +export async function handlePackageProcessing( + request: Request, + getEnv: (name: string) => string | undefined +): Promise { + const headers = { "Cache-Control": "no-store" }; + const secret = getEnv("EXTENSION_PACKAGER_SECRET"); + if (!secret) + return Response.json( + { error: { message: "Package processor is not configured" } }, + { status: 503, headers } + ); + const supplied = Buffer.from(request.headers.get("Authorization") ?? ""); + const expected = Buffer.from(`Bearer ${secret}`); + if ( + supplied.length !== expected.length || + !timingSafeEqual(supplied, expected) + ) + return new Response(null, { status: 401, headers }); + if (request.method !== "POST") + return new Response(null, { + status: 405, + headers: { ...headers, Allow: "POST" } + }); + const signal = AbortSignal.any([request.signal, AbortSignal.timeout(45000)]); + try { + const bytes = await boundedBytes(request.body, 64 * 1024); + let input; + try { + input = ProcessingInputSchema.parse( + JSON.parse(new TextDecoder().decode(bytes)) + ); + } catch { + return Response.json( + { error: { message: "Invalid processing request" } }, + { status: 422, headers } + ); + } + const result = await processPackage( + input, + b2ObjectStore(getEnv, "packages", signal), + new NodePackageImages(), + signal + ); + return Response.json({ result }, { headers }); + } catch (error) { + if (error instanceof PackageError && [413, 422].includes(error.status)) + return Response.json( + { error: { code: error.code, message: error.message } }, + { status: 422, headers } + ); + logError("extensions/packages", "Package processing failed", { + error: error instanceof Error ? error.message : "Unknown error" + }); + return Response.json( + { error: { message: "Package processing is temporarily unavailable" } }, + { status: 503, headers } + ); + } +} diff --git a/services/extension-packager/src/images.ts b/services/extension-packager/src/images.ts new file mode 100644 index 00000000..44cce599 --- /dev/null +++ b/services/extension-packager/src/images.ts @@ -0,0 +1,49 @@ +import sharp from "sharp"; +import { sanitizeSvg } from "@zeropress/svg-hush/node"; +import type { + PackageImageProcessor, + IconVariant +} from "@fossbilling/extension-packaging/images"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; + +export class NodePackageImages implements PackageImageProcessor { + async prepare( + icon: Uint8Array, + isSvg: boolean + ): Promise { + try { + if ( + isSvg || + (await sharp(icon, { limitInputPixels: 16 * 1024 * 1024 }).metadata()) + .format === "svg" + ) + return [{ format: "svg", bytes: new Uint8Array(sanitizeSvg(icon)) }]; + const variants: IconVariant[] = []; + for (const format of ["avif", "png"] as const) { + const bytes = new Uint8Array( + await sharp(icon, { limitInputPixels: 16 * 1024 * 1024 }) + .rotate() + .resize({ + width: 256, + height: 256, + fit: "inside", + withoutEnlargement: true + }) + .toFormat(format) + .timeout({ seconds: 10 }) + .toBuffer() + ); + if (bytes.length > 1024 * 1024) + throw new Error("Prepared icon exceeds size limit"); + variants.push({ format, bytes }); + } + return variants; + } catch { + throw new PackageError( + "INVALID_ICON", + "Package icon could not be processed", + 422 + ); + } + } +} diff --git a/services/extension-packager/test/processor.test.ts b/services/extension-packager/test/processor.test.ts new file mode 100644 index 00000000..ed4c85fc --- /dev/null +++ b/services/extension-packager/test/processor.test.ts @@ -0,0 +1,348 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + TextReader, + TextWriter, + Uint8ArrayReader, + Uint8ArrayWriter, + ZipReader, + ZipWriter +} from "@zip.js/zip.js"; +import sharp from "sharp"; +import { createServer } from "node:http"; +import { once } from "node:events"; +import { handlePackageProcessing } from "../src/handler"; +import { HttpPackageProcessor } from "@fossbilling/extension-packaging/adapters/http"; +import { sha256 } from "@fossbilling/extension-packaging/checksum"; +import type { ProcessingInput } from "@fossbilling/extension-packaging/processor"; + +const encode = (text: string) => new Uint8Array(new TextEncoder().encode(text)); + +const manifest = { + manifest_version: 1 as const, + id: "processor-example", + type: "module" as const, + name: "Processor Example", + version: "1.0.0", + description: "Local processor test", + author: { name: "Developer" }, + license: "MIT", + requires: { fossbilling: ">=0.7.0", php: ">=8.3" } +}; +const config: Record = { + EXTENSION_PACKAGER_SECRET: "local-test-secret", + EXTENSION_PACKAGES_S3_ENDPOINT: "https://s3.example.test", + EXTENSION_PACKAGES_S3_REGION: "test-region", + EXTENSION_PACKAGES_S3_BUCKET: "private-packages", + EXTENSION_PACKAGES_S3_PREFIX: "test", + EXTENSION_PACKAGES_S3_ACCESS_KEY_ID: "test-key", + EXTENSION_PACKAGES_S3_SECRET_ACCESS_KEY: "test-secret" +}; +const options = { useWebWorkers: false, useCompressionStream: true }; +const svg = ` + +
HTML
 + + +`; +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); +async function archive(iconName = "icon.svg", icon = encode(svg)) { + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + await writer.add("manifest.json", new TextReader("replaced metadata")); + await writer.add( + "Service.php", + new TextReader("; checksum: string } + >(); + const key = (id: string) => "/private-packages/test/packages/" + id; + const handler = (request: Request) => + handlePackageProcessing(request, (name) => config[name]); + const fetch = vi.fn(async (request: Request) => { + const url = new URL(request.url); + if (url.hostname === "processor.example.test") return handler(request); + if (url.hostname === "127.0.0.1") return nativeFetch(request); + if (url.hostname !== "s3.example.test") + throw new Error("Unexpected network request"); + expect(request.cache).toBe("no-store"); + if (request.method === "PUT") { + objects.set(url.pathname, { + bytes: new Uint8Array(await request.arrayBuffer()), + checksum: request.headers.get("x-amz-meta-sha256")! + }); + return new Response(null); + } + const value = objects.get(url.pathname); + return value + ? new Response(value.bytes, { + headers: { "x-amz-meta-sha256": value.checksum } + }) + : new Response(null, { status: 404 }); + }); + vi.stubGlobal("fetch", fetch); + const processor = new HttpPackageProcessor( + "https://processor.example.test/process", + config.EXTENSION_PACKAGER_SECRET + ); + const input = async ( + bytes: Uint8Array + ): Promise => { + const id = crypto.randomUUID(), + checksum = await sha256(bytes); + objects.set(key(id), { bytes, checksum }); + return { + manifest, + source: { id, sha256: checksum }, + archive: crypto.randomUUID(), + icons: [crypto.randomUUID(), crypto.randomUUID()] + }; + }; + return { objects, key, fetch, handler, processor, input }; +} +describe("Node package processor over the generic HTTP contract", () => { + it("stores the prepared ZIP privately, sanitizes SVG, and rebuilds from the same original source", async () => { + const f = fixture(), + input = await f.input(await archive()); + const result = await f.processor.process(input); + const bytes = f.objects.get(f.key(input.archive))!.bytes; + expect(await sha256(bytes)).toBe(result.archive.sha256); + const reader = new ZipReader(new Uint8ArrayReader(bytes), options); + const files: Record = {}; + for (const entry of await reader.getEntries()) + if (!entry.directory) + files[entry.filename] = await entry.getData(new TextWriter()); + await reader.close(); + expect(JSON.parse(files["manifest.json"])).toEqual(manifest); + expect(files["Service.php"]).toBe( + " { + const f = fixture(); + const original = new Uint8Array( + await sharp({ + create: { + width: 512, + height: 256, + channels: 4, + background: "#2364ff80" + } + }) + .toFormat(format) + .toBuffer() + ); + const result = await f.processor.process( + await f.input(await archive("icon." + format, original)) + ); + expect(result.icons.map((i) => i.format)).toEqual(["avif", "png"]); + for (const icon of result.icons) { + const bytes = f.objects.get(f.key(icon.id))!.bytes; + const info = await sharp(bytes).metadata(); + expect(info).toMatchObject({ + format: icon.format === "avif" ? "heif" : "png", + width: 256, + height: 128 + }); + expect(await sha256(bytes)).toBe(icon.sha256); + } + } + ); + it("sanitizes SVG masquerading as a raster icon and rejects invalid images before writing outputs", async () => { + const f = fixture(); + const result = await f.processor.process( + await f.input(await archive("icon.png")) + ); + expect(result.icons[0].format).toBe("svg"); + const sanitized = f.objects.get(f.key(result.icons[0].id))!.bytes; + expect(new TextDecoder().decode(sanitized)).not.toMatch(/')], + [ + "icon.svg", + encode( + '" + ) + ], + ["icon.png", encode("not an image")] + ] as const) { + const input = await f.input(await archive(name, bytes)); + await expect(f.processor.process(input)).rejects.toMatchObject({ + code: "INVALID_ICON", + status: 422 + }); + expect(f.objects.has(f.key(input.archive))).toBe(false); + } + }); + it("rejects unauthenticated and invalid requests before storage access", async () => { + const f = fixture(); + expect( + ( + await f.handler( + new Request("https://processor.example.test", { + method: "POST", + body: "{}" + }) + ) + ).status + ).toBe(401); + expect(f.fetch).not.toHaveBeenCalled(); + const response = await f.handler( + new Request("https://processor.example.test", { + method: "POST", + headers: { + Authorization: "Bearer " + config.EXTENSION_PACKAGER_SECRET + }, + body: "{}" + }) + ); + expect(response.status).toBe(422); + expect(f.fetch).not.toHaveBeenCalled(); + }); + it("rejects invalid archives and changed source bytes without writing outputs", async () => { + const f = fixture(), + input = await f.input(encode("not a ZIP")); + await expect(f.processor.process(input)).rejects.toMatchObject({ + code: "INVALID_PACKAGE", + status: 422 + }); + expect(f.objects.size).toBe(1); + const valid = await f.input(await archive()); + f.objects.get(f.key(valid.source.id))!.bytes = new Uint8Array([1, 2, 3]); + vi.spyOn(console, "error").mockImplementation(() => {}); + await expect(f.processor.process(valid)).rejects.toMatchObject({ + status: 503 + }); + expect(f.objects.has(f.key(valid.archive))).toBe(false); + }); + it("processes a near-limit ZIP over a real local HTTP connection with a small JSON response", async () => { + const f = fixture(); + const writer = new ZipWriter(new Uint8ArrayWriter(), options); + for (let i = 0; i < 500; i++) { + const bytes = new Uint8Array(33000); + crypto.getRandomValues(bytes.subarray(0, 9900)); + await writer.add(`files/file-${i}.bin`, new Uint8ArrayReader(bytes)); + } + const source = await writer.close(); + expect(source.length).toBeLessThanOrEqual(5 * 1024 * 1024); + expect(source.length).toBeGreaterThan(4.5 * 1024 * 1024); + const input = await f.input(source); + const server = createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + const response = await f.handler( + new Request("http://localhost/process", { + method: "POST", + headers: { Authorization: String(req.headers.authorization) }, + body: Buffer.concat(chunks) + }) + ); + res.writeHead(response.status, Object.fromEntries(response.headers)); + res.end(new Uint8Array(await response.arrayBuffer())); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + try { + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("Missing HTTP port"); + const processor = new HttpPackageProcessor( + `http://127.0.0.1:${address.port}/process`, + config.EXTENSION_PACKAGER_SECRET, + true + ); + const result = await processor.process(input); + expect(result.archive.bytes).toBeGreaterThan(4.5 * 1024 * 1024); + expect(result.icons).toEqual([]); + expect(JSON.stringify(result).length).toBeLessThan(1024); + expect(await sha256(f.objects.get(f.key(input.archive))!.bytes)).toBe( + result.archive.sha256 + ); + const corrected = await processor.process({ + ...input, + manifest: { ...manifest, description: "Rebuilt after a large package" }, + archive: crypto.randomUUID(), + icons: undefined + }); + expect(corrected.archive.sha256).not.toBe(result.archive.sha256); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())) + ); + } + }, 30000); + it("fails closed on malformed runner responses and leaves timeouts retryable", async () => { + const input = await fixture().input(await archive()); + const processor = new HttpPackageProcessor( + "https://processor.example.test/process", + "secret" + ); + vi.stubGlobal("fetch", async () => + Response.json({ + result: { + archive: { bytes: 20, sha256: "0".repeat(64) }, + icons: [ + { + id: crypto.randomUUID(), + format: "svg", + bytes: 10, + sha256: "0".repeat(64) + } + ] + } + }) + ); + await expect(processor.process(input)).rejects.toMatchObject({ + status: 503 + }); + const controller = new AbortController(); + vi.spyOn(AbortSignal, "timeout").mockReturnValue(controller.signal); + vi.stubGlobal( + "fetch", + (request: Request) => + new Promise((_resolve, reject) => + request.signal.addEventListener( + "abort", + () => reject(request.signal.reason), + { once: true } + ) + ) + ); + const timedOut = expect(processor.process(input)).rejects.toMatchObject({ + status: 503 + }); + controller.abort(new DOMException("Processor timed out", "TimeoutError")); + await timedOut; + }); +}); diff --git a/services/extension-packager/tsconfig.json b/services/extension-packager/tsconfig.json new file mode 100644 index 00000000..e0d52a35 --- /dev/null +++ b/services/extension-packager/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "include": [ + "src/**/*.ts", + "test/**/*.ts", + "netlify/**/*.ts", + "scripts/**/*.ts" + ] +} diff --git a/src/lib/adapters/cloudflare/package-images.ts b/src/lib/adapters/cloudflare/package-images.ts deleted file mode 100644 index 9b08c0ea..00000000 --- a/src/lib/adapters/cloudflare/package-images.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { sanitizeSvg } from "@zeropress/svg-hush/workerd"; -import type { - PackageImageProcessor, - IconVariant -} from "../../../services/extensions/v2/packages/images"; -import { PackageError } from "../../../services/extensions/v2/packages/errors"; -import { boundedBytes } from "../../../services/extensions/v2/packages/store"; - -export class CloudflarePackageImages implements PackageImageProcessor { - constructor(private readonly images?: ImagesBinding) {} - async prepare( - icon: Uint8Array, - isSvg: boolean - ): Promise { - const images = this.images; - if (isSvg) { - try { - return [{ format: "svg", bytes: new Uint8Array(sanitizeSvg(icon)) }]; - } catch { - throw new PackageError( - "INVALID_ICON", - "Package SVG icon could not be sanitized", - 422 - ); - } - } - if (!images) - throw new PackageError( - "PACKAGES_DISABLED", - "Image processing is not configured", - 503 - ); - const variants: IconVariant[] = []; - try { - const info = await images.info(new Blob([icon]).stream()); - // Check decoded format too: a misleading filename must not bypass sanitization. - if (info.format === "image/svg+xml") - return [{ format: "svg", bytes: new Uint8Array(sanitizeSvg(icon)) }]; - for (const format of ["avif", "png"] as const) { - const output = await images - .input(new Blob([icon]).stream()) - .transform({ - width: 256, - height: 256, - fit: "scale-down" - }) - .output({ format: `image/${format}`, anim: false }); - variants.push({ - format, - bytes: await boundedBytes(output.image(), 1024 * 1024) - }); - } - } catch { - throw new PackageError( - "INVALID_ICON", - "Package icon could not be processed", - 422 - ); - } - return variants; - } -} diff --git a/test/tsconfig.json b/test/tsconfig.json deleted file mode 100644 index 5e1bd461..00000000 --- a/test/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../tsconfig.json", - "compilerOptions": { - "moduleResolution": "bundler", - "types": ["@cloudflare/vitest-plugin"] - }, - "include": ["./**/*.ts", "../worker-configuration.d.ts"] -} diff --git a/tsconfig.json b/tsconfig.json index 7583443e..2eb32ffb 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,10 +6,7 @@ "strict": true, "skipLibCheck": true, "lib": ["ESNext"], - "types": [ - "./worker-configuration.d.ts", - "@cloudflare/vitest-plugin", - "node" - ] - } + "types": ["node"] + }, + "include": ["*.ts"] } From 9d8221c2a4bef1eb2f170c7f55e1d160bb771e67 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Tue, 10 Nov 2026 12:09:50 +0000 Subject: [PATCH 2/4] Skip branch deployment until Preview resources are configured --- .github/workflows/ci.yml | 10 ++++++++-- README.md | 4 +++- services/api/README.md | 7 +++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a517e10..0b799dc8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,6 +90,12 @@ jobs: API_PREVIEW_RESOURCES: ${{ vars.API_PREVIEW_RESOURCES }} - name: Validate Deployment Artifact run: npm exec -- wrangler deploy --dry-run --config services/api/dist/wrangler.json + - name: Explain Unconfigured Previews + if: github.event_name == 'push' && github.ref_type == 'branch' && github.ref != 'refs/heads/main' && vars.API_PREVIEW_RESOURCES == '' + run: | + echo '::notice::Preview migrations and deployment are skipped until API_PREVIEW_RESOURCES is configured.' + echo '### Preview deployment skipped' >> "$GITHUB_STEP_SUMMARY" + echo 'Configure the API_PREVIEW_RESOURCES repository variable with isolated staging resources to enable branch Previews. Validation and artifact checks still run.' >> "$GITHUB_STEP_SUMMARY" - name: Upload Deployment Artifact if: github.event_name == 'push' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 @@ -103,7 +109,7 @@ jobs: migrations: name: Apply API Migrations needs: [validation, packager-node24, build] - if: github.event_name == 'push' && github.ref_type == 'branch' + if: github.event_name == 'push' && github.ref_type == 'branch' && (github.ref == 'refs/heads/main' || vars.API_PREVIEW_RESOURCES != '') runs-on: ubuntu-latest timeout-minutes: 15 environment: @@ -150,7 +156,7 @@ jobs: preview: name: Cloudflare Preview needs: [validation, packager-node24, build, migrations] - if: ${{ !cancelled() && github.event_name == 'push' && github.ref_type == 'branch' && github.ref != 'refs/heads/main' }} + if: ${{ !cancelled() && github.event_name == 'push' && github.ref_type == 'branch' && github.ref != 'refs/heads/main' && vars.API_PREVIEW_RESOURCES != '' }} permissions: contents: read actions: read diff --git a/README.md b/README.md index 700a391f..e765e76f 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,9 @@ as `fossbilling.org`. [CI/CD](.github/workflows/ci.yml) validates the workspaces builds a self-contained Worker artifact, including deployment configuration and SQL. - Pushes to `main` migrate production D1 and deploy the tested artifact. -- Other branch pushes migrate staging D1 and deploy Previews, with stable branch and +- Other branch pushes migrate staging D1 and deploy Previews once `API_PREVIEW_RESOURCES` + is configured; until then, validation and artifact checks run with an explicit Preview skip summary. + Previews have stable branch and immutable commit URLs recorded on matching PRs. - PR runs validate without deploying. The [Cloudflare Events workflow](.github/workflows/cloudflare-events.yml) updates PR comments when a PR opens later or a run finishes, including failed attempts. diff --git a/services/api/README.md b/services/api/README.md index 551bfbb1..e11d53e6 100644 --- a/services/api/README.md +++ b/services/api/README.md @@ -168,7 +168,9 @@ from [FOSSBilling/.workflows](https://github.com/FOSSBilling/.workflows), matchi Validation and artifact builds run on branch pushes and PRs. A push can deploy only after checks, the Node.js 24 packager test and D1 migrations succeed. `main` targets -production; other branches target Previews. PR runs, including forks, never receive +production; other branches target Previews once `API_PREVIEW_RESOURCES` is configured. +Until then, branch builds report that Preview migrations and deployment are skipped; +validation and artifact checks still run. PR runs, including forks, never receive deployment credentials or apply remote migrations. The artifact contains the compiled Worker, `wrangler.json`, and SQL migrations. @@ -201,7 +203,8 @@ Migration jobs serialize access to the shared staging databases. The build refuses production resource IDs and duplicated IDs. These are operational identifiers, not credentials. The variable is optional for local production bundle - checks, but required for Preview migrations and deployment. Previews share these + checks. When absent, CI skips Preview migrations and deployment; a malformed or + production resource configuration still fails the build. Previews share these staging resources; they do not automatically receive separate D1/KV/R2 data. 3. Configure Preview runtime secrets in the API Worker's **Previews Base** settings. From e785a236da0f2526599d32afbb2107fbb2077d81 Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Tue, 10 Nov 2026 12:23:44 +0000 Subject: [PATCH 3/4] Address package runtime and deployment review feedback --- .github/workflows/ci.yml | 27 ++---- README.md | 4 +- .../extension-packaging/src/adapters/b2.ts | 6 +- .../extension-packaging/src/adapters/http.ts | 28 +++++-- .../extension-packaging/src/adapters/sigv4.ts | 32 +++---- .../extension-packaging/src/processor.ts | 3 + libraries/extension-packaging/src/urls.ts | 5 +- .../extension-packaging/test/b2-store.test.ts | 33 ++++++++ .../extension-packaging/test/zip.test.ts | 11 +++ services/api/README.md | 5 +- services/api/drizzle.central-alerts.config.ts | 2 +- services/api/drizzle.extensions.config.ts | 2 +- .../lib/adapters/cloudflare/packaging-jobs.ts | 16 +--- .../src/services/extensions/v2/PACKAGING.md | 7 +- .../services/extensions/v2/packages/build.ts | 13 ++- .../extensions/v2/packages/processor.ts | 20 +++++ .../services/extensions/v2/routes/packages.ts | 15 +++- services/api/test/deployment.node.test.ts | 25 +++--- .../services/extensions/v2/packages.test.ts | 36 +++++++- services/extension-packager/README.md | 4 +- services/extension-packager/netlify.toml | 2 +- .../scripts/extension-packager-dev.ts | 8 +- services/extension-packager/src/handler.ts | 8 +- services/extension-packager/src/images.ts | 14 +++- .../extension-packager/test/processor.test.ts | 84 +++++++++++++++---- 25 files changed, 293 insertions(+), 117 deletions(-) create mode 100644 services/api/src/services/extensions/v2/packages/processor.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b799dc8..3123f3e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ permissions: concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: validation: @@ -43,23 +43,6 @@ jobs: if: ${{ !cancelled() && steps.dependencies.outcome == 'success' }} run: npm test - packager-node24: - name: Packager on Node.js 24 - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Checkout Code - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 - - name: Setup Node.js - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f - with: - node-version: "24" - cache: npm - - name: Install Dependencies - run: npm ci - - name: Test Packager - run: npm test --workspace @fossbilling/extension-packager - build: name: Build API runs-on: ubuntu-latest @@ -108,7 +91,7 @@ jobs: migrations: name: Apply API Migrations - needs: [validation, packager-node24, build] + needs: [validation, build] if: github.event_name == 'push' && github.ref_type == 'branch' && (github.ref == 'refs/heads/main' || vars.API_PREVIEW_RESOURCES != '') runs-on: ubuntu-latest timeout-minutes: 15 @@ -155,7 +138,7 @@ jobs: preview: name: Cloudflare Preview - needs: [validation, packager-node24, build, migrations] + needs: [validation, build, migrations] if: ${{ !cancelled() && github.event_name == 'push' && github.ref_type == 'branch' && github.ref != 'refs/heads/main' && vars.API_PREVIEW_RESOURCES != '' }} permissions: contents: read @@ -166,7 +149,7 @@ jobs: operation: deploy mode: preview build-result: ${{ needs.build.result }} - checks-result: ${{ needs.validation.result == 'success' && needs.packager-node24.result == 'success' && needs.migrations.result == 'success' && 'success' || 'failure' }} + checks-result: ${{ needs.validation.result == 'success' && needs.migrations.result == 'success' && 'success' || 'failure' }} pr-comment: true pr-base-branch: main worker-name: api @@ -181,7 +164,7 @@ jobs: deploy: name: Deploy Production - needs: [validation, packager-node24, build, migrations] + needs: [validation, build, migrations] if: github.event_name == 'push' && github.ref == 'refs/heads/main' uses: FOSSBilling/.workflows/.github/workflows/cloudflare-workers.yml@384ecc11e315c77cf3d6abb947e859ce62b38cf5 # main with: diff --git a/README.md b/README.md index e765e76f..7a2f2b76 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ publication or library build step. Hosting configuration belongs to its service. ## Development -Use Node.js 24 or later. The packager runs on Node.js 24; CI also tests the whole -repository on Node.js 26. Native dependencies are installed for the build host. +Use Node.js 24 or later. CI and the Netlify packager use Node.js 26. Native +dependencies are installed for the build host. ```sh npm ci diff --git a/libraries/extension-packaging/src/adapters/b2.ts b/libraries/extension-packaging/src/adapters/b2.ts index 60ff135f..5c49d5a5 100644 --- a/libraries/extension-packaging/src/adapters/b2.ts +++ b/libraries/extension-packaging/src/adapters/b2.ts @@ -2,7 +2,7 @@ import { PACKAGE_LIMITS } from "../limits"; import { sha256 } from "../checksum"; import { PackageError } from "../errors"; import { boundedBytes, type PackageStore } from "../store"; -import { signS3Request } from "./sigv4"; +import { signS3Request, type S3RequestInit } from "./sigv4"; // Includes generated manifest and ZIP headers as well as the source contents. const MAX_STORED_BYTES = PACKAGE_LIMITS.expandedBytes + 1024 * 1024; @@ -69,7 +69,7 @@ export function b2ObjectStore( origin ); } - async function request(url: URL, init: RequestInit) { + async function request(url: URL, init: S3RequestInit) { try { return await fetch( await signS3Request( @@ -88,6 +88,7 @@ export function b2ObjectStore( ) ); } catch { + signal?.throwIfAborted(); throw unavailable("Package storage request failed"); } } @@ -142,6 +143,7 @@ export function b2ObjectStore( try { bytes = await boundedBytes(response.body, MAX_STORED_BYTES); } catch { + signal?.throwIfAborted(); throw unavailable( "Stored package could not be read within its size limit" ); diff --git a/libraries/extension-packaging/src/adapters/http.ts b/libraries/extension-packaging/src/adapters/http.ts index da263296..f4afb4d0 100644 --- a/libraries/extension-packaging/src/adapters/http.ts +++ b/libraries/extension-packaging/src/adapters/http.ts @@ -14,17 +14,18 @@ export class HttpPackageProcessor implements PackageProcessor { private readonly secret: string, local = false ) { - this.url = new URL(url); + const target = URL.canParse(url) ? new URL(url) : null; if ( + !target || !secret || - this.url.username || - this.url.password || - this.url.hash || + target.username || + target.password || + target.hash || !( - this.url.protocol === "https:" || + target.protocol === "https:" || (local && - this.url.protocol === "http:" && - ["localhost", "127.0.0.1", "[::1]"].includes(this.url.hostname)) + target.protocol === "http:" && + ["localhost", "127.0.0.1", "[::1]"].includes(target.hostname)) ) ) throw new PackageError( @@ -32,6 +33,7 @@ export class HttpPackageProcessor implements PackageProcessor { "Package processor is not configured correctly", 503 ); + this.url = target; } async process(input: ProcessingInput) { try { @@ -58,6 +60,12 @@ export class HttpPackageProcessor implements PackageProcessor { body.error.message.slice(0, 1000), 422 ); + if (response.status === 503 && body.error?.code === "SOURCE_UNAVAILABLE") + throw new PackageError( + "SOURCE_UNAVAILABLE", + "Source ZIP is unavailable or failed checksum verification", + 503 + ); if (!response.ok) throw new Error("Package processor request failed"); const result = ProcessingResultSchema.parse(body.result); const allowed: readonly string[] = input.icons ?? []; @@ -69,7 +77,11 @@ export class HttpPackageProcessor implements PackageProcessor { throw new Error("Package processor returned unexpected output IDs"); return result; } catch (error) { - if (error instanceof PackageError && error.status === 422) throw error; + if ( + error instanceof PackageError && + (error.status === 422 || error.code === "SOURCE_UNAVAILABLE") + ) + throw error; logError("extensions/packages", "Package processor request failed", { objectId: input.archive, error: error instanceof Error ? error.message : "Unknown error" diff --git a/libraries/extension-packaging/src/adapters/sigv4.ts b/libraries/extension-packaging/src/adapters/sigv4.ts index 37bad7d6..5df63299 100644 --- a/libraries/extension-packaging/src/adapters/sigv4.ts +++ b/libraries/extension-packaging/src/adapters/sigv4.ts @@ -19,12 +19,10 @@ function toHex(bytes: ArrayBuffer): string { .join(""); } -async function sha256Hex( - data: Uint8Array | string -): Promise { - const bytes = - typeof data === "string" ? new TextEncoder().encode(data) : data; - return toHex(await crypto.subtle.digest("SHA-256", bytes)); +async function sha256Hex(data: string): Promise { + return toHex( + await crypto.subtle.digest("SHA-256", new TextEncoder().encode(data)) + ); } async function hmac( @@ -64,19 +62,15 @@ function canonicalQueryString(searchParams: URLSearchParams): string { .join("&"); } -function bodyBytes( - body: RequestInit["body"] -): Uint8Array | string | null { - if (body == null) return null; - if (typeof body === "string") return body; - if (body instanceof Uint8Array) return new Uint8Array(body); - return null; +// PUT callers provide validated bytes and their checksum; other operations have no body. +export interface S3RequestInit extends Omit { + body?: Uint8Array; } /** Sign a request with SigV4 and return a new `Request` carrying the auth headers. */ export async function signS3Request( url: URL | string, - init: RequestInit, + init: S3RequestInit, credentials: S3Credentials, now: Date = new Date() ): Promise { @@ -89,11 +83,11 @@ export async function signS3Request( .replace(/\.\d+Z$/, "Z"); const dateStamp = amzDate.slice(0, 8); - let payloadHash = headers.get("x-amz-content-sha256"); - if (!payloadHash) { - const bytes = bodyBytes(init.body); - payloadHash = bytes == null ? EMPTY_SHA256 : await sha256Hex(bytes); - } + const payloadHash = + headers.get("x-amz-content-sha256") ?? + (init.body == null ? EMPTY_SHA256 : null); + if (!payloadHash) + throw new Error("S3 uploads require an explicit payload checksum"); headers.set("x-amz-date", amzDate); headers.set("x-amz-content-sha256", payloadHash); diff --git a/libraries/extension-packaging/src/processor.ts b/libraries/extension-packaging/src/processor.ts index f2454625..cd262be1 100644 --- a/libraries/extension-packaging/src/processor.ts +++ b/libraries/extension-packaging/src/processor.ts @@ -4,6 +4,8 @@ import { PACKAGE_LIMITS } from "./limits"; export const MAX_PACKAGE_OBJECT_BYTES = PACKAGE_LIMITS.expandedBytes + 1024 * 1024; +// Matches the sanitizer's maximum prepared SVG output. +export const MAX_PACKAGE_ICON_BYTES = 4 * 1024 * 1024; const checksum = z.string().regex(/^[a-f0-9]{64}$/); const output = z .object({ @@ -30,6 +32,7 @@ export const ProcessingResultSchema = z .array( output .extend({ + bytes: output.shape.bytes.max(MAX_PACKAGE_ICON_BYTES), id: z.uuid(), format: z.enum(["svg", "avif", "png"]) }) diff --git a/libraries/extension-packaging/src/urls.ts b/libraries/extension-packaging/src/urls.ts index a77762d1..84f08b62 100644 --- a/libraries/extension-packaging/src/urls.ts +++ b/libraries/extension-packaging/src/urls.ts @@ -1,13 +1,12 @@ import { z } from "zod"; -// Restricts to http(s) — z.string().url() alone accepts any scheme, +// Restricts to http(s) — z.url() alone accepts any scheme, // including javascript:/data:, which is unsafe for fields a consumer may // render as a link or image src. export const httpUrl = () => z - .string() - .max(2048) .url() + .max(2048) .refine((value) => /^https?:\/\//i.test(value), { message: "must use http or https" }); diff --git a/libraries/extension-packaging/test/b2-store.test.ts b/libraries/extension-packaging/test/b2-store.test.ts index 7a5ab8e9..df9bc82f 100644 --- a/libraries/extension-packaging/test/b2-store.test.ts +++ b/libraries/extension-packaging/test/b2-store.test.ts @@ -20,6 +20,39 @@ const store = (overrides: Record = {}) => afterEach(() => vi.unstubAllGlobals()); describe("B2 object storage", () => { + it("preserves a caller deadline while reading from B2", async () => { + const controller = new AbortController(); + let started!: () => void; + const fetching = new Promise((resolve) => { + started = resolve; + }); + vi.stubGlobal( + "fetch", + (request: Request) => + new Promise((_resolve, reject) => { + request.signal.addEventListener( + "abort", + () => reject(request.signal.reason), + { once: true } + ); + started(); + }) + ); + const storage = b2ObjectStore( + (name) => config[name.replace("EXTENSION_PACKAGES_S3_", "")], + "packages", + controller.signal + ); + const deadline = new DOMException( + "Processor deadline exceeded", + "TimeoutError" + ); + const read = expect(storage.get(objectId)).rejects.toBe(deadline); + await fetching; + controller.abort(deadline); + await read; + }); + it("signs ZIP writes and verifies downloaded bytes", async () => { const bytes = new Uint8Array(new TextEncoder().encode("synthetic ZIP")); const checksum = await sha256(bytes); diff --git a/libraries/extension-packaging/test/zip.test.ts b/libraries/extension-packaging/test/zip.test.ts index 35eb402e..d0e3519b 100644 --- a/libraries/extension-packaging/test/zip.test.ts +++ b/libraries/extension-packaging/test/zip.test.ts @@ -48,6 +48,17 @@ async function contents(bytes: Uint8Array) { } describe("directory package builder", () => { + it("stops ZIP processing when its caller aborts", async () => { + const source = await zip([["Service.php", " { for (const path of [ "a/".repeat(20000) + "file.php", diff --git a/services/api/README.md b/services/api/README.md index e11d53e6..133df719 100644 --- a/services/api/README.md +++ b/services/api/README.md @@ -110,7 +110,8 @@ Package storage configuration is listed in [PACKAGING.md](src/services/extension - `EXTENSIONS_RETENTION_MODE`: `dry-run` (report only) or `compact` (prune old revision bodies). See [extensions v2 README](src/services/extensions/v2/README.md#revisions-and-retention). - `M2M_ISSUER`, `M2M_JWKS_URL`, `M2M_AUDIENCE`: Issuer, JWKS endpoint, and expected audience for machine-to-machine access tokens minted by the central auth service (`FOSSBilling/auth`). Defaults live in `wrangler.jsonc`; preview environments need them pointed at the issuing preview. See [extensions v2 README](src/services/extensions/v2/README.md#machine-publishing). -Only extensions v2 consumes these. For the assertion format and the rotation procedure, see [its README](src/services/extensions/v2/README.md#authentication). +The extension-specific settings apply to extensions v2; Versions, Previews and +Stats also use `GITHUB_TOKEN`. For the assertion format and the rotation procedure, see [its README](src/services/extensions/v2/README.md#authentication). ## Development @@ -167,7 +168,7 @@ from [FOSSBilling/.workflows](https://github.com/FOSSBilling/.workflows), matchi `fossbilling.org`'s Preview URLs, deployment summaries and PR comment history. Validation and artifact builds run on branch pushes and PRs. A push can deploy only -after checks, the Node.js 24 packager test and D1 migrations succeed. `main` targets +after workspace validation, the build and D1 migrations succeed. `main` targets production; other branches target Previews once `API_PREVIEW_RESOURCES` is configured. Until then, branch builds report that Preview migrations and deployment are skipped; validation and artifact checks still run. PR runs, including forks, never receive diff --git a/services/api/drizzle.central-alerts.config.ts b/services/api/drizzle.central-alerts.config.ts index 0afe4f17..c27a05b9 100644 --- a/services/api/drizzle.central-alerts.config.ts +++ b/services/api/drizzle.central-alerts.config.ts @@ -4,6 +4,6 @@ import { defineConfig } from "drizzle-kit"; // migrations_dir instead of using drizzle-kit's own D1 driver to apply. export default defineConfig({ dialect: "sqlite", - schema: "./src/services/central-alerts/v1/db/schema", + schema: "./src/services/central-alerts/v1/db/schema.ts", out: "./src/services/central-alerts/v1/db/migrations" }); diff --git a/services/api/drizzle.extensions.config.ts b/services/api/drizzle.extensions.config.ts index 02e1c473..a7078d22 100644 --- a/services/api/drizzle.extensions.config.ts +++ b/services/api/drizzle.extensions.config.ts @@ -7,6 +7,6 @@ import { defineConfig } from "drizzle-kit"; // Cloudflare credentials are needed just to generate a migration. export default defineConfig({ dialect: "sqlite", - schema: "./src/services/extensions/v2/db/schema", + schema: "./src/services/extensions/v2/db/schema.ts", out: "./src/services/extensions/v2/db/migrations" }); diff --git a/services/api/src/lib/adapters/cloudflare/packaging-jobs.ts b/services/api/src/lib/adapters/cloudflare/packaging-jobs.ts index 74a2291d..ed03821b 100644 --- a/services/api/src/lib/adapters/cloudflare/packaging-jobs.ts +++ b/services/api/src/lib/adapters/cloudflare/packaging-jobs.ts @@ -12,7 +12,7 @@ import { } from "../../../services/extensions/v2/packages/jobs"; import { PackageError } from "@fossbilling/extension-packaging/errors"; import { configuredPackageStore } from "../../../services/extensions/v2/packages/store"; -import { HttpPackageProcessor } from "@fossbilling/extension-packaging/adapters/http"; +import { configuredPackageProcessor } from "../../../services/extensions/v2/packages/processor"; export function packagePreparation( env: CloudflareBindings @@ -27,22 +27,10 @@ export function packagePreparation( "Package storage or download URL is not configured", 503 ); - const processorUrl = getEnv("EXTENSION_PACKAGER_URL"); - const processorSecret = getEnv("EXTENSION_PACKAGER_SECRET"); - if (!processorUrl || !processorSecret) - throw new PackageError( - "PACKAGES_DISABLED", - "Package processor is not configured", - 503 - ); return { repository: new PackageBuildDatabase(env.DB_EXTENSIONS), store, - processor: new HttpPackageProcessor( - processorUrl, - processorSecret, - getEnv("ENVIRONMENT") !== "production" - ), + processor: configuredPackageProcessor(getEnv), baseUrl }; } diff --git a/services/api/src/services/extensions/v2/PACKAGING.md b/services/api/src/services/extensions/v2/PACKAGING.md index 384febe0..f7e51255 100644 --- a/services/api/src/services/extensions/v2/PACKAGING.md +++ b/services/api/src/services/extensions/v2/PACKAGING.md @@ -106,7 +106,10 @@ directory-only changes such as README edits keep the existing package. Approval publishes the revision and its artifacts in one D1 transaction. Older approved artifacts retain their URLs and checksums. -Object reservations precede uploads. Unique constraints deduplicate concurrent +Object reservations precede uploads. Before external processing, the API reserves at most +17 MiB for the archive and 4 MiB for each possible icon; the archive reservation shrinks +to its actual size after processing. This headroom keeps interrupted runner writes +accounted for within the owner's quota. Unique constraints deduplicate concurrent builds; failed writes remain recorded for cleanup. Cleanup claims unreferenced objects before deletion, and builds cannot reference claimed objects. The per-account allowance is 256 objects / 512 MiB, including abandoned reservations @@ -135,7 +138,7 @@ SVGs are sanitized with Cloudflare svg-hush through the pinned `@zeropress/svg-h Node adapter and served directly. SVG inputs are bounded by the ZIP expansion limit and the sanitizer's built-in input/output limits. Raster inputs use Sharp in the Node packager to produce AVIF and PNG variants within 256 × 256 pixels, preserving proportions and -transparency without enlarging small images. Raster inputs are limited to 16 megapixels. +transparency without enlarging small images. Raster inputs are limited to 16,777,216 pixels. Requests prefer SVG when present, otherwise AVIF when explicitly accepted, with PNG fallback. They read stored images directly and never reopen a ZIP. diff --git a/services/api/src/services/extensions/v2/packages/build.ts b/services/api/src/services/extensions/v2/packages/build.ts index c2faaf53..7290301a 100644 --- a/services/api/src/services/extensions/v2/packages/build.ts +++ b/services/api/src/services/extensions/v2/packages/build.ts @@ -1,6 +1,7 @@ import { sha256 } from "@fossbilling/extension-packaging/checksum"; import { MAX_PACKAGE_OBJECT_BYTES, + MAX_PACKAGE_ICON_BYTES, type PackageProcessor } from "@fossbilling/extension-packaging/processor"; import type { Manifest } from "../schemas/manifest"; @@ -86,12 +87,18 @@ export async function buildStoredPackage(input: { : [crypto.randomUUID(), crypto.randomUUID()]; // Reserve worst-case output space before an external runner can write to B2. // Unused/failed reservations remain discoverable by the normal cleanup job. - for (const id of [objectId, ...(iconIds ?? [])]) + await repository.reserveObject( + owner, + manifest.id, + objectId, + MAX_PACKAGE_OBJECT_BYTES + ); + for (const id of iconIds ?? []) await repository.reserveObject( owner, manifest.id, id, - MAX_PACKAGE_OBJECT_BYTES + MAX_PACKAGE_ICON_BYTES ); const built = await input.processor.process({ manifest, @@ -104,7 +111,7 @@ export async function buildStoredPackage(input: { for (const icon of built.icons) { const bytes = await boundedBytes( await store.get(icon.id, icon.sha256), - MAX_PACKAGE_OBJECT_BYTES + MAX_PACKAGE_ICON_BYTES ); if (bytes.length !== icon.bytes || (await sha256(bytes)) !== icon.sha256) throw new PackageError( diff --git a/services/api/src/services/extensions/v2/packages/processor.ts b/services/api/src/services/extensions/v2/packages/processor.ts new file mode 100644 index 00000000..e895216a --- /dev/null +++ b/services/api/src/services/extensions/v2/packages/processor.ts @@ -0,0 +1,20 @@ +import { HttpPackageProcessor } from "@fossbilling/extension-packaging/adapters/http"; +import { PackageError } from "@fossbilling/extension-packaging/errors"; + +export function configuredPackageProcessor( + getEnv: (name: string) => string | undefined +) { + const url = getEnv("EXTENSION_PACKAGER_URL"); + const secret = getEnv("EXTENSION_PACKAGER_SECRET"); + if (!url || !secret) + throw new PackageError( + "PACKAGES_DISABLED", + "Package processor is not configured", + 503 + ); + return new HttpPackageProcessor( + url, + secret, + getEnv("ENVIRONMENT") !== "production" + ); +} diff --git a/services/api/src/services/extensions/v2/routes/packages.ts b/services/api/src/services/extensions/v2/routes/packages.ts index 8f403dad..2da74192 100644 --- a/services/api/src/services/extensions/v2/routes/packages.ts +++ b/services/api/src/services/extensions/v2/routes/packages.ts @@ -1,3 +1,4 @@ +import { configuredPackageProcessor } from "../packages/processor"; import { PackagingJobsDatabase } from "../db/packaging-jobs"; import { dispatchPackagingJob, type PackagingJob } from "../packages/jobs"; import { sha256 } from "@fossbilling/extension-packaging/checksum"; @@ -112,7 +113,19 @@ export function registerPackageRoutes(app: ExtensionsV2App) { app.use("/packages", availability); app.use("/packages/*", availability); app.use("/packaging-jobs/*", availability); - app.on("POST", "/packages", paceContentIp()); + app.on( + "POST", + "/packages", + async (c, next) => { + try { + configuredPackageProcessor(getPlatform(c).getEnv); + } catch (error) { + return failure(c, error); + } + await next(); + }, + paceContentIp() + ); const boundUpload: MiddlewareHandler = async (c, next) => { try { const body = await boundedBytes(c.req.raw.body, 6 * 1024 * 1024); diff --git a/services/api/test/deployment.node.test.ts b/services/api/test/deployment.node.test.ts index 7492cb9e..6b1497c7 100644 --- a/services/api/test/deployment.node.test.ts +++ b/services/api/test/deployment.node.test.ts @@ -100,22 +100,25 @@ describe("API deployment artifact", () => { ).toBe(false); expect(read(folder, "wrangler.json").previews.d1_databases).toBeUndefined(); }); - it("rejects production resource reuse and duplicate staging databases before writing an artifact", () => { + it("rejects production resources and duplicated staging IDs before writing an artifact", () => { const folder = fixture(); const source: Unstable_RawConfig = experimental_readRawConfig({ config: join(folder, "wrangler.jsonc") }).rawConfig; - for (const DB_EXTENSIONS of [ - source.d1_databases![0].database_id, - source.d1_databases![0].database_id!.toUpperCase(), - resources.DB_CENTRAL_ALERTS - ]) { + const overrides = [ + { DB_EXTENSIONS: source.d1_databases![0].database_id }, + { DB_EXTENSIONS: source.d1_databases![0].database_id!.toUpperCase() }, + { DB_EXTENSIONS: resources.DB_CENTRAL_ALERTS }, + { AUTH_KV: source.kv_namespaces![0].id }, + { CACHE_KV: source.kv_namespaces![1].id }, + { DOWNLOAD_BUCKET: source.r2_buckets![0].bucket_name }, + { EXTENSIONS_FRONTEND: source.services![0].service }, + { AUTH_KV: resources.CACHE_KV } + ]; + for (const override of overrides) { expect(() => - prepareDeployment( - folder, - JSON.stringify({ ...resources, DB_EXTENSIONS }) - ) - ).toThrow("distinct non-production resources"); + prepareDeployment(folder, JSON.stringify({ ...resources, ...override })) + ).toThrow(); expect(existsSync(join(folder, "dist/wrangler.json"))).toBe(false); } }); diff --git a/services/api/test/services/extensions/v2/packages.test.ts b/services/api/test/services/extensions/v2/packages.test.ts index b6aa2c20..875f4c6e 100644 --- a/services/api/test/services/extensions/v2/packages.test.ts +++ b/services/api/test/services/extensions/v2/packages.test.ts @@ -673,7 +673,7 @@ describe("queued package preparation", () => { expect(await jobs.get(job.id)).toMatchObject({ status: "queued", attempts: 1, - error: { code: "PACKAGE_UNAVAILABLE", status: 503 } + error: { code: "SOURCE_UNAVAILABLE", status: 503 } }); b2.objects.set(key, staged); expect( @@ -1579,6 +1579,40 @@ describe("stored directory packages", () => { Object.assign(env, { ENVIRONMENT: "local" }); }); + it("rejects missing or malformed processor configuration before accepting an upload", async () => { + for (const overrides of [ + { EXTENSION_PACKAGER_URL: "" }, + { EXTENSION_PACKAGER_URL: "not a URL" }, + { + EXTENSION_PACKAGER_URL: "https://processor.example.test/process", + EXTENSION_PACKAGER_SECRET: "" + } + ]) { + Object.assign(env, overrides); + const response = await upload(); + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ + error: { code: "PACKAGES_DISABLED" } + }); + } + expect( + await db.prepare("SELECT count(*) AS count FROM package_jobs").first() + ).toEqual({ count: 0 }); + expect( + await db.prepare("SELECT count(*) AS count FROM package_objects").first() + ).toEqual({ count: 0 }); + }); + + it("admits a small package with 30 MiB of remaining storage headroom", async () => { + await db + .prepare( + "INSERT INTO package_objects (id,created_by,bytes) VALUES (?,'owner',?)" + ) + .bind(crypto.randomUUID(), 482 * 1024 * 1024) + .run(); + expect((await upload()).status).toBe(200); + }); + it("checks account ownership, upload bounds, invalid archives and disabled storage", async () => { expect((await upload(manifest(), new Uint8Array([1, 2, 3]))).status).toBe( 422 diff --git a/services/extension-packager/README.md b/services/extension-packager/README.md index 305d2492..9f5b258a 100644 --- a/services/extension-packager/README.md +++ b/services/extension-packager/README.md @@ -49,9 +49,9 @@ Create a dedicated Netlify project connected to this repository: Paths in `netlify.toml` are relative to the repository-root base directory, following [Netlify's monorepo configuration](https://docs.netlify.com/build/configure-builds/monorepos/). -The config selects Node.js 24 and keeps the native Sharp and sanitizer dependencies +The config selects Node.js 26 and keeps the native Sharp and sanitizer dependencies external so Netlify includes their binaries and WASM files. Verify these files and -Node.js 24 in the deployed artifact/runtime before enabling processing. +Node.js 26 in the deployed artifact/runtime before enabling processing. Set the seven variables listed above through Netlify's dashboard, CLI or API for the function runtime. Use production credentials and a separate randomly generated shared diff --git a/services/extension-packager/netlify.toml b/services/extension-packager/netlify.toml index e33daf7f..09ce8d7d 100644 --- a/services/extension-packager/netlify.toml +++ b/services/extension-packager/netlify.toml @@ -2,7 +2,7 @@ publish = "services/extension-packager/netlify/public" [build.environment] - NODE_VERSION = "24" + NODE_VERSION = "26" [functions] directory = "services/extension-packager/netlify/functions" diff --git a/services/extension-packager/scripts/extension-packager-dev.ts b/services/extension-packager/scripts/extension-packager-dev.ts index fe558806..20db0ed2 100644 --- a/services/extension-packager/scripts/extension-packager-dev.ts +++ b/services/extension-packager/scripts/extension-packager-dev.ts @@ -1,8 +1,9 @@ +import { logError } from "@fossbilling/logging"; import { createServer } from "node:http"; import { Readable } from "node:stream"; import { handlePackageProcessing } from "../src/handler"; -const port = Number(process.env.EXTENSION_PACKAGER_PORT ?? 8790); +const port = Number(process.env.EXTENSION_PACKAGER_PORT || 8790); createServer(async (incoming, outgoing) => { const controller = new AbortController(); outgoing.on("close", () => { @@ -24,7 +25,10 @@ createServer(async (incoming, outgoing) => { ); outgoing.writeHead(response.status, Object.fromEntries(response.headers)); outgoing.end(new Uint8Array(await response.arrayBuffer())); - } catch { + } catch (error) { + logError("extensions/packages", "Local packager request failed", { + error: error instanceof Error ? error.message : "Unknown error" + }); outgoing.writeHead(500); outgoing.end(); } diff --git a/services/extension-packager/src/handler.ts b/services/extension-packager/src/handler.ts index 0c178f10..80836d17 100644 --- a/services/extension-packager/src/handler.ts +++ b/services/extension-packager/src/handler.ts @@ -33,7 +33,8 @@ export async function handlePackageProcessing( }); const signal = AbortSignal.any([request.signal, AbortSignal.timeout(45000)]); try { - const bytes = await boundedBytes(request.body, 64 * 1024); + // A 64 KiB manifest string can expand in UTF-8, alongside request metadata. + const bytes = await boundedBytes(request.body, 256 * 1024); let input; try { input = ProcessingInputSchema.parse( @@ -58,6 +59,11 @@ export async function handlePackageProcessing( { error: { code: error.code, message: error.message } }, { status: 422, headers } ); + if (error instanceof PackageError && error.code === "SOURCE_UNAVAILABLE") + return Response.json( + { error: { code: error.code, message: error.message } }, + { status: 503, headers } + ); logError("extensions/packages", "Package processing failed", { error: error instanceof Error ? error.message : "Unknown error" }); diff --git a/services/extension-packager/src/images.ts b/services/extension-packager/src/images.ts index 44cce599..e30d1113 100644 --- a/services/extension-packager/src/images.ts +++ b/services/extension-packager/src/images.ts @@ -38,7 +38,19 @@ export class NodePackageImages implements PackageImageProcessor { variants.push({ format, bytes }); } return variants; - } catch { + } catch (error) { + // libvips reports processing deadlines and allocation failures as errors. + if ( + error instanceof Error && + /timeout:|out of memory|could not allocate|unable to allocate/i.test( + error.message + ) + ) + throw new PackageError( + "PACKAGE_UNAVAILABLE", + "Image processing is temporarily unavailable", + 503 + ); throw new PackageError( "INVALID_ICON", "Package icon could not be processed", diff --git a/services/extension-packager/test/processor.test.ts b/services/extension-packager/test/processor.test.ts index ed4c85fc..0fec73f4 100644 --- a/services/extension-packager/test/processor.test.ts +++ b/services/extension-packager/test/processor.test.ts @@ -204,6 +204,53 @@ describe("Node package processor over the generic HTTP contract", () => { expect(f.objects.has(f.key(input.archive))).toBe(false); } }); + it("accepts a valid manifest whose UTF-8 processing request exceeds 64 KiB", async () => { + const f = fixture(); + const input = await f.input(await archive()); + input.manifest.requires.modules = Object.fromEntries( + Array.from({ length: 30 }, (_, i) => [ + `dependency_${i}`, + ">=1.0.0" + "\u2003".repeat(990) + ]) + ); + expect(JSON.stringify(input.manifest).length).toBeLessThan(64 * 1024); + expect(Buffer.byteLength(JSON.stringify(input))).toBeGreaterThan(64 * 1024); + await expect(f.processor.process(input)).resolves.toMatchObject({ + archive: { bytes: expect.any(Number) } + }); + }); + it.each([ + "timeout: 10% complete", + "Could not allocate enough memory for raw output" + ])("keeps image processing failure retryable: %s", async (message) => { + const f = fixture(); + const png = new Uint8Array( + await sharp({ + create: { width: 1, height: 1, channels: 4, background: "red" } + }) + .png() + .toBuffer() + ); + const input = await f.input(await archive("icon.png", png)); + vi.spyOn(sharp.prototype, "toBuffer").mockRejectedValueOnce( + new Error(message) + ); + vi.spyOn(console, "error").mockImplementation(() => {}); + await expect(f.processor.process(input)).rejects.toMatchObject({ + status: 503 + }); + expect(f.objects.has(f.key(input.archive))).toBe(false); + }); + it("preserves a missing source error through the HTTP processor", async () => { + const f = fixture(); + const input = await f.input(await archive()); + f.objects.delete(f.key(input.source.id)); + await expect(f.processor.process(input)).rejects.toMatchObject({ + code: "SOURCE_UNAVAILABLE", + status: 503 + }); + expect(f.objects.has(f.key(input.archive))).toBe(false); + }); it("rejects unauthenticated and invalid requests before storage access", async () => { const f = fixture(); expect( @@ -308,24 +355,25 @@ describe("Node package processor over the generic HTTP contract", () => { "https://processor.example.test/process", "secret" ); - vi.stubGlobal("fetch", async () => - Response.json({ - result: { - archive: { bytes: 20, sha256: "0".repeat(64) }, - icons: [ - { - id: crypto.randomUUID(), - format: "svg", - bytes: 10, - sha256: "0".repeat(64) - } - ] - } - }) - ); - await expect(processor.process(input)).rejects.toMatchObject({ - status: 503 - }); + for (const result of [ + { archive: { bytes: -1, sha256: "0".repeat(64) }, icons: [] }, + { + archive: { bytes: 20, sha256: "0".repeat(64) }, + icons: [ + { + id: crypto.randomUUID(), + format: "svg", + bytes: 10, + sha256: "0".repeat(64) + } + ] + } + ]) { + vi.stubGlobal("fetch", async () => Response.json({ result })); + await expect(processor.process(input)).rejects.toMatchObject({ + status: 503 + }); + } const controller = new AbortController(); vi.spyOn(AbortSignal, "timeout").mockReturnValue(controller.signal); vi.stubGlobal( From 087445db0f4e1d0916f1cf624c241828a668aa9d Mon Sep 17 00:00:00 2001 From: Adam Daley Date: Tue, 10 Nov 2026 12:33:10 +0000 Subject: [PATCH 4/4] Tighten review regressions and Netlify runtime selection --- README.md | 5 +++-- services/api/test/deployment.node.test.ts | 4 +++- services/extension-packager/README.md | 11 ++++++++--- services/extension-packager/netlify.toml | 3 ++- services/extension-packager/test/processor.test.ts | 2 +- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7a2f2b76..01a1ca97 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,9 @@ publication or library build step. Hosting configuration belongs to its service. ## Development -Use Node.js 24 or later. CI and the Netlify packager use Node.js 26. Native -dependencies are installed for the build host. +Use Node.js 24 or later. CI tests the workspaces on Node.js 26. Hosting runtime +selection belongs to each service's deployment configuration. Native dependencies +are installed for the build host. ```sh npm ci diff --git a/services/api/test/deployment.node.test.ts b/services/api/test/deployment.node.test.ts index 6b1497c7..68bf68f3 100644 --- a/services/api/test/deployment.node.test.ts +++ b/services/api/test/deployment.node.test.ts @@ -118,7 +118,9 @@ describe("API deployment artifact", () => { for (const override of overrides) { expect(() => prepareDeployment(folder, JSON.stringify({ ...resources, ...override })) - ).toThrow(); + ).toThrow( + "API_PREVIEW_RESOURCES must use distinct non-production resources" + ); expect(existsSync(join(folder, "dist/wrangler.json"))).toBe(false); } }); diff --git a/services/extension-packager/README.md b/services/extension-packager/README.md index 9f5b258a..bf0b01a5 100644 --- a/services/extension-packager/README.md +++ b/services/extension-packager/README.md @@ -49,9 +49,14 @@ Create a dedicated Netlify project connected to this repository: Paths in `netlify.toml` are relative to the repository-root base directory, following [Netlify's monorepo configuration](https://docs.netlify.com/build/configure-builds/monorepos/). -The config selects Node.js 26 and keeps the native Sharp and sanitizer dependencies -external so Netlify includes their binaries and WASM files. Verify these files and -Node.js 26 in the deployed artifact/runtime before enabling processing. +The Netlify configuration currently selects Node.js 24, a stable Lambda runtime. +[AWS lists Node.js 26 as public preview](https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html), +so it is not yet the production hosting target. The code is tested on Node.js 26; +this runtime selection belongs only to the Netlify adapter. + +Native Sharp and sanitizer dependencies remain external so Netlify includes their +binaries and WASM files. Verify those files and the deployed runtime before +enabling processing. Set the seven variables listed above through Netlify's dashboard, CLI or API for the function runtime. Use production credentials and a separate randomly generated shared diff --git a/services/extension-packager/netlify.toml b/services/extension-packager/netlify.toml index 09ce8d7d..2800c5c4 100644 --- a/services/extension-packager/netlify.toml +++ b/services/extension-packager/netlify.toml @@ -2,7 +2,8 @@ publish = "services/extension-packager/netlify/public" [build.environment] - NODE_VERSION = "26" + # Lambda Node 26 is currently public preview; application CI uses Node 26. + NODE_VERSION = "24" [functions] directory = "services/extension-packager/netlify/functions" diff --git a/services/extension-packager/test/processor.test.ts b/services/extension-packager/test/processor.test.ts index 0fec73f4..c7c5ca50 100644 --- a/services/extension-packager/test/processor.test.ts +++ b/services/extension-packager/test/processor.test.ts @@ -100,7 +100,7 @@ function fixture() { checksum = await sha256(bytes); objects.set(key(id), { bytes, checksum }); return { - manifest, + manifest: structuredClone(manifest), source: { id, sha256: checksum }, archive: crypto.randomUUID(), icons: [crypto.randomUUID(), crypto.randomUUID()]