From 2bfbaa63ced2a5168c97adf7e26d8d4b6f050d6b Mon Sep 17 00:00:00 2001 From: "Ronald A. Richardson" Date: Sat, 24 Oct 2026 20:48:49 +0800 Subject: [PATCH 1/4] Cloud instance identity, per-company tenancy and the cutover freeze Fleetbase Cloud registers one instance per deployment (kind cloud) that acts for each cloud company's organization, and the service can freeze publishing and checkout during the cutover. - POST /v1/instances/register/cloud: signed with the shared platform secret (HMAC-SHA256 over timestamp, single-use nonce and body; 5-minute window) instead of a registration code. The instance belongs to REGISTRY_CLOUD_ORGANIZATION and is marked meta.platform. Re-registration follows D17: previous_token, else same organization and host, else the placeholder the importer left; a new host is a separate deployment's instance. - X-Registry-Tenant (company uuid) lets the platform instance act for that company's organization on every instance route: licences in /me, installs and seats, checkout and hand-offs. A company seen for the first time gets an organization (X-Registry-Tenant-Name). Other instances never act for anyone. - installs.organization_uuid: one install row and seat per company on the cloud instance; existing rows take their instance's organization. - GET /v1/instances/me/tenant-installs: what every company has installed, for the cloud API to record locally after the import. - Package gate: the platform instance may download a paid package while a company holds an active seat on it (the install report claims the seat before the download). - registry:maintenance on|off|status (or REGISTRY_MAINTENANCE=true): publisher writes, review approve and publish, and both checkouts answer 503 with the message and Retry-After. Reads, installs, package downloads and Stripe webhooks keep working. --- .env.example | 11 + app/Actions/ActivateLicence.php | 10 +- app/Actions/DeactivateLicence.php | 8 +- app/Console/Commands/ManageMaintenance.php | 48 ++++ .../Instance/InstallController.php | 12 +- .../Instance/InstanceController.php | 160 +++++++++-- .../Instance/PurchaseController.php | 9 +- .../Middleware/RefuseDuringMaintenance.php | 31 +++ app/Http/Middleware/ResolveInstanceTenant.php | 35 +++ .../Middleware/VerifyPlatformSignature.php | 25 ++ .../Instance/RegisterCloudInstanceRequest.php | 29 ++ app/Models/Install.php | 20 +- app/Models/Instance.php | 45 +++ app/Services/LicenceGate.php | 17 +- app/Support/CloudPlatform.php | 102 +++++++ app/Support/Maintenance.php | 72 +++++ app/Support/PlatformSignature.php | 95 +++++++ bootstrap/app.php | 6 + config/registry.php | 27 ++ ...dd_organization_uuid_to_installs_table.php | 36 +++ routes/api.php | 22 +- tests/Feature/Commerce/CloudCheckoutTest.php | 25 ++ tests/Feature/Instance/CloudInstanceTest.php | 260 ++++++++++++++++++ tests/Feature/MaintenanceTest.php | 79 ++++++ .../Packages/CloudPlatformGateTest.php | 53 ++++ tests/Helpers/cloud.php | 62 +++++ tests/Pest.php | 1 + 27 files changed, 1250 insertions(+), 50 deletions(-) create mode 100644 app/Console/Commands/ManageMaintenance.php create mode 100644 app/Http/Middleware/RefuseDuringMaintenance.php create mode 100644 app/Http/Middleware/ResolveInstanceTenant.php create mode 100644 app/Http/Middleware/VerifyPlatformSignature.php create mode 100644 app/Http/Requests/Instance/RegisterCloudInstanceRequest.php create mode 100644 app/Support/CloudPlatform.php create mode 100644 app/Support/Maintenance.php create mode 100644 app/Support/PlatformSignature.php create mode 100644 database/migrations/2026_10_26_000401_add_organization_uuid_to_installs_table.php create mode 100644 tests/Feature/Commerce/CloudCheckoutTest.php create mode 100644 tests/Feature/Instance/CloudInstanceTest.php create mode 100644 tests/Feature/MaintenanceTest.php create mode 100644 tests/Feature/Packages/CloudPlatformGateTest.php create mode 100644 tests/Helpers/cloud.php diff --git a/.env.example b/.env.example index 67314e8..b54449f 100644 --- a/.env.example +++ b/.env.example @@ -90,6 +90,17 @@ IMPORT_SOURCE_AWS_BUCKET= # IMPORT_SOURCE_AWS_SECRET_ACCESS_KEY= # IMPORT_SOURCE_AWS_DEFAULT_REGION= +# Fleetbase Cloud: the cloud API registers itself with a request signed by this secret (the +# same value as REGISTRY_CLOUD_PLATFORM_SECRET in the cloud API, 32+ characters). Empty: off. +REGISTRY_CLOUD_PLATFORM_SECRET= +# Organization (slug) that owns the cloud instance, and the host the cloud API registers with. +REGISTRY_CLOUD_ORGANIZATION=fleetbase +REGISTRY_CLOUD_HOST= + +# Cutover freeze: publishing and checkout answer 503 (or `php artisan registry:maintenance on`). +REGISTRY_MAINTENANCE=false +# REGISTRY_MAINTENANCE_MESSAGE= + # Registry REGISTRY_VERSION=2.0.0-dev REGISTRY_CATALOG_CACHE_SECONDS=60 diff --git a/app/Actions/ActivateLicence.php b/app/Actions/ActivateLicence.php index 9c9d719..2cd78bc 100644 --- a/app/Actions/ActivateLicence.php +++ b/app/Actions/ActivateLicence.php @@ -15,7 +15,9 @@ * extension, creating the automatic free licence on first install of a free extension. * * Seat = instance (D4). Re-activating an instance that already holds a seat only moves - * the activation to the new release. + * the activation to the new release. The cloud instance acting for a company uses that + * company's organization's licences (`Instance::licensingOrganizationUuid()`), so each cloud + * company holds its own seat on the one cloud instance. */ class ActivateLicence { @@ -25,7 +27,7 @@ public function __invoke(Instance $instance, Extension $extension, ?Release $rel $licences = Licence::query() ->active() ->where('extension_uuid', $extension->uuid) - ->where('organization_uuid', $instance->organization_uuid) + ->where('organization_uuid', $instance->licensingOrganizationUuid()) ->orderByRaw('case when kind = ? then 0 else 1 end', [Licence::KIND_PAID]) ->orderBy('created_at') ->lockForUpdate() @@ -82,7 +84,7 @@ protected function freeLicence(Instance $instance, Extension $extension): Licenc $revoked = Licence::query() ->where('extension_uuid', $extension->uuid) - ->where('organization_uuid', $instance->organization_uuid) + ->where('organization_uuid', $instance->licensingOrganizationUuid()) ->where('status', Licence::STATUS_REVOKED) ->exists(); @@ -92,7 +94,7 @@ protected function freeLicence(Instance $instance, Extension $extension): Licenc return Licence::create([ 'extension_uuid' => $extension->uuid, - 'organization_uuid' => $instance->organization_uuid, + 'organization_uuid' => $instance->licensingOrganizationUuid(), 'kind' => Licence::KIND_FREE, 'max_activations' => null, 'status' => Licence::STATUS_ACTIVE, diff --git a/app/Actions/DeactivateLicence.php b/app/Actions/DeactivateLicence.php index 07071de..62a502b 100644 --- a/app/Actions/DeactivateLicence.php +++ b/app/Actions/DeactivateLicence.php @@ -7,8 +7,8 @@ use App\Models\LicenceActivation; /** - * Releases every seat an instance holds on licences for an extension. - * Returns the number of activations closed. + * Releases every seat an instance holds on licences for an extension; for the cloud instance + * acting for a company, only that company's seats. Returns the number of activations closed. */ class DeactivateLicence { @@ -17,7 +17,9 @@ public function __invoke(Instance $instance, Extension $extension): int return LicenceActivation::query() ->where('instance_uuid', $instance->uuid) ->whereNull('deactivated_at') - ->whereHas('licence', fn ($q) => $q->where('extension_uuid', $extension->uuid)) + ->whereHas('licence', fn ($q) => $q + ->where('extension_uuid', $extension->uuid) + ->when($instance->tenant(), fn ($q, $tenant) => $q->where('organization_uuid', $tenant->uuid))) ->update(['deactivated_at' => now(), 'updated_at' => now()]); } } diff --git a/app/Console/Commands/ManageMaintenance.php b/app/Console/Commands/ManageMaintenance.php new file mode 100644 index 0000000..7591fa4 --- /dev/null +++ b/app/Console/Commands/ManageMaintenance.php @@ -0,0 +1,48 @@ +argument('action')); + if (! in_array($action, ['on', 'off', 'status'], true)) { + $this->error("Unknown action [{$action}]: use on, off or status."); + + return self::INVALID; + } + + if ($action === 'on') { + Maintenance::enable($this->option('message') ?: null); + } elseif ($action === 'off') { + Maintenance::disable(); + } + + $status = Maintenance::status(); + $this->line(sprintf('Maintenance: <%s>%s%s', $status['active'] ? 'comment' : 'info', $status['active'] ? 'ON' : 'off', $status['since'] ? " since {$status['since']}" : '')); + if ($status['active']) { + $this->line('Message: '.$status['message']); + } + if ($action === 'off' && $status['forced']) { + $this->warn('REGISTRY_MAINTENANCE=true still forces it on; unset it and restart the tasks.'); + + return self::FAILURE; + } + + return self::SUCCESS; + } +} diff --git a/app/Http/Controllers/Instance/InstallController.php b/app/Http/Controllers/Instance/InstallController.php index 9878e53..52de628 100644 --- a/app/Http/Controllers/Instance/InstallController.php +++ b/app/Http/Controllers/Instance/InstallController.php @@ -18,7 +18,8 @@ use Illuminate\Support\Facades\DB; /** - * Install telemetry from instances. One `installs` row per install attempt: + * Install telemetry from instances. One `installs` row per install attempt and organization (an + * instance's own; the cloud instance installs for the company it acts for): * `installing` opens a row, `installed` or `failed` closes it. Once a release is * installed, earlier installed rows for the same extension become `uninstalled`, so an * instance counts once per extension in the catalog's install counts. Analytics tell the @@ -64,9 +65,12 @@ public function store(ReportInstallRequest $request, ActivateLicence $activate, $activation = $status === Install::STATUS_FAILED ? null : $activate($instance, $extension, $release); - [$install, $created] = DB::transaction(function () use ($instance, $extension, $release, $status, $request) { + $organization = $instance->licensingOrganizationUuid(); + + [$install, $created] = DB::transaction(function () use ($instance, $organization, $extension, $release, $status, $request) { $open = Install::query() ->where('instance_uuid', $instance->uuid) + ->where('organization_uuid', $organization) ->where('extension_uuid', $extension->uuid) ->where('status', Install::STATUS_INSTALLING) ->orderByDesc('id') @@ -79,6 +83,7 @@ public function store(ReportInstallRequest $request, ActivateLicence $activate, $install = $open ?? new Install([ 'instance_uuid' => $instance->uuid, + 'organization_uuid' => $organization, 'extension_uuid' => $extension->uuid, 'started_at' => now(), ]); @@ -87,6 +92,7 @@ public function store(ReportInstallRequest $request, ActivateLicence $activate, $replaced = $status === Install::STATUS_INSTALLED ? Install::query() ->where('instance_uuid', $instance->uuid) + ->where('organization_uuid', $organization) ->where('extension_uuid', $extension->uuid) ->where('status', Install::STATUS_INSTALLED) ->when($install->exists, fn ($q) => $q->whereKeyNot($install->getKey())) @@ -144,6 +150,7 @@ public function destroy(Request $request, string $extension, DeactivateLicence $ $installs = Install::query() ->where('instance_uuid', $instance->uuid) + ->where('organization_uuid', $instance->licensingOrganizationUuid()) ->where('extension_uuid', $model->uuid) ->whereIn('status', [Install::STATUS_INSTALLED, Install::STATUS_INSTALLING]) ->orderByDesc('id') @@ -194,6 +201,7 @@ protected function hasInstalled(Instance $instance, Extension $extension): bool { return Install::query() ->where('instance_uuid', $instance->uuid) + ->where('organization_uuid', $instance->licensingOrganizationUuid()) ->where('extension_uuid', $extension->uuid) ->where('status', Install::STATUS_INSTALLED) ->exists(); diff --git a/app/Http/Controllers/Instance/InstanceController.php b/app/Http/Controllers/Instance/InstanceController.php index b0fd6ad..68c4880 100644 --- a/app/Http/Controllers/Instance/InstanceController.php +++ b/app/Http/Controllers/Instance/InstanceController.php @@ -3,10 +3,12 @@ namespace App\Http\Controllers\Instance; use App\Http\Controllers\Controller; +use App\Http\Requests\Instance\RegisterCloudInstanceRequest; use App\Http\Requests\Instance\RegisterInstanceRequest; use App\Http\Resources\ExtensionReferenceResource; use App\Http\Resources\InstanceResource; use App\Http\Resources\LicenceResource; +use App\Http\Resources\OrganizationResource; use App\Http\Resources\ReleaseResource; use App\Models\Event; use App\Models\Install; @@ -16,6 +18,7 @@ use App\Models\LicenceActivation; use App\Models\Organization; use App\Models\Release; +use App\Support\CloudPlatform; use App\Support\Compat; use App\Support\Semver; use Illuminate\Http\JsonResponse; @@ -28,9 +31,9 @@ class InstanceController extends Controller * POST /v1/instances/register — exchanges a registration code for an instance token. * The token (`flbi_…`) is returned once; only its SHA-256 hash is stored. * - * Registering an instance that is already registered rotates it: the same row is updated and - * handed a new token (the old one stops working), so its installs and licence seats stay on - * one instance, and the response is 200 with `rotated: true`. The instance is matched by + * Registering an instance that is already registered rotates it (D17): the same row is updated + * and handed a new token (the old one stops working), so its installs and licence seats stay + * on one instance, and the response is 200 with `rotated: true`. The instance is matched by * * 1. `previous_token`, the instance's current token, when it belongs to an active instance in * the code's organization; else @@ -58,25 +61,84 @@ public function register(RegisterInstanceRequest $request): JsonResponse if ($previous === null && ($previous = Instance::findByHost($organization->uuid, $request->input('host')))) { $matched = 'host'; } - $rotated = $matched !== null; - - [$instance, $plain] = DB::transaction(function () use ($request, $code, $organization, $previous, $rotated) { - $attributes = [ - 'name' => $request->input('name'), - 'host' => $request->input('host'), - 'kind' => $request->input('kind'), - 'console_version' => $request->input('console_version'), - 'api_version' => $request->input('api_version'), - 'status' => Instance::STATUS_ACTIVE, - 'last_seen_at' => now(), - ]; - if ($rotated) { + [$instance, $plain] = $this->issue($organization, [ + 'name' => $request->input('name'), + 'host' => $request->input('host'), + 'kind' => $request->input('kind'), + 'console_version' => $request->input('console_version'), + 'api_version' => $request->input('api_version'), + ], $previous, $matched !== null, $code->account_uuid, fn (Instance $instance) => $code->forceFill(['instance_uuid' => $instance->uuid])->save()); + + Event::record($matched ? 'instance.reregistered' : 'instance.registered', $instance, $code->account, array_filter([ + 'code' => $code->uuid, + 'matched_by' => $matched, + 'replaced' => $previous && ! $matched ? $previous->public_id : null, + ]), $request->ip()); + + return $this->registered($request, $instance, $plain, $matched !== null); + } + + /** + * POST /v1/instances/register/cloud — the cloud console's API registers itself on boot, + * signed with the platform secret (`platform` middleware) instead of a registration code. + * + * One instance per cloud deployment, kind `cloud`, owned by `registry.cloud.organization`; it + * acts for each cloud company through `X-Registry-Tenant`. Idempotent across restarts and + * replicas by the same rotation rules (D17): it is matched by `previous_token`, else by the + * same organization and host, else it adopts the placeholder `registry:import` created that + * never had a token. Matched: 200 with `rotated: true`; otherwise a new instance, 201. + */ + public function registerCloud(RegisterCloudInstanceRequest $request): JsonResponse + { + $organization = CloudPlatform::organization(); + + $previous = $request->filled('previous_token') ? Instance::findByToken((string) $request->input('previous_token')) : null; + $matched = $previous !== null && $previous->isPlatform() && $previous->organization_uuid === $organization->uuid ? 'token' : null; + if ($matched === null) { + $previous = CloudPlatform::instance($organization, $request->input('host'), anyHost: false); + $matched = match (true) { + $previous === null => null, + Instance::normalizeHost($previous->host) === Instance::normalizeHost($request->input('host')) => 'host', + default => 'placeholder', + }; + } + + [$instance, $plain] = $this->issue($organization, [ + 'name' => $request->input('name') ?: 'Fleetbase Cloud', + 'host' => $request->input('host'), + 'kind' => Instance::KIND_CLOUD, + 'console_version' => $request->input('console_version'), + 'api_version' => $request->input('api_version'), + 'meta' => array_merge($previous?->meta ?? [], ['platform' => true]), + ], $matched ? $previous : null, $matched !== null); + + Event::record($matched ? 'instance.reregistered' : 'instance.registered', $instance, null, array_filter([ + 'platform' => true, + 'matched_by' => $matched, + ]), $request->ip()); + + return $this->registered($request, $instance, $plain, $matched !== null); + } + + /** + * Rotate `$previous` (when `$rotate`) or create the instance, and issue its token. + * + * @param array $attributes + * @param (callable(Instance): mixed)|null $issued runs in the same transaction + * @return array{0: Instance, 1: string} + */ + protected function issue(Organization $organization, array $attributes, ?Instance $previous, bool $rotate, ?string $registeredBy = null, ?callable $issued = null): array + { + return DB::transaction(function () use ($organization, $attributes, $previous, $rotate, $registeredBy, $issued) { + $attributes += ['status' => Instance::STATUS_ACTIVE, 'last_seen_at' => now()]; + + if ($rotate) { $instance = tap($previous)->update($attributes); } else { $instance = Instance::create($attributes + [ 'organization_uuid' => $organization->uuid, - 'registered_by_account_uuid' => $code->account_uuid, + 'registered_by_account_uuid' => $registeredBy, ]); if ($previous) { @@ -85,17 +147,16 @@ public function register(RegisterInstanceRequest $request): JsonResponse } $plain = $instance->issueToken(); - $code->forceFill(['instance_uuid' => $instance->uuid])->save(); + if ($issued) { + $issued($instance); + } return [$instance, $plain]; }); + } - Event::record($rotated ? 'instance.reregistered' : 'instance.registered', $instance, $code->account, array_filter([ - 'code' => $code->uuid, - 'matched_by' => $matched, - 'replaced' => $previous && ! $rotated ? $previous->public_id : null, - ]), $request->ip()); - + protected function registered(Request $request, Instance $instance, string $plain, bool $rotated): JsonResponse + { return response()->json([ 'token' => $plain, 'token_type' => 'Bearer', @@ -108,6 +169,7 @@ public function register(RegisterInstanceRequest $request): JsonResponse * GET /v1/instances/me — the instance, its organization and the organization's * licences with seat usage, whether this instance holds a seat, and the receipt of a * bought licence (so a console can link it after a re-registration or a cache clear). + * The cloud instance acting for a company gets that company's licences, and `tenant`. */ public function me(Request $request): JsonResponse { @@ -115,7 +177,7 @@ public function me(Request $request): JsonResponse $instance = $request->user(); $licences = Licence::query() - ->where('organization_uuid', $instance->organization_uuid) + ->where('organization_uuid', $instance->licensingOrganizationUuid()) ->with(['extension', 'purchase']) ->withCount('activeActivations') ->addSelect(['activated_here' => LicenceActivation::query() @@ -127,10 +189,11 @@ public function me(Request $request): JsonResponse ->orderByDesc('created_at') ->get(); - return response()->json([ + return response()->json(array_filter([ 'instance' => (new InstanceResource($instance->load('organization')))->resolve($request), + 'tenant' => $instance->tenant() ? (new OrganizationResource($instance->tenant()))->resolve($request) : null, 'licences' => LicenceResource::collection($licences)->resolve($request), - ]); + ], fn ($value) => $value !== null)); } /** @@ -148,6 +211,7 @@ public function updates(Request $request): JsonResponse $installs = Install::query() ->where('instance_uuid', $instance->uuid) + ->where('organization_uuid', $instance->licensingOrganizationUuid()) ->where('status', Install::STATUS_INSTALLED) ->with(['extension', 'release.engineDist']) ->orderBy('created_at') @@ -186,6 +250,46 @@ public function updates(Request $request): JsonResponse ]); } + /** + * GET /v1/instances/me/tenant-installs — every extension installed on the cloud instance, per + * cloud company, 500 a page (`?page=`, `?per_page=` up to 500). The cloud console's API reads it once after the + * cutover import (`registry:cloud-sync-installs`) to record locally what each company had + * installed, so their consoles keep loading those engines. Cloud instance only. + */ + public function tenantInstalls(Request $request): JsonResponse + { + /** @var Instance $instance */ + $instance = $request->user(); + if (! $instance->isPlatform()) { + return response()->error('Only the cloud instance lists installs per company.', 403); + } + + $page = Install::query() + ->where('instance_uuid', $instance->uuid) + ->where('status', Install::STATUS_INSTALLED) + ->whereNotNull('organization_uuid') + ->with(['organization', 'extension.packages', 'release']) + ->orderBy('id') + ->paginate(min(500, max(1, (int) $request->input('per_page', 500)))); + + $data = collect($page->items()) + ->filter(fn (Install $install) => $install->extension !== null && $install->organization !== null) + ->map(fn (Install $install) => [ + 'company_uuid' => $install->organization->source_uuid, + 'organization' => $install->organization->public_id, + 'extension' => (new ExtensionReferenceResource($install->extension))->resolve($request), + 'packages' => $install->extension->packages->mapWithKeys(fn ($package) => [$package->kind => $package->name])->all(), + 'release' => $install->release ? ['id' => $install->release->public_id, 'version' => $install->release->version] : null, + 'installed_at' => $install->installed_at, + ]) + ->values(); + + return response()->json([ + 'data' => $data, + 'meta' => ['current_page' => $page->currentPage(), 'last_page' => $page->lastPage(), 'total' => $page->total()], + ]); + } + /** * Retire an instance that re-registered into another organization: revoke it, release the * seats it held on its old organization's licences and move its open installs to the new @@ -203,6 +307,6 @@ protected function replace(Instance $previous, Instance $instance): void Install::query() ->where('instance_uuid', $previous->uuid) ->whereIn('status', [Install::STATUS_INSTALLING, Install::STATUS_INSTALLED]) - ->update(['instance_uuid' => $instance->uuid, 'updated_at' => now()]); + ->update(['instance_uuid' => $instance->uuid, 'organization_uuid' => $instance->organization_uuid, 'updated_at' => now()]); } } diff --git a/app/Http/Controllers/Instance/PurchaseController.php b/app/Http/Controllers/Instance/PurchaseController.php index c8dde59..5947b9f 100644 --- a/app/Http/Controllers/Instance/PurchaseController.php +++ b/app/Http/Controllers/Instance/PurchaseController.php @@ -23,8 +23,9 @@ class PurchaseController extends Controller { /** * POST /v1/instances/me/purchases/checkout — a hosted Checkout Session for the instance's - * organization. The return and cancel URLs must be on the instance's registered host or its - * domain. Answers 201 with the hand-off id and Stripe's URL. + * organization (the company's, when the cloud instance acts for one). The return and cancel + * URLs must be on the instance's registered host or its domain. Answers 201 with the hand-off + * id and Stripe's URL. */ public function checkout(StartCheckoutRequest $request, CheckoutSessions $sessions): JsonResponse { @@ -46,7 +47,7 @@ public function checkout(StartCheckoutRequest $request, CheckoutSessions $sessio $checkout = $sessions->start( $extension, - $instance->organization, + $instance->licensingOrganization(), Checkout::SOURCE_INSTANCE, $request->input('return_url'), $request->input('cancel_url') ?? $request->input('return_url'), @@ -69,7 +70,7 @@ public function show(Request $request, string $handoff): CheckoutResource $checkout = Checkout::query() ->where('public_id', $handoff) - ->where('organization_uuid', $instance->organization_uuid) + ->where('organization_uuid', $instance->licensingOrganizationUuid()) ->with(['extension', 'organization', 'purchase.extension', 'purchase.licence']) ->first(); diff --git a/app/Http/Middleware/RefuseDuringMaintenance.php b/app/Http/Middleware/RefuseDuringMaintenance.php new file mode 100644 index 0000000..d5a8df0 --- /dev/null +++ b/app/Http/Middleware/RefuseDuringMaintenance.php @@ -0,0 +1,31 @@ +method(), ['GET', 'HEAD', 'OPTIONS'], true)) { + return $next($request); + } + + if (! Maintenance::active()) { + return $next($request); + } + + return response() + ->error(Maintenance::message(), 503, ['maintenance' => true]) + ->header('Retry-After', (string) max(60, (int) config('registry.maintenance.retry_after', 3600))); + } +} diff --git a/app/Http/Middleware/ResolveInstanceTenant.php b/app/Http/Middleware/ResolveInstanceTenant.php new file mode 100644 index 0000000..4b38f96 --- /dev/null +++ b/app/Http/Middleware/ResolveInstanceTenant.php @@ -0,0 +1,35 @@ +user(); + $company = trim((string) $request->header(CloudPlatform::TENANT_HEADER, '')); + + if ($instance instanceof Instance && $instance->isPlatform() && $company !== '') { + $tenant = CloudPlatform::tenant($company, rawurldecode((string) $request->header(CloudPlatform::TENANT_NAME_HEADER, ''))); + if ($tenant === null) { + return response()->error('X-Registry-Tenant must be the cloud company uuid.', 422); + } + + $instance->actAs($tenant); + } + + return $next($request); + } +} diff --git a/app/Http/Middleware/VerifyPlatformSignature.php b/app/Http/Middleware/VerifyPlatformSignature.php new file mode 100644 index 0000000..4c5ee22 --- /dev/null +++ b/app/Http/Middleware/VerifyPlatformSignature.php @@ -0,0 +1,25 @@ +error($refusal, PlatformSignature::enabled() ? 401 : 403); + } + + return $next($request); + } +} diff --git a/app/Http/Requests/Instance/RegisterCloudInstanceRequest.php b/app/Http/Requests/Instance/RegisterCloudInstanceRequest.php new file mode 100644 index 0000000..e352d7d --- /dev/null +++ b/app/Http/Requests/Instance/RegisterCloudInstanceRequest.php @@ -0,0 +1,29 @@ + ['nullable', 'string', 'max:191'], + 'host' => ['required', 'string', 'max:255'], + 'console_version' => ['nullable', 'string', 'max:64'], + 'api_version' => ['nullable', 'string', 'max:64'], + 'previous_token' => ['nullable', 'string', 'max:128'], + ]; + } +} diff --git a/app/Models/Install.php b/app/Models/Install.php index a8daace..6d3a575 100644 --- a/app/Models/Install.php +++ b/app/Models/Install.php @@ -32,7 +32,7 @@ class Install extends Model protected $table = 'installs'; protected $fillable = [ - 'instance_uuid', 'extension_uuid', 'release_uuid', 'previous_release_uuid', 'status', 'installed_by', 'error', + 'instance_uuid', 'organization_uuid', 'extension_uuid', 'release_uuid', 'previous_release_uuid', 'status', 'installed_by', 'error', 'started_at', 'finished_at', 'installed_at', 'uninstalled_at', 'replaced_at', ]; @@ -51,11 +51,29 @@ protected static function newFactory(): InstallFactory return InstallFactory::new(); } + /** + * An install is for its instance's organization unless it says otherwise (the cloud instance + * installs for the company it acts for). + */ + protected static function booted(): void + { + static::creating(function (Install $install) { + if ($install->organization_uuid === null && $install->instance_uuid !== null) { + $install->organization_uuid = Instance::withTrashed()->where('uuid', $install->instance_uuid)->value('organization_uuid'); + } + }); + } + public function instance(): BelongsTo { return $this->belongsTo(Instance::class, 'instance_uuid', 'uuid'); } + public function organization(): BelongsTo + { + return $this->belongsTo(Organization::class, 'organization_uuid', 'uuid'); + } + public function extension(): BelongsTo { return $this->belongsTo(Extension::class, 'extension_uuid', 'uuid'); diff --git a/app/Models/Instance.php b/app/Models/Instance.php index 1574517..03f5527 100644 --- a/app/Models/Instance.php +++ b/app/Models/Instance.php @@ -50,6 +50,12 @@ class Instance extends Model implements AuthenticatableContract protected $casts = ['last_seen_at' => 'datetime', 'meta' => 'array']; + /** + * The cloud company's organization this request acts for (cloud platform instances only; see + * `CloudPlatform`). Never persisted. + */ + protected ?Organization $tenant = null; + protected static function newFactory(): InstanceFactory { return InstanceFactory::new(); @@ -183,6 +189,45 @@ public function isActive(): bool return $this->status === self::STATUS_ACTIVE; } + /** + * The Fleetbase Cloud instance: registered with the platform secret (or created for it by the + * importer). It serves every cloud company, so it may act for their organizations. + */ + public function isPlatform(): bool + { + return $this->kind === self::KIND_CLOUD && ($this->meta['platform'] ?? false) === true; + } + + /** + * Act for a cloud company's organization for the rest of this request: its licences, installs + * and checkouts. Only the platform instance can; null goes back to its own organization. + */ + public function actAs(?Organization $organization): static + { + $this->tenant = $organization !== null && $this->isPlatform() ? $organization : null; + + return $this; + } + + public function tenant(): ?Organization + { + return $this->tenant; + } + + /** + * The organization whose licences and installs this request uses: the tenant when the platform + * instance acts for one, else the instance's own. + */ + public function licensingOrganizationUuid(): string + { + return $this->tenant?->uuid ?? $this->organization_uuid; + } + + public function licensingOrganization(): ?Organization + { + return $this->tenant ?? $this->organization; + } + public function touchLastSeen(?string $ip = null): void { $meta = $this->meta ?? []; diff --git a/app/Services/LicenceGate.php b/app/Services/LicenceGate.php index a3fc69c..8208872 100644 --- a/app/Services/LicenceGate.php +++ b/app/Services/LicenceGate.php @@ -50,7 +50,7 @@ public function check(Request $request, Extension $extension): array } $organizations = $this->organizationsFor($credential); - if (in_array($extension->organization_uuid, $organizations, true) || $this->hasActiveLicence($extension, $organizations)) { + if (in_array($extension->organization_uuid, $organizations, true) || $this->hasActiveLicence($extension, $organizations) || $this->platformHoldsSeat($credential, $extension)) { return ['status' => self::STATUS_ALLOWED, 'message' => null]; } @@ -72,6 +72,21 @@ public function hasActiveLicence(Extension $extension, array $organizationUuids) ->exists(); } + /** + * The cloud instance downloads for whichever company it is installing for, and Composer and + * npm cannot name that company. It may read a paid package while any company holds an active + * seat on it: the install report that claims the seat comes before the download. + */ + protected function platformHoldsSeat(Token|Instance $credential, Extension $extension): bool + { + return $credential instanceof Instance + && $credential->isPlatform() + && $credential->activations() + ->whereNull('deactivated_at') + ->whereHas('licence', fn ($q) => $q->active()->where('extension_uuid', $extension->uuid)) + ->exists(); + } + protected function resolve(string $plain, Request $request): Token|Instance|null { if (str_starts_with($plain, Token::PREFIX_PERSONAL)) { diff --git a/app/Support/CloudPlatform.php b/app/Support/CloudPlatform.php new file mode 100644 index 0000000..c586099 --- /dev/null +++ b/app/Support/CloudPlatform.php @@ -0,0 +1,102 @@ +where('slug', $slug)->first() + ?? Organization::create(['slug' => $slug, 'name' => 'Fleetbase Cloud', 'is_personal' => false, 'meta' => ['source' => 'cloud-platform']]); + } + + /** + * The active cloud instance for a host: the one registered from that host (D17), else the + * placeholder `registry:import` left that never got a token, else (`$anyHost`, for the + * importer when it is given no host) the most recently seen cloud instance of the + * organization. Null when there is none. + */ + public static function instance(Organization $organization, ?string $host, bool $anyHost = true): ?Instance + { + $instances = Instance::query() + ->where('organization_uuid', $organization->uuid) + ->where('kind', Instance::KIND_CLOUD) + ->where('status', Instance::STATUS_ACTIVE) + ->orderByDesc('last_seen_at') + ->orderByDesc('id') + ->get() + ->filter(fn (Instance $instance) => $instance->isPlatform()); + + $normalized = Instance::normalizeHost($host); + + return ($normalized !== null ? $instances->first(fn (Instance $i) => Instance::normalizeHost($i->host) === $normalized) : null) + ?? $instances->first(fn (Instance $i) => $i->token_hash === null) + ?? ($anyHost && $normalized === null ? $instances->first() : null); + } + + /** + * The organization a cloud company acts as: the one imported from that company or created + * the first time it is seen. Null for a value that is not a uuid. + */ + public static function tenant(string $companyUuid, ?string $name = null): ?Organization + { + $companyUuid = strtolower(trim($companyUuid)); + if (! Str::isUuid($companyUuid)) { + return null; + } + + $organization = Organization::query()->where('source_uuid', $companyUuid)->first(); + if ($organization) { + return $organization; + } + + $name = trim((string) $name) !== '' ? Str::limit(trim((string) $name), 191, '') : 'Company '.Str::limit($companyUuid, 8, ''); + + $organization = (new Organization)->forceFill([ + 'slug' => static::uniqueSlug($name), + 'name' => $name, + 'is_personal' => false, + // Same key the importer gives a company's organization, so a later import finds it. + 'source_uuid' => $companyUuid, + 'meta' => ['source' => 'cloud', 'company_uuid' => $companyUuid], + ]); + $organization->save(); + + return $organization; + } + + protected static function uniqueSlug(string $name): string + { + $base = Str::slug($name) ?: 'company'; + $slug = $base; + for ($n = 2; Organization::withTrashed()->where('slug', $slug)->exists(); $n++) { + $slug = $base.'-'.$n; + } + + return $slug; + } +} diff --git a/app/Support/Maintenance.php b/app/Support/Maintenance.php new file mode 100644 index 0000000..9f62e5b --- /dev/null +++ b/app/Support/Maintenance.php @@ -0,0 +1,72 @@ + static::active(), + 'forced' => (bool) config('registry.maintenance.enabled'), + 'message' => static::message(), + 'since' => $stored['since'] ?? null, + ]; + } + + public static function enable(?string $message = null): void + { + Cache::forever(self::CACHE_KEY, ['message' => $message, 'since' => now()->toIso8601String()]); + } + + public static function disable(): void + { + Cache::forget(self::CACHE_KEY); + } + + /** + * @return array{message: ?string, since: ?string}|null + */ + protected static function stored(): ?array + { + try { + $value = Cache::get(self::CACHE_KEY); + } catch (\Throwable) { + return null; + } + + return is_array($value) ? $value : null; + } +} diff --git a/app/Support/PlatformSignature.php b/app/Support/PlatformSignature.php new file mode 100644 index 0000000..4355483 --- /dev/null +++ b/app/Support/PlatformSignature.php @@ -0,0 +1,95 @@ +header(self::TIMESTAMP_HEADER, ''); + $nonce = (string) $request->header(self::NONCE_HEADER, ''); + $signature = (string) $request->header(self::SIGNATURE_HEADER, ''); + + if (! ctype_digit($timestamp) || ! preg_match('/^[A-Za-z0-9_-]{16,64}$/', $nonce) || ! str_starts_with($signature, 'v1=')) { + return 'The request is not signed with the platform secret.'; + } + + $tolerance = max(30, (int) config('registry.cloud.signature_tolerance', 300)); + if (abs(time() - (int) $timestamp) > $tolerance) { + return 'The signed request has expired. Check the clock on the calling server.'; + } + + $expected = static::sign($secret, $timestamp, $nonce, $request->getContent()); + if (! hash_equals($expected, substr($signature, 3))) { + return 'The request is not signed with the platform secret.'; + } + + // A replayed request would rotate the instance token again and hand the caller the new one. + if (! Cache::add('registry:platform-nonce:'.hash('sha256', $nonce), true, $tolerance * 2)) { + return 'This signed request was already used.'; + } + + return null; + } + + public static function sign(string $secret, string $timestamp, string $nonce, string $body): string + { + return hash_hmac('sha256', $timestamp.'.'.$nonce.'.'.$body, $secret); + } + + /** + * Headers for a body signed now (tests and tooling). + * + * @return array + */ + public static function headers(string $secret, string $body, ?int $timestamp = null, ?string $nonce = null): array + { + $timestamp = (string) ($timestamp ?? time()); + $nonce ??= bin2hex(random_bytes(16)); + + return [ + self::TIMESTAMP_HEADER => $timestamp, + self::NONCE_HEADER => $nonce, + self::SIGNATURE_HEADER => 'v1='.static::sign($secret, $timestamp, $nonce, $body), + ]; + } + + protected static function secret(): ?string + { + $secret = config('registry.cloud.platform_secret'); + + return is_string($secret) && strlen(trim($secret)) > 0 ? $secret : null; + } +} diff --git a/bootstrap/app.php b/bootstrap/app.php index ff0015e..0c612f2 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -5,7 +5,10 @@ use App\Http\Middleware\EnsureStaff; use App\Http\Middleware\EnsureTokenScope; use App\Http\Middleware\HandleInertiaRequests; +use App\Http\Middleware\RefuseDuringMaintenance; +use App\Http\Middleware\ResolveInstanceTenant; use App\Http\Middleware\StatefulSession; +use App\Http\Middleware\VerifyPlatformSignature; use Illuminate\Auth\AuthenticationException; use Illuminate\Cookie\Middleware\EncryptCookies; use Illuminate\Database\Eloquent\ModelNotFoundException; @@ -34,6 +37,9 @@ 'licence' => EnsurePackageLicence::class, 'staff' => EnsureStaff::class, 'stateful' => StatefulSession::class, + 'maintenance' => RefuseDuringMaintenance::class, + 'tenant' => ResolveInstanceTenant::class, + 'platform' => VerifyPlatformSignature::class, ]); // The website's cookie session must start before `auth:token,web` resolves the caller. diff --git a/config/registry.php b/config/registry.php index 502ca5e..b31942c 100644 --- a/config/registry.php +++ b/config/registry.php @@ -75,6 +75,33 @@ 'source_disk' => env('REGISTRY_IMPORT_SOURCE_DISK', 'import_source'), ], + 'cloud' => [ + // Shared with the cloud console's API (REGISTRY_CLOUD_PLATFORM_SECRET there too). It signs + // `POST /v1/instances/register/cloud`; empty turns cloud registration off. + 'platform_secret' => env('REGISTRY_CLOUD_PLATFORM_SECRET'), + + // Organization that owns the cloud instance (slug); created on first use. Licences stay + // with each cloud company's organization: the instance only acts for them. + 'organization' => env('REGISTRY_CLOUD_ORGANIZATION', 'fleetbase'), + + // Host the cloud API registers with (its APP_URL), e.g. `api.fleetbase.io`. The importer + // attaches cloud installs to the instance of this host so the first registration + // rotates onto it. + 'host' => env('REGISTRY_CLOUD_HOST'), + + // Seconds a signed registration request stays valid. + 'signature_tolerance' => (int) env('REGISTRY_CLOUD_SIGNATURE_TOLERANCE', 300), + ], + + 'maintenance' => [ + // Refuse publishing and checkout with 503 (cutover freeze). `registry:maintenance on` + // sets it at runtime without a deploy; this variable forces it on. + 'enabled' => (bool) env('REGISTRY_MAINTENANCE', false), + 'message' => env('REGISTRY_MAINTENANCE_MESSAGE', 'The Fleetbase extensions registry is being upgraded. Publishing and purchases are paused and will be back shortly.'), + // Retry-After header on refused requests, in seconds. + 'retry_after' => (int) env('REGISTRY_MAINTENANCE_RETRY_AFTER', 3600), + ], + 'storage' => [ // Disk that holds tarballs, prebuilt dist archives, icons and screenshots. 'disk' => env('REGISTRY_STORAGE_DISK', 's3'), diff --git a/database/migrations/2026_10_26_000401_add_organization_uuid_to_installs_table.php b/database/migrations/2026_10_26_000401_add_organization_uuid_to_installs_table.php new file mode 100644 index 0000000..30b9f8c --- /dev/null +++ b/database/migrations/2026_10_26_000401_add_organization_uuid_to_installs_table.php @@ -0,0 +1,36 @@ +uuid('organization_uuid')->nullable()->after('instance_uuid'); + // Named: the default name is longer than MySQL's 64 characters. + $table->index(['instance_uuid', 'organization_uuid', 'extension_uuid', 'status'], 'installs_instance_organization_extension_status_index'); + }); + + DB::table('installs')->whereNull('organization_uuid')->update([ + 'organization_uuid' => DB::table('instances')->select('organization_uuid')->whereColumn('instances.uuid', 'installs.instance_uuid')->limit(1), + ]); + } + + public function down(): void + { + Schema::table('installs', function (Blueprint $table) { + $table->dropIndex('installs_instance_organization_extension_status_index'); + $table->dropColumn('organization_uuid'); + }); + } +}; diff --git a/routes/api.php b/routes/api.php index 9660471..fdeb95f 100644 --- a/routes/api.php +++ b/routes/api.php @@ -41,7 +41,10 @@ | Package routes (/npm, /composer) sit outside /v1 because npm and Composer | clients address them as registry roots. | Instance routes: Authorization: Bearer flbi_… (instance token), except -| `instances/register`, which takes a registration code in the body. +| `instances/register`, which takes a registration code in the body, and +| `instances/register/cloud`, signed with the cloud platform secret. The cloud instance acts +| for one cloud company per request (`X-Registry-Tenant`). +| Publishing and checkout answer 503 while `registry:maintenance` is on (the cutover freeze). | Publisher routes: personal token (or website session) with the `publish` scope and the | owner, admin or publisher role in the listing's organization. That includes a listing's | analytics (`extensions/{slug}/stats`); `organizations/{slug}/stats` sums every listing of @@ -115,11 +118,12 @@ Route::get('organizations/{slug}/stats', [OrganizationStatsController::class, 'show'])->name('organizations.stats'); Route::get('organizations/{slug}/stats.csv', [OrganizationStatsController::class, 'export'])->name('organizations.stats.export'); - Route::post('purchases/checkout', [CheckoutController::class, 'checkout'])->name('purchases.checkout'); + Route::post('purchases/checkout', [CheckoutController::class, 'checkout'])->middleware('maintenance')->name('purchases.checkout'); Route::get('purchases/{handoff}', [CheckoutController::class, 'show'])->name('purchases.show'); }); - Route::prefix('publisher')->name('publisher.')->middleware(['auth:token,web', 'throttle:publish', 'scope:publish'])->group(function () { + // Frozen (503) while `registry:maintenance` is on, reads excepted. + Route::prefix('publisher')->name('publisher.')->middleware(['auth:token,web', 'throttle:publish', 'scope:publish', 'maintenance:write'])->group(function () { Route::get('extensions', [PublisherExtensionController::class, 'index'])->name('extensions.index'); Route::post('extensions', [PublisherExtensionController::class, 'store'])->name('extensions.store'); Route::get('packages/availability', [PublisherExtensionController::class, 'availability'])->name('packages.availability'); @@ -148,9 +152,9 @@ Route::get('releases/{release}', [ReviewController::class, 'show'])->name('releases.show'); Route::get('releases/{release}/files/{kind}', [ReviewController::class, 'download'])->where('kind', 'engine|api|dist')->name('releases.files.download'); Route::post('releases/{release}/claim', [ReviewController::class, 'claim'])->name('releases.claim'); - Route::post('releases/{release}/approve', [ReviewController::class, 'approve'])->name('releases.approve'); + Route::post('releases/{release}/approve', [ReviewController::class, 'approve'])->middleware('maintenance')->name('releases.approve'); Route::post('releases/{release}/reject', [ReviewController::class, 'reject'])->name('releases.reject'); - Route::post('releases/{release}/publish', [ReviewController::class, 'publish'])->name('releases.publish'); + Route::post('releases/{release}/publish', [ReviewController::class, 'publish'])->middleware('maintenance')->name('releases.publish'); Route::post('extensions/{slug}/suspend', [ReviewController::class, 'suspend'])->name('extensions.suspend'); Route::post('extensions/{slug}/unsuspend', [ReviewController::class, 'unsuspend'])->name('extensions.unsuspend'); Route::get('licences', [AdminLicenceController::class, 'index'])->name('licences.index'); @@ -168,13 +172,17 @@ // Instances authenticate with flbi_ tokens only: never a session or CSRF. Route::prefix('instances')->name('instances.')->withoutMiddleware('stateful')->group(function () { Route::post('register', [InstanceController::class, 'register'])->middleware('throttle:instance-register')->name('register'); + // The cloud console's API, signed with the platform secret instead of a registration code. + Route::post('register/cloud', [InstanceController::class, 'registerCloud'])->middleware(['throttle:instance-register', 'platform'])->name('register.cloud'); - Route::middleware(['auth:instance', 'throttle:instances'])->group(function () { + // `tenant`: the cloud instance acts for the company in X-Registry-Tenant. + Route::middleware(['auth:instance', 'throttle:instances', 'tenant'])->group(function () { Route::get('me', [InstanceController::class, 'me'])->name('me'); Route::get('me/updates', [InstanceController::class, 'updates'])->name('me.updates'); + Route::get('me/tenant-installs', [InstanceController::class, 'tenantInstalls'])->name('me.tenant-installs'); Route::post('me/installs', [InstallController::class, 'store'])->name('me.installs.store'); Route::delete('me/installs/{extension}', [InstallController::class, 'destroy'])->where('extension', '.+')->name('me.installs.destroy'); - Route::post('me/purchases/checkout', [PurchaseController::class, 'checkout'])->name('me.purchases.checkout'); + Route::post('me/purchases/checkout', [PurchaseController::class, 'checkout'])->middleware('maintenance')->name('me.purchases.checkout'); Route::get('me/purchases/{handoff}', [PurchaseController::class, 'show'])->name('me.purchases.show'); }); }); diff --git a/tests/Feature/Commerce/CloudCheckoutTest.php b/tests/Feature/Commerce/CloudCheckoutTest.php new file mode 100644 index 0000000..be80355 --- /dev/null +++ b/tests/Feature/Commerce/CloudCheckoutTest.php @@ -0,0 +1,25 @@ + 'https://api.fleetbase.test']); + actingAsInstance($instance); + $acme = cloudCompany('Acme'); + + actingForCompany($acme); + $handoff = $this->postJson('/v1/instances/me/purchases/checkout', ['extension' => $listing->slug, 'return_url' => 'https://console.fleetbase.test/extensions/purchased']) + ->assertCreated() + ->assertJsonPath('organization.id', $acme->public_id) + ->json('handoff'); + + expect(Checkout::sole()->organization_uuid)->toBe($acme->uuid) + ->and(stripe()->sent('post', '/v1/checkout/sessions')[0]['params']['metadata']['organization_uuid'])->toBe($acme->uuid); + + $this->getJson("/v1/instances/me/purchases/{$handoff}")->assertOk(); + + // Another company cannot read Acme's hand-off. + actingForCompany(cloudCompany('Globex')); + $this->getJson("/v1/instances/me/purchases/{$handoff}")->assertNotFound(); +}); diff --git a/tests/Feature/Instance/CloudInstanceTest.php b/tests/Feature/Instance/CloudInstanceTest.php new file mode 100644 index 0000000..27df996 --- /dev/null +++ b/tests/Feature/Instance/CloudInstanceTest.php @@ -0,0 +1,260 @@ + CLOUD_PLATFORM_SECRET, 'registry.cloud.organization' => 'fleetbase']); +}); + +/** + * A published free extension with one published release. + * + * @return array{0: Extension, 1: Release} + */ +function cloudFreeListing(): array +{ + $extension = Extension::factory()->published()->free()->create(); + $release = Release::factory()->create(['extension_uuid' => $extension->uuid, 'version' => '1.0.0', 'status' => Release::STATUS_PUBLISHED, 'published_at' => now()]); + + return [$extension, $release]; +} + +it('registers the cloud instance with a request signed by the platform secret', function () { + $response = registerCloudInstance()->assertCreated() + ->assertJsonPath('rotated', false) + ->assertJsonPath('instance.kind', 'cloud') + ->assertJsonPath('instance.host', 'https://api.fleetbase.test') + ->assertJsonPath('instance.organization.slug', 'fleetbase'); + + $instance = Instance::sole(); + expect($instance->isPlatform())->toBeTrue() + ->and($instance->name)->toBe('Fleetbase Cloud') + ->and($instance->console_version)->toBe('0.8.0') + ->and($instance->registered_by_account_uuid)->toBeNull() + ->and(Event::where('kind', 'instance.registered')->sole()->payload)->toBe(['platform' => true]); + + $this->withHeader('Authorization', 'Bearer '.$response->json('token'))->getJson('/v1/instances/me') + ->assertOk()->assertJsonPath('instance.id', $instance->public_id); +}); + +it('refuses cloud registration without a valid platform signature', function (TestResponse $response, int $status, string $message) { + expect($response->assertStatus($status)->json('errors.0'))->toBe($message) + ->and(Instance::count())->toBe(0); +})->with([ + 'no secret configured' => [function () { + config(['registry.cloud.platform_secret' => null]); + + return registerCloudInstance(); + }, 403, 'Cloud registration is not enabled on this registry.'], + 'unsigned' => [fn () => registerCloudInstance([], null), 401, 'The request is not signed with the platform secret.'], + 'wrong secret' => [fn () => registerCloudInstance([], 'another-secret-another-secret-another'), 401, 'The request is not signed with the platform secret.'], + 'expired' => [fn () => registerCloudInstance([], null, PlatformSignature::headers(CLOUD_PLATFORM_SECRET, json_encode(cloudRegistrationBody()), time() - 600)), 401, 'The signed request has expired. Check the clock on the calling server.'], + 'body changed after signing' => [fn () => registerCloudInstance(['host' => 'https://evil.test'], null, PlatformSignature::headers(CLOUD_PLATFORM_SECRET, json_encode(cloudRegistrationBody()))), 401, 'The request is not signed with the platform secret.'], +]); + +it('refuses a replayed signed request', function () { + $body = cloudRegistrationBody(); + $headers = PlatformSignature::headers(CLOUD_PLATFORM_SECRET, json_encode($body)); + + $this->withHeaders($headers)->postJson('/v1/instances/register/cloud', $body)->assertCreated(); + $this->withHeaders($headers)->postJson('/v1/instances/register/cloud', $body) + ->assertUnauthorized()->assertJsonPath('errors.0', 'This signed request was already used.'); + + expect(Instance::count())->toBe(1); +}); + +it('checks the signature before the body and requires the host', function () { + registerCloudInstance(['host' => null], null)->assertUnauthorized(); + registerCloudInstance(['host' => null])->assertUnprocessable(); +}); + +it('is idempotent: the same organization and host rotates the one cloud instance (D17)', function () { + $first = registerCloudInstance()->assertCreated(); + $instance = Instance::sole(); + + // A restart without the stored token (another replica, a cleared cache): matched by host. + $second = registerCloudInstance(['host' => 'HTTPS://API.fleetbase.test/', 'console_version' => '0.8.1'])->assertOk() + ->assertJsonPath('rotated', true) + ->assertJsonPath('instance.id', $instance->public_id); + + // With its current token: matched by token even from another host. + registerCloudInstance(['host' => 'https://api-2.fleetbase.test', 'previous_token' => $second->json('token')])->assertOk() + ->assertJsonPath('instance.id', $instance->public_id); + + expect(Instance::count())->toBe(1) + ->and($instance->fresh()->console_version)->toBe('0.8.0') + ->and(Instance::findByToken($first->json('token')))->toBeNull() + ->and(Instance::findByToken($second->json('token')))->toBeNull() + ->and(Event::where('kind', 'instance.reregistered')->orderBy('id')->get()->pluck('payload.matched_by')->all())->toBe(['host', 'token']); +}); + +it('keeps installs and seats when the cloud instance re-registers', function () { + $token = registerCloudInstance()->assertCreated()->json('token'); + [$extension, $release] = cloudFreeListing(); + $company = cloudCompany(); + + actingForCompany($company); + $this->withHeader('Authorization', 'Bearer '.$token) + ->postJson('/v1/instances/me/installs', ['extension' => $extension->public_id, 'version' => '1.0.0', 'status' => 'installed'])->assertCreated(); + + $rotated = registerCloudInstance()->assertOk()->json('token'); + + $this->withHeader('Authorization', 'Bearer '.$rotated)->getJson('/v1/instances/me/updates') + ->assertOk()->assertJsonPath('data.0.extension.id', $extension->public_id); + expect(LicenceActivation::whereNull('deactivated_at')->count())->toBe(1); +}); + +it('creates a separate instance for another host, as another cloud deployment', function () { + registerCloudInstance()->assertCreated(); + registerCloudInstance(['host' => 'https://api.staging.fleetbase.test'])->assertCreated()->assertJsonPath('rotated', false); + + expect(Instance::count())->toBe(2) + ->and(Instance::all()->every(fn (Instance $i) => $i->isPlatform()))->toBeTrue(); +}); + +it('adopts the placeholder the importer left, whatever its host', function () { + $placeholder = cloudPlatformInstance(['host' => null, 'token_hash' => null, 'meta' => ['platform' => true, 'source' => 'import']]); + + registerCloudInstance()->assertOk() + ->assertJsonPath('rotated', true) + ->assertJsonPath('instance.id', $placeholder->public_id) + ->assertJsonPath('instance.host', 'https://api.fleetbase.test'); + + expect($placeholder->fresh()->meta)->toBe(['platform' => true, 'source' => 'import']) + ->and(Event::where('kind', 'instance.reregistered')->sole()->payload['matched_by'])->toBe('placeholder'); +}); + +it('never rotates a self-hosted instance or another organization through the cloud endpoint', function () { + $selfHosted = Instance::factory()->create(['host' => 'https://api.fleetbase.test']); + $token = $selfHosted->issueToken(); + + registerCloudInstance(['previous_token' => $token])->assertCreated(); + + expect(Instance::findByToken($token)?->is($selfHosted))->toBeTrue() + ->and(Instance::count())->toBe(2); +}); + +it('acts for the cloud company named in X-Registry-Tenant', function () { + $instance = cloudPlatformInstance(); + actingAsInstance($instance); + [$extension] = cloudFreeListing(); + $acme = cloudCompany('Acme'); + $licence = Licence::create(['extension_uuid' => $extension->uuid, 'organization_uuid' => $acme->uuid, 'kind' => Licence::KIND_PAID, 'max_activations' => 5, 'status' => Licence::STATUS_ACTIVE]); + Licence::create(['extension_uuid' => $extension->uuid, 'organization_uuid' => $instance->organization_uuid, 'kind' => Licence::KIND_FREE, 'status' => Licence::STATUS_ACTIVE]); + + actingForCompany($acme); + $this->getJson('/v1/instances/me')->assertOk() + ->assertJsonPath('instance.organization.slug', 'fleetbase') + ->assertJsonPath('tenant.id', $acme->public_id) + ->assertJsonCount(1, 'licences') + ->assertJsonPath('licences.0.id', $licence->public_id); +}); + +it('creates the organization of a cloud company seen for the first time', function () { + actingAsInstance(cloudPlatformInstance()); + $company = '7f0c2c3e-0d1b-4f43-9a7e-3b1f9d0c1a22'; + + actingForCompany($company, 'Globex Logística'); + $this->getJson('/v1/instances/me')->assertOk()->assertJsonPath('tenant.name', 'Globex Logística'); + $this->getJson('/v1/instances/me')->assertOk(); + + $organization = Organization::where('source_uuid', $company)->sole(); + expect($organization->slug)->toBe('globex-logistica') + ->and($organization->is_personal)->toBeFalse() + ->and($organization->meta)->toBe(['source' => 'cloud', 'company_uuid' => $company]); + + actingForCompany('not-a-uuid'); + $this->getJson('/v1/instances/me')->assertUnprocessable(); +}); + +it('ignores X-Registry-Tenant from any instance but the cloud one', function () { + $organization = Organization::factory()->create(); + actingAsInstance(Instance::factory()->create(['organization_uuid' => $organization->uuid, 'kind' => Instance::KIND_CLOUD])); + + actingForCompany(cloudCompany()); + $this->getJson('/v1/instances/me')->assertOk()->assertJsonMissingPath('tenant'); + expect(Organization::count())->toBe(2); +}); + +it('keeps one install and one seat per company on the cloud instance', function () { + $instance = cloudPlatformInstance(); + actingAsInstance($instance); + [$extension] = cloudFreeListing(); + [$acme, $globex] = [cloudCompany('Acme'), cloudCompany('Globex')]; + + foreach ([$acme, $globex] as $company) { + actingForCompany($company); + $this->postJson('/v1/instances/me/installs', ['extension' => $extension->public_id, 'version' => '1.0.0', 'status' => 'installing'])->assertCreated(); + $this->postJson('/v1/instances/me/installs', ['extension' => $extension->public_id, 'version' => '1.0.0', 'status' => 'installed'])->assertOk(); + } + + $installs = Install::where('instance_uuid', $instance->uuid)->where('status', Install::STATUS_INSTALLED)->get(); + expect($installs->pluck('organization_uuid')->sort()->values()->all())->toBe(collect([$acme->uuid, $globex->uuid])->sort()->values()->all()) + ->and(Licence::where('extension_uuid', $extension->uuid)->pluck('organization_uuid')->sort()->values()->all())->toBe(collect([$acme->uuid, $globex->uuid])->sort()->values()->all()) + ->and(LicenceActivation::where('instance_uuid', $instance->uuid)->whereNull('deactivated_at')->count())->toBe(2); + + // Acme uninstalls: Globex keeps its install and seat. + actingForCompany($acme); + $this->deleteJson('/v1/instances/me/installs/'.$extension->public_id)->assertOk()->assertJsonPath('deactivated', 1); + $this->getJson('/v1/instances/me/updates')->assertOk()->assertJsonCount(0, 'data'); + + actingForCompany($globex); + $this->getJson('/v1/instances/me/updates')->assertOk()->assertJsonCount(1, 'data'); + expect(LicenceActivation::whereNull('deactivated_at')->sole()->licence->organization_uuid)->toBe($globex->uuid); +}); + +it('refuses a paid install for a company without a licence', function () { + actingAsInstance(cloudPlatformInstance()); + $extension = Extension::factory()->published()->paid()->create(); + Release::factory()->create(['extension_uuid' => $extension->uuid, 'version' => '1.0.0', 'status' => Release::STATUS_PUBLISHED, 'published_at' => now()]); + $buyer = cloudCompany(); + Licence::create(['extension_uuid' => $extension->uuid, 'organization_uuid' => $buyer->uuid, 'kind' => Licence::KIND_PAID, 'max_activations' => 5, 'status' => Licence::STATUS_ACTIVE]); + + actingForCompany(cloudCompany()); + $this->postJson('/v1/instances/me/installs', ['extension' => $extension->public_id, 'version' => '1.0.0', 'status' => 'installing'])->assertStatus(403); + + actingForCompany($buyer); + $this->postJson('/v1/instances/me/installs', ['extension' => $extension->public_id, 'version' => '1.0.0', 'status' => 'installing'])->assertCreated(); +}); + +it('lists installs per company for the cloud instance only', function () { + $instance = cloudPlatformInstance(); + [$extension, $release] = cloudFreeListing(); + $extension->packages()->create(['kind' => 'npm', 'name' => '@acme/thing-engine']); + $acme = cloudCompany('Acme'); + Install::factory()->create(['instance_uuid' => $instance->uuid, 'organization_uuid' => $acme->uuid, 'extension_uuid' => $extension->uuid, 'release_uuid' => $release->uuid]); + Install::factory()->uninstalled()->create(['instance_uuid' => $instance->uuid, 'organization_uuid' => cloudCompany()->uuid, 'extension_uuid' => $extension->uuid]); + + actingAsInstance($instance); + $this->getJson('/v1/instances/me/tenant-installs')->assertOk() + ->assertJsonCount(1, 'data') + ->assertJsonPath('data.0.company_uuid', $acme->source_uuid) + ->assertJsonPath('data.0.extension.id', $extension->public_id) + ->assertJsonPath('data.0.packages', ['npm' => '@acme/thing-engine']) + ->assertJsonPath('data.0.release.version', '1.0.0') + ->assertJsonPath('meta.total', 1); + $this->getJson('/v1/instances/me/tenant-installs?per_page=0&page=2')->assertOk()->assertJsonCount(0, 'data')->assertJsonPath('meta.last_page', 1); + + actingAsInstance(Instance::factory()->create()); + $this->getJson('/v1/instances/me/tenant-installs')->assertForbidden(); +}); + +it('records installs for the organization of a self-hosted instance', function () { + $instance = Instance::factory()->create(); + actingAsInstance($instance); + [$extension] = cloudFreeListing(); + + actingForCompany(cloudCompany()); + $this->postJson('/v1/instances/me/installs', ['extension' => $extension->public_id, 'version' => '1.0.0', 'status' => 'installed'])->assertCreated(); + + expect(Install::sole()->organization_uuid)->toBe($instance->organization_uuid); +}); diff --git a/tests/Feature/MaintenanceTest.php b/tests/Feature/MaintenanceTest.php new file mode 100644 index 0000000..bbadb25 --- /dev/null +++ b/tests/Feature/MaintenanceTest.php @@ -0,0 +1,79 @@ +assertStatus(503) + ->assertJsonPath('errors.0', $message) + ->assertJsonPath('maintenance', true) + ->assertHeader('Retry-After', '3600'); +} + +it('freezes publishing and checkout while maintenance is on, and nothing else', function () { + [$account, , $listing] = publisher(); + $default = config('registry.maintenance.message'); + + $this->artisan('registry:maintenance', ['action' => 'on'])->expectsOutputToContain('Maintenance: ON')->assertSuccessful(); + + actingAsAccount($account, [Token::SCOPE_ALL]); + assertFrozen($this->postJson('/v1/publisher/extensions', ['name' => 'New thing']), $default); + assertFrozen($this->patchJson("/v1/publisher/extensions/{$listing->slug}", ['subtitle' => 'x']), $default); + assertFrozen($this->postJson("/v1/publisher/extensions/{$listing->slug}/releases", []), $default); + assertFrozen($this->postJson('/v1/purchases/checkout', ['extension' => $listing->slug]), $default); + $this->getJson('/v1/publisher/extensions')->assertOk(); + $this->getJson("/v1/publisher/extensions/{$listing->slug}")->assertOk(); + + actingAsAccount(staffAccount()); + $release = Release::factory()->create(['extension_uuid' => $listing->uuid, 'status' => Release::STATUS_IN_REVIEW]); + assertFrozen($this->postJson("/v1/admin/releases/{$release->public_id}/approve", []), $default); + assertFrozen($this->postJson("/v1/admin/releases/{$release->public_id}/publish", []), $default); + $this->getJson('/v1/admin/releases')->assertOk(); + + actingAsInstance(Instance::factory()->create()); + assertFrozen($this->postJson('/v1/instances/me/purchases/checkout', ['extension' => $listing->slug, 'return_url' => 'https://console.example/x']), $default); + $this->getJson('/v1/instances/me')->assertOk(); + + // Catalog, downloads and Stripe webhooks for checkouts already paid keep working. + $this->getJson('/v1/catalog/extensions')->assertOk(); + $this->call('POST', '/v1/webhooks/stripe', [], [], [], ['CONTENT_TYPE' => 'application/json', 'HTTP_ACCEPT' => 'application/json'], '{}') + ->assertJsonMissingPath('maintenance'); + + $this->artisan('registry:maintenance', ['action' => 'off'])->expectsOutputToContain('Maintenance: off')->assertSuccessful(); + actingAsAccount($account, [Token::SCOPE_ALL]); + $this->patchJson("/v1/publisher/extensions/{$listing->slug}", ['subtitle' => 'Back on'])->assertOk(); + expect(Extension::find($listing->uuid)->subtitle)->toBe('Back on'); +}); + +it('shows the message given when maintenance was turned on', function () { + [$account, , $listing] = publisher(); + + $this->artisan('registry:maintenance', ['action' => 'on', '--message' => 'Back at 14:00 UTC.']) + ->expectsOutputToContain('Message: Back at 14:00 UTC.')->assertSuccessful(); + + actingAsAccount($account, [Token::SCOPE_ALL]); + assertFrozen($this->patchJson("/v1/publisher/extensions/{$listing->slug}", ['subtitle' => 'x']), 'Back at 14:00 UTC.'); + expect(Maintenance::status())->toMatchArray(['active' => true, 'forced' => false, 'message' => 'Back at 14:00 UTC.']); +}); + +it('stays on while REGISTRY_MAINTENANCE forces it', function () { + config(['registry.maintenance.enabled' => true]); + [$account, , $listing] = publisher(); + + $this->artisan('registry:maintenance', ['action' => 'off']) + ->expectsOutputToContain('REGISTRY_MAINTENANCE=true still forces it on') + ->assertFailed(); + + actingAsAccount($account, [Token::SCOPE_ALL]); + $this->patchJson("/v1/publisher/extensions/{$listing->slug}", ['subtitle' => 'x'])->assertStatus(503); +}); + +it('rejects an unknown maintenance action', function () { + $this->artisan('registry:maintenance', ['action' => 'maybe'])->assertExitCode(2); + expect(Maintenance::active())->toBeFalse(); +}); diff --git a/tests/Feature/Packages/CloudPlatformGateTest.php b/tests/Feature/Packages/CloudPlatformGateTest.php new file mode 100644 index 0000000..97859b9 --- /dev/null +++ b/tests/Feature/Packages/CloudPlatformGateTest.php @@ -0,0 +1,53 @@ +extension->packages()->create(['kind' => ExtensionPackage::KIND_COMPOSER, 'name' => 'cloud/pro-api']); + makeRelease($npm, '1.0.0'); + makeRelease($composer, '1.0.0'); + + return $npm->extension; +} + +it('lets the cloud instance download a paid package while a company holds a seat on it', function () { + $extension = cloudPaidExtension(); + $instance = cloudPlatformInstance(); + actingAsInstance($instance); + + $this->get('/npm/@cloud%2fpro-engine')->assertForbidden(); + $this->get('/composer/dist/cloud/pro-api/1.0.0.tar')->assertForbidden(); + + $licence = Licence::create(['extension_uuid' => $extension->uuid, 'organization_uuid' => cloudCompany()->uuid, 'kind' => Licence::KIND_PAID, 'max_activations' => 5, 'status' => Licence::STATUS_ACTIVE]); + $activation = LicenceActivation::create(['licence_uuid' => $licence->uuid, 'instance_uuid' => $instance->uuid, 'activated_at' => now()]); + + $this->get('/npm/@cloud%2fpro-engine')->assertOk(); + $this->get('/composer/dist/cloud/pro-api/1.0.0.tar')->assertOk(); + + // A released seat or a revoked licence closes it again. + $activation->forceFill(['deactivated_at' => now()])->save(); + $this->get('/npm/@cloud%2fpro-engine')->assertForbidden(); + + $activation->forceFill(['deactivated_at' => null])->save(); + $licence->forceFill(['status' => Licence::STATUS_REVOKED])->save(); + $this->get('/npm/@cloud%2fpro-engine')->assertForbidden(); +}); + +it('keeps the seat rule to the cloud instance', function () { + $extension = cloudPaidExtension(); + $instance = Instance::factory()->create(['kind' => Instance::KIND_CLOUD]); + $licence = Licence::create(['extension_uuid' => $extension->uuid, 'organization_uuid' => cloudCompany()->uuid, 'kind' => Licence::KIND_PAID, 'max_activations' => 5, 'status' => Licence::STATUS_ACTIVE]); + LicenceActivation::create(['licence_uuid' => $licence->uuid, 'instance_uuid' => $instance->uuid, 'activated_at' => now()]); + + actingAsInstance($instance); + $this->get('/npm/@cloud%2fpro-engine')->assertForbidden(); +}); diff --git a/tests/Helpers/cloud.php b/tests/Helpers/cloud.php new file mode 100644 index 0000000..3f56280 --- /dev/null +++ b/tests/Helpers/cloud.php @@ -0,0 +1,62 @@ +create(array_merge([ + 'organization_uuid' => CloudPlatform::organization()->uuid, + 'name' => 'Fleetbase Cloud', + 'kind' => Instance::KIND_CLOUD, + 'host' => 'https://api.fleetbase.test', + 'meta' => ['platform' => true], + ], $attributes)); +} + +/** + * The organization of a cloud company, as the importer creates it. + */ +function cloudCompany(?string $name = null): Organization +{ + $organization = Organization::factory()->create(['name' => $name ?? 'Company '.Str::random(6)]); + $organization->forceFill(['source_uuid' => (string) Str::uuid()])->save(); + + return $organization; +} + +/** + * Send instance requests as the cloud instance acting for a company. + */ +function actingForCompany(Organization|string $company, ?string $name = null): void +{ + test()->withHeader(CloudPlatform::TENANT_HEADER, $company instanceof Organization ? $company->source_uuid : $company); + if ($name !== null) { + test()->withHeader(CloudPlatform::TENANT_NAME_HEADER, rawurlencode($name)); + } +} + +function cloudRegistrationBody(array $overrides = []): array +{ + return array_merge(['name' => 'Fleetbase Cloud', 'host' => 'https://api.fleetbase.test', 'console_version' => '0.8.0', 'api_version' => '1.6.80'], $overrides); +} + +/** + * `POST /v1/instances/register/cloud` signed with the platform secret. + */ +function registerCloudInstance(array $body = [], ?string $secret = CLOUD_PLATFORM_SECRET, array $headers = []): TestResponse +{ + $body = cloudRegistrationBody($body); + $signed = $secret !== null ? PlatformSignature::headers($secret, json_encode($body)) : []; + + return test()->withHeaders(array_merge($signed, $headers))->postJson('/v1/instances/register/cloud', $body); +} diff --git a/tests/Pest.php b/tests/Pest.php index 391eb9a..24efed3 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -18,6 +18,7 @@ require_once __DIR__.'/Helpers/stripe.php'; require_once __DIR__.'/Helpers/commerce.php'; require_once __DIR__.'/Helpers/analytics.php'; +require_once __DIR__.'/Helpers/cloud.php'; pest()->extend(TestCase::class) ->use(RefreshDatabase::class) From 4ab9435caacdae5bf6c96bcb3c644e96d1b2b196 Mon Sep 17 00:00:00 2001 From: "Ronald A. Richardson" Date: Sat, 24 Oct 2026 20:49:04 +0800 Subject: [PATCH 2/4] Importer: cloud installs on the cloud instance, registry:import --verify - Cloud installs and paid licences' seats attach to the one cloud instance (CloudPlatform), each install for its company's organization, instead of a synthetic instance per company. The instance is found by --cloud-host (default REGISTRY_CLOUD_HOST) or created as a tokenless placeholder that the cloud API's first registration rotates onto. Per-company instances from earlier runs are revoked and their seats move to it. - --verify compares source and target and exits 1 on drift: a dry run against the target must create or update nothing, every extension, release, purchase, licence and install has its row except the ones the importer skips, and a seeded sample of rows matches field by field, tarballs by SHA-256 of the source object, the stored copy and its record. - Re-runs stay "unchanged" on MySQL, which stores JSON with its keys reordered (found in the cutover rehearsal: every extension, release and purchase counted as updated). - mysql:// sources take ?unix_socket= and ?ssl_ca= (a TLS-only source database). - IMPORT_SOURCE_DRIVER=local reads the cloud objects from a folder (rehearsals). --- app/Console/Commands/ImportRegistry.php | 78 +++++- app/Import/ImportVerifier.php | 267 ++++++++++++++++++++ app/Import/RegistryImporter.php | 190 ++++++++++---- app/Import/SourceDatabase.php | 3 + config/filesystems.php | 10 +- tests/Feature/Import/RegistryImportTest.php | 159 +++++++++++- 6 files changed, 645 insertions(+), 62 deletions(-) create mode 100644 app/Import/ImportVerifier.php diff --git a/app/Console/Commands/ImportRegistry.php b/app/Console/Commands/ImportRegistry.php index c4a58b3..9c12f27 100644 --- a/app/Console/Commands/ImportRegistry.php +++ b/app/Console/Commands/ImportRegistry.php @@ -3,6 +3,7 @@ namespace App\Console\Commands; use App\Import\ImportReport; +use App\Import\ImportVerifier; use App\Import\RegistryImporter; use App\Import\SourceDatabase; use App\Models\Event; @@ -16,6 +17,10 @@ class ImportRegistry extends Command protected $signature = 'registry:import {--from= : mysql://user:pass@host:3306/db, sqlite:///path/to/file.sqlite, or a path to a .sql / .sql.gz dump} {--dry-run : Map everything and print counts and anomalies, then roll back; no files are copied} + {--verify : Compare source and target after an import (pending changes, counts, a sample of rows and tarballs); exits 1 on drift} + {--sample=25 : Rows per table compared field by field with --verify} + {--seed= : Sample seed for --verify (default: today\'s date)} + {--cloud-host= : Host the cloud API registers with; cloud installs attach to its instance (default: registry.cloud.host)} {--source-disk= : Filesystem disk holding the cloud `files` objects (default: registry.import.source_disk)} {--report= : Also write the full report, every anomaly included, as JSON to this path} {--examples=10 : Anomaly examples printed per type}'; @@ -31,6 +36,10 @@ public function handle(): int return self::INVALID; } + if ($this->option('verify')) { + return $this->verify(new SourceDatabase($from)); + } + $dryRun = (bool) $this->option('dry-run'); $sourceDisk = $this->option('source-disk') ?: config('registry.import.source_disk'); $targetDisk = config('registry.storage.disk'); @@ -41,7 +50,7 @@ public function handle(): int try { $connection = $source->open(); - $report = (new RegistryImporter($connection, $sourceDisk, $targetDisk, $dryRun))->run(); + $report = (new RegistryImporter($connection, $sourceDisk, $targetDisk, $dryRun, $this->cloudHost()))->run(); } catch (\Throwable $e) { $this->error('Import failed: '.$e->getMessage()); @@ -71,6 +80,73 @@ public function handle(): int return self::SUCCESS; } + /** + * `--verify`: exit 0 when the target matches the source, 1 on drift (see ImportVerifier). + */ + protected function verify(SourceDatabase $source): int + { + $this->line('Verifying '.$source->describe().' against this registry'); + + try { + $verifier = (new ImportVerifier( + $source->open(), + $this->option('source-disk') ?: config('registry.import.source_disk'), + config('registry.storage.disk'), + $this->cloudHost(), + max(0, (int) $this->option('sample')), + $this->option('seed') !== null ? (int) $this->option('seed') : null, + ))->run(); + } catch (\Throwable $e) { + $this->error('Verify failed: '.$e->getMessage()); + + return self::FAILURE; + } finally { + $source->close(); + } + + $this->newLine(); + $this->line('Pending changes (a dry run against this registry)'); + $this->line($verifier->pending === [] ? ' none: the import is current' : ''); + if ($verifier->pending !== []) { + $this->table(['Table', 'Would create', 'Would update'], collect($verifier->pending)->map(fn ($c, $t) => [$t, $c['created'], $c['updated']])->values()->all()); + } + + $this->line('Counts (target rows matched on source_uuid)'); + $this->table(['Table', 'Source', 'Target', 'Missing', 'Skipped by importer', 'Drift'], collect($verifier->counts)->map(fn ($c) => [ + $c['table'], $c['source'], $c['target'], $c['missing'], $c['skipped'], $c['drift'] ? 'yes' : 'no', + ])->all()); + + $this->line('Sample ('.collect($verifier->sampled)->map(fn ($n, $t) => "{$t} {$n}")->implode(', ').')'); + if ($verifier->mismatches === []) { + $this->line(' every sampled row matches'); + } else { + $this->table(['Table', 'Source uuid', 'Field', 'Source', 'Target'], collect($verifier->mismatches)->map(fn ($m) => [ + $m['table'], $m['uuid'], $m['field'], json_encode($m['source']), json_encode($m['target']), + ])->all()); + } + + $cloud = $verifier->cloudInstance; + $this->line('Cloud instance: '.($cloud ? "{$cloud->public_id} ({$cloud->host}, ".($cloud->token_hash ? 'registered' : 'waiting for the cloud API to register').')' : 'none')); + + $this->newLine(); + if ($verifier->drift()) { + $this->error('Drift between source and target.'); + + return self::FAILURE; + } + + $this->info('No drift.'); + + return self::SUCCESS; + } + + protected function cloudHost(): ?string + { + $host = $this->option('cloud-host') ?: config('registry.cloud.host'); + + return is_string($host) && trim($host) !== '' ? trim($host) : null; + } + protected function printReport(ImportReport $report, int $examples): void { $this->newLine(); diff --git a/app/Import/ImportVerifier.php b/app/Import/ImportVerifier.php new file mode 100644 index 0000000..6291884 --- /dev/null +++ b/app/Import/ImportVerifier.php @@ -0,0 +1,267 @@ + [target model, label] */ + public const MAPPINGS = [ + 'extensions' => ['registry_extensions', Extension::class], + 'releases' => ['registry_extension_bundles', Release::class], + 'purchases' => ['registry_extension_purchases', Purchase::class], + 'licences' => ['registry_extension_purchases', Licence::class], + 'installs' => ['registry_extension_installs', Install::class], + ]; + + /** @var list */ + public array $counts = []; + + /** @var array */ + public array $pending = []; + + /** @var list */ + public array $mismatches = []; + + /** @var array table => rows sampled */ + public array $sampled = []; + + public ?ImportReport $dryRun = null; + + public ?Instance $cloudInstance = null; + + public function __construct( + protected Connection $source, + protected string $sourceDisk, + protected string $targetDisk, + protected ?string $cloudHost = null, + protected int $sample = 25, + protected ?int $seed = null, + ) {} + + public function run(): static + { + $this->dryRun = (new RegistryImporter($this->source, $this->sourceDisk, $this->targetDisk, true, $this->cloudHost))->run(); + + foreach ($this->dryRun->counts as $table => $counts) { + $created = (int) ($counts[ImportReport::CREATED] ?? 0); + $updated = (int) ($counts[ImportReport::UPDATED] ?? 0); + if ($created + $updated > 0) { + $this->pending[$table] = ['created' => $created, 'updated' => $updated]; + } + } + + $this->cloudInstance = Instance::query()->where('kind', Instance::KIND_CLOUD)->where('status', Instance::STATUS_ACTIVE) + ->get()->first(fn (Instance $instance) => $instance->isPlatform()); + + foreach (self::MAPPINGS as $table => [$sourceTable, $model]) { + $this->compare($table, $sourceTable, $model); + } + + return $this; + } + + public function drift(): bool + { + return $this->pending !== [] + || $this->mismatches !== [] + || collect($this->counts)->contains('drift', true); + } + + /** + * @param class-string $model + */ + protected function compare(string $table, string $sourceTable, string $model): void + { + $rows = $this->sourceRows($sourceTable); + $targets = collect(); + foreach ($rows->keys()->chunk(500) as $chunk) { + $targets = $targets->merge($this->query($model)->whereIn('source_uuid', $chunk->all())->get()->keyBy('source_uuid')); + } + + $skipped = (int) ($this->dryRun->counts[$table][ImportReport::SKIPPED] ?? 0); + if ($table === 'licences') { + $skipped = (int) ($this->dryRun->counts['purchases'][ImportReport::SKIPPED] ?? 0); + } + + $missing = $rows->count() - $targets->count(); + $this->counts[] = [ + 'table' => $table, + 'source' => $rows->count(), + 'target' => $targets->count(), + 'missing' => $missing, + 'skipped' => $skipped, + 'drift' => $missing > $skipped, + ]; + + // Same rows for the same seed (default: today), so a rerun after a fix checks them again. + $sample = $targets->keys()->shuffle(crc32(($this->seed ?? date('Ymd')).':'.$table))->take(max(0, $this->sample)); + $this->sampled[$table] = $sample->count(); + + foreach ($sample as $uuid) { + $this->{'check'.ucfirst($table)}($rows->get($uuid), $targets->get($uuid)); + } + } + + protected function checkExtensions(object $row, Extension $extension): void + { + $hasBundles = $this->source->table('registry_extension_bundles')->where('extension_uuid', $row->uuid)->exists(); + $status = RegistryImporter::EXTENSION_STATUSES[$row->status ?? ''] ?? Extension::STATUS_DRAFT; + if (! $hasBundles && trim((string) ($row->package_name ?? '')) === '') { + $status = Extension::STATUS_DRAFT; + } + + $publisher = ($row->publisher_uuid ?? null) ?: ($row->company_uuid ?? null); + + $this->expect('extensions', $row->uuid, 'name', $row->name, $extension->name); + $this->expect('extensions', $row->uuid, 'status', $status, $extension->status); + $this->expect('extensions', $row->uuid, 'deleted', $this->filled($row->deleted_at ?? null), $extension->trashed()); + $this->expect('extensions', $row->uuid, 'publisher', $publisher, $extension->organization()->withTrashed()->value('source_uuid')); + } + + protected function checkReleases(object $bundle, Release $release): void + { + $meta = $this->json($bundle->meta ?? null) ?? []; + $version = ltrim((string) (data_get($meta['extension.json'] ?? null, 'version') ?: ($bundle->version ?? null) ?: data_get($meta['package.json'] ?? null, 'version') ?: ''), 'vV'); + + $this->expect('releases', $bundle->uuid, 'version', $version, $release->version); + $this->expect('releases', $bundle->uuid, 'extension', $bundle->extension_uuid, $release->extension()->withTrashed()->value('source_uuid')); + + if ($release->status === Release::STATUS_YANKED && filled($release->meta['import_yank_reason'] ?? null)) { + return; + } + + $this->expect('releases', $bundle->uuid, 'status', RegistryImporter::RELEASE_STATUSES[$bundle->status ?? ''] ?? Release::STATUS_UPLOADED, $release->status); + + $file = File::withTrashed()->find($release->meta['source_bundle_file_uuid'] ?? null); + $source = $this->source->table('files')->where('uuid', $bundle->bundle_uuid)->first(); + if (! $file || ! $source) { + $this->expect('releases', $bundle->uuid, 'tarball', 'copied', 'no file record'); + + return; + } + + $sourceHash = $this->hash($this->sourceDisk, $source->path); + $targetHash = $this->hash($file->disk, $file->path); + $this->expect('releases', $bundle->uuid, 'tarball sha256 (source → copy)', $sourceHash, $targetHash); + $this->expect('releases', $bundle->uuid, 'tarball sha256 (copy → record)', $targetHash, $file->checksum ? substr($file->checksum, 7) : null); + } + + protected function checkPurchases(object $row, Purchase $purchase): void + { + $buyer = filled($row->purchaser_uuid ?? null) && filled($row->purchaser_type ?? null) ? $row->purchaser_uuid : ($row->company_uuid ?? null); + + $this->expect('purchases', $row->uuid, 'extension', $row->extension_uuid, $purchase->extension()->withTrashed()->value('source_uuid')); + $this->expect('purchases', $row->uuid, 'buyer', $buyer, $purchase->buyer()->withTrashed()->value('source_uuid')); + $this->expect('purchases', $row->uuid, 'deleted', $this->filled($row->deleted_at ?? null), $purchase->trashed()); + if (($row->locked_price ?? null) !== null) { + $this->expect('purchases', $row->uuid, 'amount', (int) $row->locked_price, (int) $purchase->amount_total); + } + } + + protected function checkLicences(object $row, Licence $licence): void + { + $this->expect('licences', $row->uuid, 'kind', Licence::KIND_PAID, $licence->kind); + $this->expect('licences', $row->uuid, 'status', $this->filled($row->deleted_at ?? null) ? Licence::STATUS_REVOKED : Licence::STATUS_ACTIVE, $licence->status); + $this->expect('licences', $row->uuid, 'organization', $licence->purchase()->withTrashed()->first()?->buyer_organization_uuid, $licence->organization_uuid); + } + + protected function checkInstalls(object $row, Install $install): void + { + $this->expect('installs', $row->uuid, 'instance', $this->cloudInstance?->uuid ?? 'the cloud instance', $install->instance_uuid); + $this->expect('installs', $row->uuid, 'company', $row->company_uuid ?? null, $install->organization()->withTrashed()->value('source_uuid')); + $this->expect('installs', $row->uuid, 'extension', $row->extension_uuid, Extension::withTrashed()->where('uuid', $install->extension_uuid)->value('source_uuid')); + if ($this->filled($row->deleted_at ?? null)) { + $this->expect('installs', $row->uuid, 'status', Install::STATUS_UNINSTALLED, $install->status); + } + } + + protected function expect(string $table, string $uuid, string $field, mixed $source, mixed $target): void + { + if ($source !== $target) { + $this->mismatches[] = compact('table', 'uuid', 'field', 'source', 'target'); + } + } + + /** + * @return Collection + */ + protected function sourceRows(string $table): Collection + { + return $this->source->getSchemaBuilder()->hasTable($table) + ? $this->source->table($table)->orderBy('id')->get()->keyBy('uuid') + : collect(); + } + + /** + * @param class-string $model + */ + protected function query(string $model): Builder + { + return in_array(SoftDeletes::class, class_uses_recursive($model), true) ? $model::withTrashed() : $model::query(); + } + + protected function hash(string $disk, ?string $path): ?string + { + if (blank($path)) { + return null; + } + + try { + $stream = Storage::disk($disk)->readStream($path); + } catch (\Throwable) { + return null; + } + if (! is_resource($stream)) { + return null; + } + + $context = hash_init('sha256'); + hash_update_stream($context, $stream); + fclose($stream); + + return hash_final($context); + } + + protected function filled(mixed $date): bool + { + return $date !== null && $date !== '' && ! str_starts_with((string) $date, '0000-00-00'); + } + + protected function json(mixed $value): ?array + { + if (is_array($value)) { + return $value; + } + $decoded = is_string($value) ? json_decode($value, true) : null; + if (is_string($decoded)) { + $decoded = json_decode($decoded, true); + } + + return is_array($decoded) ? $decoded : null; + } +} diff --git a/app/Import/RegistryImporter.php b/app/Import/RegistryImporter.php index dbc268f..a31dc5d 100644 --- a/app/Import/RegistryImporter.php +++ b/app/Import/RegistryImporter.php @@ -13,12 +13,14 @@ use App\Models\Install; use App\Models\Instance; use App\Models\Licence; +use App\Models\LicenceActivation; use App\Models\Membership; use App\Models\Model; use App\Models\Organization; use App\Models\Price; use App\Models\Purchase; use App\Models\Release; +use App\Support\CloudPlatform; use App\Support\Compat; use App\Support\Semver; use Illuminate\Database\Connection; @@ -44,6 +46,27 @@ class RegistryImporter { public const DEFAULT_PAID_SEATS = 5; + /** `registry_extensions.status` → `extensions.status`; anything else imports as a draft. */ + public const EXTENSION_STATUSES = [ + 'pending' => Extension::STATUS_DRAFT, + 'awaiting_review' => Extension::STATUS_IN_REVIEW, + 'in_review' => Extension::STATUS_IN_REVIEW, + 'approved' => Extension::STATUS_APPROVED, + 'rejected' => Extension::STATUS_REJECTED, + 'published' => Extension::STATUS_PUBLISHED, + 'unpublished' => Extension::STATUS_UNPUBLISHED, + ]; + + /** `registry_extension_bundles.status` → `releases.status`; anything else imports as uploaded. */ + public const RELEASE_STATUSES = [ + 'pending' => Release::STATUS_UPLOADED, + 'awaiting_review' => Release::STATUS_IN_REVIEW, + 'in_review' => Release::STATUS_IN_REVIEW, + 'approved' => Release::STATUS_APPROVED, + 'published' => Release::STATUS_PUBLISHED, + 'rejected' => Release::STATUS_REJECTED, + ]; + protected ImportReport $report; /** @var Collection */ @@ -100,14 +123,19 @@ class RegistryImporter /** @var array source bundle uuid => release */ protected array $releaseMap = []; - /** @var array company uuid => synthetic instance */ - protected array $instances = []; + /** The cloud instance every cloud install and seat attaches to. */ + protected ?Instance $cloudInstance = null; + /** + * @param string|null $cloudHost host the cloud API registers with (`registry.cloud.host`), so + * its first registration rotates onto the imported instance + */ public function __construct( protected Connection $source, protected string $sourceDisk, protected string $targetDisk, protected bool $dryRun = false, + protected ?string $cloudHost = null, ) { $this->report = new ImportReport($dryRun); } @@ -592,15 +620,7 @@ protected function packageNames(object $row, Collection $bundles): array protected function mapExtensionStatus(object $row): string { - $status = match ($row->status ?? null) { - 'pending' => Extension::STATUS_DRAFT, - 'awaiting_review', 'in_review' => Extension::STATUS_IN_REVIEW, - 'approved' => Extension::STATUS_APPROVED, - 'rejected' => Extension::STATUS_REJECTED, - 'published' => Extension::STATUS_PUBLISHED, - 'unpublished' => Extension::STATUS_UNPUBLISHED, - default => null, - }; + $status = self::EXTENSION_STATUSES[$row->status ?? ''] ?? null; if ($status === null) { $this->report->anomaly('unknown_status', ['table' => 'registry_extensions', 'uuid' => $row->uuid, 'status' => $row->status ?? null, 'mapped_to' => Extension::STATUS_DRAFT]); @@ -831,14 +851,7 @@ protected function importRelease(object $bundle, array $meta, string $version, E protected function mapReleaseStatus(object $bundle): string { - $status = match ($bundle->status ?? null) { - 'pending' => Release::STATUS_UPLOADED, - 'awaiting_review', 'in_review' => Release::STATUS_IN_REVIEW, - 'approved' => Release::STATUS_APPROVED, - 'published' => Release::STATUS_PUBLISHED, - 'rejected' => Release::STATUS_REJECTED, - default => null, - }; + $status = self::RELEASE_STATUSES[$bundle->status ?? ''] ?? null; if ($status === null) { $this->report->anomaly('unknown_status', ['table' => 'registry_extension_bundles', 'uuid' => $bundle->uuid, 'status' => $bundle->status ?? null, 'mapped_to' => Release::STATUS_UPLOADED]); @@ -962,9 +975,10 @@ protected function purchaserKind(object $row): ?string // --------------------------------------------------------------------- /** - * One synthetic cloud instance per company; soft-deleted installs become - * `uninstalled` rows. Live installs claim a licence seat on the company's - * instance (the paid licence from the purchase, or an automatic free licence). + * Cloud installs attach to the one cloud instance (kind `cloud`, see `CloudPlatform`), one + * row per company: each install is for the company's organization, and soft-deleted installs + * become `uninstalled` rows. Live installs claim a seat on the cloud instance for the + * company's organization (its paid licence from the purchase, or an automatic free licence). */ protected function importInstalls(): void { @@ -972,13 +986,14 @@ protected function importInstalls(): void $deactivate = app(DeactivateLicence::class); $groups = $this->installs->groupBy(fn ($i) => ($i->company_uuid ?? '').'|'.($i->extension_uuid ?? '')); + $instance = $groups->isNotEmpty() ? $this->cloudInstance() : null; foreach ($groups as $rows) { $first = $rows->first(); $extension = $this->extensionMap[$first->extension_uuid ?? ''] ?? null; - $instance = $this->syntheticInstance($first->company_uuid ?? null); + $organization = $this->companyOrganizations[$first->company_uuid ?? ''] ?? null; - if (! $extension || ! $instance) { + if (! $extension || ! $organization) { foreach ($rows as $row) { $this->report->anomaly('missing_reference', [ 'table' => 'registry_extension_installs', 'uuid' => $row->uuid, @@ -991,6 +1006,7 @@ protected function importInstalls(): void continue; } + $instance->actAs($organization); $release = $this->installedRelease($first->extension_uuid, $extension); $live = $rows->filter(fn ($r) => ! $this->date($r->deleted_at ?? null)) ->sortByDesc(fn ($r) => [(string) ($r->created_at ?? ''), (int) ($r->id ?? 0)]); @@ -1008,6 +1024,7 @@ protected function importInstalls(): void $install = $this->save(Install::query()->where('source_uuid', $row->uuid)->first() ?? new Install, [ 'source_uuid' => $row->uuid, 'instance_uuid' => $instance->uuid, + 'organization_uuid' => $organization->uuid, 'extension_uuid' => $extension->uuid, 'release_uuid' => $release?->uuid, 'status' => $installed ? Install::STATUS_INSTALLED : Install::STATUS_UNINSTALLED, @@ -1033,7 +1050,7 @@ protected function importInstalls(): void continue; } - $licencesBefore = Licence::query()->where('extension_uuid', $extension->uuid)->where('organization_uuid', $instance->organization_uuid)->count(); + $licencesBefore = Licence::query()->where('extension_uuid', $extension->uuid)->where('organization_uuid', $organization->uuid)->count(); try { $activation = $activate($instance, $extension, $release); @@ -1043,7 +1060,7 @@ protected function importInstalls(): void continue; } - if (Licence::query()->where('extension_uuid', $extension->uuid)->where('organization_uuid', $instance->organization_uuid)->count() > $licencesBefore) { + if (Licence::query()->where('extension_uuid', $extension->uuid)->where('organization_uuid', $organization->uuid)->count() > $licencesBefore) { $this->report->count('licences', ImportReport::CREATED); } @@ -1054,34 +1071,81 @@ protected function importInstalls(): void $this->report->count('licence_activations', ImportReport::UNCHANGED); } } + + $instance?->actAs(null); } - protected function syntheticInstance(?string $companyUuid): ?Instance + /** + * The cloud instance: the one registered from `$cloudHost` (or the placeholder an earlier run + * created), else a placeholder in the cloud organization with that host and no token. The + * cloud API's first registration rotates onto it (D17), so imported installs and seats are + * already there when the new console starts. Instances from earlier runs that made one + * synthetic instance per company are retired into it. + */ + protected function cloudInstance(): Instance { - if ($companyUuid === null || $companyUuid === '') { - return null; + if ($this->cloudInstance) { + return $this->cloudInstance; } - if (isset($this->instances[$companyUuid])) { - return $this->instances[$companyUuid]; - } + $organization = CloudPlatform::organization(); + $instance = CloudPlatform::instance($organization, $this->cloudHost); - $organization = $this->companyOrganizations[$companyUuid] ?? null; - if (! $organization) { - return null; + if ($instance) { + $this->report->count('instances', ImportReport::UNCHANGED); + } else { + $instance = $this->save(new Instance, [ + 'organization_uuid' => $organization->uuid, + 'name' => 'Fleetbase Cloud', + 'host' => $this->cloudHost, + 'kind' => Instance::KIND_CLOUD, + 'status' => Instance::STATUS_ACTIVE, + 'meta' => ['platform' => true, 'source' => 'import'], + ], 'instances'); } - $company = $this->companies->get($companyUuid); - $instance = Instance::withTrashed()->where('source_uuid', $companyUuid)->where('kind', Instance::KIND_CLOUD)->first() ?? new Instance; + $this->retireSyntheticInstances($instance); - return $this->instances[$companyUuid] = $this->save($instance, [ - 'source_uuid' => $companyUuid, - 'organization_uuid' => $organization->uuid, - 'name' => ($company->name ?? $organization->name).' (Fleetbase Cloud)', - 'kind' => Instance::KIND_CLOUD, - 'status' => Instance::STATUS_ACTIVE, - 'meta' => array_filter(['synthetic' => true, 'source' => 'companies', 'company_public_id' => $company->public_id ?? null]), - ], 'instances'); + return $this->cloudInstance = $instance; + } + + /** + * Earlier importer runs created one synthetic cloud instance per company (`meta.synthetic`). + * Their seats move to the cloud instance (dropped where it already holds that licence) and + * they are revoked; their install rows move when this run updates them by `source_uuid`. + */ + protected function retireSyntheticInstances(Instance $cloud): void + { + $synthetic = Instance::query() + ->where('kind', Instance::KIND_CLOUD) + ->where('status', Instance::STATUS_ACTIVE) + ->whereKeyNot($cloud->getKey()) + ->get() + ->filter(fn (Instance $instance) => ($instance->meta['synthetic'] ?? false) === true); + + foreach ($synthetic as $instance) { + $held = LicenceActivation::query()->where('instance_uuid', $cloud->uuid)->whereNull('deactivated_at')->pluck('licence_uuid'); + + LicenceActivation::query() + ->where('instance_uuid', $instance->uuid) + ->whereNull('deactivated_at') + ->whereIn('licence_uuid', $held) + ->update(['deactivated_at' => now(), 'updated_at' => now()]); + + $moved = LicenceActivation::query() + ->where('instance_uuid', $instance->uuid) + ->whereNull('deactivated_at') + ->update(['instance_uuid' => $cloud->uuid, 'updated_at' => now()]); + + if ($moved > 0) { + $this->report->count('licence_activations', ImportReport::UPDATED, $moved); + } + + $this->save($instance, [ + 'status' => Instance::STATUS_REVOKED, + 'meta' => array_merge($instance->meta ?? [], ['retired_into' => $cloud->public_id]), + ], 'instances'); + } } /** @@ -1201,6 +1265,8 @@ protected function save(EloquentModel $model, array $attributes, string $table): */ protected function persist(EloquentModel $model, string $table, bool $created = false): EloquentModel { + $this->keepEquivalentJson($model); + $outcome = match (true) { $created || ! $model->exists => ImportReport::CREATED, $model->isDirty() => ImportReport::UPDATED, @@ -1216,6 +1282,40 @@ protected function persist(EloquentModel $model, string $table, bool $created = return $model; } + /** + * MySQL stores JSON with its object keys reordered, and Eloquent compares JSON attributes with + * key order, so a re-run would count every row with a JSON column as updated. Put back the + * stored value of any JSON attribute that only differs in key order. + */ + protected function keepEquivalentJson(EloquentModel $model): void + { + if (! $model->exists) { + return; + } + + foreach (array_keys($model->getDirty()) as $key) { + if ($model->hasCast($key, ['array', 'json', 'object', 'collection', 'encrypted:array', 'encrypted:json']) + && $this->sortedKeys($model->getAttribute($key)) === $this->sortedKeys($model->getOriginal($key))) { + $model->setAttribute($key, $model->getOriginal($key)); + } + } + } + + protected function sortedKeys(mixed $value): mixed + { + if (is_object($value)) { + $value = json_decode(json_encode($value), true); + } + if (! is_array($value)) { + return $value; + } + if (! array_is_list($value)) { + ksort($value); + } + + return array_map(fn ($item) => $this->sortedKeys($item), $value); + } + /** * A slug unique among the model's rows (including soft-deleted ones). * diff --git a/app/Import/SourceDatabase.php b/app/Import/SourceDatabase.php index 6cf622c..3401b56 100644 --- a/app/Import/SourceDatabase.php +++ b/app/Import/SourceDatabase.php @@ -94,6 +94,9 @@ protected function connectionConfig(): array 'password' => isset($parts['pass']) ? urldecode($parts['pass']) : '', 'charset' => $query['charset'] ?? 'utf8mb4', 'collation' => $query['collation'] ?? 'utf8mb4_unicode_ci', + // ?unix_socket=/path (a local rehearsal), ?ssl_ca=/path/to/rds-ca.pem (a TLS-only database) + 'unix_socket' => $query['unix_socket'] ?? '', + 'options' => isset($query['ssl_ca']) ? [\PDO::MYSQL_ATTR_SSL_CA => $query['ssl_ca']] : [], 'prefix' => '', 'strict' => false, ]; diff --git a/config/filesystems.php b/config/filesystems.php index fd94504..fa7c4b8 100644 --- a/config/filesystems.php +++ b/config/filesystems.php @@ -60,8 +60,14 @@ 'report' => false, ], - // Read-only view of the cloud API's file bucket, used by `registry:import`. - 'import_source' => [ + // Read-only view of the cloud API's file bucket, used by `registry:import`. A local folder + // with IMPORT_SOURCE_DRIVER=local (cutover rehearsals, scripts/rehearse-cutover.sh). + 'import_source' => env('IMPORT_SOURCE_DRIVER', 's3') === 'local' ? [ + 'driver' => 'local', + 'root' => env('IMPORT_SOURCE_ROOT', storage_path('app/import-source')), + 'throw' => false, + 'report' => false, + ] : [ 'driver' => 's3', 'key' => env('IMPORT_SOURCE_AWS_ACCESS_KEY_ID', env('AWS_ACCESS_KEY_ID')), 'secret' => env('IMPORT_SOURCE_AWS_SECRET_ACCESS_KEY', env('AWS_SECRET_ACCESS_KEY')), diff --git a/tests/Feature/Import/RegistryImportTest.php b/tests/Feature/Import/RegistryImportTest.php index 5632a8e..ed48c6b 100644 --- a/tests/Feature/Import/RegistryImportTest.php +++ b/tests/Feature/Import/RegistryImportTest.php @@ -18,6 +18,7 @@ use App\Models\Purchase; use App\Models\Release; use Database\Seeders\CategorySeeder; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Storage; const FIXTURE_DUMP = __DIR__.'/../../Fixtures/import/cloud-registry.sql'; @@ -104,7 +105,7 @@ function anomalyItems(array $report, string $type): array 'extension_packages' => ['created' => 5, 'skipped' => 1], 'releases' => ['created' => 4, 'skipped' => 2], 'purchases' => ['created' => 2], - 'instances' => ['created' => 3], + 'instances' => ['created' => 1], 'installs' => ['created' => 4], 'licence_activations' => ['created' => 2], ]); @@ -194,30 +195,36 @@ function anomalyItems(array $report, string $type): array ->and((int) $refunded->amount_total)->toBe(4900) ->and(Licence::where('purchase_uuid', $refunded->uuid)->value('status'))->toBe(Licence::STATUS_REVOKED); - // Installs: one synthetic cloud instance per company, deleted installs uninstalled. - $gammaInstance = Instance::where('source_uuid', 'c0000000-0000-4000-8000-000000000003')->sole(); - expect($gammaInstance->kind)->toBe(Instance::KIND_CLOUD) - ->and($gammaInstance->token_hash)->toBeNull() - ->and($gammaInstance->meta['synthetic'])->toBeTrue(); + // Installs: every cloud install on the one cloud instance, for its company's organization; + // deleted installs uninstalled. The instance waits for the cloud API to register (no token). + $cloud = Instance::sole(); + $beta = Organization::where('source_uuid', 'c0000000-0000-4000-8000-000000000002')->sole(); + $gamma = Organization::where('source_uuid', 'c0000000-0000-4000-8000-000000000003')->sole(); + expect($cloud->kind)->toBe(Instance::KIND_CLOUD) + ->and($cloud->isPlatform())->toBeTrue() + ->and($cloud->token_hash)->toBeNull() + ->and($cloud->organization->slug)->toBe('fleetbase') + ->and(Install::where('instance_uuid', '!=', $cloud->uuid)->exists())->toBeFalse(); $routeInstalls = Install::with('release')->where('extension_uuid', $route->uuid)->orderBy('started_at')->get(); expect($routeInstalls->pluck('status')->all())->toBe(['uninstalled', 'installed']) + ->and($routeInstalls->pluck('organization_uuid')->unique()->all())->toBe([$gamma->uuid]) ->and($routeInstalls[1]->release->version)->toBe('0.3.0'); - // Live installs hold a seat: the paid licence for Beta, an automatic free licence for Gamma. - $betaInstance = Instance::where('source_uuid', 'c0000000-0000-4000-8000-000000000002')->sole(); - expect(LicenceActivation::where('instance_uuid', $betaInstance->uuid)->whereNull('deactivated_at')->sole()->licence_uuid)->toBe($paid->licence->uuid); + // Live installs hold a seat on the cloud instance: Beta's paid licence, Gamma's automatic free licence. + $betaSeat = LicenceActivation::whereNull('deactivated_at')->whereHas('licence', fn ($q) => $q->where('organization_uuid', $beta->uuid))->sole(); + expect($betaSeat->licence_uuid)->toBe($paid->licence->uuid) + ->and($betaSeat->instance_uuid)->toBe($cloud->uuid); $free = Licence::where('extension_uuid', $route->uuid)->sole(); expect($free->kind)->toBe(Licence::KIND_FREE) ->and($free->max_activations)->toBeNull() - ->and($free->organization_uuid)->toBe($gammaInstance->organization_uuid) - ->and($free->activeActivations()->sole()->instance_uuid)->toBe($gammaInstance->uuid); + ->and($free->organization_uuid)->toBe($gamma->uuid) + ->and($free->activeActivations()->sole()->instance_uuid)->toBe($cloud->uuid); // Acme installed its own paid extension without a purchase: install kept, no seat. - $acmeInstance = Instance::where('source_uuid', 'c0000000-0000-4000-8000-000000000001')->sole(); - expect(Install::where('instance_uuid', $acmeInstance->uuid)->value('status'))->toBe('installed') - ->and(LicenceActivation::where('instance_uuid', $acmeInstance->uuid)->count())->toBe(0); + expect(Install::where('organization_uuid', $acme->uuid)->value('status'))->toBe('installed') + ->and(LicenceActivation::whereHas('licence', fn ($q) => $q->where('organization_uuid', $acme->uuid))->count())->toBe(0); expect(Event::where('kind', 'registry.imported')->sole()->payload['counts']['extensions']['created'])->toBe(5); @@ -291,6 +298,36 @@ function anomalyItems(array $report, string $type): array expect(Release::where('version', '0.3.0')->sole()->yanked_at->toDateTimeString())->toBe($yankedAt); }); +it('re-runs as unchanged when the database reorders JSON keys, as MySQL does', function () { + runImport(); + + // MySQL returns JSON objects with their keys sorted by length, then bytes. + $reorder = function (?string $json) use (&$reorder) { + $sort = function ($value) use (&$sort) { + if (! is_array($value)) { + return $value; + } + if (! array_is_list($value)) { + uksort($value, fn ($a, $b) => [strlen($a), $a] <=> [strlen($b), $b]); + } + + return array_map($sort, $value); + }; + + return $json === null ? null : json_encode($sort(json_decode($json, true))); + }; + foreach (['extensions' => ['tags', 'meta'], 'releases' => ['manifest', 'compat', 'meta'], 'purchases' => ['price_snapshot', 'meta'], 'instances' => ['meta']] as $table => $columns) { + foreach (DB::table($table)->get() as $row) { + DB::table($table)->where('id', $row->id)->update(collect($columns)->mapWithKeys(fn ($c) => [$c => $reorder($row->{$c})])->all()); + } + } + + $second = runImport(); + foreach ($second['counts'] as $table => $counts) { + expect($counts['updated'] ?? 0)->toBe(0, "{$table} updated rows on the second run"); + } +}); + it('picks up a tarball that becomes retrievable on a later run', function () { runImport(); expect(Release::where('version', '0.3.0')->value('status'))->toBe(Release::STATUS_YANKED); @@ -340,8 +377,102 @@ function anomalyItems(array $report, string $type): array expect($report['counts']['extensions']['created'])->toBe(5); }); +it('reads socket and TLS settings from a mysql:// source', function () { + $source = new class('mysql://importer:p%40ss@db.internal:3307/fleetbase?unix_socket=/tmp/my.sock&ssl_ca=/etc/ssl/rds.pem') extends SourceDatabase + { + public function config(): array + { + return $this->connectionConfig(); + } + }; + + expect($source->config())->toMatchArray([ + 'driver' => 'mysql', 'host' => 'db.internal', 'port' => 3307, 'database' => 'fleetbase', + 'username' => 'importer', 'password' => 'p@ss', 'unix_socket' => '/tmp/my.sock', + 'options' => [PDO::MYSQL_ATTR_SSL_CA => '/etc/ssl/rds.pem'], + ])->and($source->describe())->toBe('mysql://importer:***@db.internal:3307/fleetbase?unix_socket=/tmp/my.sock&ssl_ca=/etc/ssl/rds.pem'); +}); + it('fails cleanly without a source or with an unreadable one', function () { $this->artisan('registry:import')->expectsOutputToContain('--from is required')->assertExitCode(2); $this->artisan('registry:import', ['--from' => '/nope/missing.sql'])->expectsOutputToContain('SQL dump not found')->assertFailed(); $this->artisan('registry:import', ['--from' => 'mysql://user@/'])->expectsOutputToContain('Expected a DSN like')->assertFailed(); }); + +it('attaches cloud installs to the instance of the cloud host, which the cloud API then registers onto', function () { + config(['registry.cloud.platform_secret' => CLOUD_PLATFORM_SECRET]); + + runImport(['--cloud-host' => 'api.fleetbase.test']); + $cloud = Instance::sole(); + expect($cloud->host)->toBe('api.fleetbase.test'); + + $token = registerCloudInstance(['host' => 'https://api.fleetbase.test'])->assertOk() + ->assertJsonPath('rotated', true) + ->assertJsonPath('instance.id', $cloud->public_id) + ->json('token'); + + // Beta's console sees its purchase, with the seat on this instance. + $beta = Organization::where('source_uuid', 'c0000000-0000-4000-8000-000000000002')->sole(); + $this->withHeaders(['Authorization' => 'Bearer '.$token, 'X-Registry-Tenant' => $beta->source_uuid]) + ->getJson('/v1/instances/me')->assertOk() + ->assertJsonPath('tenant.id', $beta->public_id) + ->assertJsonPath('licences.0.extension.slug', 'fleet-insights') + ->assertJsonPath('licences.0.activated_here', true); + + // A later import run finds the registered instance again. + $second = runImport(['--cloud-host' => 'api.fleetbase.test']); + expect(Instance::count())->toBe(1) + ->and($second['counts']['instances'])->toBe(['unchanged' => 1]); +}); + +it('retires the per-company instances of earlier import runs into the cloud instance', function () { + $gamma = Organization::factory()->create(); + $gamma->forceFill(['source_uuid' => 'c0000000-0000-4000-8000-000000000003'])->save(); + $old = Instance::factory()->create(['organization_uuid' => $gamma->uuid, 'kind' => Instance::KIND_CLOUD, 'token_hash' => null, 'meta' => ['synthetic' => true, 'source' => 'companies']]); + $extension = Extension::factory()->create(); + $licence = Licence::create(['extension_uuid' => $extension->uuid, 'organization_uuid' => $gamma->uuid, 'kind' => Licence::KIND_FREE, 'status' => Licence::STATUS_ACTIVE]); + $seat = LicenceActivation::create(['licence_uuid' => $licence->uuid, 'instance_uuid' => $old->uuid, 'activated_at' => now()]); + + runImport(); + + $cloud = Instance::get()->first(fn (Instance $i) => $i->isPlatform()); + expect($old->fresh()->status)->toBe(Instance::STATUS_REVOKED) + ->and($old->fresh()->meta['retired_into'])->toBe($cloud->public_id) + ->and($seat->fresh()->instance_uuid)->toBe($cloud->uuid) + ->and($seat->fresh()->deactivated_at)->toBeNull(); +}); + +it('verifies a clean import: no pending changes, no missing rows, sampled rows and tarballs match', function () { + runImport(); + + $this->artisan('registry:import', ['--from' => FIXTURE_DUMP, '--verify' => true, '--seed' => 7]) + ->expectsOutputToContain('none: the import is current') + ->expectsOutputToContain('every sampled row matches') + ->expectsOutputToContain('waiting for the cloud API to register') + ->expectsOutputToContain('No drift.') + ->assertSuccessful(); +}); + +it('exits non-zero when the target drifted from the source', function (Closure $tamper, string $expected) { + runImport(); + $tamper(); + + $this->artisan('registry:import', ['--from' => FIXTURE_DUMP, '--verify' => true, '--sample' => 100]) + ->expectsOutputToContain($expected) + ->expectsOutputToContain('Drift between source and target.') + ->assertFailed(); +})->with([ + 'a licence lost' => [fn () => Licence::whereNotNull('source_uuid')->where('kind', 'paid')->where('status', 'active')->first()->forceDelete(), 'Would create'], + 'an install moved off the cloud instance' => [fn () => Install::where('status', 'installed')->first()->forceFill(['instance_uuid' => Instance::factory()->create()->uuid])->save(), 'instance'], + 'a release renamed' => [fn () => Release::where('version', '1.1.0')->update(['version' => '1.1.1']), 'version'], + 'a tarball changed in storage' => [fn () => Storage::disk('registry')->put(File::where('type', File::TYPE_SOURCE_BUNDLE)->first()->path, 'tampered'), 'tarball sha256'], + 'an extension deleted' => [fn () => Extension::where('slug', 'route-planner')->forceDelete(), 'yes'], +]); + +it('verifies with nothing imported yet as drift', function () { + $this->artisan('registry:import', ['--from' => FIXTURE_DUMP, '--verify' => true]) + ->expectsOutputToContain('Drift between source and target.') + ->assertFailed(); + + expect(Extension::withTrashed()->count())->toBe(0); +}); From 5302bf8d3af0436a40e89347f5dba700b674468d Mon Sep 17 00:00:00 2001 From: "Ronald A. Richardson" Date: Sat, 24 Oct 2026 22:22:13 +0800 Subject: [PATCH 3/4] Importer: dumps from a filesystem disk, a 512 MB memory limit with the peak printed - --from=disk:path downloads a dump from a configured disk (the cloud files bucket the importer already reads), so the production service needs no network path to the cloud database. - The source tables are held in memory: the rehearsal at three times today's scale needed 134 MB, over PHP's default 128 MB. --memory-limit (default 512M, never lowered) and the peak in the output and the JSON report. --- app/Console/Commands/ImportRegistry.php | 27 ++++++++++++-- app/Import/SourceDatabase.php | 39 +++++++++++++++++++-- tests/Feature/Import/RegistryImportTest.php | 10 ++++++ 3 files changed, 70 insertions(+), 6 deletions(-) diff --git a/app/Console/Commands/ImportRegistry.php b/app/Console/Commands/ImportRegistry.php index 9c12f27..e760813 100644 --- a/app/Console/Commands/ImportRegistry.php +++ b/app/Console/Commands/ImportRegistry.php @@ -15,7 +15,7 @@ class ImportRegistry extends Command { protected $signature = 'registry:import - {--from= : mysql://user:pass@host:3306/db, sqlite:///path/to/file.sqlite, or a path to a .sql / .sql.gz dump} + {--from= : mysql://user:pass@host:3306/db, sqlite:///path/to/file.sqlite, a path to a .sql / .sql.gz dump, or disk:path for a dump on a filesystem disk (import_source:cutover/cloud.sql.gz)} {--dry-run : Map everything and print counts and anomalies, then roll back; no files are copied} {--verify : Compare source and target after an import (pending changes, counts, a sample of rows and tarballs); exits 1 on drift} {--sample=25 : Rows per table compared field by field with --verify} @@ -23,12 +23,15 @@ class ImportRegistry extends Command {--cloud-host= : Host the cloud API registers with; cloud installs attach to its instance (default: registry.cloud.host)} {--source-disk= : Filesystem disk holding the cloud `files` objects (default: registry.import.source_disk)} {--report= : Also write the full report, every anomaly included, as JSON to this path} - {--examples=10 : Anomaly examples printed per type}'; + {--examples=10 : Anomaly examples printed per type} + {--memory-limit=512M : PHP memory limit for the run (the source tables are held in memory); never lowered}'; protected $description = 'Import extensions, releases, accounts, purchases and installs from the cloud registry-bridge tables'; public function handle(): int { + $this->raiseMemoryLimit((string) $this->option('memory-limit')); + $from = (string) $this->option('from'); if ($from === '') { $this->error('--from is required: a mysql:// or sqlite:// DSN, or a path to a SQL dump.'); @@ -69,8 +72,11 @@ public function handle(): int ], '127.0.0.1'); } + $peak = (int) round(memory_get_peak_usage(true) / 1048576); + $this->line("Peak memory: {$peak} MB of ".ini_get('memory_limit')); + if ($path = $this->option('report')) { - file_put_contents($path, json_encode($report->toArray(), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)); + file_put_contents($path, json_encode([...$report->toArray(), 'peak_memory_mb' => $peak], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)); $this->line("Full report written to {$path}"); } @@ -140,6 +146,21 @@ protected function verify(SourceDatabase $source): int return self::SUCCESS; } + /** + * Raise PHP's memory limit to `$limit` (e.g. 512M); a higher or unlimited one is kept. + */ + protected function raiseMemoryLimit(string $limit): void + { + $bytes = fn (string $value) => (int) $value * match (strtoupper(substr(trim($value), -1))) { + 'G' => 1024 ** 3, 'M' => 1024 ** 2, 'K' => 1024, default => 1, + }; + + $current = (string) ini_get('memory_limit'); + if ($limit !== '' && $current !== '-1' && $bytes($limit) > $bytes($current)) { + ini_set('memory_limit', $limit); + } + } + protected function cloudHost(): ?string { $host = $this->option('cloud-host') ?: config('registry.cloud.host'); diff --git a/app/Import/SourceDatabase.php b/app/Import/SourceDatabase.php index 3401b56..ec03ab1 100644 --- a/app/Import/SourceDatabase.php +++ b/app/Import/SourceDatabase.php @@ -5,6 +5,7 @@ use Illuminate\Database\Connection; use Illuminate\Support\Facades\Config; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Storage; use RuntimeException; /** @@ -14,6 +15,8 @@ * mysql://user:pass@host:3306/fleetbase (also mariadb://) * sqlite:///absolute/path/snapshot.sqlite * /path/to/dump.sql or dump.sql.gz (loaded into a temporary SQLite file) + * import_source:cutover/cloud.sql.gz (a dump on a filesystem disk, e.g. the cloud bucket the + * importer reads; downloaded, then loaded as above) */ class SourceDatabase { @@ -28,6 +31,8 @@ class SourceDatabase protected ?string $temporaryFile = null; + protected ?string $downloadedDump = null; + /** @var array rows loaded per table, when the source is a dump */ public array $dumpRowCounts = []; @@ -56,10 +61,38 @@ public function close(): void { DB::purge(self::CONNECTION); - if ($this->temporaryFile && is_file($this->temporaryFile)) { - @unlink($this->temporaryFile); + foreach ([$this->temporaryFile, $this->downloadedDump] as $file) { + if ($file && is_file($file)) { + @unlink($file); + } } $this->temporaryFile = null; + $this->downloadedDump = null; + } + + /** + * `disk:path` names a dump on a configured filesystem disk: download it to a temporary file and + * return that file's path. Null for anything else. + */ + protected function diskDump(): ?string + { + if (! preg_match('#^([A-Za-z0-9_-]+):(?!//)(.+)$#', $this->from, $m) || ! is_array(config('filesystems.disks.'.$m[1]))) { + return null; + } + + [, $disk, $path] = $m; + $stream = Storage::disk($disk)->readStream($path); + if (! is_resource($stream)) { + throw new RuntimeException("SQL dump not found on the {$disk} disk: {$path}"); + } + + $this->downloadedDump = tempnam(sys_get_temp_dir(), 'registry-dump-').(str_ends_with($path, '.gz') ? '.sql.gz' : '.sql'); + $out = fopen($this->downloadedDump, 'wb'); + stream_copy_to_stream($stream, $out); + fclose($out); + fclose($stream); + + return $this->downloadedDump; } protected function isDsn(): bool @@ -103,7 +136,7 @@ protected function connectionConfig(): array } $this->temporaryFile = tempnam(sys_get_temp_dir(), 'registry-import-').'.sqlite'; - $this->dumpRowCounts = (new SqlDumpLoader(self::TABLES))->load($this->from, $this->temporaryFile); + $this->dumpRowCounts = (new SqlDumpLoader(self::TABLES))->load($this->diskDump() ?? $this->from, $this->temporaryFile); return ['driver' => 'sqlite', 'database' => $this->temporaryFile, 'prefix' => '', 'foreign_key_constraints' => false]; } diff --git a/tests/Feature/Import/RegistryImportTest.php b/tests/Feature/Import/RegistryImportTest.php index ed48c6b..e7562a6 100644 --- a/tests/Feature/Import/RegistryImportTest.php +++ b/tests/Feature/Import/RegistryImportTest.php @@ -377,6 +377,16 @@ function anomalyItems(array $report, string $type): array expect($report['counts']['extensions']['created'])->toBe(5); }); +it('reads a dump from a filesystem disk', function () { + $this->source->put('registry-cutover/cloud-registry.sql', file_get_contents(FIXTURE_DUMP)); + + $report = runImport(['--from' => 'import_source:registry-cutover/cloud-registry.sql', '--dry-run' => true]); + + expect($report['counts']['extensions']['created'])->toBe(5); + $this->artisan('registry:import', ['--from' => 'import_source:registry-cutover/missing.sql', '--dry-run' => true]) + ->expectsOutputToContain('SQL dump not found on the import_source disk')->assertFailed(); +}); + it('reads socket and TLS settings from a mysql:// source', function () { $source = new class('mysql://importer:p%40ss@db.internal:3307/fleetbase?unix_socket=/tmp/my.sock&ssl_ca=/etc/ssl/rds.pem') extends SourceDatabase { From 9746fb3928e49cb43b02fea66992c9e098dce627 Mon Sep 17 00:00:00 2001 From: "Ronald A. Richardson" Date: Sat, 24 Oct 2026 22:22:13 +0800 Subject: [PATCH 4/4] Cutover runbook and rehearsal - docs/cutover.md: the five steps (freeze, import, infra, deploy, unfreeze) with exact commands, preparation, go/no-go checklist, smoke tests, rollback per step, decisions it depends on, and the timings from a local rehearsal. - scripts/fake-cloud-dump.php: a seeded fake of the cloud registry tables and their objects, at any scale, with every anomaly class the importer reports. No real data. - scripts/rehearse-cutover.sh: times the data path end to end against it (two dry runs, import, verify, second import, verify, dump, dry run from the dump) on MySQL and S3 or local folders. - scripts/cloud-registry-dump.sh: dumps exactly the cloud rows the importer reads, in one transaction, for --from=import_source:. - README: the cloud instance API, maintenance, and the new import options. --- .gitignore | 1 + README.md | 28 +- docs/cutover.md | 483 +++++++++++++++++++++++++++++++++ scripts/cloud-registry-dump.sh | 32 +++ scripts/fake-cloud-dump.php | 275 +++++++++++++++++++ scripts/rehearse-cutover.sh | 110 ++++++++ 6 files changed, 927 insertions(+), 2 deletions(-) create mode 100644 docs/cutover.md create mode 100755 scripts/cloud-registry-dump.sh create mode 100644 scripts/fake-cloud-dump.php create mode 100755 scripts/rehearse-cutover.sh diff --git a/.gitignore b/.gitignore index e5987c4..1d74c9b 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ Thumbs.db /playwright-report /test-results /storage/e2e +/storage/rehearsal diff --git a/README.md b/README.md index 7927c68..8bde3dc 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Phase 1, Workstreams A to C, and Phase 2 Workstream E: | Website (Workstream F): Inertia (React) catalog, account, Studio and review console, light and dark | done | | Commerce (Phase 4, K to M): Stripe Connect Express, prices mirrored to Stripe, Checkout from instances and the website, webhooks, payouts and billing APIs and pages, staff finance and refunds | done | | Publisher analytics: listing and organization stats APIs with CSV, Studio Insights | done | +| Cloud cutover (Phase 5, O): the cloud instance (signed registration, per-company tenancy), cutover freeze, importer on one cloud instance with `--verify`, [runbook](docs/cutover.md) and rehearsal scripts | done | | CLI v2 (G), staff finance view | later workstreams | ## Stack @@ -422,6 +423,26 @@ disables a token. | `GET /v1/instances/me/updates?channel=` | Every extension installed on the instance: the installed release, the newest compatible published release, `update_available`. `channel=beta` includes beta releases | | `POST /v1/instances/me/installs` | Body: `extension` (slug, `ext_` id or an npm/Composer package name), `version` or `release` (`rel_` id), `status` (`installing`, `installed`, `failed`), optional `error`, `installed_by`. Returns `{install, activation}` | | `DELETE /v1/instances/me/installs/{extension}` | Reports an uninstall (scoped npm names work in the path). Returns `{install, deactivated}` | +| `POST /v1/instances/register/cloud` | Fleetbase Cloud only. No bearer token and no code: signed with the platform secret (below). Body: `host` (required), optional `name`, `console_version`, `api_version`, `previous_token`. Same response as `register` | +| `GET /v1/instances/me/tenant-installs?page=&per_page=` | Cloud instance only: every installed extension per cloud company (`company_uuid`, `extension`, `packages`, `release`), 500 a page | + +**Fleetbase Cloud.** The cloud console's API registers one instance per deployment (kind `cloud`, +`meta.platform`) in the organization `REGISTRY_CLOUD_ORGANIZATION`, on boot and without a code: +`X-Registry-Timestamp` (unix seconds, within 5 minutes), `X-Registry-Nonce` (16 to 64 characters, +single use) and `X-Registry-Signature: v1=`. A missing secret answers 403, a bad, expired or replayed +signature 401. Re-registration rotates the same instance by `previous_token`, else by host (D17), +else it adopts the tokenless instance `registry:import` prepared. That instance serves every cloud +company: `X-Registry-Tenant: ` on any instance route makes it act for that company's +organization (licences in `me`, install rows and seats, checkouts and hand-offs); an unknown company +gets an organization named from `X-Registry-Tenant-Name`. Other instances never act for anyone. The +package gate lets the cloud instance download a paid package while one of its companies holds an +active seat on it. + +**Maintenance.** `php artisan registry:maintenance on [--message=]` (or `REGISTRY_MAINTENANCE=true`) +answers 503 `{errors: [message], maintenance: true}` with `Retry-After` on publishing (publisher +writes, review approve and publish) and on both checkouts, for the cutover window +([docs/cutover.md](docs/cutover.md)). Catalog, packages, installs and Stripe webhooks keep working. Re-registration: @@ -704,11 +725,14 @@ php artisan registry:import --from=mysql://reader:secret@cloud-replica:3306/flee | Option | Meaning | | --- | --- | -| `--from` | `mysql://user:pass@host:3306/db` (or `mariadb://`), `sqlite:///path/file.sqlite`, or a path to a `.sql` / `.sql.gz` dump. Dumps are loaded into a temporary SQLite file without a MySQL server; mysqldump's extended inserts and one-tuple-per-line dumps both work, and statements for other tables are skipped | +| `--from` | `mysql://user:pass@host:3306/db` (or `mariadb://`; `?unix_socket=` and `?ssl_ca=` accepted), `sqlite:///path/file.sqlite`, a path to a `.sql` / `.sql.gz` dump, or `disk:path` for a dump on a filesystem disk (`import_source:registry-cutover/cloud.sql.gz`). Dumps are loaded into a temporary SQLite file without a MySQL server; mysqldump's extended inserts and one-tuple-per-line dumps both work, and statements for other tables are skipped. `scripts/cloud-registry-dump.sh` dumps exactly the rows the importer reads | | `--dry-run` | Runs the whole mapping in a transaction that is rolled back, so the counts and anomalies are exactly what a real run would produce | | `--source-disk` | Filesystem disk holding the cloud `files` objects. Default `REGISTRY_IMPORT_SOURCE_DISK` (`import_source`, an S3 disk configured by `IMPORT_SOURCE_AWS_*`) | | `--report` | Also writes the full report, every anomaly included, as JSON | | `--examples` | Anomaly examples printed per type (default 10) | +| `--memory-limit` | PHP memory limit for the run (default `512M`, never lowered; the source tables are held in memory). The peak is printed and kept in the report | +| `--cloud-host` | Host the cloud API registers with (default `REGISTRY_CLOUD_HOST`); cloud installs attach to that host's cloud instance | +| `--verify` | Compares source and target instead of importing; exits 1 on drift. A dry run against the target must create or update nothing; every extension, release, purchase, licence and install has its row except the ones the importer skips; a seeded sample of rows (`--sample`, default 25; `--seed`, default today) matches field by field, tarballs by SHA-256 of the source object, the copy and its record | What it writes: @@ -721,7 +745,7 @@ What it writes: | `registry_extensions` | `extensions`, `prices`, `extension_packages` | Package names from the current bundle's `package.json` and `composer.json`, then older bundles, then the Phase 0 `package_name` column; listings with no bundle and no package name become drafts; `self_managed` maps to hosting `self_hosted`; icon and screenshots copied | | `registry_extension_bundles` | `releases` | Version from `extension.json`; `bundle_number` kept in `manifest`; status pending → uploaded, awaiting_review → in_review, approved, published, rejected; `compat` from `extension.json` when present. The bundle archive is copied to `tarballs/imported/…` as a `source_bundle` file with a SHA-256 checksum; a bundle whose file is missing imports as `yanked` | | `registry_extension_purchases` | `purchases`, `licences` | One paid licence per purchase with 5 seats; a soft-deleted purchase revokes its licence | -| `registry_extension_installs` | `instances`, `installs`, `licence_activations` | One synthetic cloud instance per company (no token); soft-deleted installs become `uninstalled`; live installs claim a seat on the paid licence or an automatic free licence | +| `registry_extension_installs` | `instances`, `installs`, `licence_activations` | One cloud instance for all companies (the cloud host's, else a tokenless placeholder its first registration adopts); each install is for its company's organization; soft-deleted installs become `uninstalled`; live installs claim a seat on the company's paid licence or an automatic free licence. Per-company instances from earlier runs are revoked and their seats moved | Idempotency: every imported row stores the uuid of its source row in `source_uuid`, and re-runs match on it and update in place; an unchanged source produces only "unchanged" diff --git a/docs/cutover.md b/docs/cutover.md new file mode 100644 index 0000000..4c7a453 --- /dev/null +++ b/docs/cutover.md @@ -0,0 +1,483 @@ +# Cutover runbook: cloud onto the Registry Service + +One deployment in five steps (implementation plan, "Data migration and cutover", revised 24 Oct 2026): +**freeze, import, infra, deploy, unfreeze**. v1 is removed from registry-bridge 0.2.0, so the cloud +console cannot fall back to its old `registry_*` tables; rollback is redeploying the previous release +of every component together, which works because this release only reads those tables. + +Planned window: **60 minutes**, of which the import itself must stay under 30. In the local rehearsal +below the window's data work (steps 2 and 4: dumps, import, verifies, second import) takes **about 4 +minutes** on a fake larger than today's cloud registry and about 21 at three times that, so the window +is dominated by deploys and checks, not by the data. Allow twice the rehearsal times on the service's +0.5 vCPU task. + +Contents: [Roles and conventions](#roles-and-conventions) · [Before the window](#before-the-window) +· [Go/no-go](#gono-go) · [Step 1 Freeze](#step-1-freeze) · [Step 2 Import](#step-2-import) · [Step 3 +Infra](#step-3-infra) · [Step 4 Deploy](#step-4-deploy) · [Step 5 Unfreeze](#step-5-unfreeze) · +[Smoke tests](#smoke-tests) · [Rollback](#rollback) · [Rehearsal and timings](#rehearsal-and-timings) +· [After the window](#after-the-window) + +## How it fits together + +- **The service** (`fleetbase/registry` `v2.0.0`) runs in the `production` stack of + `fleetbase/registry-infra` beside Verdaccio, on `registry.fleetbase.io`. It is deployed and frozen + (`registry:maintenance on`) before the window, with an empty database. +- **The cloud console's API** (fleetbase `v0.8.0` with registry-bridge `0.2.0`) registers itself with + the service on boot as the one cloud instance (`registry:init` → `registry:cloud-register`), signed + with the platform secret both sides share. That instance acts for each cloud company + (`X-Registry-Tenant`): licences, installs and checkouts belong to the company's organization. +- **The importer** (`registry:import`) maps the cloud tables onto the service and attaches every cloud + install and paid seat to that cloud instance. It creates the instance first, without a token; the + API's first registration rotates onto it by host (D17), so installs and seats are already in place + when the new console starts. +- **The import runs twice.** Once in step 2 (the bulk of the data, and the copy of every tarball), + and once more in step 4, after the old API is gone, to pick up anything the old console or + Verdaccio wrote meanwhile. The importer is idempotent; the second run takes about a minute. This is + what makes the freeze safe even though the running cloud API (bridge 0.1.x) has no maintenance + switch (see [Decisions](#decisions-this-runbook-depends-on)). +- **After the second import** the API records locally what each company had installed + (`registry:cloud-sync-installs`), which is what makes each company's console load its engines. + +## Roles and conventions + +| Role | Who | Does | +| --- | --- | --- | +| Lead | backend engineer | runs the steps, calls go/no-go | +| Infra | DevOps | Pulumi, ECS, deploys | +| Checker | second engineer or staff reviewer | smoke tests in three reference organizations, Stripe dashboard | + +Set these once in the shell every command below runs from (values in angle brackets come from the +pre-flight table): + +```bash +export AWS_PROFILE=fleetbase-oss AWS_REGION=ap-southeast-1 +export PULUMI_BACKEND_URL=https://api.pulumi.com # the local CLI may point at another backend +export SVC_CLUSTER=fleetbase-registry-production-service +export CLOUD_CLUSTER= CLOUD_SERVICE= CLOUD_CONTAINER= +export CLOUD_FILES_BUCKET= +export CUTOVER=$(date -u +%Y%m%d) + +# Run a command in the service's app container, in its queue container (imports: one worker +# shares the task's 1 GB instead of nginx and php-fpm), or in the cloud API. +svc() { + local task; task=$(aws ecs list-tasks --cluster "$SVC_CLUSTER" --service-name app --query 'taskArns[0]' --output text) + aws ecs execute-command --cluster "$SVC_CLUSTER" --task "$task" --container app --interactive --command "$*" +} +svcq() { + local task; task=$(aws ecs list-tasks --cluster "$SVC_CLUSTER" --service-name queue --query 'taskArns[0]' --output text) + aws ecs execute-command --cluster "$SVC_CLUSTER" --task "$task" --container queue --interactive --command "$*" +} +cloud() { + local task; task=$(aws ecs list-tasks --cluster "$CLOUD_CLUSTER" --service-name "$CLOUD_SERVICE" --query 'taskArns[0]' --output text) + aws ecs execute-command --cluster "$CLOUD_CLUSTER" --task "$task" --container "$CLOUD_CONTAINER" --interactive --command "$*" +} +``` + +Long commands run under `nohup` inside the container and are followed with `tail`, so a dropped +session does not stop them. Every import prints its peak memory; it runs with a 512 MB PHP limit +(`--memory-limit`), and the rehearsal peaked well below that (see the timings). + +## Before the window + +Do these in the week before. Each has a check; all of them feed the [go/no-go](#gono-go). + +**Pre-flight values** (fill in and paste into the cutover channel): + +| Value | Where it comes from | +| --- | --- | +| Cloud API cluster, service, container | `fleetbase/infra` production stack | +| Cloud API host (`APP_URL`) | the cloud API environment, e.g. `https://api.fleetbase.io` | +| Cloud files bucket | the cloud API's S3 bucket; also `service.import_source_bucket` | +| Staff organization slug | the organization Fleetbase's own cloud company imports as (see the dry-run report) | +| Three reference organizations | one with a paid purchase, one with several free installs, one publisher; their company uuids | +| Previous release tags | cloud API and console images now in production; registry-bridge 0.1.x | + +**1. Service infrastructure** (registry-infra, `production` stack). `service.enabled` is set before the +window, not in step 3: the import in step 2 writes into the production service's database, so it has to +exist. Its listener rules only take the service's own paths (`/v1/*`, `/npm/*`, `/composer/*`, the +website pages), which Verdaccio never served, so nothing changes for current clients. + +```bash +cd registry-infra && pulumi stack select production +pulumi config set --path 'service.enabled' true +pulumi config set --path 'service.import_source_bucket' "$CLOUD_FILES_BUCKET" +pulumi config set --path --secret 'service_configuration.REGISTRY_CLOUD_PLATFORM_SECRET' "$(openssl rand -hex 32)" +pulumi config set --path 'service_configuration.REGISTRY_CLOUD_HOST' '' +pulumi config set --path 'service_configuration.REGISTRY_CLOUD_ORGANIZATION' '' +pulumi config set --path 'service_configuration.IMPORT_SOURCE_AWS_DEFAULT_REGION' ap-southeast-1 +# Stripe live mode: STRIPE_KEY, STRIPE_SECRET, STRIPE_WEBHOOK_SECRET as secrets, the same way. +pulumi preview && pulumi up +``` + +Check: `pulumi stack output` shows the service; `curl -fsS https://registry.fleetbase.io/up` answers 200. + +**2. Deploy the service and freeze it.** Push the `v2.0.0` tag of `fleetbase/registry` to +`deploy/production` (the workflow builds the image, runs migrations and deploys `app` and `queue`). +Then, before anything else reaches it: + +```bash +svc php artisan registry:maintenance on --message="Extensions are moving to the new Fleetbase registry. Publishing and purchases are paused until the move is finished." +svc php artisan registry:maintenance status # Maintenance: ON +svc php artisan migrate:status # every migration Ran +``` + +The freeze is stored in the service's database cache, so it holds across tasks and deploys until step 5. + +**3. Cloud API configuration, staged** (`fleetbase/infra`, not deployed until step 4). The new release +reads these: + +| Variable | Value | +| --- | --- | +| `REGISTRY_SERVICE_URL` | `https://registry.fleetbase.io` | +| `REGISTRY_CLOUD_AUTO_REGISTER` | `true` | +| `REGISTRY_CLOUD_PLATFORM_SECRET` | the same value as the service's (compare `sha256sum` fingerprints, never the secret) | +| `REGISTRY_MAINTENANCE` | `true` (the console comes up frozen; removed in step 5) | +| `APP_URL` | unchanged; its host must equal the service's `REGISTRY_CLOUD_HOST` | + +`registry:init` must stay in the API's start-up command (it registers the instance), and the scheduler +must run (it re-registers within five minutes if the token is ever refused). + +**4. Two clean dry runs on recent snapshots.** On the `qa` stack, or against production while it is +still empty, using the same dump path as the window (step 2). A run is clean when it exits 0, the +anomaly report has no class the team has not signed off, and its duration fits the budget. Record both +in the cutover channel with their dates. + +```bash +cloud "sh -c 'CLOUD_DB_HOST= CLOUD_DB_USER= MYSQL_PWD= CLOUD_DB_NAME=fleetbase /path/to/cloud-registry-dump.sh /tmp/cloud-registry-dryrun.sql.gz && aws s3 cp /tmp/cloud-registry-dryrun.sql.gz s3://$CLOUD_FILES_BUCKET/registry-cutover/'" +svcq "sh -c 'nohup php artisan registry:import --from=import_source:registry-cutover/cloud-registry-dryrun.sql.gz --dry-run --report=/tmp/dry-run.json > /tmp/dry-run.log 2>&1 &'" +svcq tail -n 80 /tmp/dry-run.log +``` + +`scripts/cloud-registry-dump.sh` is in this repository; run it wherever the cloud database is reachable +and `mysqldump` and the AWS CLI exist (the bastion, or the cloud API task if its image has them; the +`cloud …` lines below assume the latter, so adapt them to the bastion otherwise). It dumps only what the importer reads, in one +consistent transaction. The dump holds password hashes of registry users: keep it in that private +bucket and delete it after the cutover. + +**5. qa end to end.** On the `qa` stacks: the qa cloud console registers itself on boot +(`registry:cloud-register` reports `current` on the next run), Explore, Installed and Purchased show the +imported data for three qa organizations, an install and an uninstall work, and a test-mode purchase +completes with its webhook. + +**6. Old clients that use Verdaccio as an npm registry.** Verdaccio proxies every package to npmjs, and +consoles set up before v0.8 point the `@fleetbase` scope at `https://registry.fleetbase.io`. The service +never proxies npm (D10), so after step 3 those clients get 404 for every `@fleetbase/*` package, +including the core ones. Measure it from the Verdaccio access logs for the last 14 days: + +```bash +aws logs filter-log-events --log-group-name --start-time $(( ($(date +%s) - 14*86400) * 1000 )) \ + --filter-pattern '"GET /@fleetbase"' --query 'events[].message' --output text | wc -l +``` + +If that is not close to zero, decide before the window between flipping `route_all` anyway (with a +published upgrade note) and keeping Verdaccio for npm reads only (skip step 3's `route_all` and +scale-down; see [Decisions](#decisions-this-runbook-depends-on)). + +**7. Releases ready.** Tags built and images pushed: registry `v2.0.0`, registry-bridge `0.2.0`, +ember-core `0.4.0`, fleetbase `v0.8.0` (API and console), `@fleetbase/cli@2.0.0` published under the +`next` dist-tag. Previous production tags written down for rollback. + +**8. Announce** the window (status page, console banner on the current release if available, email to +publishers): publishing and purchases pause, installed extensions keep working. + +## Go/no-go + +Called by the lead, 30 minutes before the window and again at each step's gate. Any "no" stops the +cutover before step 1, or triggers [rollback](#rollback) after it. + +| # | Check | Evidence | +| --- | --- | --- | +| 1 | All roll-up PRs merged into `main`; tags built (registry, bridge, ember-core, fleetbase, CLI under `next`) | release links | +| 2 | Service deployed, healthy and frozen | `/up` 200; `registry:maintenance status` ON; `migrate:status` all Ran | +| 3 | Platform secret set on both sides, same fingerprint; `REGISTRY_CLOUD_HOST` = cloud API host | `sha256sum` of each, compared | +| 4 | Two clean dry runs on recent snapshots, each well under 30 minutes | dates, durations, anomaly sign-off | +| 5 | qa exercised end to end (registration, Explore, Installed, Purchased, install, test purchase) | checklist from the qa run | +| 6 | Verdaccio npm-read traffic measured and the `route_all` decision recorded | log count, decision | +| 7 | Three reference organizations chosen, their expected installs and licences written down | SQL below, run on the cloud database | +| 8 | Rollback ready: previous tags, the registry-infra values to restore, people on call | this page | +| 9 | Stripe: live webhook endpoint `https://registry.fleetbase.io/v1/webhooks/stripe` created, secret in the service | Stripe dashboard | +| 10 | Window announced; lead, infra and checker present | channel | + +Expected values for the reference organizations, on the cloud database: + +```sql +SELECT company_uuid, COUNT(*) AS live_installs FROM registry_extension_installs +WHERE deleted_at IS NULL AND company_uuid IN ('', '', '') GROUP BY company_uuid; +SELECT company_uuid, COUNT(*) AS purchases FROM registry_extension_purchases +WHERE deleted_at IS NULL AND company_uuid IN ('', '', '') GROUP BY company_uuid; +``` + +## Step 1 Freeze + +*Gate: go/no-go items 1 to 10. Time: 5 minutes.* + +1. Post the start in the channel; status page to "maintenance". +2. Confirm the service is frozen (it has been since it was deployed): + + ```bash + svc php artisan registry:maintenance status + ``` + +3. The running cloud API (bridge 0.1.x) has no maintenance switch, so purchases, installs and + Verdaccio publishing are not technically blocked until step 4 replaces it. That is safe: anything + written to the old tables until then is imported by the second run in step 4. Keep the window short + and the announcement clear. + +## Step 2 Import + +*Gate: two clean dry runs on recent snapshots; each under 30 minutes. Time: 10 minutes (rehearsal: +about 5 with the in-window dry run).* + +1. Dump the cloud tables and hand the dump to the service: + + ```bash + cloud "sh -c 'CLOUD_DB_HOST= CLOUD_DB_USER= MYSQL_PWD= CLOUD_DB_NAME=fleetbase /path/to/cloud-registry-dump.sh /tmp/cloud-registry-$CUTOVER-1.sql.gz && aws s3 cp /tmp/cloud-registry-$CUTOVER-1.sql.gz s3://$CLOUD_FILES_BUCKET/registry-cutover/'" + ``` + +2. A last dry run on exactly this dump (no files are copied, nothing is written): + + ```bash + svcq "sh -c 'nohup php artisan registry:import --from=import_source:registry-cutover/cloud-registry-$CUTOVER-1.sql.gz --dry-run --report=/tmp/dry-run-final.json > /tmp/dry-run-final.log 2>&1 &'" + svcq tail -n 80 /tmp/dry-run-final.log + ``` + + Compare its counts and anomalies with the last rehearsal; stop on a new anomaly class. + +3. The import (copies every tarball, icon and screenshot into the service bucket): + + ```bash + svcq "sh -c 'nohup php artisan registry:import --from=import_source:registry-cutover/cloud-registry-$CUTOVER-1.sql.gz --report=/tmp/import-1.json > /tmp/import-1.log 2>&1 &'" + svcq tail -n 80 /tmp/import-1.log # ends with "Import complete." + ``` + +4. Verify. Exits 0 when the dry run against the imported data would change nothing, every extension, + release, purchase, licence and install has its row (except those the importer skips, which are in + the anomaly report), and a sample of 50 rows per table matches field by field, tarball bytes + included: + + ```bash + svcq php artisan registry:import --from=import_source:registry-cutover/cloud-registry-$CUTOVER-1.sql.gz --verify --sample=50 + ``` + + The last line reads `Cloud instance: inst_… (, waiting for the cloud API to + register)`. + +5. The staff organization (reserved `@fleetbase/*` and `fleetbase/*` names): + + ```bash + svc php artisan registry:staff + ``` + +## Step 3 Infra + +*Gate: qa stack exercised end to end; the `route_all` decision from go/no-go item 6. Time: 5 minutes.* + +`service.enabled` is already on (see "Before the window"). Send every path on `registry.fleetbase.io` +to the service, then park Verdaccio. Its bucket, task definition and image stay for 90 days. + +```bash +cd registry-infra && pulumi stack select production +pulumi config set --path 'service.route_all' true +pulumi config set --path 'verdaccio.desired_count' 0 +pulumi preview # expect: listener rule(s) changed, the Verdaccio ECS service's desiredCount 1 → 0 +pulumi up +``` + +Checks: + +```bash +curl -fsS https://registry.fleetbase.io/up +curl -fsS 'https://registry.fleetbase.io/v1/catalog/extensions?limit=1' | jq .meta.total # published count from the import report +curl -fsS https://registry.fleetbase.io/composer/packages.json | jq '."available-packages" | length' +aws ecs describe-services --cluster fleetbase-registry-production --services app --query 'services[0].[desiredCount,runningCount]' # [0, 0] +``` + +## Step 4 Deploy + +*Gate: smoke test of Explore, install and purchase on the three reference organizations (below). Time: +25 minutes.* + +1. **Cloud API and console**: deploy fleetbase `v0.8.0` (ember-core 0.4.0, registry-bridge 0.2.0) with + the staged configuration, the usual way for production. Wait for the ECS deployment to reach steady + state, so no task of the old API is left writing to the old tables. + +2. **Registration**: each task runs `registry:init` on boot. Confirm the instance is registered and + kept its imported data: + + ```bash + cloud php artisan registry:cloud-register --json + # {"status": "current", "summary": {"id": "inst_…", "kind": "cloud", ...}} + ``` + + The `id` must be the `inst_…` the verify printed in step 2 (`Cloud instance: inst_…`): the API + rotated onto the imported instance (by host, or by adopting it while it had no token). A + different id means the imported instance already held a token from another host (an earlier + registration against this database) and the API got a new one: stop and roll back step 4; the + installs and seats are on the other instance. Go/no-go item 3 (`REGISTRY_CLOUD_HOST` equals the + API host, on an empty database) prevents it. + + If it reads `failed`: the message names the cause (a secret mismatch is "not signed with the platform + secret", a clock skew is "expired"). Fix the configuration and run `cloud php artisan + registry:cloud-register`. + +3. **Second import**, from a dump taken now (picks up whatever the old API or Verdaccio wrote since + step 2): + + ```bash + cloud "sh -c 'CLOUD_DB_HOST= CLOUD_DB_USER= MYSQL_PWD= CLOUD_DB_NAME=fleetbase /path/to/cloud-registry-dump.sh /tmp/cloud-registry-$CUTOVER-2.sql.gz && aws s3 cp /tmp/cloud-registry-$CUTOVER-2.sql.gz s3://$CLOUD_FILES_BUCKET/registry-cutover/'" + svcq "sh -c 'nohup php artisan registry:import --from=import_source:registry-cutover/cloud-registry-$CUTOVER-2.sql.gz --report=/tmp/import-2.json > /tmp/import-2.log 2>&1 &'" + svcq tail -n 40 /tmp/import-2.log + svcq php artisan registry:import --from=import_source:registry-cutover/cloud-registry-$CUTOVER-2.sql.gz --verify --sample=50 + ``` + +4. **Installed extensions per company**, recorded on the cloud API so each console loads its engines: + + ```bash + cloud php artisan registry:cloud-sync-installs --dry-run + cloud php artisan registry:cloud-sync-installs + # Created + Unchanged = installed rows on the service; Skipped = companies deleted from the cloud API + ``` + +5. **CLI**: `npm dist-tag add @fleetbase/cli@2.0.0 latest`. + +6. Run the [smoke tests](#smoke-tests), frozen part. + +## Step 5 Unfreeze + +*Gate: the frozen smoke tests passed; Stripe webhooks verified in live mode. Time: 10 minutes.* + +1. Service: + + ```bash + svc php artisan registry:maintenance off + ``` + +2. Cloud API: remove `REGISTRY_MAINTENANCE` from its configuration and roll the tasks (a new + deployment of the same release). Then `cloud php artisan registry:maintenance status` reads `off`. + +3. **Stripe live check**: buy a listing priced for this purpose (Stripe Checkout needs at least $0.50; + a $1 staff listing, or a 100% promotion code on a real one), from a staff organization in the cloud + console. The hand-off completes, the licence appears under Purchased with "activated here" after the + install, the webhook shows 200 in the Stripe dashboard. Refund it from the staff finance view. + +4. Run the [smoke tests](#smoke-tests), unfrozen part. Status page back to operational; announce. + +## Smoke tests + +**Frozen (end of step 4)**, in each of the three reference organizations, in the cloud console: + +- Every Extensions page shows the yellow "Extensions are paused" banner with the message. +- Explore lists the catalog with categories and counts; a listing's panel opens with versions. +- Installed shows the organization's extensions with the versions the cloud tables had; their engines + appear in the sidebar after a reload. +- Purchased shows every licence the organization bought (one per purchase), receipts where Stripe had + them. +- Install, uninstall and Buy are refused with the maintenance message. + +And on the service, the reference organizations' installs and paid licences match the cloud numbers +from go/no-go item 7. Open `svc php artisan tinker` and paste: + +```php +App\Models\Organization::whereIn('source_uuid', ['', '', ''])->get() + ->map(fn ($o) => [$o->name, App\Models\Install::where('organization_uuid', $o->uuid)->where('status', 'installed')->count(), App\Models\Licence::where('organization_uuid', $o->uuid)->where('kind', 'paid')->count()]); +``` + +**Unfrozen (end of step 5)**, in one reference organization: + +- Install a free extension: the installer modal runs to "installed"; uninstall it again. +- The purchase in step 5 completed (above). +- `npm view --registry https://registry.fleetbase.io/npm/` and + `composer show -a --repository https://registry.fleetbase.io/composer` answer. +- A publisher can open the Studio on `https://registry.fleetbase.io/studio` and the review queue loads + for staff. + +## Rollback + +Rollback is redeploying the previous release of every component together. The old cloud tables were +only read, so the previous cloud API resumes exactly where it stopped. + +| From | Do | +| --- | --- | +| Step 1 or 2 | Nothing has changed for clients. Leave the service frozen; `registry:import` can run again later. | +| Step 3 | `pulumi config set --path 'service.route_all' false && pulumi config set --path 'verdaccio.desired_count' 1 && pulumi up`; Verdaccio is back in about two minutes. | +| Step 4 | Redeploy the previous cloud API and console tags, then roll back step 3. The service stays up and frozen; its cloud instance is unused. | +| Step 5 | As step 4, but first `svc php artisan registry:maintenance on`. Purchases and installs made on the service after the unfreeze are not in the old tables: list them from the staff finance view (`/review/finance`) and settle them by hand (refund, or re-grant in the old console). | + +Never drop or truncate the old `registry_*` tables in this release (Workstream P removes them after a +release cycle without reads). + +## Rehearsal and timings + +`scripts/rehearse-cutover.sh` runs the data path of steps 2 and 4 end to end against a seeded fake +cloud database (`scripts/fake-cloud-dump.php`, no real data): two dry runs, the import, `--verify`, a +second import, `--verify` again, then the dump the runbook takes (`scripts/cloud-registry-dump.sh`) +and a dry run reading it. + +```bash +SCALE=1 S3_ENDPOINT= MYSQL_PORT=33308 MYSQL_SOCKET=/tmp/mysql.sock scripts/rehearse-cutover.sh +``` + +At `SCALE=1` the fake is larger than today's cloud registry (tens of extensions, a small community): +80 extensions, 453 bundles with 134 MB of tarballs, 600 purchases, 9,000 installs across 2,000 +companies, 825 registry users, 120 developer accounts, and every anomaly class the importer reports +(12 missing tarballs, a bundle without a version, a package name used twice, a listing without bundles, +duplicate versions, a user without email, paid installs without a purchase). + +Run 2026-10-24 on an Apple-silicon laptop shared with other builds (load average 15 to 28 throughout, +so these are pessimistic for the CPU and optimistic for nothing else): PHP 8.3 without Xdebug, MySQL +8.1 over a Unix socket (about 0.4 ms a prepared query, close to an in-VPC RDS round trip; loopback TCP +on macOS is 3 to 7 times slower and was not used), and local folders standing in for both S3 buckets +(no MinIO image could be pulled). S3-to-S3 copies in production add the transfer time of the +tarballs: 134 MB at 50 to 100 MB/s is 2 to 3 seconds. Scale 3 is 240 extensions, 1,433 bundles with 428 +MB of tarballs, 1,800 purchases and 27,000 installs across 6,000 companies. + +| Step | Scale 1 | Scale 3 | +| --- | --- | --- | +| Dump (`cloud-registry-dump.sh`) | 2 s | 2 s | +| Dry run, cold | 1 min 41 s | 6 min 56 s | +| Dry run, again | 2 min 53 s | 6 min 29 s | +| Import | 2 min 11 s | 9 min 38 s | +| Verify | 31 s | 3 min 11 s | +| Second import (nothing new) | 1 min 3 s | 4 min 32 s | +| Verify again (other sample) | 34 s | 3 min 27 s | +| Dry run from the dump | 1 min 46 s | 5 min 8 s | +| Peak memory (PHP) | 72 MB | 134 MB | + +What the scale 1 import wrote: 2,114 organizations, 799 accounts, 80 extensions with 156 package names, +447 releases (6 skipped: duplicate versions and bundles without a version), 675 files, 600 purchases +and their 600 paid licences, 6,069 automatic free licences, 9,000 installs and 6,560 seats, all on one +cloud instance. The second import changed nothing (every table "unchanged") and both verifies exited 0. + +**Budget for the window** at scale 1: step 2 (dump, import, verify) about 3 minutes, step 4 (dump, +second import, verify) under 2; doubled for the 0.5 vCPU task, about 10 minutes of the 60. The +in-window dry run in step 2 is optional (two clean ones precede the window): skip it if the window is +tight. A dry run can take longer than the import because the rolled-back transaction is undone row by +row. The default 128 MB PHP limit would have failed at scale 3, so the importer now raises its own +limit to 512 MB (`--memory-limit`) and prints its peak. + +The rehearsal found and fixed one defect before it could reach production: on MySQL every re-import +counted extensions, releases and purchases as updated (MySQL reorders JSON object keys), so `--verify` +would have reported drift on every run. It also caught a migration index name over MySQL's 64 +characters that SQLite accepted. + +## After the window + +- Delete the dumps: `aws s3 rm s3://$CLOUD_FILES_BUCKET/registry-cutover/ --recursive`. +- Keep Verdaccio parked (bucket and image) for 90 days, then archive (Workstream P). +- Watch for a week: failed registrations (`registry:cloud-register` in the scheduler log), install + failures by stage, webhook failures, 503s after the unfreeze. +- Publishers re-verify payout details on the website (Connect accounts were copied from + `companies.stripe_connect_id`; `payout_status` refreshes on the first webhook). + +## Decisions this runbook depends on + +1. **Freeze on the running cloud API.** Bridge 0.1.x has no maintenance switch, so the old console + cannot be frozen in step 1. This runbook relies on the second import in step 4 instead. If the + window must also stop purchases on the old console, backport the bridge's `REGISTRY_MAINTENANCE` + setting to a 0.1.x hotfix and deploy it before the window. +2. **`route_all` and Verdaccio as an npm proxy** (go/no-go item 6). Flip it and break pre-0.8 clients + that read `@fleetbase/*` through registry.fleetbase.io, or keep Verdaccio for npm reads (no + `route_all`, no scale-down) until those clients upgrade. +3. **Order of `service.enabled`.** It moves before the window (the plan has it in step 3), because the + import writes into the production service. +4. **Reference organizations** and the staff organization slug (`REGISTRY_CLOUD_ORGANIZATION`). diff --git a/scripts/cloud-registry-dump.sh b/scripts/cloud-registry-dump.sh new file mode 100755 index 0000000..5593086 --- /dev/null +++ b/scripts/cloud-registry-dump.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Dumps exactly the cloud rows `registry:import` reads (docs/cutover.md, steps 2 and 4), consistently +# (one transaction, no table locks), gzipped: +# +# CLOUD_DB_HOST=… CLOUD_DB_USER=… MYSQL_PWD=… CLOUD_DB_NAME=fleetbase scripts/cloud-registry-dump.sh cloud-registry.sql.gz +# +# Run it where the cloud database is reachable (a cloud API task or the bastion), then copy the file +# to the cloud files bucket the registry reads (`aws s3 cp … s3:///registry-cutover/`) and +# import it with `--from=import_source:registry-cutover/`. The dump holds password hashes of +# the registry users: keep it in that private bucket and delete it after the cutover. +# +# Optional: CLOUD_DB_PORT (3306), CLOUD_DB_SSL_CA (a CA bundle when the server requires TLS). +set -euo pipefail + +OUT="${1:?usage: cloud-registry-dump.sh }" +DB="${CLOUD_DB_NAME:-fleetbase}" +ARGS=(-h "${CLOUD_DB_HOST:?set CLOUD_DB_HOST}" -P "${CLOUD_DB_PORT:-3306}" -u "${CLOUD_DB_USER:?set CLOUD_DB_USER}" + --single-transaction --skip-lock-tables --no-tablespaces --protocol=tcp) +[ -n "${CLOUD_DB_SSL_CA:-}" ] && ARGS+=(--ssl-ca="$CLOUD_DB_SSL_CA") +# mysqldump 8 adds GTID statements unless told not to; MariaDB's dump does not know the option. +mysqldump --help 2>/dev/null | grep -q 'set-gtid-purged' && ARGS+=(--set-gtid-purged=OFF) + +{ + # The registry tables and the small lookup tables, whole. + mysqldump "${ARGS[@]}" "$DB" registry_extensions registry_extension_bundles registry_extension_purchases \ + registry_extension_installs registry_users registry_developer_accounts companies categories + # Only the users and files the registry tables reference (the importer reads nothing else). + mysqldump "${ARGS[@]}" "$DB" users --where="uuid IN (SELECT user_uuid FROM registry_users) OR uuid IN (SELECT created_by_uuid FROM registry_extensions) OR uuid IN (SELECT created_by_uuid FROM registry_extension_bundles)" + mysqldump "${ARGS[@]}" "$DB" files --where="uuid IN (SELECT icon_uuid FROM registry_extensions) OR uuid IN (SELECT bundle_uuid FROM registry_extension_bundles) OR subject_uuid IN (SELECT uuid FROM registry_extensions)" +} | gzip -9 > "$OUT" + +echo "$OUT: $(du -h "$OUT" | cut -f1)" diff --git a/scripts/fake-cloud-dump.php b/scripts/fake-cloud-dump.php new file mode 100644 index 0000000..3b959dd --- /dev/null +++ b/scripts/fake-cloud-dump.php @@ -0,0 +1,275 @@ + max(1, (int) round($base * $scale)); +$counts = [ + 'categories' => 12, + 'developers' => $n(120), + 'companies' => $n(2000), + 'extensions' => $n(80), + 'purchases' => $n(600), + 'installs' => $n(9000), +]; + +// --------------------------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------------------------- + +/** A UUID v4 from the seeded generator, so the same seed gives the same dump. */ +function fakeUuid(): string +{ + $r = fn (int $max = 0xFFFF) => mt_rand(0, $max); + + return sprintf('%04x%04x-%04x-4%03x-%04x-%04x%04x%04x', $r(), $r(), $r(), $r(0xFFF), $r(0x3FFF) | 0x8000, $r(), $r(), $r()); +} + +function pick(array $items) +{ + return $items[mt_rand(0, count($items) - 1)]; +} + +function chance(float $p): bool +{ + return mt_rand() / mt_getrandmax() < $p; +} + +function ts(int $daysAgo, int $jitter = 0): string +{ + return date('Y-m-d H:i:s', strtotime('2026-10-20 03:00:00') - $daysAgo * 86400 - mt_rand(0, max(0, $jitter))); +} + +function sqlValue($value): string +{ + if ($value === null) { + return 'NULL'; + } + if (is_bool($value)) { + return $value ? '1' : '0'; + } + if (is_int($value)) { + return (string) $value; + } + if (is_array($value)) { + $value = json_encode($value, JSON_UNESCAPED_SLASHES); + } + + return "'".str_replace(['\\', "'", "\n", "\r"], ['\\\\', "\\'", '\\n', '\\r'], (string) $value)."'"; +} + +function writeObject(string $dir, string $path, int $bytes): int +{ + $file = rtrim($dir, '/').'/'.$path; + @mkdir(dirname($file), 0777, true); + $h = fopen($file, 'wb'); + $left = $bytes; + while ($left > 0) { + $chunk = min($left, 65536); + fwrite($h, random_bytes($chunk)); + $left -= $chunk; + } + fclose($h); + + return $bytes; +} + +$words = ['fleet', 'route', 'cargo', 'dock', 'yard', 'pallet', 'driver', 'parcel', 'freight', 'depot', 'transit', 'ledger', 'invoice', 'telematics', 'fuel', 'shift', 'dispatch', 'track', 'customs', 'cold']; +$suffix = ['insights', 'planner', 'sync', 'hub', 'pro', 'connect', 'manager', 'tools', 'bridge', 'scan', 'board', 'flow']; + +// --------------------------------------------------------------------------------------------- +// Rows +// --------------------------------------------------------------------------------------------- + +$rows = array_fill_keys(['companies', 'users', 'files', 'categories', 'registry_developer_accounts', 'registry_users', 'registry_extensions', 'registry_extension_bundles', 'registry_extension_purchases', 'registry_extension_installs'], []); +$id = array_fill_keys(array_keys($rows), 0); +$add = function (string $table, array $row) use (&$rows, &$id) { + $rows[$table][] = ['id' => ++$id[$table]] + $row; +}; +$bcrypt = '$2y$10$'.str_repeat('a', 53); // never a real password + +for ($i = 0; $i < $counts['categories']; $i++) { + $name = ucfirst($words[$i]).' '.ucfirst($suffix[$i % count($suffix)]); + $add('categories', ['uuid' => fakeUuid(), 'public_id' => 'category_'.$i, 'name' => $name, 'internal_id' => strtolower(str_replace(' ', '-', $name)), 'description' => null, 'tags' => null, 'icon' => null, 'for' => 'extension_category', 'slug' => strtolower(str_replace(' ', '-', $name)), 'created_at' => ts(900), 'updated_at' => ts(900)]); +} +$categoryUuids = array_column($rows['categories'], 'uuid'); + +$companies = []; +for ($i = 0; $i < $counts['companies']; $i++) { + $uuid = fakeUuid(); + $owner = fakeUuid(); + $name = ucfirst(pick($words)).' '.pick(['Logistics', 'Freight', 'Transport', 'Express', 'Couriers', 'Haulage']).' '.($i + 1); + $companies[] = ['uuid' => $uuid, 'owner' => $owner]; + $add('companies', ['uuid' => $uuid, 'public_id' => 'company_'.$i, 'stripe_connect_id' => $i < 40 ? 'acct_fake'.str_pad((string) $i, 8, '0', STR_PAD_LEFT) : null, 'owner_uuid' => $owner, 'name' => $name, 'website_url' => null, 'description' => null, 'slug' => strtolower(str_replace(' ', '-', $name)), 'created_at' => ts(1000, 86400 * 600), 'updated_at' => ts(10), 'deleted_at' => $i % 250 === 249 ? ts(30) : null]); + $add('users', ['uuid' => $owner, 'public_id' => 'user_'.$i, 'company_uuid' => $uuid, 'username' => 'owner'.$i, 'email' => $i % 500 === 499 ? null : "owner{$i}@company{$i}.example", 'password' => $bcrypt, 'name' => 'Owner '.$i, 'timezone' => 'UTC', 'status' => 'active', 'email_verified_at' => ts(900), 'created_at' => ts(900), 'updated_at' => ts(10), 'deleted_at' => null]); +} + +// Publishing companies: the first 25 publish; each has a registry user (one row per token). +$publishers = array_slice($companies, 0, min(25, count($companies))); +foreach ($publishers as $j => $company) { + $add('registry_users', ['uuid' => fakeUuid(), 'account_type' => 'cloud', 'developer_account_uuid' => null, 'company_uuid' => $company['uuid'], 'user_uuid' => $company['owner'], 'public_id' => 'ru_'.$j, 'token' => bin2hex(random_bytes(8)), 'registry_token' => null, 'scope' => null, 'expires_at' => null, 'last_used_at' => null, 'name' => null, 'meta' => null, 'revoked' => 0, 'created_at' => ts(700), 'updated_at' => ts(700), 'deleted_at' => null]); +} +// Cloud users who logged in to the registry to buy (registry_users of buying companies). +for ($j = 0; $j < $n(800); $j++) { + $company = $companies[mt_rand(25, count($companies) - 1)]; + $add('registry_users', ['uuid' => fakeUuid(), 'account_type' => 'cloud', 'developer_account_uuid' => null, 'company_uuid' => $company['uuid'], 'user_uuid' => $company['owner'], 'public_id' => 'rub_'.$j, 'token' => bin2hex(random_bytes(8)), 'registry_token' => null, 'scope' => null, 'expires_at' => null, 'last_used_at' => null, 'name' => null, 'meta' => null, 'revoked' => 0, 'created_at' => ts(500), 'updated_at' => ts(500), 'deleted_at' => null]); +} + +$developers = []; +for ($i = 0; $i < $counts['developers']; $i++) { + $uuid = fakeUuid(); + $developers[] = $uuid; + $add('registry_developer_accounts', ['uuid' => $uuid, 'username' => 'dev'.$i, 'email' => "dev{$i}@indie.example", 'password' => $bcrypt, 'name' => 'Developer '.$i, 'avatar_url' => null, 'github_username' => chance(0.5) ? 'dev-gh-'.$i : null, 'website' => null, 'bio' => null, 'email_verified_at' => $i % 7 === 6 ? null : ts(300), 'verification_token' => null, 'status' => $i % 7 === 6 ? 'pending_verification' : 'active', 'created_at' => ts(300, 86400 * 200), 'updated_at' => ts(10), 'deleted_at' => null]); +} + +// Extensions, their bundles and the objects behind them. +$extensions = []; +$objectBytes = 0; +$missingObjects = 0; +for ($i = 0; $i < $counts['extensions']; $i++) { + $uuid = fakeUuid(); + $byDeveloper = $i % 3 === 2; + $publisher = $byDeveloper ? pick($developers) : $publishers[$i % count($publishers)]['uuid']; + $slug = $words[$i % count($words)].'-'.$suffix[intdiv($i, count($words)) % count($suffix)].($i >= count($words) * count($suffix) ? '-'.$i : ''); + $scope = $byDeveloper ? 'dev'.array_search($publisher, $developers) : 'pub'.($i % count($publishers)); + $paid = $i % 10 < 3; + $status = match (true) { + $i % 20 === 19 => 'pending', + $i % 20 === 18 => 'rejected', + $i % 20 === 17 => 'awaiting_review', + $i % 40 === 16 => 'unpublished', + default => 'published', + }; + $noBundles = $i % 40 === 39; // a listing that never uploaded anything + $deleted = $i % 25 === 24; + $npmName = '@'.$scope.'/'.$slug.'-engine'; + if ($i === 5 && count($extensions) > 1) { + $npmName = $extensions[1]['npm']; // a package name claimed twice + } + + $iconUuid = fakeUuid(); + $iconPath = "uploads/{$scope}/icons/{$slug}.png"; + $add('files', ['uuid' => $iconUuid, 'public_id' => 'file_icon_'.$i, 'company_uuid' => null, 'subject_uuid' => $uuid, 'disk' => 's3', 'path' => $iconPath, 'subject_type' => 'Fleetbase\\RegistryBridge\\Models\\RegistryExtension', 'bucket' => 'flb-cloud', 'original_filename' => $slug.'.png', 'type' => 'extension_icon', 'content_type' => 'image/png', 'file_size' => 2048, 'created_at' => ts(600), 'updated_at' => ts(600), 'deleted_at' => null]); + $objectBytes += writeObject($filesDir, $iconPath, 2048); + for ($s = 0; $s < 2; $s++) { + $path = "uploads/{$scope}/screens/{$slug}-{$s}.png"; + $add('files', ['uuid' => fakeUuid(), 'public_id' => "file_shot_{$i}_{$s}", 'company_uuid' => null, 'subject_uuid' => $uuid, 'disk' => 's3', 'path' => $path, 'subject_type' => 'Fleetbase\\RegistryBridge\\Models\\RegistryExtension', 'bucket' => 'flb-cloud', 'original_filename' => basename($path), 'type' => 'extension_screenshot', 'content_type' => 'image/png', 'file_size' => 40960, 'created_at' => ts(600), 'updated_at' => ts(600), 'deleted_at' => null]); + $objectBytes += writeObject($filesDir, $path, 40960); + } + + $bundleUuids = []; + $bundleCount = $noBundles ? 0 : mt_rand(3, 9); + for ($b = 0; $b < $bundleCount; $b++) { + $version = '1.'.intdiv($b, 3).'.'.($b % 3); + $fileUuid = fakeUuid(); + $path = "uploads/{$scope}/extensions/{$slug}-{$version}.tar.gz"; + $bytes = (int) ($bundleKb * 1024 * (0.5 + mt_rand() / mt_getrandmax())); + $missing = ($i * 7 + $b) % 50 === 0; // ~2% of tarballs are gone from the bucket + $add('files', ['uuid' => $fileUuid, 'public_id' => "file_b_{$i}_{$b}", 'company_uuid' => null, 'subject_uuid' => $uuid, 'disk' => 's3', 'path' => $path, 'subject_type' => 'Fleetbase\\RegistryBridge\\Models\\RegistryExtension', 'bucket' => 'flb-cloud', 'original_filename' => basename($path), 'type' => 'extension_bundle', 'content_type' => 'application/gzip', 'file_size' => $bytes, 'created_at' => ts(500 - $b * 20), 'updated_at' => ts(500 - $b * 20), 'deleted_at' => null]); + if ($missing) { + $missingObjects++; + } else { + $objectBytes += writeObject($filesDir, $path, $bytes); + } + + $bundleUuid = fakeUuid(); + $bundleUuids[] = $bundleUuid; + $last = $b === $bundleCount - 1; + $bundleStatus = $last ? ($status === 'published' || $status === 'unpublished' ? 'published' : ($status === 'rejected' ? 'rejected' : 'pending')) : (chance(0.1) ? 'rejected' : 'published'); + $meta = [ + 'extension.json' => array_filter(['name' => ucfirst(str_replace('-', ' ', $slug)), 'version' => $version, 'compat' => $b > 2 ? ['console' => '>=0.7.0', 'api' => '^1.6'] : null]), + 'package.json' => ['name' => $npmName, 'version' => $version], + 'composer.json' => ['name' => $scope.'/'.$slug.'-api'], + ]; + if ($i % 30 === 3 && $b === 0) { + $meta = null; // a bundle without any version + } + $add('registry_extension_bundles', ['uuid' => $bundleUuid, 'public_id' => "bundle_{$i}_{$b}", 'bundle_id' => strtoupper(substr(md5($bundleUuid), 0, 14)), 'company_uuid' => $byDeveloper ? null : $publisher, 'created_by_uuid' => null, 'extension_uuid' => $uuid, 'bundle_uuid' => $fileUuid, 'bundle_number' => (string) ($b + 1), 'version' => $meta === null ? null : $version, 'status' => $bundleStatus, 'meta' => $meta, 'created_at' => ts(500 - $b * 20), 'updated_at' => ts(499 - $b * 20), 'deleted_at' => null]); + } + if ($bundleCount > 3 && $i % 15 === 4) { + // the same version uploaded twice + $last = end($rows['registry_extension_bundles']); + $add('registry_extension_bundles', ['uuid' => fakeUuid()] + array_diff_key($last, ['id' => 1, 'uuid' => 1]) + ['created_at' => ts(50)]); + } + + $price = $paid ? 1900 + 1000 * mt_rand(0, 8) : null; + $add('registry_extensions', ['uuid' => $uuid, 'company_uuid' => $byDeveloper ? null : $publisher, 'publisher_type' => $byDeveloper ? 'developer' : 'cloud', 'publisher_uuid' => $publisher, 'created_by_uuid' => null, 'registry_user_uuid' => null, 'current_bundle_uuid' => $bundleUuids ? end($bundleUuids) : null, 'next_bundle_uuid' => null, 'icon_uuid' => $iconUuid, 'category_uuid' => pick($categoryUuids), 'public_id' => 'ext_'.$i, 'stripe_product_id' => $paid ? 'prod_fake'.$i : null, 'name' => ucfirst(str_replace('-', ' ', $slug)), 'subtitle' => 'Fake listing '.$i, 'self_managed' => $i % 13 === 12 ? 1 : 0, 'payment_required' => $paid ? 1 : 0, 'price' => $price, 'sale_price' => $paid && $i % 4 === 0 ? $price - 500 : null, 'on_sale' => 0, 'subscription_required' => 0, 'currency' => 'USD', 'slug' => $slug, 'version' => null, 'fa_icon' => null, 'description' => str_repeat('Lorem ipsum dolor sit amet. ', 20), 'promotional_text' => null, 'website_url' => null, 'repo_url' => null, 'support_url' => null, 'privacy_policy_url' => null, 'tos_url' => null, 'copyright' => null, 'primary_language' => 'php', 'tags' => ['fake', pick($words)], 'languages' => ['en'], 'meta' => null, 'core_extension' => 0, 'status' => $status, 'package_name' => $noBundles && $i % 80 !== 39 ? $npmName : null, 'published_at' => $status === 'published' ? ts(400) : null, 'accepted_at' => null, 'rejected_at' => null, 'created_at' => ts(600, 86400 * 100), 'updated_at' => ts(20), 'deleted_at' => $deleted ? ts(60) : null]); + $extensions[] = ['uuid' => $uuid, 'paid' => $paid, 'live' => $status === 'published' && ! $deleted && ! $noBundles, 'npm' => $npmName, 'price' => $price]; +} + +$live = array_values(array_filter($extensions, fn ($e) => $e['live'])); +$livePaid = array_values(array_filter($live, fn ($e) => $e['paid'])); +$liveFree = array_values(array_filter($live, fn ($e) => ! $e['paid'])); + +// Purchases: companies (and a few developers) buying paid listings; some refunded (deleted). +$owned = []; +for ($i = 0; $i < $counts['purchases'] && $livePaid; $i++) { + $extension = pick($livePaid); + $developer = $i % 50 === 49; + $buyer = $developer ? pick($developers) : $companies[mt_rand(25, count($companies) - 1)]['uuid']; + if (! $developer) { + $owned[$buyer][$extension['uuid']] = true; + } + $add('registry_extension_purchases', ['uuid' => fakeUuid(), 'purchaser_uuid' => $buyer, 'purchaser_type' => $developer ? 'Fleetbase\\RegistryBridge\\Models\\RegistryDeveloperAccount' : 'Fleetbase\\Models\\Company', 'company_uuid' => $developer ? null : $buyer, 'extension_uuid' => $extension['uuid'], 'stripe_checkout_session_id' => 'cs_fake_'.$i, 'stripe_payment_intent_id' => 'pi_fake_'.$i, 'is_subcription' => 0, 'locked_price' => $extension['price'], 'subscription_billing_period' => null, 'subscription_model' => null, 'meta' => null, 'created_at' => ts(300, 86400 * 250), 'updated_at' => ts(10), 'deleted_at' => $i % 40 === 39 ? ts(20) : null]); +} + +// Installs: free listings widely, paid listings where the company bought them, plus a few anomalies. +for ($i = 0; $i < $counts['installs'] && $live; $i++) { + $company = $companies[mt_rand(25, count($companies) - 1)]['uuid']; + $ownedHere = array_keys($owned[$company] ?? []); + $extension = $ownedHere && chance(0.6) ? ['uuid' => pick($ownedHere)] : ($liveFree ? pick($liveFree) : pick($live)); + if ($i % 400 === 399 && $livePaid) { + $extension = pick($livePaid); // paid install without a purchase + } + $add('registry_extension_installs', ['uuid' => fakeUuid(), 'company_uuid' => $company, 'extension_uuid' => $extension['uuid'], 'meta' => null, 'created_at' => ts(250, 86400 * 200), 'updated_at' => ts(5), 'deleted_at' => chance(0.15) ? ts(5, 86400 * 30) : null]); +} + +// --------------------------------------------------------------------------------------------- +// Dump +// --------------------------------------------------------------------------------------------- + +$fixture = file_get_contents(__DIR__.'/../tests/Fixtures/import/cloud-registry.sql'); +$schema = implode("\n", array_filter(explode("\n", $fixture), fn ($line) => ! str_starts_with($line, 'INSERT INTO'))); +$schema = preg_replace('/^-- Dump completed.*$/m', '', $schema); +$schema = str_replace('/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;', '', $schema); + +@mkdir(dirname($out), 0777, true); +$h = fopen($out, 'wb'); +fwrite($h, "-- Fake cloud registry dump for cutover rehearsals (scripts/fake-cloud-dump.php, seed {$seed}, scale {$scale}). Not real data.\n"); +fwrite($h, $schema."\n"); +foreach ($rows as $table => $tableRows) { + foreach (array_chunk($tableRows, 500) as $chunk) { + $columns = array_keys($chunk[0]); + fwrite($h, 'INSERT INTO `'.$table.'` (`'.implode('`,`', $columns).'`) VALUES '.implode(',', array_map(fn ($row) => '('.implode(',', array_map('sqlValue', array_values($row))).')', $chunk)).";\n"); + } +} +fwrite($h, "/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;\n-- Dump completed\n"); +fclose($h); + +$summary = array_map('count', $rows) + ['objects_mb' => round($objectBytes / 1048576, 1), 'missing_tarballs' => $missingObjects, 'dump_mb' => round(filesize($out) / 1048576, 1)]; +echo json_encode($summary, JSON_PRETTY_PRINT)."\n"; diff --git a/scripts/rehearse-cutover.sh b/scripts/rehearse-cutover.sh new file mode 100755 index 0000000..c212ba9 --- /dev/null +++ b/scripts/rehearse-cutover.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Rehearses the import half of the cutover (docs/cutover.md) against a seeded fake cloud dump and +# times each step: two dry runs, the import, --verify, a second import (must change nothing), +# --verify again, then the dump path the runbook uses (cloud-registry-dump.sh, then a dry run that +# reads the dump). Everything runs locally against MySQL and an S3-compatible store (MinIO); it never +# touches AWS. +# +# docker run -d --name rehearsal-mysql -e MYSQL_ROOT_PASSWORD=root -p 33307:3306 mysql:8.4 +# docker run -d --name rehearsal-minio -e MINIO_ROOT_USER=minio -e MINIO_ROOT_PASSWORD=miniosecret -p 19000:9000 minio/minio server /data +# SCALE=1 scripts/rehearse-cutover.sh +# +# Settings (environment): SCALE (dump size, default 1), SEED (42), MYSQL_HOST (127.0.0.1), +# MYSQL_PORT (33307), MYSQL_SOCKET (empty; a Unix socket is used when set, which is closer to an +# in-VPC round trip than macOS loopback TCP), MYSQL_ROOT_PASSWORD (root), S3_ENDPOINT (http://127.0.0.1:19000; empty for +# local folders as both the cloud bucket and service storage), S3_KEY (minio), S3_SECRET +# (miniosecret). Results: storage/rehearsal/timings-.json. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SCALE="${SCALE:-1}" +SEED="${SEED:-42}" +STATE="$ROOT/storage/rehearsal/scale-$SCALE" +MYSQL_HOST="${MYSQL_HOST:-127.0.0.1}" +MYSQL_PORT="${MYSQL_PORT:-33307}" +MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-root}" +MYSQL_SOCKET="${MYSQL_SOCKET:-}" +SOURCE_DB="cloud_rehearsal" +TARGET_DB="registry_rehearsal" +S3_ENDPOINT="${S3_ENDPOINT-http://127.0.0.1:19000}" + +# Production images have no Xdebug; a loaded one slows every call +export APP_ENV=production APP_DEBUG=false XDEBUG_MODE=off +export APP_KEY="${APP_KEY:-base64:$(php -r 'echo base64_encode(random_bytes(32));')}" +export DB_CONNECTION=mysql DB_HOST="$MYSQL_HOST" DB_PORT="$MYSQL_PORT" DB_SOCKET="$MYSQL_SOCKET" DB_DATABASE="$TARGET_DB" DB_USERNAME=root DB_PASSWORD="$MYSQL_ROOT_PASSWORD" +export CACHE_STORE=database QUEUE_CONNECTION=sync MAIL_MAILER=log LOG_LEVEL=warning +if [ -n "$S3_ENDPOINT" ]; then + export REGISTRY_STORAGE_DISK=s3 AWS_ACCESS_KEY_ID="${S3_KEY:-minio}" AWS_SECRET_ACCESS_KEY="${S3_SECRET:-miniosecret}" AWS_DEFAULT_REGION=us-east-1 + export AWS_BUCKET=registry-rehearsal AWS_ENDPOINT="$S3_ENDPOINT" AWS_USE_PATH_STYLE_ENDPOINT=true + export IMPORT_SOURCE_AWS_BUCKET=cloud-rehearsal IMPORT_SOURCE_AWS_ENDPOINT="$S3_ENDPOINT" IMPORT_SOURCE_AWS_USE_PATH_STYLE_ENDPOINT=true +else + export REGISTRY_STORAGE_DISK=local IMPORT_SOURCE_DRIVER=local IMPORT_SOURCE_ROOT="$STATE/files" +fi +export REGISTRY_CLOUD_HOST="https://api.fleetbase.test" + +MYSQL=(mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -uroot "-p$MYSQL_ROOT_PASSWORD" --protocol=tcp) +export CLOUD_DB_HOST="$MYSQL_HOST" CLOUD_DB_PORT="$MYSQL_PORT" CLOUD_DB_USER=root CLOUD_DB_NAME="$SOURCE_DB" MYSQL_PWD="$MYSQL_ROOT_PASSWORD" +FROM="mysql://root:$MYSQL_ROOT_PASSWORD@$MYSQL_HOST:$MYSQL_PORT/$SOURCE_DB${MYSQL_SOCKET:+?unix_socket=$MYSQL_SOCKET}" +cd "$ROOT" +php artisan config:clear >/dev/null + +now() { php -r 'echo microtime(true);'; } +TIMINGS=() +step() { + local name="$1"; shift + echo "==> $name" + local started; started="$(now)" + local code=0 + "$@" > "$STATE/$name.log" 2>&1 || code=$? + local seconds; seconds="$(php -r "printf('%.1f', $(now) - $started);")" + echo " ${seconds}s (exit $code)" + TIMINGS+=("{\"step\": \"$name\", \"seconds\": $seconds, \"exit\": $code}") +} + +rm -rf "$STATE" && mkdir -p "$STATE" +[ -z "$S3_ENDPOINT" ] && rm -rf "$ROOT"/storage/app/private/{tarballs,icons,screenshots}/imported + +echo "==> Fake cloud dump (scale $SCALE, seed $SEED)" +php scripts/fake-cloud-dump.php --out="$STATE/cloud.sql" --files="$STATE/files" --scale="$SCALE" --seed="$SEED" | tee "$STATE/dump.json" + +echo "==> Source database and bucket" +"${MYSQL[@]}" -e "DROP DATABASE IF EXISTS $SOURCE_DB; CREATE DATABASE $SOURCE_DB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; DROP DATABASE IF EXISTS $TARGET_DB; CREATE DATABASE $TARGET_DB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" 2>/dev/null +"${MYSQL[@]}" "$SOURCE_DB" < "$STATE/cloud.sql" 2>/dev/null +if [ -z "$S3_ENDPOINT" ]; then + echo " local folders: $STATE/files -> storage/app/private" +else + # shellcheck disable=SC2016 # PHP code, not shell + php -r ' +require "vendor/autoload.php"; +$app = require "bootstrap/app.php"; +$app->make(Illuminate\Contracts\Console\Kernel::class)->bootstrap(); +foreach (["registry-rehearsal" => "s3", "cloud-rehearsal" => "import_source"] as $bucket => $disk) { + $client = Illuminate\Support\Facades\Storage::disk($disk)->getClient(); + if (! $client->doesBucketExist($bucket)) { $client->createBucket(["Bucket" => $bucket]); } +} +$dir = $argv[1]; +$it = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir, FilesystemIterator::SKIP_DOTS)); +$n = 0; +foreach ($it as $file) { + $path = substr($file->getPathname(), strlen($dir) + 1); + Illuminate\Support\Facades\Storage::disk("import_source")->writeStream($path, fopen($file->getPathname(), "rb")); + $n++; +} +echo " uploaded $n objects\n"; +' "$STATE/files" +fi + +echo "==> Fresh registry database" +php artisan migrate:fresh --seed --force > "$STATE/migrate.log" 2>&1 || { tail -20 "$STATE/migrate.log"; exit 1; } + +step dry-run-1 php artisan registry:import --from="$FROM" --dry-run --report="$STATE/dry-run-1.json" +step dry-run-2 php artisan registry:import --from="$FROM" --dry-run --report="$STATE/dry-run-2.json" +step import php artisan registry:import --from="$FROM" --report="$STATE/import.json" +step verify php artisan registry:import --from="$FROM" --verify +step import-again php artisan registry:import --from="$FROM" --report="$STATE/import-again.json" +step verify-again php artisan registry:import --from="$FROM" --verify --seed=1 +step cloud-dump scripts/cloud-registry-dump.sh "$STATE/cloud-registry.sql.gz" +step dry-run-from-dump php artisan registry:import --from="$STATE/cloud-registry.sql.gz" --dry-run --report="$STATE/dry-run-from-dump.json" + +printf '{"scale": %s, "seed": %s, "dump": %s, "steps": [%s]}\n' "$SCALE" "$SEED" "$(cat "$STATE/dump.json")" "$(IFS=,; echo "${TIMINGS[*]}")" > "$ROOT/storage/rehearsal/timings-$SCALE.json" +echo "Timings: storage/rehearsal/timings-$SCALE.json; logs and reports: $STATE"