From 0a5061ac75433cce257e81b89190bbd72b4e2655 Mon Sep 17 00:00:00 2001 From: Justin Persaud Date: Thu, 22 Oct 2026 11:21:34 -0400 Subject: [PATCH 1/4] Give the production Worker its previous builds bucket Tabs still running an older production build can then load its chunks after a deploy, as on staging. --- web/cloudflare-worker/tests/worker.test.ts | 22 +++++++++++++++++----- web/cloudflare-worker/wrangler.jsonc | 6 ++++-- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/web/cloudflare-worker/tests/worker.test.ts b/web/cloudflare-worker/tests/worker.test.ts index 09997a2c11b4..9d83ed351213 100644 --- a/web/cloudflare-worker/tests/worker.test.ts +++ b/web/cloudflare-worker/tests/worker.test.ts @@ -335,10 +335,12 @@ describe('test-staging worker', () => { }); describe('production worker', () => { - const {server, cleanup} = createWorkerHarness('production'); + const {server, addPreviousBuildFiles, cleanup} = createWorkerHarness('production'); + const PREVIOUS_CHUNK_PATH = '/main-aaaaaaaaaaaaaaaa.bundle.js'; before(async () => { await server.listen(); + await addPreviousBuildFiles([{path: PREVIOUS_CHUNK_PATH.slice(1), contents: 'window.previousBuildChunk = true;', contentType: 'text/javascript'}]); }); after(async () => { @@ -357,12 +359,22 @@ describe('production worker', () => { assertTemplateInlineScriptsAllowed(csp); }); - test('returns 404 for a missing chunk without a previous builds bucket', async () => { - // Given production has no PREVIOUS_BUILDS binding yet - // When a chunk the current build doesn't have is requested + test('serves a chunk the current build no longer has from a previous build', async () => { + // Given a tab still running an older production build, whose chunk is in the production bucket + // When the tab lazy-loads it after a newer deploy + const response = await server.fetch(PREVIOUS_CHUNK_PATH); + + // Then it loads from the previous build + assert.equal(response.status, 200); + assert.equal(await response.text(), 'window.previousBuildChunk = true;'); + }); + + test('returns 404 for a chunk no build has', async () => { + // Given a chunk that is neither in the current build nor in the bucket + // When it is requested const response = await server.fetch('/main-ffffffffffffffff.bundle.js'); - // Then it is a 404 rather than an error from the missing binding + // Then it is a 404, so the tab gets a ChunkLoadError and recovers assert.equal(response.status, 404); }); }); diff --git a/web/cloudflare-worker/wrangler.jsonc b/web/cloudflare-worker/wrangler.jsonc index cee869f80266..eb7db520dab9 100644 --- a/web/cloudflare-worker/wrangler.jsonc +++ b/web/cloudflare-worker/wrangler.jsonc @@ -28,14 +28,14 @@ // Every response needs security headers, and HTML needs a per-request CSP nonce, including SPA navigations. "run_worker_first": true, }, - // Files from previous staging builds, served when a tab running an older build requests a file the current build + // Files from previous builds, served when a tab running an older build requests a file the current build // doesn't have. `wrangler dev` uses an empty local bucket. "r2_buckets": [{"binding": "PREVIOUS_BUILDS", "bucket_name": "new-expensify-web-assets-staging"}], "vars": { "ENVIRONMENT": "staging", }, // Routes are managed in Cloudflare, not here: syncing routes needs zone access, and the CI deploy token can only - // edit the test-staging and staging Workers. + // edit these Workers. "env": { // Serves a staging build on test-staging.new.expensify.com, a hostname nobody else uses. "test-staging": { @@ -61,8 +61,10 @@ "ENVIRONMENT": "staging", }, }, + // Serves nothing on new.expensify.com until new.expensify.com/* is routed to this Worker. "production": { "name": "new-expensify-assets", + "r2_buckets": [{"binding": "PREVIOUS_BUILDS", "bucket_name": "new-expensify-web-assets-production"}], "observability": { "enabled": true, "head_sampling_rate": 0.01, From c29cc992db28145bd2d11ca08719faa5b0f21eb1 Mon Sep 17 00:00:00 2001 From: Justin Persaud Date: Thu, 22 Oct 2026 11:21:38 -0400 Subject: [PATCH 2/4] Deploy production web builds to the production Cloudflare Worker The Worker deploy job now runs for production too: it uploads the build to the production previous builds bucket, deploys the production Worker and checks the version on WEB_PRODUCTION_WORKER_HOST. Until that variable is set to WEB_PRODUCTION_HOST at switchover, webDeploy keeps checking WEB_PRODUCTION_HOST, so each host is checked once, by the job that serves it. --- .github/workflows/deploy.yml | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d2cebadd8ba8..7588a1bb1989 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -769,9 +769,9 @@ jobs: CF_API_KEY: ${{ secrets.CLOUDFLARE_TOKEN }} HOST: ${{ env.DEPLOY_ENV == 'production' && vars.WEB_PRODUCTION_HOST || vars.WEB_STAGING_HOST }} - # The staging host is verified by webDeployStagingWorker, which deploys after this job. + # webDeployWorker verifies staging, and verifies production once WEB_PRODUCTION_WORKER_HOST is set to WEB_PRODUCTION_HOST. - name: Verify deploy - if: ${{ env.DEPLOY_ENV == 'production' }} + if: ${{ env.DEPLOY_ENV == 'production' && vars.WEB_PRODUCTION_WORKER_HOST != vars.WEB_PRODUCTION_HOST }} run: | APP_VERSION=$(jq -r .version < package.json) ./.github/scripts/verifyDeploy.sh "$HOST" "$APP_VERSION" @@ -836,11 +836,11 @@ jobs: env: HOST: https://test-staging.new.expensify.com - webDeployStagingWorker: - name: Deploy Web to the staging Cloudflare Worker + webDeployWorker: + name: Deploy Web to the Cloudflare Worker # After webDeploy, so the S3 copy of this build is in place before the Worker's version check. needs: [prep, webBuild, buildStorybook, webDeploy] - if: ${{ always() && needs.webBuild.result == 'success' && needs.prep.outputs.DEPLOY_ENV == 'staging' }} + if: ${{ always() && needs.webBuild.result == 'success' }} runs-on: blacksmith-2vcpu-ubuntu-2404 steps: - name: Checkout @@ -876,7 +876,7 @@ jobs: fi aws configure set default.s3.preferred_transfer_client classic # Re-uploads every file, so files that stay the same across builds don't hit the bucket's 60-day expiry while still in use. - aws s3 cp dist s3://new-expensify-web-assets-staging/ --recursive \ + aws s3 cp dist "s3://new-expensify-web-assets-$DEPLOY_ENV/" --recursive \ --endpoint-url "https://$CLOUDFLARE_ACCOUNT_ID.r2.cloudflarestorage.com" \ --no-progress \ --exclude '*.br' --exclude '*.map' @@ -900,7 +900,7 @@ jobs: - name: Deploy to Cloudflare Workers working-directory: web/cloudflare-worker - run: npx wrangler deploy --env staging + run: npx wrangler deploy --env "$DEPLOY_ENV" env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_WORKERS_DEPLOY_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} @@ -908,10 +908,15 @@ jobs: - name: Verify deploy run: | + if [[ -z "$HOST" ]]; then + echo "::error::No host to verify this deploy on. Set the WEB_PRODUCTION_WORKER_HOST repository variable." + exit 1 + fi APP_VERSION=$(jq -r .version < package.json) ./.github/scripts/verifyDeploy.sh "$HOST" "$APP_VERSION" env: - HOST: ${{ vars.WEB_STAGING_HOST }} + # Until production switches over, the production Worker is only served on WEB_PRODUCTION_WORKER_HOST. + HOST: ${{ env.DEPLOY_ENV == 'production' && vars.WEB_PRODUCTION_WORKER_HOST || vars.WEB_STAGING_HOST }} buildStorybook: name: Build storybook docs @@ -960,7 +965,7 @@ jobs: iosSubmit, webBuild, webDeploy, - webDeployStagingWorker, + webDeployWorker, victoryChartRendererBuild, ] steps: @@ -976,7 +981,7 @@ jobs: name: Post a Slack message when the deploy is cancelled manually runs-on: blacksmith-2vcpu-ubuntu-2404 if: ${{ cancelled() }} - needs: [androidBuild, androidUploadGooglePlay, androidSubmit, iosBuild, iosUploadTestflight, iosSubmit, webBuild, webDeploy, webDeployStagingWorker] + needs: [androidBuild, androidUploadGooglePlay, androidSubmit, iosBuild, iosUploadTestflight, iosSubmit, webBuild, webDeploy, webDeployWorker] steps: - name: Check whether a newer deploy run exists id: check @@ -1015,7 +1020,7 @@ jobs: ANDROID_RESULT: ${{ steps.platformResults.outputs.ANDROID_RESULT }} IOS_RESULT: ${{ steps.platformResults.outputs.IOS_RESULT }} WEB_RESULT: ${{ steps.platformResults.outputs.WEB_RESULT }} - needs: [androidBuild, androidUploadGooglePlay, androidSubmit, iosBuild, iosUploadTestflight, iosSubmit, webBuild, webDeploy, webDeployStagingWorker, victoryChartRendererBuild] + needs: [androidBuild, androidUploadGooglePlay, androidSubmit, iosBuild, iosUploadTestflight, iosSubmit, webBuild, webDeploy, webDeployWorker, victoryChartRendererBuild] if: ${{ always() }} steps: # Determine effective result for each platform @@ -1050,11 +1055,11 @@ jobs: echo "IOS_RESULT=${{ needs.iosUploadTestflight.result }}" >> "$GITHUB_OUTPUT" fi - # Web: propagate build failure even when deploy is skipped. Staging is only deployed once its Worker is too. + # Web: propagate build failure even when deploy is skipped. Web is only deployed once its Worker is too. if [ "${{ needs.webBuild.result }}" == "failure" ]; then echo "WEB_RESULT=failure" >> "$GITHUB_OUTPUT" - elif [ "${{ env.DEPLOY_ENV }}" == "staging" ] && [ "${{ needs.webDeployStagingWorker.result }}" != "success" ]; then - echo "WEB_RESULT=${{ needs.webDeployStagingWorker.result }}" >> "$GITHUB_OUTPUT" + elif [ "${{ needs.webDeployWorker.result }}" != "success" ]; then + echo "WEB_RESULT=${{ needs.webDeployWorker.result }}" >> "$GITHUB_OUTPUT" else echo "WEB_RESULT=${{ needs.webDeploy.result }}" >> "$GITHUB_OUTPUT" fi From f4cb23a7895ae59ec86e35d35167e8dfcad9292e Mon Sep 17 00:00:00 2001 From: Justin Persaud Date: Thu, 22 Oct 2026 11:21:41 -0400 Subject: [PATCH 3/4] Document production Worker deploys --- web/cloudflare-worker/README.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/web/cloudflare-worker/README.md b/web/cloudflare-worker/README.md index c2cd4be84b00..dc32426c00b3 100644 --- a/web/cloudflare-worker/README.md +++ b/web/cloudflare-worker/README.md @@ -108,14 +108,14 @@ The script diffs status, caching and security headers for a fixed set of paths. | Environment | Worker | Route | CSP | | --- | --- | --- | --- | | `test-staging` | `test-staging-new-expensify-assets` | `test-staging.new.expensify.com/*` (managed in Cloudflare, not in `wrangler.jsonc`) | Staging | -| `staging` | `staging-new-expensify-assets` | None until switchover, then `staging.new.expensify.com/*` (managed in Cloudflare, not in `wrangler.jsonc`) | Staging | -| `production` | `new-expensify-assets` | None until switchover, then `new.expensify.com/*` | Production | +| `staging` | `staging-new-expensify-assets` | `staging.new.expensify.com/*` (managed in Cloudflare, not in `wrangler.jsonc`) | Staging | +| `production` | `new-expensify-assets` | The host in the `WEB_PRODUCTION_WORKER_HOST` repository variable until switchover, then `new.expensify.com/*` (managed in Cloudflare, not in `wrangler.jsonc`) | Production | -Every staging deploy also deploys the same build to the `staging` Worker (the `webDeployStagingWorker` job in `.github/workflows/deploy.yml`) and the `test-staging` Worker (`webDeployTestStaging`). Both use `CLOUDFLARE_WORKERS_DEPLOY_TOKEN`, which can only edit those two Workers. That token can't read zones, so routes aren't in `wrangler.jsonc`: wrangler would try to sync them and fail. +Every staging and production deploy also deploys the same build to that environment's Worker (the `webDeployWorker` job in `.github/workflows/deploy.yml`). Staging deploys also deploy it to the `test-staging` Worker (`webDeployTestStaging`). They use `CLOUDFLARE_WORKERS_DEPLOY_TOKEN`, which can only edit these three Workers. That token can't read zones, so routes aren't in `wrangler.jsonc`: wrangler would try to sync them and fail. -The `staging` deploy runs after the S3 upload and is required: if it fails, or `staging.new.expensify.com/version.json` doesn't report the new version, the staging deploy fails. The `test-staging` deploy is a trial, so its failures don't affect the staging deploy. +The Worker deploy runs after `webDeploy` and is required: if it fails, or the Worker doesn't report the new version in `/version.json`, the deploy fails. Staging is checked on `WEB_STAGING_HOST` and production on `WEB_PRODUCTION_WORKER_HOST`. While `WEB_PRODUCTION_WORKER_HOST` differs from `WEB_PRODUCTION_HOST`, the `webDeploy` job also checks `WEB_PRODUCTION_HOST`; setting both to `new.expensify.com` at switchover leaves the check to the Worker job. The `test-staging` deploy is a trial, so its failures don't affect the staging deploy. -`test-staging` serves a staging build on a hostname nobody else uses. `staging` serves nothing until `staging.new.expensify.com/*` is pointed at its Worker in Cloudflare. +`test-staging` serves a staging build on a hostname nobody else uses. To deploy one by hand, put a staging build in `dist/` and run: @@ -127,11 +127,9 @@ CI also adds Storybook to `dist/docs` before deploying, so it is served at `/doc ### Files from previous builds -Each Worker version only contains the files uploaded with it, but a tab still running an older build keeps requesting that build's chunks after a deploy. So every staging deploy also uploads its build (without `*.br`, `*.map` and Storybook) to the R2 bucket `new-expensify-web-assets-staging`. Every file is re-uploaded each time, so a file still used by recent builds never reaches the bucket's 60-day expiry. When the current build doesn't have a file that a script or image requests, the `test-staging` and `staging` Workers look it up in that bucket (the `PREVIOUS_BUILDS` binding) and return `404` only if it isn't there either. The current build always takes priority, and navigations never read from the bucket. +Each Worker version only contains the files uploaded with it, but a tab still running an older build keeps requesting that build's chunks after a deploy. So every staging and production deploy also uploads its build (without `*.br`, `*.map` and Storybook) to that environment's R2 bucket, `new-expensify-web-assets-staging` or `new-expensify-web-assets-production`. Every file is re-uploaded each time, so a file still used by recent builds never reaches the bucket's 60-day expiry. When the current build doesn't have a file that a script or image requests, the Worker looks it up in its environment's bucket (the `PREVIOUS_BUILDS` binding; `test-staging` uses the staging one) and returns `404` only if it isn't there either. The current build always takes priority, and navigations never read from the bucket. -The bucket and its 60-day lifecycle rule are managed in Terraform. The upload uses `CLOUDFLARE_R2_WEB_ASSETS_ACCESS_KEY_ID` and `CLOUDFLARE_R2_WEB_ASSETS_SECRET_ACCESS_KEY`, an R2 token that can only read and write that bucket. `wrangler dev` uses an empty local bucket. - -Staging and production deploys will run `wrangler deploy --env staging|production` from CI, with `CLOUDFLARE_ACCOUNT_ID` and a scoped `CLOUDFLARE_API_TOKEN` supplied as environment secrets. +The buckets and their 60-day lifecycle rule are managed in Terraform. The upload uses `CLOUDFLARE_R2_WEB_ASSETS_ACCESS_KEY_ID` and `CLOUDFLARE_R2_WEB_ASSETS_SECRET_ACCESS_KEY`, an R2 token that can only read and write those buckets. `wrangler dev` uses an empty local bucket. Workers Logs and traces are sampled in `staging` and `production` (`head_sampling_rate`), because the Worker runs for every asset request. `test-staging` logs and traces every request. From b875dbcb67848e14e3acac9fc99004e09d3eb9ac Mon Sep 17 00:00:00 2001 From: Justin Persaud Date: Thu, 22 Oct 2026 11:53:47 -0400 Subject: [PATCH 4/4] Verify production on WEB_PRODUCTION_HOST after the Worker deploys Both jobs have deployed by the time the Worker job runs, so checking new.expensify.com there works before and after it is routed to the Worker, without a separate host for the production Worker. --- .github/workflows/deploy.yml | 16 ++-------------- web/cloudflare-worker/README.md | 4 ++-- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7588a1bb1989..9c1e209c32ae 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -769,14 +769,7 @@ jobs: CF_API_KEY: ${{ secrets.CLOUDFLARE_TOKEN }} HOST: ${{ env.DEPLOY_ENV == 'production' && vars.WEB_PRODUCTION_HOST || vars.WEB_STAGING_HOST }} - # webDeployWorker verifies staging, and verifies production once WEB_PRODUCTION_WORKER_HOST is set to WEB_PRODUCTION_HOST. - - name: Verify deploy - if: ${{ env.DEPLOY_ENV == 'production' && vars.WEB_PRODUCTION_WORKER_HOST != vars.WEB_PRODUCTION_HOST }} - run: | - APP_VERSION=$(jq -r .version < package.json) - ./.github/scripts/verifyDeploy.sh "$HOST" "$APP_VERSION" - env: - HOST: ${{ vars.WEB_PRODUCTION_HOST }} + # The host is verified by webDeployWorker, which deploys after this job. webDeployTestStaging: name: Deploy Web to test-staging.new.expensify.com @@ -908,15 +901,10 @@ jobs: - name: Verify deploy run: | - if [[ -z "$HOST" ]]; then - echo "::error::No host to verify this deploy on. Set the WEB_PRODUCTION_WORKER_HOST repository variable." - exit 1 - fi APP_VERSION=$(jq -r .version < package.json) ./.github/scripts/verifyDeploy.sh "$HOST" "$APP_VERSION" env: - # Until production switches over, the production Worker is only served on WEB_PRODUCTION_WORKER_HOST. - HOST: ${{ env.DEPLOY_ENV == 'production' && vars.WEB_PRODUCTION_WORKER_HOST || vars.WEB_STAGING_HOST }} + HOST: ${{ env.DEPLOY_ENV == 'production' && vars.WEB_PRODUCTION_HOST || vars.WEB_STAGING_HOST }} buildStorybook: name: Build storybook docs diff --git a/web/cloudflare-worker/README.md b/web/cloudflare-worker/README.md index dc32426c00b3..02ed14cba170 100644 --- a/web/cloudflare-worker/README.md +++ b/web/cloudflare-worker/README.md @@ -109,11 +109,11 @@ The script diffs status, caching and security headers for a fixed set of paths. | --- | --- | --- | --- | | `test-staging` | `test-staging-new-expensify-assets` | `test-staging.new.expensify.com/*` (managed in Cloudflare, not in `wrangler.jsonc`) | Staging | | `staging` | `staging-new-expensify-assets` | `staging.new.expensify.com/*` (managed in Cloudflare, not in `wrangler.jsonc`) | Staging | -| `production` | `new-expensify-assets` | The host in the `WEB_PRODUCTION_WORKER_HOST` repository variable until switchover, then `new.expensify.com/*` (managed in Cloudflare, not in `wrangler.jsonc`) | Production | +| `production` | `new-expensify-assets` | None until switchover, then `new.expensify.com/*` (managed in Cloudflare, not in `wrangler.jsonc`) | Production | Every staging and production deploy also deploys the same build to that environment's Worker (the `webDeployWorker` job in `.github/workflows/deploy.yml`). Staging deploys also deploy it to the `test-staging` Worker (`webDeployTestStaging`). They use `CLOUDFLARE_WORKERS_DEPLOY_TOKEN`, which can only edit these three Workers. That token can't read zones, so routes aren't in `wrangler.jsonc`: wrangler would try to sync them and fail. -The Worker deploy runs after `webDeploy` and is required: if it fails, or the Worker doesn't report the new version in `/version.json`, the deploy fails. Staging is checked on `WEB_STAGING_HOST` and production on `WEB_PRODUCTION_WORKER_HOST`. While `WEB_PRODUCTION_WORKER_HOST` differs from `WEB_PRODUCTION_HOST`, the `webDeploy` job also checks `WEB_PRODUCTION_HOST`; setting both to `new.expensify.com` at switchover leaves the check to the Worker job. The `test-staging` deploy is a trial, so its failures don't affect the staging deploy. +The Worker deploy runs after `webDeploy` and is required: if it fails, or `WEB_STAGING_HOST` or `WEB_PRODUCTION_HOST` doesn't report the new version in `/version.json`, the deploy fails. The version is checked there because both jobs have deployed by then, whichever one serves the host. The `test-staging` deploy is a trial, so its failures don't affect the staging deploy. `test-staging` serves a staging build on a hostname nobody else uses.