Restore back local-dist flow and e2e based on serialized data

.env.docker.example

@@ -1,8 +1,18 @@
+MB_PORT=4300
 CLIENT_PORT=4400
 API_PORT=4500
 
-MB_INSTANCE_URL="https://shoppy.metabaseapp.com"
-METABASE_JWT_SHARED_SECRET="your_secret_here"
+PREMIUM_EMBEDDING_TOKEN="<enterprise_token>"
+METABASE_JWT_SHARED_SECRET="ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
+
+METABASE_ADMIN_EMAIL: "admin@example.com"
+METABASE_ADMIN_PASSWORD="foobarbaz"
+
+METABASE_APP_DB_HOST: "shoppy_app_db"
+METABASE_APP_DB_PORT: "5432"
+METABASE_APP_DB: "shoppy_app"
+METABASE_APP_DB_USER: "shoppy_app"
+METABASE_APP_DB_PASSWORD: "foobarbaz"
 
 SHOPPY_DB_HOST="shoppy_db"
 SHOPPY_DB_PORT="5432"
@@ -11,3 +21,14 @@ SHOPPY_DB_USER="shoppy"
 SHOPPY_DB_PASSWORD="foobarbaz"
 
 WATCH="false"
+
+# for development
+# MB_RUN_MODE="dev"
+# METASTORE_DEV_SERVER_URL=""
+
+# Datadog RUM (optional client-side monitoring; leave empty to disable).
+DATADOG_APPLICATION_ID=""
+DATADOG_CLIENT_TOKEN=""
+DATADOG_SITE=""
+DATADOG_SERVICE=""
+DATADOG_ENV=""

.github/workflows/deploy-production.yml

@@ -27,10 +27,6 @@ jobs:
           role-session-name: GitHub_to_AWS_via_FederatedOIDC
           aws-region: us-east-1
 
-      - name: Get runner public IP
-        id: runner-ip
-        run: echo "ip=$(curl -s https://checkip.amazonaws.com)" >> "$GITHUB_OUTPUT"
-
       - name: Ensure that Esno executable is ready
         run: yarn install --cwd api --frozen-lockfile
 

.github/workflows/e2e-tests.yml

@@ -14,20 +14,29 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 15
     name: e2e-tests
+    env:
+      # The local Metabase runs EE in dev mode and validates its token against the metastore.
+      PREMIUM_EMBEDDING_TOKEN: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}
+      MB_RUN_MODE: dev
+      METASTORE_DEV_SERVER_URL: ${{ secrets.METASTORE_DEV_SERVER_URL }}
     permissions:
       contents: read
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Prepare env
+      - name: Run Shoppy in Docker (local Metabase seeded from serialized data)
         run: |
           cp .env.docker.example .env.docker
-          # The api signs SSO JWTs that production Metabase must trust, so use the production secret.
-          sed -i "s|^METABASE_JWT_SHARED_SECRET=.*|METABASE_JWT_SHARED_SECRET=\"${{ secrets.SHOPPY_PROD_JWT_SHARED_SECRET }}\"|" .env.docker
+          yarn docker:e2e:up --wait
 
-      - name: Run Shoppy (client + api + warehouse) against production Metabase
-        run: yarn docker:e2e:up --wait
+      - name: Dump container logs on failure
+        if: failure()
+        run: |
+          echo "=== shoppy-metabase-1 ===" && docker logs shoppy-metabase-1 2>&1 || true
+          echo "=== shoppy-api-1 ===" && docker logs shoppy-api-1 2>&1 || true
+          echo "=== shoppy-client-1 ===" && docker logs shoppy-client-1 2>&1 || true
+          echo "=== container statuses ===" && docker ps -a 2>&1 || true
 
       - name: Install Chrome v111
         uses: browser-actions/setup-chrome@v1
@@ -43,13 +52,6 @@ jobs:
         id: run-e2e-tests
         run: cd e2e && yarn cypress:run
 
-      - name: Dump container logs on failure
-        if: failure()
-        run: |
-          echo "=== shoppy-client-1 ===" && docker logs shoppy-client-1 2>&1 || true
-          echo "=== shoppy-api-1 ===" && docker logs shoppy-api-1 2>&1 || true
-          echo "=== container statuses ===" && docker ps -a 2>&1 || true
-
       - name: Upload Cypress Artifacts upon failure
         uses: actions/upload-artifact@v4
         if: ${{ steps.run-e2e-tests.outcome != 'success' }}

.gitignore

@@ -5,6 +5,10 @@ node_modules
 .pnp
 .pnp.js
 
+# local dist
+local-dist/*
+!local-dist/.gitkeep
+
 # testing
 coverage
 cypress
@@ -13,6 +17,11 @@ cypress
 dist
 build
 
+# db dumps
+*.sql
+# but allow migration files
+!api/drizzle/**/*.sql
+
 # misc
 .DS_Store
 .env.development.local

Dockerfile

@@ -39,6 +39,13 @@ RUN yarn --frozen-lockfile
 # Copy source code last (changes most frequently)
 COPY --exclude=./api --exclude=./metabase . .
 
+RUN if [ -d "./local-dist/embedding-sdk" ]; then \
+      echo "Local embedding-sdk dist is found in ./local-dist/embedding-sdk, installing it..."; \
+      yarn add file:./local-dist/embedding-sdk; \
+    else \
+      echo "Local embedding-sdk dist is not found in ./local-dist/embedding-sdk, skipping copy"; \
+    fi
+
 RUN if [ "$WATCH" != "true" ]; then \
       echo "WATCH env is not set; running production yarn build..."; \
       yarn build; \

README.md

@@ -32,13 +32,22 @@ This demo uses the data from the hosted Metabase Cloud instance and provides a h
 
 ### Using Docker
 
-- Clone `.env.docker.example` to `.env.docker`. Set `MB_INSTANCE_URL` to your Metabase instance and `METABASE_JWT_SHARED_SECRET` to that instance's JWT shared secret.
-- Run `yarn docker:up` for a production build, or `WATCH=true yarn docker:up` for a dev build with watch.
-  - The command launches the Shoppy DWH, API and Client, pointing at the configured Metabase instance.
+- Clone `.env.docker.example` to `.env.docker` and set the proper `PREMIUM_EMBEDDING_TOKEN` value.
+- Run Docker via `yarn docker:up` for the `production` build or `WATCH=true yarn docker:up` for the development build with the `watch` support.
+  - The command launches a local MB instance, the Shoppy DWH, Shoppy API and Shoppy Client. The local Metabase is seeded from the committed serialized data (`./metabase/metabase_data.tar.gz`), so no database dump is needed.
   - Visit `http://localhost:4400`.
 - To stop containers run `yarn docker:down`.
 - To remove containers and images completely run `yarn docker:rm`.
 
+#### Local development (For Metabase developers)
+
+To run the containers with a locally built `metabase.jar` and/or a locally built Embedding SDK:
+
+- Copy a locally built `metabase.jar` to `./local-dist/metabase.jar`. Without it, the `metabase.jar` from the Docker image is used.
+- Copy a locally built Embedding SDK package to `./local-dist/embedding-sdk`. Without it, the `@metabase/embedding-sdk-react` version from `package.json` is used.
+- Run `yarn docker:local-dist:up` to start the containers using the local dist from `./local-dist`.
+- To remove containers and images completely run `yarn docker:rm`.
+
 ### Using an existing running MB instance
 
 - Place the metabase repository in `../metabase`
@@ -65,11 +74,11 @@ If you cannot use the hosted JWT server, you can run the JWT server locally.
 
 ### Running e2e tests (For Metabase developers)
 
-The e2e tests run the client, api and warehouse locally (via Docker) against the production Metabase instance, and drive the local client with Cypress.
+The e2e tests run the local Shoppy stack (a local Metabase seeded from the committed serialized data) and drive it with Cypress.
 
-- In `.env.docker`, set `METABASE_JWT_SHARED_SECRET` to the production instance's JWT shared secret (from 1Password) so the api's SSO tokens are trusted, and `MB_INSTANCE_URL` to that instance.
-- Start the stack: `yarn docker:e2e:up --wait`.
-- Run the tests: `cd e2e && yarn cypress:run` (headless) or `yarn cypress:open` (Cypress UI).
-- Stop the stack: `yarn docker:down`.
+- Clone `.env.docker.example` to `.env.docker` and set the proper `PREMIUM_EMBEDDING_TOKEN` value.
+- Run `yarn docker:e2e:up --wait` to start all required containers.
+- Run the tests: `cd e2e && yarn cypress:run` (headless) or `cd e2e && yarn cypress:open` (Cypress UI).
+- To stop containers run `yarn docker:rm`.
 
-CI runs the same suite in `.github/workflows/e2e-tests.yml`, injecting the production secret from the `SHOPPY_PROD_JWT_SHARED_SECRET` GitHub secret.
+CI runs the same suite in `.github/workflows/e2e-tests.yml`.

api/src/constants/env.ts

@@ -14,6 +14,3 @@ export const FRONTEND_URL =
   process.env.FRONTEND_URL ?? `http://localhost:${process.env.CLIENT_PORT}`
 
 export const { VERCEL_ENV } = process.env
-
-// This variable is used in a local (Docker) environment only
-export const METABASE_ADMIN_EMAIL = process.env.METABASE_ADMIN_EMAIL

docker-compose.local-dist.yml

@@ -0,0 +1,9 @@
+services:
+  metabase:
+    volumes:
+      - type: bind
+        source: ./local-dist/metabase.jar
+        target: /app/metabase.jar
+        read_only: true
+        bind:
+          create_host_path: false

docker-compose.yml

@@ -1,5 +1,3 @@
-# Shoppy stack: client + api + warehouse. Metabase is the production instance (MB_INSTANCE_URL);
-# the client talks to it same-origin via `/mb`, which the api proxies onward (see api/src/main.ts).
 services:
   shoppy_db:
     image: postgres:latest
@@ -16,8 +14,25 @@ services:
     expose:
       - "${SHOPPY_DB_PORT}"
 
+  shoppy_app_db:
+    image: postgres:latest
+    environment:
+      POSTGRES_PORT: "${METABASE_APP_DB_PORT}"
+      POSTGRES_DB: "${METABASE_APP_DB}"
+      POSTGRES_USER: "${METABASE_APP_DB_USER}"
+      POSTGRES_PASSWORD: "${METABASE_APP_DB_PASSWORD}"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${METABASE_APP_DB_USER}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    expose:
+      - "${METABASE_APP_DB_PORT}"
+
   api:
     depends_on:
+      shoppy_app_db:
+        condition: service_healthy
       shoppy_db:
         condition: service_healthy
     build:
@@ -30,8 +45,9 @@ services:
       CLIENT_PORT: "${CLIENT_PORT}"
       IS_LOCAL_DB: "true"
       DB_URL: "postgres://${SHOPPY_DB_USER}:${SHOPPY_DB_PASSWORD}@${SHOPPY_DB_HOST}:${SHOPPY_DB_PORT}/${SHOPPY_DB}"
-      METABASE_INSTANCE_URL: "${MB_INSTANCE_URL}"
+      METABASE_INSTANCE_URL: "http://metabase:${MB_PORT}"
       METABASE_JWT_SHARED_SECRET: "${METABASE_JWT_SHARED_SECRET}"
+      METABASE_ADMIN_EMAIL: "${METABASE_ADMIN_EMAIL}"
     healthcheck:
       test: curl --fail -X GET -I "http://localhost:${API_PORT}/" || exit 1
       interval: 15s
@@ -40,15 +56,63 @@ services:
     ports:
       - "${API_PORT}:${API_PORT}"
 
+  metabase:
+    depends_on:
+      shoppy_app_db:
+        condition: service_healthy
+      shoppy_db:
+        condition: service_healthy
+      api:
+        condition: service_healthy
+    image: metabase/metabase-enterprise:v1.63.x
+    entrypoint: ["sh", "/app/entrypoint.sh"]
+    environment:
+      MB_CONFIG_FILE_PATH: "./app/config.yml"
+      MB_JETTY_PORT: "${MB_PORT}"
+      MB_EDITION: "ee"
+      MB_SITE_URL: "http://localhost:${MB_PORT}/"
+      MB_PREMIUM_EMBEDDING_TOKEN: "${PREMIUM_EMBEDDING_TOKEN}"
+      MB_RUN_MODE: "${MB_RUN_MODE}"
+      METASTORE_DEV_SERVER_URL: "${METASTORE_DEV_SERVER_URL}"
+      MB_JWT_IDENTITY_PROVIDER_URI: "http://localhost:${API_PORT}/sso/metabase"
+      MB_JWT_SHARED_SECRET: "${METABASE_JWT_SHARED_SECRET}"
+      MB_WAREHOUSE_ALLOWED_NETWORKS: "allow-all"
+      MB_DB_TYPE: "postgres"
+      MB_DB_HOST: "${METABASE_APP_DB_HOST}"
+      MB_DB_PORT: "${METABASE_APP_DB_PORT}"
+      MB_DB_USER: "${METABASE_APP_DB_USER}"
+      MB_DB_PASS: "${METABASE_APP_DB_PASSWORD}"
+      MB_DB_DBNAME: "${METABASE_APP_DB}"
+      METABASE_ADMIN_EMAIL: "${METABASE_ADMIN_EMAIL}"
+      METABASE_ADMIN_PASSWORD: "${METABASE_ADMIN_PASSWORD}"
+      SHOPPY_DB_HOST: "${SHOPPY_DB_HOST}"
+      SHOPPY_DB_PORT: "${SHOPPY_DB_PORT}"
+      SHOPPY_DB: "${SHOPPY_DB}"
+      SHOPPY_DB_USER: "${SHOPPY_DB_USER}"
+      SHOPPY_DB_PASSWORD: "${SHOPPY_DB_PASSWORD}"
+    healthcheck:
+      test: curl --fail -X GET -I "http://localhost:${MB_PORT}/api/health" || exit 1
+      interval: 15s
+      timeout: 5s
+      retries: 10
+    ports:
+      - "${MB_PORT}:${MB_PORT}"
+    volumes:
+      - ./metabase/entrypoint.sh:/app/entrypoint.sh
+      - ./metabase/config.yml:/app/config.yml
+      - ./metabase/metabase_data.tar.gz:/app/metabase_data.tar.gz
+
   client:
     depends_on:
+      metabase:
+        condition: service_healthy
       api:
         condition: service_healthy
     build:
       context: .
       dockerfile: Dockerfile
       args:
-        VITE_APP_METABASE_INSTANCE_URL: "/mb"
+        VITE_APP_METABASE_INSTANCE_URL: "http://localhost:${MB_PORT}"
         VITE_APP_BACKEND_URL: "http://localhost:${API_PORT}"
         VITE_DATADOG_APPLICATION_ID: "${DATADOG_APPLICATION_ID}"
         VITE_DATADOG_CLIENT_TOKEN: "${DATADOG_CLIENT_TOKEN}"
@@ -58,7 +122,7 @@ services:
         WATCH: "${WATCH}"
     environment:
       CLIENT_PORT: "${CLIENT_PORT}"
-      VITE_APP_METABASE_INSTANCE_URL: "/mb"
+      VITE_APP_METABASE_INSTANCE_URL: "http://localhost:${MB_PORT}"
       VITE_APP_BACKEND_URL: "http://localhost:${API_PORT}"
       VITE_APP_DOCKER_OVERRIDE_BACKEND_URL: "http://api:${API_PORT}"
       WATCH: "${WATCH}"

e2e/package.json

@@ -9,6 +9,7 @@
   },
   "scripts": {
     "cypress:open": "env-cmd -f ../.env.docker cypress open --config-file ./support/cypress.config.js --e2e",
-    "cypress:run": "env-cmd -f ../.env.docker cypress run --config-file ./support/cypress.config.js"
+    "cypress:run": "env-cmd -f ../.env.docker cypress run --config-file ./support/cypress.config.js",
+    "cypress:synthetic": "cypress run --config-file ./support/cypress.synthetic.config.js --browser chrome"
   }
 }

e2e/support/helpers/sign-in.js

@@ -0,0 +1,59 @@
+// TODO (Kelvin 2026-07-07) bandage, not a fix (EMB-2059) — see compatibility.cy.spec.js.
+
+export const METABASE_URL = "http://localhost:4300"
+
+// JWT_SHARED_SECRET also appears in .env.docker.example, so it's not a real secret.
+const JWT_SHARED_SECRET =
+  "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
+const ADMIN_EMAIL = "shoppy@metabase.com"
+
+function base64url(bytes) {
+  let binary = ""
+  new Uint8Array(bytes).forEach((byte) => {
+    binary += String.fromCharCode(byte)
+  })
+  return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "")
+}
+
+async function getSignedJwtForAdmin() {
+  const encoder = new TextEncoder()
+  const header = { alg: "HS256", typ: "JWT" }
+  const payload = {
+    email: ADMIN_EMAIL,
+    first_name: "Shoppy",
+    last_name: "Admin",
+    exp: Math.floor(Date.now() / 1000) + 600,
+  }
+  const signingInput = `${base64url(encoder.encode(JSON.stringify(header)))}.${base64url(
+    encoder.encode(JSON.stringify(payload)),
+  )}`
+
+  const key = await crypto.subtle.importKey(
+    "raw",
+    encoder.encode(JWT_SHARED_SECRET),
+    { name: "HMAC", hash: "SHA-256" },
+    false,
+    ["sign"],
+  )
+  const signature = await crypto.subtle.sign(
+    "HMAC",
+    key,
+    encoder.encode(signingInput),
+  )
+
+  return `${signingInput}.${base64url(signature)}`
+}
+
+export function signInAsAdmin() {
+  return cy
+    .wrap(getSignedJwtForAdmin())
+    .then((jwt) =>
+      cy.request({
+        method: "GET",
+        url: `${METABASE_URL}/auth/sso`,
+        qs: { jwt },
+        headers: { "X-Metabase-Client": "embedding-sdk-react" },
+      }),
+    )
+    .then(({ body }) => body.id)
+}