Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 2 additions & 16 deletions .env.docker.example
Original file line number Diff line number Diff line change
@@ -1,18 +1,8 @@
MB_PORT=4300
CLIENT_PORT=4400
API_PORT=4500

PREMIUM_EMBEDDING_TOKEN="<enterprise_token>"
METABASE_JWT_SHARED_SECRET="ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"

METABASE_ADMIN_EMAIL: "admin@example.com"
METABASE_ADMIN_PASSWORD="foobarbaz"

METABASE_APP_DB_HOST: "shoppy_app_db"
METABASE_APP_DB_PORT: "5432"
METABASE_APP_DB: "shoppy_app"
METABASE_APP_DB_USER: "shoppy_app"
METABASE_APP_DB_PASSWORD: "foobarbaz"
MB_INSTANCE_URL="https://shoppy.metabaseapp.com"
METABASE_JWT_SHARED_SECRET="your_secret_here"

SHOPPY_DB_HOST="shoppy_db"
SHOPPY_DB_PORT="5432"
Expand All @@ -21,7 +11,3 @@ SHOPPY_DB_USER="shoppy"
SHOPPY_DB_PASSWORD="foobarbaz"

WATCH="false"

# for development
# MB_RUN_MODE="dev"
# METASTORE_DEV_SERVER_URL=""
2 changes: 1 addition & 1 deletion .env.example
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
PORT=4173
VITE_APP_METABASE_INSTANCE_URL=https://shoppy.coredev.metabase.com
VITE_APP_METABASE_INSTANCE_URL=https://shoppy.metabaseapp.com
VITE_APP_BACKEND_URL=https://embedded-analytics-sdk-demo.metabase.com
95 changes: 16 additions & 79 deletions .github/workflows/deploy-production.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,104 +20,41 @@ jobs:
- name: Checkout code
uses: actions/checkout@v4

- name: Tailscale
uses: tailscale/github-action@v2
with:
oauth-client-id: ${{ secrets.TAILSCALE_OAUTH_CLIENT_ID }}
oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }}
tags: tag:ci
version: 1.50.1
sha256sum: d9fe6b480fb5078f0aa57dace686898dda7e2a768884271159faa74846bfb576

- name: Create OIDC Token
id: create-oidc-token
shell: bash
run: |
export OIDC_URL_WITH_AUDIENCE="$ACTIONS_ID_TOKEN_REQUEST_URL&audience=${{ secrets.K8S_AUDIENCE }}"
IDTOKEN=$(curl -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
-H "Accept: application/json; api-version=2.0" "$OIDC_URL_WITH_AUDIENCE" \
| jq -r .value)
echo "::add-mask::${IDTOKEN}"
echo "idToken=${IDTOKEN}" >>$GITHUB_OUTPUT

- name: Setup Kube Context
uses: azure/k8s-set-context@v2
with:
method: kubeconfig
kubeconfig: |
kind: Config
apiVersion: v1
current-context: default
clusters:
- name: default
cluster:
certificate-authority-data: ${{ secrets.K8S_CERTIFICATE_AUTHORITY_DATA }}
server: ${{ secrets.K8S_SERVER }}
users:
- name: oidc-token
user:
token: ${{ steps.create-oidc-token.outputs.IDTOKEN }}
contexts:
- name: default
context:
cluster: default
namespace: ${{secrets.K8S_NAMESPACE}}
user: oidc-token

- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.IAM_ROLE }}
role-session-name: GitHub_to_AWS_via_FederatedOIDC
aws-region: us-east-1

- name: Setup psql client
- name: Get runner public IP
id: runner-ip
run: echo "ip=$(curl -s https://checkip.amazonaws.com)" >> "$GITHUB_OUTPUT"

- name: Authorize runner IP on the warehouse security group
run: |
sudo apt-get update
sudo apt-get install -y postgresql-client
aws ec2 authorize-security-group-ingress \
--group-id ${{ secrets.SHOPPY_DB_SECURITY_GROUP_ID }} \
--protocol tcp --port ${{ vars.SHOPPY_DB_PORT }} \
--cidr ${{ steps.runner-ip.outputs.ip }}/32

- name: Ensure that Esno executable is ready
run: yarn install --cwd api --frozen-lockfile

- name: Reinitialize Shoppy's DWH DB
env:
DB_URL: postgres://${{ secrets.SHOPPY_DB_USER }}:${{ secrets.SHOPPY_DB_PASSWORD }}@${{ secrets.SHOPPY_DB_HOST }}:${{ vars.SHOPPY_DB_PORT }}/${{ vars.SHOPPY_DB }}
DB_URL: postgres://${{ secrets.SHOPPY_DB_USER }}:${{ secrets.SHOPPY_DB_PASSWORD }}@${{ secrets.SHOPPY_DB_HOST }}:${{ vars.SHOPPY_DB_PORT }}/${{ vars.SHOPPY_DB }}?sslmode=no-verify
run: |
cd api &&
yarn db:initialize

- name: Download PostgreSQL dump from S3
run: |
aws s3 cp s3://${{ secrets.METABASE_APP_DB_S3_BUCKET }}/${{ vars.METABASE_APP_DB_S3_KEY }} dump.sql

- name: Pause Shoppy's Metabase Instance (set pause.enabled=true)
run: |
kubectl patch metabase hosting-shoppy \
--type=merge \
-p '{"spec":{"pause":{"enabled":true}}}'

- name: Drop Shoppy's Metabase Instance app database and restore
run: |
export PGPASSWORD='${{ secrets.APP_DB_PASSWORD }}'
export PGUSER=${{ secrets.APP_DB_USER }}
export PGDATABASE=${{ secrets.APP_DB_DATABASE }}
export PGHOST=${{ secrets.APP_DB_HOST }}
psql -d ${{ secrets.APP_DB_DATABASE }} \
-c "DROP DATABASE IF EXISTS ${{ secrets.APP_DB_SHOPPY_DATABASE }} WITH (FORCE);"
psql -d ${{ secrets.APP_DB_DATABASE }} \
-c "CREATE DATABASE ${{ secrets.APP_DB_SHOPPY_DATABASE }} WITH OWNER ${{ secrets.APP_DB_USER }};"
pg_restore -d ${{ secrets.APP_DB_SHOPPY_DATABASE }} dump.sql

- name: Update Shoppy's Metabase Instance version and unpause
run: |
kubectl patch metabase hosting-shoppy \
--type=merge \
-p '{"spec":{"image":{"repo": "${{ secrets.MB_IMAGE_REPO }}", "name": "${{ vars.MB_IMAGE_NAME }}", "version":"${{ vars.MB_IMAGE_VERSION }}"},"pause":{"enabled":false}}}'

- name: Verify Shoppy's Metabase Instance deployment
- name: Revoke runner IP from the warehouse security group
if: always()
run: |
kubectl get metabase hosting-shoppy -o jsonpath='{.spec.image.version}'
echo "Metabase updated to version ${{ vars.MB_IMAGE_VERSION }} and unpaused"
aws ec2 revoke-security-group-ingress \
--group-id ${{ secrets.SHOPPY_DB_SECURITY_GROUP_ID }} \
--protocol tcp --port ${{ vars.SHOPPY_DB_PORT }} \
--cidr ${{ steps.runner-ip.outputs.ip }}/32

deploy-to-vercel:
needs: [ deploy ]
Expand Down
39 changes: 0 additions & 39 deletions .github/workflows/dump.yml

This file was deleted.

43 changes: 14 additions & 29 deletions .github/workflows/e2e-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,44 +12,22 @@ on:
jobs:
e2e-tests:
runs-on: ubuntu-22.04
timeout-minutes: 10
timeout-minutes: 15
name: e2e-tests
env:
PREMIUM_EMBEDDING_TOKEN: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}
MB_RUN_MODE: dev
METASTORE_DEV_SERVER_URL: ${{ secrets.METASTORE_DEV_SERVER_URL }}
METABASE_APP_DB_USER: ${{ secrets.APP_DB_USER }}
METABASE_APP_DB: ${{ secrets.APP_DB_SHOPPY_DATABASE }}
permissions:
id-token: write
contents: read

steps:
- uses: actions/checkout@v4

- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.IAM_ROLE }}
role-session-name: GitHub_to_AWS_via_FederatedOIDC
aws-region: us-east-1

- name: Download App DB PostgreSQL dump from S3
- name: Prepare env
run: |
aws s3 cp s3://${{ secrets.METABASE_APP_DB_S3_BUCKET }}/${{ vars.METABASE_APP_DB_S3_KEY }} ./metabase/metabase_dump.sql
cp .env.docker.example .env.docker
# The api signs SSO JWTs that production Metabase must trust, so use the production secret.
sed -i "s|^METABASE_JWT_SHARED_SECRET=.*|METABASE_JWT_SHARED_SECRET=\"${{ secrets.SHOPPY_PROD_JWT_SHARED_SECRET }}\"|" .env.docker

- name: Run Shoppy in Docker
run: |
cp .env.docker.example .env.docker &&
yarn docker:e2e:up --wait

- name: Dump container logs on failure
if: failure()
run: |
echo "=== shoppy-metabase-1 logs ===" && docker logs shoppy-metabase-1 2>&1 || true
echo "=== shoppy-api-1 logs ===" && docker logs shoppy-api-1 2>&1 || true
echo "=== shoppy-client-1 logs ===" && docker logs shoppy-client-1 2>&1 || true
echo "=== Container statuses ===" && docker ps -a 2>&1 || true
- name: Run Shoppy (client + api + warehouse) against production Metabase
run: yarn docker:e2e:up --wait

- name: Install Chrome v111
uses: browser-actions/setup-chrome@v1
Expand All @@ -65,6 +43,13 @@ jobs:
id: run-e2e-tests
run: cd e2e && yarn cypress:run

- name: Dump container logs on failure
if: failure()
run: |
echo "=== shoppy-client-1 ===" && docker logs shoppy-client-1 2>&1 || true
echo "=== shoppy-api-1 ===" && docker logs shoppy-api-1 2>&1 || true
echo "=== container statuses ===" && docker ps -a 2>&1 || true

- name: Upload Cypress Artifacts upon failure
uses: actions/upload-artifact@v4
if: ${{ steps.run-e2e-tests.outcome != 'success' }}
Expand Down
9 changes: 0 additions & 9 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,6 @@ node_modules
.pnp
.pnp.js

# local dist
local-dist/*
!local-dist/.gitkeep

# testing
coverage
cypress
Expand All @@ -17,11 +13,6 @@ cypress
dist
build

# db dumps
*.sql
# but allow migration files
!api/drizzle/**/*.sql

# misc
.DS_Store
.env.development.local
Expand Down
7 changes: 0 additions & 7 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,6 @@ RUN yarn --frozen-lockfile
# Copy source code last (changes most frequently)
COPY --exclude=./api --exclude=./metabase . .

RUN if [ -d "./local-dist/embedding-sdk" ]; then \
echo "Local embedding-sdk dist is found in ./local-dist/embedding-sdk, installing it..."; \
yarn add file:./local-dist/embedding-sdk; \
else \
echo "Local embedding-sdk dist is not found in ./local-dist/embedding-sdk, skipping copy"; \
fi

RUN if [ "$WATCH" != "true" ]; then \
echo "WATCH env is not set; running production yarn build..."; \
yarn build; \
Expand Down
38 changes: 9 additions & 29 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,24 +32,13 @@ This demo uses the data from the hosted Metabase Cloud instance and provides a h

### Using Docker

- Clone `.env.docker.example` to `.env.docker` and set the proper `PREMIUM_EMBEDDING_TOKEN` value.
- Run Docker via `yarn docker:up` for the `production` build or `WATCH=true yarn docker:up` for the development build with the `watch` support.
- The command launches containers with the local MB instance, Shoppy DWH, Shoppy API and Shoppy Client.
- Clone `.env.docker.example` to `.env.docker`. Set `MB_INSTANCE_URL` to your Metabase instance and `METABASE_JWT_SHARED_SECRET` to that instance's JWT shared secret.
- Run `yarn docker:up` for a production build, or `WATCH=true yarn docker:up` for a dev build with watch.
- The command launches the Shoppy DWH, API and Client, pointing at the configured Metabase instance.
- Visit `http://localhost:4400`.
- To stop containers run `yarn docker:down`.
- To remove containers and images completely run `yarn docker:rm`.

#### Local development (For Metabase developers)

For a local development the App DB dump of the Shoppy's Metabase Instance must be downloaded.

See the [Getting the App DB dump](#getting-the-app-db-dump-of-the-shoppys-metabase-instance) section.

- To run containers with a locally built `metabase.jar`, copy it to the `./local-dist` folder as `./local-dist/metabase.jar`.
- To run containers with a locally built Embedding SDK package, copy it to the `./local-dist` folder as `./local-dist/embedding-sdk`.
- Run `yarn docker:local-dist:up` to start containers and use locally built dist from the `./local-dist` folder.
- To remove containers and images completely run `yarn docker:rm`.

### Using an existing running MB instance

- Place the metabase repository in `../metabase`
Expand All @@ -76,20 +65,11 @@ If you cannot use the hosted JWT server, you can run the JWT server locally.

### Running e2e tests (For Metabase developers)

To run e2e tests locally, the App DB dump of the Shoppy's Metabase Instance must be downloaded.

See the [Getting the App DB dump](#getting-the-app-db-dump-of-the-shoppys-metabase-instance) section.

Then run `yarn docker:e2e:up` to start all required containers.
After containers are up, run `cd e2e && yarn cypress:open` to run Cypress.
To stop containers run `yarn docker:rm`.


### Getting the App DB dump of the Shoppy's Metabase Instance
The e2e tests run the client, api and warehouse locally (via Docker) against the production Metabase instance, and drive the local client with Cypress.

For a local development or for running e2e locally an App DB dump of the Shoppy's Metabase Instance must be placed to the `./metabase/metabase_dump.sql`
- In `.env.docker`, set `METABASE_JWT_SHARED_SECRET` to the production instance's JWT shared secret (from 1Password) so the api's SSO tokens are trusted, and `MB_INSTANCE_URL` to that instance.
- Start the stack: `yarn docker:e2e:up --wait`.
- Run the tests: `cd e2e && yarn cypress:run` (headless) or `yarn cypress:open` (Cypress UI).
- Stop the stack: `yarn docker:down`.

You can get it by:
- Enabling the `Tailscale` and logging in using your work email address.
- Running `pg_dump "postgres://{{ username }}:{{ password }}@{{ host }}:{{ port }}/{{ database }}" > ./metabase/metabase_dump.sql` command.
- See the `Shoppy Coredev Appdb` record in `1password` for credentials.
CI runs the same suite in `.github/workflows/e2e-tests.yml`, injecting the production secret from the `SHOPPY_PROD_JWT_SHARED_SECRET` GitHub secret.
2 changes: 1 addition & 1 deletion api/.env.example
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
PORT=3003
FRONTEND_URL=http://localhost:3004
METABASE_INSTANCE_URL=https://shoppy.coredev.metabase.com
METABASE_INSTANCE_URL=https://shoppy.metabaseapp.com
METABASE_JWT_SHARED_SECRET=your_secret_here
DB_URL=your_postgres_url_here
9 changes: 4 additions & 5 deletions api/src/constants/users.ts
Original file line number Diff line number Diff line change
@@ -1,32 +1,31 @@
import { User } from "../types/user"
import { METABASE_ADMIN_EMAIL } from "./env"

export const users: User[] = [
{
firstName: "Rene",
lastName: "Mueller",
email: METABASE_ADMIN_EMAIL ?? "rene@example.com",
email: "rene@example.com",
group: "Pug & Play",
shopId: 1,
},
{
firstName: "Cecilia",
lastName: "Stark",
email: METABASE_ADMIN_EMAIL ?? "cecilia@example.com",
email: "cecilia@example.com",
group: "theStitch",
shopId: 2,
},
{
firstName: "Emily",
lastName: "Johnson",
email: METABASE_ADMIN_EMAIL ?? "emily@example.com",
email: "emily@example.com",
group: "Luminara Beauty",
shopId: 3,
},
{
firstName: "Jennifer",
lastName: "Martinez",
email: METABASE_ADMIN_EMAIL ?? "jennifer@example.com",
email: "jennifer@example.com",
group: "ProficiencyLabs",
shopId: 4,
},
Expand Down
Loading
Loading