From 4bd5c7b632cf45ad643783a61369a07ddf9e8114 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 26 Mar 2026 15:43:29 -0400 Subject: [PATCH 01/12] feat: add Kustomize deployment with base and full overlays Introduces kustomize/ directory with two deployment overlays: - base: minimal deployment (PostGIS, metadata DB, Superset web, sample data) - full: production overlay adding Redis, Celery workers/beat, and Flux GitOps Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 5 +- CLAUDE.md | 4 + README.md | 4 + kustomize/README.md | 108 ++++++++++++++++++ kustomize/base/config/superset.env | 29 +++++ kustomize/base/kustomization.yaml | 47 ++++++++ kustomize/base/namespace.yaml | 4 + kustomize/base/postgis/init.sql | 62 ++++++++++ kustomize/base/postgis/service.yaml | 11 ++ kustomize/base/postgis/statefulset.yaml | 71 ++++++++++++ .../base/postgres-metadata/examples-init.sh | 13 +++ kustomize/base/postgres-metadata/service.yaml | 11 ++ .../base/postgres-metadata/statefulset.yaml | 87 ++++++++++++++ kustomize/base/sample-data-ingest/job.yaml | 47 ++++++++ kustomize/base/secrets.yaml.example | 14 +++ .../base/superset-frontend/deployment.yaml | 37 ++++++ kustomize/base/superset-frontend/service.yaml | 10 ++ kustomize/base/superset-init/job.yaml | 63 ++++++++++ kustomize/base/superset-web/deployment.yaml | 76 ++++++++++++ kustomize/base/superset-web/service.yaml | 10 ++ kustomize/full/config/superset-env-patch.env | 8 ++ kustomize/full/flux/git-source.yaml | 10 ++ kustomize/full/flux/kustomization.yaml | 13 +++ kustomize/full/kustomization.yaml | 29 +++++ kustomize/full/redis/service.yaml | 11 ++ kustomize/full/redis/statefulset.yaml | 48 ++++++++ kustomize/full/superset-beat/deployment.yaml | 60 ++++++++++ kustomize/full/superset-web-patch.yaml | 23 ++++ .../full/superset-worker/deployment.yaml | 67 +++++++++++ wiki/Development-Guide.md | 3 + wiki/Getting-Started.md | 2 + wiki/Home.md | 1 + 32 files changed, 987 insertions(+), 1 deletion(-) create mode 100644 kustomize/README.md create mode 100644 kustomize/base/config/superset.env create mode 100644 kustomize/base/kustomization.yaml create mode 100644 kustomize/base/namespace.yaml create mode 100644 kustomize/base/postgis/init.sql create mode 100644 kustomize/base/postgis/service.yaml create mode 100644 kustomize/base/postgis/statefulset.yaml create mode 100644 kustomize/base/postgres-metadata/examples-init.sh create mode 100644 kustomize/base/postgres-metadata/service.yaml create mode 100644 kustomize/base/postgres-metadata/statefulset.yaml create mode 100644 kustomize/base/sample-data-ingest/job.yaml create mode 100644 kustomize/base/secrets.yaml.example create mode 100644 kustomize/base/superset-frontend/deployment.yaml create mode 100644 kustomize/base/superset-frontend/service.yaml create mode 100644 kustomize/base/superset-init/job.yaml create mode 100644 kustomize/base/superset-web/deployment.yaml create mode 100644 kustomize/base/superset-web/service.yaml create mode 100644 kustomize/full/config/superset-env-patch.env create mode 100644 kustomize/full/flux/git-source.yaml create mode 100644 kustomize/full/flux/kustomization.yaml create mode 100644 kustomize/full/kustomization.yaml create mode 100644 kustomize/full/redis/service.yaml create mode 100644 kustomize/full/redis/statefulset.yaml create mode 100644 kustomize/full/superset-beat/deployment.yaml create mode 100644 kustomize/full/superset-web-patch.yaml create mode 100644 kustomize/full/superset-worker/deployment.yaml diff --git a/.gitignore b/.gitignore index e5662d8f09..788f3917a4 100644 --- a/.gitignore +++ b/.gitignore @@ -137,4 +137,7 @@ scratch/ # Untracked personal Claude Code files (e.g. CLAUDE.local.md, commands/my-command.local.md) .claude/**/*.local.* -CLAUDE.local.md \ No newline at end of file +CLAUDE.local.md + +# Kustomize secrets (real values — never commit) +kustomize/**/secrets.yaml \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 74694f84f5..b1431ac976 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,6 +28,10 @@ When pulling in upstream Apache Superset changes, update the **"Based on"** fiel Wiki pages live in `wiki/` and are synced to the GitHub Wiki on merge to main via `.github/workflows/sync-wiki.yml`. Documentation sync runs automatically as a background process during `/commit-and-push` and `/merge-request` — it audits wiki pages, README.md, and inline documentation against code changes. See `.claude/skills/sync-documentation.md` for details. +## Kubernetes Deployment + +Kustomize overlays live in `kustomize/`. This is GeoSet-specific (not upstream Superset). `base/` is the dev/demo overlay analogous to Docker Compose; `full/` adds Redis, Celery, and Flux GitOps for production. + ## Important Notes - Always use context7 when I need code generation, setup or configuration steps, or diff --git a/README.md b/README.md index 1fdb778888..d38b2b750d 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,10 @@ The Dockerfile at the root of the repository uses the same Debian-based image us DOCKERFILE=Dockerfile.rhel docker compose up ``` +### Kubernetes Deployment + +Kustomize overlays are available in [`kustomize/`](./kustomize/) for deploying GeoSet to Kubernetes. The `base` overlay mirrors the Docker Compose stack (dev/demo), while `full` adds Redis, Celery workers, and Flux GitOps for production use. See [`kustomize/README.md`](./kustomize/README.md) for details. + ### Step 3 - Open GeoSet and Explore We've created an example dashboard accessible at [http://localhost:9001/superset/dashboard/geoset-example-dashboard](http://localhost:9001/superset/dashboard/geoset-example-dashboard). diff --git a/kustomize/README.md b/kustomize/README.md new file mode 100644 index 0000000000..c8876d7fbc --- /dev/null +++ b/kustomize/README.md @@ -0,0 +1,108 @@ +# GeoSet Kustomize Deployment + +Kubernetes manifests for deploying GeoSet, organized as Kustomize overlays. + +## Overlays + +| Overlay | Description | Use case | +|---------|-------------|----------| +| **base** | PostGIS, metadata DB, Superset web, sample data ingest. No Redis or Celery. | Local/dev clusters, quick demos | +| **full** | Everything in base + Redis, Celery workers, Celery beat (scheduled tasks, cache warmup, report generation), Flux GitOps | Staging/production | + +## Prerequisites + +- A Kubernetes cluster (minikube, kind, EKS, etc.) +- `kubectl` installed and configured +- Container images pushed (default: `jmeegan607/geoset:6.0.48`, `ebienstock/geoset:data-ingest-latest`) + +## Setup + +### 1. Create secrets + +Copy the example and fill in real values: + +```bash +cp kustomize/base/secrets.yaml.example kustomize/base/secrets.yaml +``` + +Edit `kustomize/base/secrets.yaml` and set: + +| Secret | Description | Required | +|--------|-------------|----------| +| `DATABASE_PASSWORD` | Superset metadata Postgres password | Yes | +| `POSTGIS_PASSWORD` | PostGIS (geospatial data) password | Yes | +| `EXAMPLES_PASSWORD` | Superset examples DB password | Yes | +| `SUPERSET_SECRET_KEY` | Flask secret key — generate with `openssl rand -base64 42` | Yes | +| `ADMIN_PASSWORD` | Superset admin user password | Yes | +| `MAPBOX_API_KEY` | Mapbox GL token for map tiles | Yes | + +> **Never commit `secrets.yaml`** — it is gitignored. Only `secrets.yaml.example` is tracked. + +### 2. Review environment variables + +Base env config lives in `base/config/superset.env`. The defaults work out of the box for most setups. Key variables: + +| Variable | Default | Notes | +|----------|---------|-------| +| `SUPERSET_CONFIG_PATH` | `superset_config_docker_light.py` (base) / `superset_config.py` (full) | Auto-switched by overlay | +| `DATABASE_HOST` | `postgres-metadata` | K8s service name | +| `POSTGIS_HOST` | `postgis` | K8s service name | +| `REDIS_HOST` | `redis` | Only used in full overlay | + +You generally don't need to change these unless you're pointing at external databases. + +### 3. Update container images (if needed) + +The image tag is controlled in one place — the `images` block in `base/kustomization.yaml`: + +```yaml +images: + - name: jmeegan607/geoset + newTag: "6.0.48" +``` + +Change `newTag` to use a different version. This applies to all manifests automatically. + +## Deploy + +### Base (dev/demo) + +```bash +kubectl apply -k kustomize/base +``` + +### Full (staging/production) + +```bash +kubectl apply -k kustomize/full +``` + +### Verify + +```bash +kubectl -n geoset get pods +kubectl -n geoset get svc +``` + +Superset web will be available on port `8088` via the `superset-web` service. To access locally: + +```bash +kubectl -n geoset port-forward svc/superset-web 8088:8088 +``` + +## Full overlay extras + +The full overlay adds on top of base: + +- **Redis** — caching backend and Celery message broker +- **Celery workers** (2 replicas) — async query execution +- **Celery beat** — scheduled tasks including cache warmup and automated report generation +- **Flux GitOps** — auto-syncs from `raft-tech/GeoSet` main branch + +It also patches `superset-web` to 2 replicas and adds a Redis readiness check to its init container. + +## Teardown + +```bash +kubectl delete -k kustomize/base # or kustomize/full +``` diff --git a/kustomize/base/config/superset.env b/kustomize/base/config/superset.env new file mode 100644 index 0000000000..719090cd74 --- /dev/null +++ b/kustomize/base/config/superset.env @@ -0,0 +1,29 @@ +# Superset runtime +PYTHONUNBUFFERED=1 +PYTHONPATH=/app/pythonpath:/app/docker/pythonpath_dev +FLASK_DEBUG=false +SUPERSET_ENV=production +SUPERSET_LOG_LEVEL=info +SUPERSET_LOAD_EXAMPLES=no + +# Superset config — light mode (no Redis/Celery) for base deployment +SUPERSET_CONFIG_PATH=/app/docker/pythonpath_dev/superset_config_docker_light.py + +# Postgres metadata DB +DATABASE_DIALECT=postgresql +DATABASE_HOST=postgres-metadata +DATABASE_PORT=5432 +DATABASE_DB=superset +DATABASE_USER=superset + +# Examples DB (same Postgres instance as metadata) +EXAMPLES_HOST=postgres-metadata +EXAMPLES_PORT=5432 +EXAMPLES_DB=examples +EXAMPLES_USER=examples + +# PostGIS (GeoSet geospatial data) +POSTGIS_HOST=postgis +POSTGIS_PORT=5432 +POSTGIS_DB=geoset +POSTGIS_USER=geoset diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml new file mode 100644 index 0000000000..18867c24dc --- /dev/null +++ b/kustomize/base/kustomization.yaml @@ -0,0 +1,47 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: geoset + +configMapGenerator: + - name: superset-env + envs: + - config/superset.env + - name: postgres-metadata-initdb + files: + - postgres-metadata/examples-init.sh + - name: postgis-initdb + files: + - postgis/init.sql + +generatorOptions: + disableNameSuffixHash: true + +images: + - name: jmeegan607/geoset + newTag: "6.0.48" + +resources: + # Cluster setup + - namespace.yaml + # NOTE: Copy secrets.yaml.example to secrets.yaml and fill in real values + # secrets.yaml is gitignored — never commit real secrets + - secrets.yaml + + # StatefulSets (data layer) + - postgis/statefulset.yaml + - postgis/service.yaml + - postgres-metadata/statefulset.yaml + - postgres-metadata/service.yaml + + # Jobs (init before app starts) + - superset-init/job.yaml + - sample-data-ingest/job.yaml + + # Deployments (app layer) + - superset-web/deployment.yaml + - superset-web/service.yaml + # superset-frontend is dev-only (webpack dev server) — the production image + # already includes built frontend assets. Uncomment for a dev overlay if needed. + # - superset-frontend/deployment.yaml + # - superset-frontend/service.yaml diff --git a/kustomize/base/namespace.yaml b/kustomize/base/namespace.yaml new file mode 100644 index 0000000000..efd9b3122b --- /dev/null +++ b/kustomize/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: geoset diff --git a/kustomize/base/postgis/init.sql b/kustomize/base/postgis/init.sql new file mode 100644 index 0000000000..c3757b9f27 --- /dev/null +++ b/kustomize/base/postgis/init.sql @@ -0,0 +1,62 @@ +CREATE EXTENSION IF NOT EXISTS postgis; + +CREATE TABLE IF NOT EXISTS census_state_boundaries ( + id SERIAL PRIMARY KEY, + state_code VARCHAR(2) NOT NULL, + state_gnis_code VARCHAR(8), + state_abbrev VARCHAR(2) NOT NULL, + full_geoid VARCHAR(14), + geoid VARCHAR(2), + legal_statistical_code VARCHAR(2), + land_area BIGINT, + water_area BIGINT, + state_name VARCHAR(100) NOT NULL, + state_boundary TEXT +); + +CREATE TABLE IF NOT EXISTS nifc_wildfire_locations ( + id SERIAL PRIMARY KEY, + fire_id INTEGER, + irwin_id TEXT, + incident_size DOUBLE PRECISION, + containment_time TIMESTAMPTZ, + percent_contained DOUBLE PRECISION, + control_time TIMESTAMPTZ, + incident_description TEXT, + discovery_acres DOUBLE PRECISION, + final_acres DOUBLE PRECISION, + fire_cause TEXT, + origin_coordinate TEXT, + dispatch_center_id TEXT, + fire_discovery_time TIMESTAMPTZ, + nifc_created_time TIMESTAMPTZ, + nifc_modified_time TIMESTAMPTZ, + estimated_cost_to_date DOUBLE PRECISION, + incident_name TEXT, + origin_fips_code CHAR(5), + origin_city_name TEXT, + origin_state_code CHAR(5), + origin_county_name TEXT, + landowner_type TEXT, + is_multijurisdictional BOOLEAN +); + +CREATE TABLE IF NOT EXISTS nhc_best_track ( + id SERIAL PRIMARY KEY, + effective_timestamp TIMESTAMPTZ NOT NULL, + min_sea_level_pressure_mb INTEGER, + max_gust_mph INTEGER, + storm_name TEXT NOT NULL, + nhc_identifier TEXT, + year INTEGER NOT NULL, + observation_point GEOGRAPHY(POINT, 4326) +); + +CREATE INDEX IF NOT EXISTS idx_nhc_identifier ON nhc_best_track (nhc_identifier); +CREATE INDEX IF NOT EXISTS idx_nhc_year ON nhc_best_track (year); + +-- Drop schemas that GeoSet doesn't use so Superset's schema picker only shows public. +DROP SCHEMA IF EXISTS tiger_data CASCADE; +DROP SCHEMA IF EXISTS tiger CASCADE; +DROP SCHEMA IF EXISTS topology CASCADE; +DROP SCHEMA IF EXISTS information_schema CASCADE; diff --git a/kustomize/base/postgis/service.yaml b/kustomize/base/postgis/service.yaml new file mode 100644 index 0000000000..e3772fcd8f --- /dev/null +++ b/kustomize/base/postgis/service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgis +spec: + clusterIP: None + selector: + app: postgis + ports: + - port: 5432 + targetPort: 5432 diff --git a/kustomize/base/postgis/statefulset.yaml b/kustomize/base/postgis/statefulset.yaml new file mode 100644 index 0000000000..7ef87ea299 --- /dev/null +++ b/kustomize/base/postgis/statefulset.yaml @@ -0,0 +1,71 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgis +spec: + serviceName: postgis + replicas: 1 + selector: + matchLabels: + app: postgis + template: + metadata: + labels: + app: postgis + spec: + containers: + - name: postgis + image: postgis/postgis:16-3.4 + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: superset-env + key: POSTGIS_DB + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: superset-env + key: POSTGIS_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: superset-secrets + key: POSTGIS_PASSWORD + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: postgis-data + mountPath: /var/lib/postgresql/data + subPath: pgdata + - name: initdb + mountPath: /docker-entrypoint-initdb.d + readinessProbe: + exec: + command: ["pg_isready", "-U", "geoset", "-d", "geoset"] + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + exec: + command: ["pg_isready", "-U", "geoset", "-d", "geoset"] + initialDelaySeconds: 30 + periodSeconds: 30 + volumes: + - name: initdb + configMap: + name: postgis-initdb + volumeClaimTemplates: + - metadata: + name: postgis-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi diff --git a/kustomize/base/postgres-metadata/examples-init.sh b/kustomize/base/postgres-metadata/examples-init.sh new file mode 100644 index 0000000000..c41d0dbcc0 --- /dev/null +++ b/kustomize/base/postgres-metadata/examples-init.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Required: Superset needs a separate "examples" database and user in the metadata +# Postgres for `superset load_examples` to work. This is the k8s equivalent of +# docker/docker-entrypoint-initdb.d/examples-init.sh in the Superset repo. +set -e +psql -v ON_ERROR_STOP=1 --username "${POSTGRES_USER}" <<-EOSQL + CREATE USER ${EXAMPLES_USER} WITH PASSWORD '${EXAMPLES_PASSWORD}'; + CREATE DATABASE ${EXAMPLES_DB}; + GRANT ALL PRIVILEGES ON DATABASE ${EXAMPLES_DB} TO ${EXAMPLES_USER}; +EOSQL +psql -v ON_ERROR_STOP=1 --username "${POSTGRES_USER}" -d "${EXAMPLES_DB}" <<-EOSQL + GRANT ALL ON SCHEMA public TO ${EXAMPLES_USER}; +EOSQL diff --git a/kustomize/base/postgres-metadata/service.yaml b/kustomize/base/postgres-metadata/service.yaml new file mode 100644 index 0000000000..63728fa124 --- /dev/null +++ b/kustomize/base/postgres-metadata/service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres-metadata +spec: + clusterIP: None + selector: + app: postgres-metadata + ports: + - port: 5432 + targetPort: 5432 diff --git a/kustomize/base/postgres-metadata/statefulset.yaml b/kustomize/base/postgres-metadata/statefulset.yaml new file mode 100644 index 0000000000..2c413360b1 --- /dev/null +++ b/kustomize/base/postgres-metadata/statefulset.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres-metadata +spec: + serviceName: postgres-metadata + replicas: 1 + selector: + matchLabels: + app: postgres-metadata + template: + metadata: + labels: + app: postgres-metadata + spec: + containers: + - name: postgres + image: postgres:16-alpine + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: superset-env + key: DATABASE_DB + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: superset-env + key: DATABASE_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: superset-secrets + key: DATABASE_PASSWORD + - name: EXAMPLES_USER + valueFrom: + configMapKeyRef: + name: superset-env + key: EXAMPLES_USER + - name: EXAMPLES_PASSWORD + valueFrom: + secretKeyRef: + name: superset-secrets + key: EXAMPLES_PASSWORD + - name: EXAMPLES_DB + valueFrom: + configMapKeyRef: + name: superset-env + key: EXAMPLES_DB + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + volumeMounts: + - name: metadata-data + mountPath: /var/lib/postgresql/data + subPath: pgdata + - name: initdb + mountPath: /docker-entrypoint-initdb.d + readinessProbe: + exec: + command: ["pg_isready", "-U", "superset", "-d", "superset"] + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + exec: + command: ["pg_isready", "-U", "superset", "-d", "superset"] + initialDelaySeconds: 30 + periodSeconds: 30 + volumes: + - name: initdb + configMap: + name: postgres-metadata-initdb + defaultMode: 0755 + volumeClaimTemplates: + - metadata: + name: metadata-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 2Gi diff --git a/kustomize/base/sample-data-ingest/job.yaml b/kustomize/base/sample-data-ingest/job.yaml new file mode 100644 index 0000000000..b91a5cf4c1 --- /dev/null +++ b/kustomize/base/sample-data-ingest/job.yaml @@ -0,0 +1,47 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: sample-data-ingest +spec: + backoffLimit: 3 + template: + metadata: + labels: + app: sample-data-ingest + spec: + restartPolicy: OnFailure + initContainers: + - name: wait-for-postgis + image: busybox:1.36 + command: + - sh + - -c + - | + until nc -z postgis 5432; do + echo "waiting for postgis..." + sleep 2 + done + containers: + - name: ingest + image: ebienstock/geoset:data-ingest-latest + env: + - name: DB_HOST + value: "postgis" + - name: DB_PORT + value: "5432" + - name: DB_NAME + value: "geoset" + - name: DB_USER + value: "geoset" + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: superset-secrets + key: POSTGIS_PASSWORD + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi diff --git a/kustomize/base/secrets.yaml.example b/kustomize/base/secrets.yaml.example new file mode 100644 index 0000000000..0bec75301e --- /dev/null +++ b/kustomize/base/secrets.yaml.example @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Secret +metadata: + name: superset-secrets + namespace: geoset +type: Opaque +stringData: + # --- CHANGE THESE BEFORE ANY REAL USE --- + DATABASE_PASSWORD: "superset" + POSTGIS_PASSWORD: "geoset" + EXAMPLES_PASSWORD: "examples" + SUPERSET_SECRET_KEY: "CHANGE_ME_TO_A_RANDOM_SECRET" + ADMIN_PASSWORD: "admin" + MAPBOX_API_KEY: "CHANGE_ME" diff --git a/kustomize/base/superset-frontend/deployment.yaml b/kustomize/base/superset-frontend/deployment.yaml new file mode 100644 index 0000000000..c03e5a7555 --- /dev/null +++ b/kustomize/base/superset-frontend/deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: superset-frontend + labels: + environment: dev-only +spec: + replicas: 1 + selector: + matchLabels: + app: superset-frontend + template: + metadata: + labels: + app: superset-frontend + component: superset + environment: dev-only + spec: + containers: + - name: frontend + image: jmeegan607/geoset + command: ["npm", "run", "dev-server", "--prefix", "/app/superset-frontend"] + ports: + - containerPort: 9000 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + readinessProbe: + httpGet: + path: / + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 10 diff --git a/kustomize/base/superset-frontend/service.yaml b/kustomize/base/superset-frontend/service.yaml new file mode 100644 index 0000000000..3c056f7cf7 --- /dev/null +++ b/kustomize/base/superset-frontend/service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: superset-frontend +spec: + selector: + app: superset-frontend + ports: + - port: 9000 + targetPort: 9000 diff --git a/kustomize/base/superset-init/job.yaml b/kustomize/base/superset-init/job.yaml new file mode 100644 index 0000000000..514d63d089 --- /dev/null +++ b/kustomize/base/superset-init/job.yaml @@ -0,0 +1,63 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: superset-init +spec: + backoffLimit: 3 + template: + metadata: + labels: + app: superset-init + component: superset + spec: + restartPolicy: OnFailure + initContainers: + - name: wait-for-db + image: busybox:1.36 + command: + - sh + - -c + - | + until nc -z postgres-metadata 5432; do + echo "waiting for postgres-metadata..." + sleep 2 + done + until nc -z postgis 5432; do + echo "waiting for postgis..." + sleep 2 + done + containers: + - name: init + image: jmeegan607/geoset + command: ["/app/docker/docker-init-geoset.sh"] + envFrom: + - configMapRef: + name: superset-env + env: + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: superset-secrets + key: DATABASE_PASSWORD + - name: EXAMPLES_PASSWORD + valueFrom: + secretKeyRef: + name: superset-secrets + key: EXAMPLES_PASSWORD + - name: SUPERSET_SECRET_KEY + valueFrom: + secretKeyRef: + name: superset-secrets + key: SUPERSET_SECRET_KEY + - name: ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: superset-secrets + key: ADMIN_PASSWORD + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi diff --git a/kustomize/base/superset-web/deployment.yaml b/kustomize/base/superset-web/deployment.yaml new file mode 100644 index 0000000000..b257083c1e --- /dev/null +++ b/kustomize/base/superset-web/deployment.yaml @@ -0,0 +1,76 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: superset-web +spec: + replicas: 1 + selector: + matchLabels: + app: superset-web + template: + metadata: + labels: + app: superset-web + component: superset + spec: + initContainers: + - name: wait-for-db + image: busybox:1.36 + command: + - sh + - -c + - | + until nc -z postgres-metadata 5432; do + echo "waiting for postgres-metadata..." + sleep 2 + done + containers: + - name: superset + image: jmeegan607/geoset + envFrom: + - configMapRef: + name: superset-env + env: + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: superset-secrets + key: DATABASE_PASSWORD + - name: EXAMPLES_PASSWORD + valueFrom: + secretKeyRef: + name: superset-secrets + key: EXAMPLES_PASSWORD + - name: SUPERSET_SECRET_KEY + valueFrom: + secretKeyRef: + name: superset-secrets + key: SUPERSET_SECRET_KEY + - name: MAPBOX_API_KEY + valueFrom: + secretKeyRef: + name: superset-secrets + key: MAPBOX_API_KEY + - name: SERVER_WORKER_AMOUNT + value: "4" + ports: + - containerPort: 8088 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + readinessProbe: + httpGet: + path: /health + port: 8088 + initialDelaySeconds: 30 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 8088 + initialDelaySeconds: 60 + periodSeconds: 30 diff --git a/kustomize/base/superset-web/service.yaml b/kustomize/base/superset-web/service.yaml new file mode 100644 index 0000000000..ae437bf64f --- /dev/null +++ b/kustomize/base/superset-web/service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: superset-web +spec: + selector: + app: superset-web + ports: + - port: 8088 + targetPort: 8088 diff --git a/kustomize/full/config/superset-env-patch.env b/kustomize/full/config/superset-env-patch.env new file mode 100644 index 0000000000..401cbe4ff3 --- /dev/null +++ b/kustomize/full/config/superset-env-patch.env @@ -0,0 +1,8 @@ +# Override: use full config (with Celery/Redis), not light mode +SUPERSET_CONFIG_PATH=/app/docker/pythonpath_dev/superset_config.py + +# Redis (required for Celery workers/beat) +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_CELERY_DB=0 +REDIS_RESULTS_DB=1 diff --git a/kustomize/full/flux/git-source.yaml b/kustomize/full/flux/git-source.yaml new file mode 100644 index 0000000000..3da07c41b8 --- /dev/null +++ b/kustomize/full/flux/git-source.yaml @@ -0,0 +1,10 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: geoset + namespace: flux-system +spec: + interval: 1m + url: https://github.com/raft-tech/GeoSet.git + ref: + branch: main diff --git a/kustomize/full/flux/kustomization.yaml b/kustomize/full/flux/kustomization.yaml new file mode 100644 index 0000000000..e10448d746 --- /dev/null +++ b/kustomize/full/flux/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: geoset + namespace: flux-system +spec: + interval: 1m + sourceRef: + kind: GitRepository + name: geoset + path: ./kustomize/full + prune: true + targetNamespace: geoset diff --git a/kustomize/full/kustomization.yaml b/kustomize/full/kustomization.yaml new file mode 100644 index 0000000000..a58d2c978b --- /dev/null +++ b/kustomize/full/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Inherit everything from base +resources: + - ../base + + # Additional services for full deployment + - redis/statefulset.yaml + - redis/service.yaml + - superset-worker/deployment.yaml + - superset-beat/deployment.yaml + +# Override the superset-env configmap to add Redis vars and switch to full config +configMapGenerator: + - name: superset-env + behavior: merge + envs: + - config/superset-env-patch.env + +generatorOptions: + disableNameSuffixHash: true + +# Patch superset-web: bump replicas to 2 and wait for Redis +patches: + - path: superset-web-patch.yaml + target: + kind: Deployment + name: superset-web diff --git a/kustomize/full/redis/service.yaml b/kustomize/full/redis/service.yaml new file mode 100644 index 0000000000..19e6f17ddc --- /dev/null +++ b/kustomize/full/redis/service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis +spec: + clusterIP: None + selector: + app: redis + ports: + - port: 6379 + targetPort: 6379 diff --git a/kustomize/full/redis/statefulset.yaml b/kustomize/full/redis/statefulset.yaml new file mode 100644 index 0000000000..8479f3401a --- /dev/null +++ b/kustomize/full/redis/statefulset.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis +spec: + serviceName: redis + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:7-alpine + ports: + - containerPort: 6379 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - name: redis-data + mountPath: /data + readinessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 15 + periodSeconds: 20 + volumeClaimTemplates: + - metadata: + name: redis-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/kustomize/full/superset-beat/deployment.yaml b/kustomize/full/superset-beat/deployment.yaml new file mode 100644 index 0000000000..bb2a1ce321 --- /dev/null +++ b/kustomize/full/superset-beat/deployment.yaml @@ -0,0 +1,60 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: superset-beat +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: superset-beat + template: + metadata: + labels: + app: superset-beat + component: superset + spec: + initContainers: + - name: wait-for-redis + image: busybox:1.36 + command: + - sh + - -c + - | + until nc -z redis 6379; do + echo "waiting for redis..." + sleep 2 + done + containers: + - name: beat + image: jmeegan607/geoset + command: ["celery", "--app=superset.tasks.celery_app:app", + "beat", "--loglevel=INFO", + "--schedule=/tmp/celerybeat-schedule"] + envFrom: + - configMapRef: + name: superset-env + env: + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: superset-secrets + key: DATABASE_PASSWORD + - name: EXAMPLES_PASSWORD + valueFrom: + secretKeyRef: + name: superset-secrets + key: EXAMPLES_PASSWORD + - name: SUPERSET_SECRET_KEY + valueFrom: + secretKeyRef: + name: superset-secrets + key: SUPERSET_SECRET_KEY + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi diff --git a/kustomize/full/superset-web-patch.yaml b/kustomize/full/superset-web-patch.yaml new file mode 100644 index 0000000000..1f4ba5e376 --- /dev/null +++ b/kustomize/full/superset-web-patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: superset-web +spec: + replicas: 2 + template: + spec: + initContainers: + - name: wait-for-db + image: busybox:1.36 + command: + - sh + - -c + - | + until nc -z postgres-metadata 5432; do + echo "waiting for postgres-metadata..." + sleep 2 + done + until nc -z redis 6379; do + echo "waiting for redis..." + sleep 2 + done diff --git a/kustomize/full/superset-worker/deployment.yaml b/kustomize/full/superset-worker/deployment.yaml new file mode 100644 index 0000000000..ea43597a25 --- /dev/null +++ b/kustomize/full/superset-worker/deployment.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: superset-worker +spec: + replicas: 2 + selector: + matchLabels: + app: superset-worker + template: + metadata: + labels: + app: superset-worker + component: superset + spec: + initContainers: + - name: wait-for-redis + image: busybox:1.36 + command: + - sh + - -c + - | + until nc -z redis 6379; do + echo "waiting for redis..." + sleep 2 + done + containers: + - name: worker + image: jmeegan607/geoset + command: ["celery", "--app=superset.tasks.celery_app:app", + "worker", "--pool=prefork", "-O", "fair", + "-c", "4", "--loglevel=INFO"] + envFrom: + - configMapRef: + name: superset-env + env: + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: superset-secrets + key: DATABASE_PASSWORD + - name: EXAMPLES_PASSWORD + valueFrom: + secretKeyRef: + name: superset-secrets + key: EXAMPLES_PASSWORD + - name: SUPERSET_SECRET_KEY + valueFrom: + secretKeyRef: + name: superset-secrets + key: SUPERSET_SECRET_KEY + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + livenessProbe: + exec: + command: + - sh + - -c + - celery --app=superset.tasks.celery_app:app inspect ping -d celery@$HOSTNAME + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 10 diff --git a/wiki/Development-Guide.md b/wiki/Development-Guide.md index 42a9b9a730..5a6c9bf751 100644 --- a/wiki/Development-Guide.md +++ b/wiki/Development-Guide.md @@ -78,6 +78,9 @@ GeoSet/ ├── sample-data/ # Demo data ingestion pipeline ├── docker/ # Docker configuration and init scripts ├── docker-compose.yml # Main stack (includes GeoSet demo data) +├── kustomize/ # Kubernetes deployment (Kustomize overlays) +│ ├── base/ # Dev/demo: PostGIS, metadata DB, Superset web +│ └── full/ # Production: adds Redis, Celery, Flux GitOps └── VERSIONING.md # GeoSet version policy and changelog ``` diff --git a/wiki/Getting-Started.md b/wiki/Getting-Started.md index bfcbaf86ac..3844324cb4 100644 --- a/wiki/Getting-Started.md +++ b/wiki/Getting-Started.md @@ -75,6 +75,8 @@ docker compose down docker compose down -v ``` +> **Kubernetes:** A Kustomize-based deployment option is also available. See [`kustomize/README.md`](../kustomize/README.md) for details. + ## Next Steps - Load the [[Sample Dashboards]] to see GeoSet in action diff --git a/wiki/Home.md b/wiki/Home.md index bcd0f7bb63..bc27ea3914 100644 --- a/wiki/Home.md +++ b/wiki/Home.md @@ -24,6 +24,7 @@ GeoSet is a geospatial data monitoring and visualization platform built on [Apac - [[Sample Dashboards]] — Loading the example Hurricane and Wildfire dashboards - [[Development Guide]] — Local dev setup, plugin architecture, contributing - [[JSON Config Spec]] — Reference for the GeoSet Map Layer JSON configuration schema +- [Kubernetes Deployment](../kustomize/README.md) — Deploying GeoSet to Kubernetes with Kustomize overlays ## Repository From fabf8cdd7982bff5a6139268b4eac1e5a4359df4 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 30 Mar 2026 09:32:48 -0400 Subject: [PATCH 02/12] feat: add MailHog, wildfire alert bootstrap, and SMTP config for full overlay Add MailHog as an in-cluster dev SMTP server and a bootstrap Job that seeds the daily Wildfire Proximity Alert via the Superset REST API. Make WEBDRIVER_BASEURL env-configurable and disable alert dry-run mode so alerts actually fire in both Docker Compose and Kustomize deployments. Co-Authored-By: Claude Opus 4.6 (1M context) --- docker/pythonpath_dev/superset_config.py | 9 +- kustomize/README.md | 3 + kustomize/full/config/superset-env-patch.env | 16 +++ kustomize/full/kustomization.yaml | 6 + kustomize/full/mailhog/deployment.yaml | 31 +++++ kustomize/full/mailhog/service.yaml | 16 +++ .../full/wildfire-alert/bootstrap-alert.sh | 122 ++++++++++++++++++ kustomize/full/wildfire-alert/job.yaml | 54 ++++++++ 8 files changed, 253 insertions(+), 4 deletions(-) create mode 100644 kustomize/full/mailhog/deployment.yaml create mode 100644 kustomize/full/mailhog/service.yaml create mode 100755 kustomize/full/wildfire-alert/bootstrap-alert.sh create mode 100644 kustomize/full/wildfire-alert/job.yaml diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index 2de8f03794..b2c6497891 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -100,11 +100,12 @@ class CeleryConfig: CELERY_CONFIG = CeleryConfig FEATURE_FLAGS = {"ALERT_REPORTS": True} -ALERT_REPORTS_NOTIFICATION_DRY_RUN = True -WEBDRIVER_BASEURL = f"http://superset_app{os.environ.get('SUPERSET_APP_ROOT', '/')}/" # When using docker compose baseurl should be http://superset_nginx{ENV{BASEPATH}}/ # noqa: E501 +ALERT_REPORTS_NOTIFICATION_DRY_RUN = False +WEBDRIVER_BASEURL = os.environ.get("WEBDRIVER_BASEURL", f"http://superset_app{os.environ.get('SUPERSET_APP_ROOT', '/')}/") # The base URL for the email report hyperlinks. -WEBDRIVER_BASEURL_USER_FRIENDLY = ( - f"http://localhost:8888/{os.environ.get('SUPERSET_APP_ROOT', '/')}/" +WEBDRIVER_BASEURL_USER_FRIENDLY = os.environ.get( + "WEBDRIVER_BASEURL_USER_FRIENDLY", + f"http://localhost:8888/{os.environ.get('SUPERSET_APP_ROOT', '/')}/", ) SQLLAB_CTAS_NO_LIMIT = True diff --git a/kustomize/README.md b/kustomize/README.md index c8876d7fbc..7763202d08 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -97,6 +97,9 @@ The full overlay adds on top of base: - **Redis** — caching backend and Celery message broker - **Celery workers** (2 replicas) — async query execution - **Celery beat** — scheduled tasks including cache warmup and automated report generation +- **MailHog** — in-cluster fake SMTP server for dev/testing alert email delivery (SMTP on port 1025, web UI on port 8025) +- **Wildfire Alert bootstrap** — one-time Job that seeds a daily "Wildfire Proximity Alert" SQL alert via the Superset REST API after init +- **SMTP / Alerts & Reports env vars** — pre-configured to use MailHog; swap env vars in `config/superset-env-patch.env` for production SMTP - **Flux GitOps** — auto-syncs from `raft-tech/GeoSet` main branch It also patches `superset-web` to 2 replicas and adds a Redis readiness check to its init container. diff --git a/kustomize/full/config/superset-env-patch.env b/kustomize/full/config/superset-env-patch.env index 401cbe4ff3..2ff0e15428 100644 --- a/kustomize/full/config/superset-env-patch.env +++ b/kustomize/full/config/superset-env-patch.env @@ -6,3 +6,19 @@ REDIS_HOST=redis REDIS_PORT=6379 REDIS_CELERY_DB=0 REDIS_RESULTS_DB=1 + +# SMTP for Alerts & Reports +# Using MailHog (in-cluster fake SMTP) for dev/testing — swap to real SMTP for production +SMTP_HOST=mailhog +SMTP_PORT=1025 +SMTP_STARTTLS=false +SMTP_SSL=false +SMTP_USER= +SMTP_PASSWORD= +SMTP_MAIL_FROM=geoset-alerts@teamraft.com + +# Alerts & Reports +ALERT_REPORTS_DRY_RUN=false + +# Webdriver for chart screenshots (must resolve from within K8s cluster) +WEBDRIVER_BASEURL=http://superset-web:8088/ diff --git a/kustomize/full/kustomization.yaml b/kustomize/full/kustomization.yaml index a58d2c978b..2055f3c4a7 100644 --- a/kustomize/full/kustomization.yaml +++ b/kustomize/full/kustomization.yaml @@ -10,6 +10,9 @@ resources: - redis/service.yaml - superset-worker/deployment.yaml - superset-beat/deployment.yaml + - wildfire-alert/job.yaml + - mailhog/deployment.yaml + - mailhog/service.yaml # Override the superset-env configmap to add Redis vars and switch to full config configMapGenerator: @@ -17,6 +20,9 @@ configMapGenerator: behavior: merge envs: - config/superset-env-patch.env + - name: wildfire-alert-bootstrap + files: + - bootstrap-alert.sh=wildfire-alert/bootstrap-alert.sh generatorOptions: disableNameSuffixHash: true diff --git a/kustomize/full/mailhog/deployment.yaml b/kustomize/full/mailhog/deployment.yaml new file mode 100644 index 0000000000..fd54da4835 --- /dev/null +++ b/kustomize/full/mailhog/deployment.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mailhog + labels: + app: mailhog +spec: + replicas: 1 + selector: + matchLabels: + app: mailhog + template: + metadata: + labels: + app: mailhog + spec: + containers: + - name: mailhog + image: mailhog/mailhog:latest + ports: + - name: smtp + containerPort: 1025 + - name: http + containerPort: 8025 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi diff --git a/kustomize/full/mailhog/service.yaml b/kustomize/full/mailhog/service.yaml new file mode 100644 index 0000000000..88f9634b64 --- /dev/null +++ b/kustomize/full/mailhog/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: mailhog + labels: + app: mailhog +spec: + selector: + app: mailhog + ports: + - name: smtp + port: 1025 + targetPort: 1025 + - name: http + port: 8025 + targetPort: 8025 diff --git a/kustomize/full/wildfire-alert/bootstrap-alert.sh b/kustomize/full/wildfire-alert/bootstrap-alert.sh new file mode 100755 index 0000000000..41c2872b15 --- /dev/null +++ b/kustomize/full/wildfire-alert/bootstrap-alert.sh @@ -0,0 +1,122 @@ +#!/bin/bash +set -euo pipefail + +# Bootstrap the wildfire proximity alert in Superset Alerts & Reports. +# Runs once after superset-init to seed the SQL alert via the REST API. + +SUPERSET_URL="${SUPERSET_URL:-http://superset-web:8088}" +ADMIN_USER="${ADMIN_USERNAME:-admin}" +ADMIN_PASS="${ADMIN_PASSWORD}" +ALERT_RECIPIENT="${WILDFIRE_ALERT_RECIPIENT:-jmeegan@teamraft.com}" +COOKIE_JAR="/tmp/superset_cookies.txt" + +echo "Waiting for Superset API to be ready..." +until curl -sf "${SUPERSET_URL}/health" > /dev/null 2>&1; do + sleep 5 +done + +# Run the rest via Python to avoid bash quoting hell +python3 << PYEOF +import json, subprocess, sys + +SUPERSET_URL = "${SUPERSET_URL}" +ADMIN_USER = "${ADMIN_USER}" +ADMIN_PASS = "${ADMIN_PASS}" +ALERT_RECIPIENT = "${ALERT_RECIPIENT}" +COOKIE_JAR = "${COOKIE_JAR}" + +def curl(*args): + """Run curl and return stdout.""" + cmd = ["curl", "-s", "-b", COOKIE_JAR, "-c", COOKIE_JAR] + list(args) + result = subprocess.run(cmd, capture_output=True, text=True) + return result.stdout + +def curl_json(*args): + """Run curl and parse JSON response.""" + raw = curl(*args) + try: + return json.loads(raw) + except json.JSONDecodeError: + print(f"Failed to parse JSON: {raw[:200]}", file=sys.stderr) + sys.exit(1) + +# Login +print("Logging in...") +login_data = curl_json( + f"{SUPERSET_URL}/api/v1/security/login", + "-H", "Content-Type: application/json", + "-d", json.dumps({"username": ADMIN_USER, "password": ADMIN_PASS, "provider": "db"}) +) +token = login_data["access_token"] +auth = f"Bearer {token}" +print("Login OK") + +# CSRF - first call establishes session, second gets token +print("Fetching CSRF token...") +curl(f"{SUPERSET_URL}/api/v1/security/csrf_token/", "-H", f"Authorization: {auth}") +csrf_data = curl_json(f"{SUPERSET_URL}/api/v1/security/csrf_token/", "-H", f"Authorization: {auth}") +csrf_token = csrf_data["result"] +print("CSRF OK") + +# Check if alert already exists +reports = curl_json(f"{SUPERSET_URL}/api/v1/report/", "-H", f"Authorization: {auth}") +existing = [r for r in reports.get("result", []) if r.get("name") == "Wildfire Proximity Alert"] +if existing: + print("Wildfire Proximity Alert already exists, skipping.") + sys.exit(0) + +# Find DART database +databases = curl_json(f"{SUPERSET_URL}/api/v1/database/", "-H", f"Authorization: {auth}") +dart_dbs = [d for d in databases["result"] if "dart" in d["database_name"].lower()] +if not dart_dbs: + print("ERROR: No 'dart' database found in Superset", file=sys.stderr) + sys.exit(1) +database_id = dart_dbs[0]["id"] +print(f"Using database ID: {database_id}") + +# Find the wildfire proximity chart +charts = curl_json(f"{SUPERSET_URL}/api/v1/chart/", "-H", f"Authorization: {auth}") +chart_matches = [c for c in charts.get("result", []) if "active wildfire locations program office" in c.get("slice_name", "").lower()] +chart_id = chart_matches[0]["id"] if chart_matches else None +if chart_id: + print(f"Using chart ID: {chart_id}") +else: + print("WARNING: No 'active wildfire locations program office' chart found") + +# Build alert payload +alert = { + "type": "Alert", + "name": "Wildfire Proximity Alert", + "description": "Daily alert when active wildfires are within 25 miles of program offices (NIFC data)", + "active": True, + "crontab": "0 8 * * *", + "timezone": "America/New_York", + "database": database_id, + "sql": 'SELECT COUNT(*) FROM _program_locations_joined_with_active_nifc_fast WHERE "Name" IS NOT NULL AND is_fully_contained = false', + "validator_type": "operator", + "validator_config_json": {"op": ">", "threshold": 0}, + "recipients": [{"type": "Email", "recipient_config_json": {"target": ALERT_RECIPIENT}}], + "report_format": "TEXT", + "force_screenshot": False, +} +if chart_id: + alert["chart"] = chart_id + +# Create alert +print("Creating Wildfire Proximity Alert...") +result = curl_json( + f"{SUPERSET_URL}/api/v1/report/", + "-X", "POST", + "-H", f"Authorization: {auth}", + "-H", "Content-Type: application/json", + "-H", f"X-CSRFToken: {csrf_token}", + "-H", f"Referer: {SUPERSET_URL}/", + "-d", json.dumps(alert) +) + +if "id" in result: + print(f"Wildfire Proximity Alert created, id: {result['id']}") +else: + print(f"ERROR: {json.dumps(result, indent=2)}", file=sys.stderr) + sys.exit(1) +PYEOF diff --git a/kustomize/full/wildfire-alert/job.yaml b/kustomize/full/wildfire-alert/job.yaml new file mode 100644 index 0000000000..ecc0fcbd11 --- /dev/null +++ b/kustomize/full/wildfire-alert/job.yaml @@ -0,0 +1,54 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: wildfire-alert-bootstrap +spec: + backoffLimit: 3 + template: + metadata: + labels: + app: wildfire-alert-bootstrap + component: superset + spec: + restartPolicy: OnFailure + initContainers: + - name: wait-for-superset + image: busybox:1.36 + command: + - sh + - -c + - | + until nc -z superset-web 8088; do + echo "waiting for superset-web..." + sleep 5 + done + containers: + - name: bootstrap + image: jmeegan607/geoset + command: ["/bin/bash", "/scripts/bootstrap-alert.sh"] + envFrom: + - configMapRef: + name: superset-env + env: + - name: ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: superset-secrets + key: ADMIN_PASSWORD + - name: WILDFIRE_ALERT_RECIPIENT + value: "jmeegan@teamraft.com" + volumeMounts: + - name: bootstrap-script + mountPath: /scripts + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 250m + memory: 128Mi + volumes: + - name: bootstrap-script + configMap: + name: wildfire-alert-bootstrap + defaultMode: 0755 From 5ae098f45bb3324bb7204cb89873b38f6ab65367 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 2 Apr 2026 11:39:08 -0400 Subject: [PATCH 03/12] feat: add Mattermost notification support and Chromium Dockerfile for alerts Add MattermostNotification backend handler (stubbed), frontend UI for Mattermost as a notification method, and Dockerfile.chromium for building worker images with Playwright/Chromium. Patch base.py to respect ALERT_REPORTS_NOTIFICATION_METHODS config. Update worker/beat deployments with chromium image tag and MAPBOX_API_KEY from secrets. Co-Authored-By: Claude Opus 4.6 (1M context) --- kustomize/full/Dockerfile.chromium | 6 +++ kustomize/full/config/superset-env-patch.env | 4 ++ kustomize/full/superset-beat/deployment.yaml | 7 +++- .../full/superset-worker/deployment.yaml | 7 +++- .../alerts/components/NotificationMethod.tsx | 6 ++- .../alerts/components/RecipientIcon.tsx | 6 +++ .../src/features/alerts/types.ts | 2 + superset/reports/models.py | 1 + superset/reports/notifications/__init__.py | 1 + superset/reports/notifications/mattermost.py | 41 +++++++++++++++++++ superset/views/base.py | 6 ++- 11 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 kustomize/full/Dockerfile.chromium create mode 100644 superset/reports/notifications/mattermost.py diff --git a/kustomize/full/Dockerfile.chromium b/kustomize/full/Dockerfile.chromium new file mode 100644 index 0000000000..a0d36b2cec --- /dev/null +++ b/kustomize/full/Dockerfile.chromium @@ -0,0 +1,6 @@ +FROM jmeegan607/geoset +USER root +RUN uv pip install --python /app/.venv/bin/python playwright && playwright install-deps && /app/.venv/bin/playwright install chromium +RUN echo 'WEBDRIVER_TYPE = "chrome"' >> /app/docker/pythonpath_dev/superset_config_docker.py && \ + echo 'ALERT_REPORTS_NOTIFICATION_METHODS = ["Mattermost"]' >> /app/docker/pythonpath_dev/superset_config_docker.py +USER superset diff --git a/kustomize/full/config/superset-env-patch.env b/kustomize/full/config/superset-env-patch.env index 2ff0e15428..7ffccef2cc 100644 --- a/kustomize/full/config/superset-env-patch.env +++ b/kustomize/full/config/superset-env-patch.env @@ -21,4 +21,8 @@ SMTP_MAIL_FROM=geoset-alerts@teamraft.com ALERT_REPORTS_DRY_RUN=false # Webdriver for chart screenshots (must resolve from within K8s cluster) +WEBDRIVER_TYPE=chrome WEBDRIVER_BASEURL=http://superset-web:8088/ + +# Mattermost webhook for alerts & reports +MATTERMOST_WEBHOOK_URL= diff --git a/kustomize/full/superset-beat/deployment.yaml b/kustomize/full/superset-beat/deployment.yaml index bb2a1ce321..92a4fd0b8b 100644 --- a/kustomize/full/superset-beat/deployment.yaml +++ b/kustomize/full/superset-beat/deployment.yaml @@ -28,7 +28,7 @@ spec: done containers: - name: beat - image: jmeegan607/geoset + image: jmeegan607/geoset:chromium command: ["celery", "--app=superset.tasks.celery_app:app", "beat", "--loglevel=INFO", "--schedule=/tmp/celerybeat-schedule"] @@ -51,6 +51,11 @@ spec: secretKeyRef: name: superset-secrets key: SUPERSET_SECRET_KEY + - name: MAPBOX_API_KEY + valueFrom: + secretKeyRef: + name: superset-secrets + key: MAPBOX_API_KEY resources: requests: cpu: 50m diff --git a/kustomize/full/superset-worker/deployment.yaml b/kustomize/full/superset-worker/deployment.yaml index ea43597a25..43b79200b2 100644 --- a/kustomize/full/superset-worker/deployment.yaml +++ b/kustomize/full/superset-worker/deployment.yaml @@ -26,7 +26,7 @@ spec: done containers: - name: worker - image: jmeegan607/geoset + image: jmeegan607/geoset:chromium command: ["celery", "--app=superset.tasks.celery_app:app", "worker", "--pool=prefork", "-O", "fair", "-c", "4", "--loglevel=INFO"] @@ -49,6 +49,11 @@ spec: secretKeyRef: name: superset-secrets key: SUPERSET_SECRET_KEY + - name: MAPBOX_API_KEY + valueFrom: + secretKeyRef: + name: superset-secrets + key: MAPBOX_API_KEY resources: requests: cpu: 500m diff --git a/superset-frontend/src/features/alerts/components/NotificationMethod.tsx b/superset-frontend/src/features/alerts/components/NotificationMethod.tsx index 4c53cce544..9f5cd2910c 100644 --- a/superset-frontend/src/features/alerts/components/NotificationMethod.tsx +++ b/superset-frontend/src/features/alerts/components/NotificationMethod.tsx @@ -325,6 +325,8 @@ export const NotificationMethod: FunctionComponent = ({ ); if (slackEnabled && !slackOptions[0]?.options.length) { updateSlackOptions(); + } else { + setMethodOptionsLoading(false); } }, []); @@ -339,7 +341,8 @@ export const NotificationMethod: FunctionComponent = ({ ((!isFeatureEnabled(FeatureFlag.AlertReportSlackV2) || useSlackV1) && method === NotificationMethodOption.Slack) || - method === NotificationMethodOption.Email, + method === NotificationMethodOption.Email || + method === NotificationMethodOption.Mattermost, ) .map(method => ({ label: @@ -522,6 +525,7 @@ export const NotificationMethod: FunctionComponent = ({ {[ NotificationMethodOption.Email, NotificationMethodOption.Slack, + NotificationMethodOption.Mattermost, ].includes(method) ? ( <>
diff --git a/superset-frontend/src/features/alerts/components/RecipientIcon.tsx b/superset-frontend/src/features/alerts/components/RecipientIcon.tsx index 229194f443..b04199e6da 100644 --- a/superset-frontend/src/features/alerts/components/RecipientIcon.tsx +++ b/superset-frontend/src/features/alerts/components/RecipientIcon.tsx @@ -52,6 +52,12 @@ export default function RecipientIcon({ type }: { type: string }) { ); recipientIconConfig.label = NotificationMethodOption.Slack; break; + case NotificationMethodOption.Mattermost: + recipientIconConfig.icon = ( + + ); + recipientIconConfig.label = NotificationMethodOption.Mattermost; + break; default: recipientIconConfig.icon = null; recipientIconConfig.label = ''; diff --git a/superset-frontend/src/features/alerts/types.ts b/superset-frontend/src/features/alerts/types.ts index 06449697c3..9fc1dea847 100644 --- a/superset-frontend/src/features/alerts/types.ts +++ b/superset-frontend/src/features/alerts/types.ts @@ -45,6 +45,7 @@ export enum NotificationMethodOption { Email = 'Email', Slack = 'Slack', SlackV2 = 'SlackV2', + Mattermost = 'Mattermost', } export type SelectValue = { @@ -162,6 +163,7 @@ export enum RecipientIconName { Email = 'Email', Slack = 'Slack', SlackV2 = 'SlackV2', + Mattermost = 'Mattermost', } export interface AlertsReportsConfig { ALERT_REPORTS_DEFAULT_WORKING_TIMEOUT: number; diff --git a/superset/reports/models.py b/superset/reports/models.py index e4cdd7c9b4..d53362af19 100644 --- a/superset/reports/models.py +++ b/superset/reports/models.py @@ -63,6 +63,7 @@ class ReportRecipientType(StrEnum): EMAIL = "Email" SLACK = "Slack" SLACKV2 = "SlackV2" + MATTERMOST = "Mattermost" class ReportState(StrEnum): diff --git a/superset/reports/notifications/__init__.py b/superset/reports/notifications/__init__.py index 938e393447..b5d58b78ca 100644 --- a/superset/reports/notifications/__init__.py +++ b/superset/reports/notifications/__init__.py @@ -17,6 +17,7 @@ from superset.reports.models import ReportRecipients from superset.reports.notifications.base import BaseNotification, NotificationContent from superset.reports.notifications.email import EmailNotification # noqa: F401 +from superset.reports.notifications.mattermost import MattermostNotification # noqa: F401 from superset.reports.notifications.slack import SlackNotification # noqa: F401 from superset.reports.notifications.slackv2 import SlackV2Notification # noqa: F401 diff --git a/superset/reports/notifications/mattermost.py b/superset/reports/notifications/mattermost.py new file mode 100644 index 0000000000..ed7ed458be --- /dev/null +++ b/superset/reports/notifications/mattermost.py @@ -0,0 +1,41 @@ +import logging + +from superset.reports.models import ReportRecipientType +from superset.reports.notifications.base import BaseNotification +from superset.reports.notifications.slack_mixin import SlackMixin +from superset.utils import json +from superset.utils.core import recipients_string_to_list + +logger = logging.getLogger(__name__) + + +class MattermostNotification(SlackMixin, BaseNotification): + """ + Sends a notification to Mattermost via incoming webhook. + Currently stubbed — logs the payload and returns success. + TODO: Wire up to real Mattermost webhook. + """ + + type = ReportRecipientType.MATTERMOST + + def _get_channels(self) -> list[str]: + recipient_str = json.loads(self._recipient.recipient_config_json)["target"] + return recipients_string_to_list(recipient_str) + + def send(self) -> None: + body = self._get_body(content=self._content) + channels = self._get_channels() + + has_screenshot = bool(self._content.screenshots) + has_csv = bool(self._content.csv) + has_pdf = bool(self._content.pdf) + + logger.info( + "[MATTERMOST STUB] Would send to channels=%s | " + "screenshot=%s | csv=%s | pdf=%s | body=%s", + channels, + has_screenshot, + has_csv, + has_pdf, + body[:200], + ) diff --git a/superset/views/base.py b/superset/views/base.py index 276ef56de8..2a51655600 100644 --- a/superset/views/base.py +++ b/superset/views/base.py @@ -385,7 +385,11 @@ def cached_common_bootstrap_data( # pylint: disable=unused-argument for k in FRONTEND_CONF_KEYS } - if app.config.get("SLACK_API_TOKEN"): + if app.config.get("ALERT_REPORTS_NOTIFICATION_METHODS"): + frontend_config["ALERT_REPORTS_NOTIFICATION_METHODS"] = app.config[ + "ALERT_REPORTS_NOTIFICATION_METHODS" + ] + elif app.config.get("SLACK_API_TOKEN"): frontend_config["ALERT_REPORTS_NOTIFICATION_METHODS"] = [ ReportRecipientType.EMAIL, ReportRecipientType.SLACK, From 7e9aff3ec19778e453941ef1594f0580b99e9618 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 2 Apr 2026 13:37:19 -0400 Subject: [PATCH 04/12] feat: wire up real Mattermost webhook, add full Docker Compose, kustomize fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Mattermost notification now posts to real webhook with inline base64 screenshot support for PNG format alerts - Add docker-compose.full.yml with Redis, Celery worker/beat - Add configMapGenerator for superset-config-overrides (WEBDRIVER_TYPE, ALERT_REPORTS_NOTIFICATION_METHODS, PLAYWRIGHT feature flag) - Mount config overrides on web, worker, and beat deployments - Set imagePullPolicy: Always and chromium tag for full overlay - Comment out base image tag (6.0.48) — full overlay overrides with chromium Co-Authored-By: Claude Opus 4.6 (1M context) --- docker-compose.full.yml | 104 ++++++++++++++++++ kustomize/base/kustomization.yaml | 6 +- .../full/config/superset_config_docker.py | 7 ++ kustomize/full/kustomization.yaml | 7 ++ kustomize/full/superset-beat/deployment.yaml | 9 ++ kustomize/full/superset-web-patch.yaml | 11 ++ .../full/superset-worker/deployment.yaml | 9 ++ superset/reports/notifications/mattermost.py | 70 ++++++++---- 8 files changed, 199 insertions(+), 24 deletions(-) create mode 100644 docker-compose.full.yml create mode 100644 kustomize/full/config/superset_config_docker.py diff --git a/docker-compose.full.yml b/docker-compose.full.yml new file mode 100644 index 0000000000..8a09c2740a --- /dev/null +++ b/docker-compose.full.yml @@ -0,0 +1,104 @@ +# +# Full GeoSet development environment. +# Adds Redis, Celery worker, and Celery beat to the base compose. +# +# USAGE: +# docker compose -f docker-compose.yml -f docker-compose.full.yml up +# +services: + # ---------- Redis (Celery broker + cache) ---------- + redis: + image: redis:7-alpine + restart: unless-stopped + volumes: + - redis_data:/data + + # ---------- Override base superset to use full config ---------- + superset-init: + environment: + SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config.py + depends_on: + db: + condition: service_started + postgis: + condition: service_healthy + redis: + condition: service_started + + superset: + environment: + SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config.py + depends_on: + superset-init: + condition: service_completed_successfully + redis: + condition: service_started + + # ---------- Celery worker (executes alerts, takes screenshots) ---------- + superset-worker: + build: + context: . + dockerfile: ${DOCKERFILE:-Dockerfile} + target: dev + args: + DEV_MODE: "true" + INCLUDE_CHROMIUM: "true" + command: ["celery", "--app=superset.tasks.celery_app:app", + "worker", "--pool=prefork", "-O", "fair", + "-c", "2", "--loglevel=INFO"] + restart: unless-stopped + env_file: + - path: docker/.env + required: true + - path: docker/.env-local + required: false + environment: + SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config.py + user: root + depends_on: + superset-init: + condition: service_completed_successfully + redis: + condition: service_started + volumes: + - ./docker:/app/docker + - ./superset:/app/superset + - ./superset-frontend:/app/superset-frontend + - superset_home:/app/superset_home + - ./tests:/app/tests + + # ---------- Celery beat (schedules alerts) ---------- + superset-beat: + build: + context: . + dockerfile: ${DOCKERFILE:-Dockerfile} + target: dev + args: + DEV_MODE: "true" + command: ["celery", "--app=superset.tasks.celery_app:app", + "beat", "--loglevel=INFO", + "--schedule=/tmp/celerybeat-schedule"] + restart: unless-stopped + env_file: + - path: docker/.env + required: true + - path: docker/.env-local + required: false + environment: + SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config.py + user: root + depends_on: + superset-init: + condition: service_completed_successfully + redis: + condition: service_started + volumes: + - ./docker:/app/docker + - ./superset:/app/superset + - ./superset-frontend:/app/superset-frontend + - superset_home:/app/superset_home + - ./tests:/app/tests + +volumes: + redis_data: + external: false diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index 18867c24dc..abfa6923b0 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -17,9 +17,9 @@ configMapGenerator: generatorOptions: disableNameSuffixHash: true -images: - - name: jmeegan607/geoset - newTag: "6.0.48" +# images: +# - name: jmeegan607/geoset +# newTag: "6.0.48" resources: # Cluster setup diff --git a/kustomize/full/config/superset_config_docker.py b/kustomize/full/config/superset_config_docker.py new file mode 100644 index 0000000000..c29e07dc7d --- /dev/null +++ b/kustomize/full/config/superset_config_docker.py @@ -0,0 +1,7 @@ + +import os + +WEBDRIVER_TYPE = "chrome" +ALERT_REPORTS_NOTIFICATION_METHODS = ["Mattermost"] +FEATURE_FLAGS = {"ALERT_REPORTS": True, "PLAYWRIGHT_REPORTS_AND_THUMBNAILS": True} +MATTERMOST_WEBHOOK_URL = os.getenv("MATTERMOST_WEBHOOK_URL", "https://mattermost.teamraft.com/hooks/q7co9uqot7g398cnzg63kfxaay") diff --git a/kustomize/full/kustomization.yaml b/kustomize/full/kustomization.yaml index 2055f3c4a7..f3238e9b5f 100644 --- a/kustomize/full/kustomization.yaml +++ b/kustomize/full/kustomization.yaml @@ -20,6 +20,9 @@ configMapGenerator: behavior: merge envs: - config/superset-env-patch.env + - name: superset-config-overrides + files: + - superset_config_docker.py=config/superset_config_docker.py - name: wildfire-alert-bootstrap files: - bootstrap-alert.sh=wildfire-alert/bootstrap-alert.sh @@ -27,6 +30,10 @@ configMapGenerator: generatorOptions: disableNameSuffixHash: true +images: + - name: jmeegan607/geoset + newTag: chromium + # Patch superset-web: bump replicas to 2 and wait for Redis patches: - path: superset-web-patch.yaml diff --git a/kustomize/full/superset-beat/deployment.yaml b/kustomize/full/superset-beat/deployment.yaml index 92a4fd0b8b..999d7cc642 100644 --- a/kustomize/full/superset-beat/deployment.yaml +++ b/kustomize/full/superset-beat/deployment.yaml @@ -29,6 +29,7 @@ spec: containers: - name: beat image: jmeegan607/geoset:chromium + imagePullPolicy: Always command: ["celery", "--app=superset.tasks.celery_app:app", "beat", "--loglevel=INFO", "--schedule=/tmp/celerybeat-schedule"] @@ -56,6 +57,10 @@ spec: secretKeyRef: name: superset-secrets key: MAPBOX_API_KEY + volumeMounts: + - name: config-overrides + mountPath: /app/docker/pythonpath_dev/superset_config_docker.py + subPath: superset_config_docker.py resources: requests: cpu: 50m @@ -63,3 +68,7 @@ spec: limits: cpu: 250m memory: 256Mi + volumes: + - name: config-overrides + configMap: + name: superset-config-overrides diff --git a/kustomize/full/superset-web-patch.yaml b/kustomize/full/superset-web-patch.yaml index 1f4ba5e376..15cea46503 100644 --- a/kustomize/full/superset-web-patch.yaml +++ b/kustomize/full/superset-web-patch.yaml @@ -21,3 +21,14 @@ spec: echo "waiting for redis..." sleep 2 done + containers: + - name: superset + imagePullPolicy: Always + volumeMounts: + - name: config-overrides + mountPath: /app/docker/pythonpath_dev/superset_config_docker.py + subPath: superset_config_docker.py + volumes: + - name: config-overrides + configMap: + name: superset-config-overrides diff --git a/kustomize/full/superset-worker/deployment.yaml b/kustomize/full/superset-worker/deployment.yaml index 43b79200b2..7afd509e69 100644 --- a/kustomize/full/superset-worker/deployment.yaml +++ b/kustomize/full/superset-worker/deployment.yaml @@ -27,6 +27,7 @@ spec: containers: - name: worker image: jmeegan607/geoset:chromium + imagePullPolicy: Always command: ["celery", "--app=superset.tasks.celery_app:app", "worker", "--pool=prefork", "-O", "fair", "-c", "4", "--loglevel=INFO"] @@ -61,6 +62,10 @@ spec: limits: cpu: "2" memory: 2Gi + volumeMounts: + - name: config-overrides + mountPath: /app/docker/pythonpath_dev/superset_config_docker.py + subPath: superset_config_docker.py livenessProbe: exec: command: @@ -70,3 +75,7 @@ spec: initialDelaySeconds: 60 periodSeconds: 60 timeoutSeconds: 10 + volumes: + - name: config-overrides + configMap: + name: superset-config-overrides diff --git a/superset/reports/notifications/mattermost.py b/superset/reports/notifications/mattermost.py index ed7ed458be..21e6aae549 100644 --- a/superset/reports/notifications/mattermost.py +++ b/superset/reports/notifications/mattermost.py @@ -1,10 +1,17 @@ +import base64 import logging +import requests +from flask import current_app + from superset.reports.models import ReportRecipientType from superset.reports.notifications.base import BaseNotification +from superset.reports.notifications.exceptions import ( + NotificationParamException, + NotificationUnprocessableException, +) from superset.reports.notifications.slack_mixin import SlackMixin from superset.utils import json -from superset.utils.core import recipients_string_to_list logger = logging.getLogger(__name__) @@ -12,30 +19,51 @@ class MattermostNotification(SlackMixin, BaseNotification): """ Sends a notification to Mattermost via incoming webhook. - Currently stubbed — logs the payload and returns success. - TODO: Wire up to real Mattermost webhook. + Embeds screenshots as inline base64 images. """ type = ReportRecipientType.MATTERMOST - def _get_channels(self) -> list[str]: - recipient_str = json.loads(self._recipient.recipient_config_json)["target"] - return recipients_string_to_list(recipient_str) + def _get_inline_image(self) -> str | None: + """Get the first available image as a base64 data URI.""" + if self._content.screenshots: + b64 = base64.b64encode(self._content.screenshots[0]).decode() + return f"data:image/png;base64,{b64}" + if self._content.pdf: + # PDF is built from screenshots — encode it as-is + # Mattermost won't render PDF inline, but we can try + # converting via the first page if possible + try: + from superset.utils.pdf import build_pdf_from_screenshots # noqa + + # The PDF bytes are already built, no way to reverse easily. + # Just skip image for PDF format — text + link is sent. + return None + except ImportError: + return None + return None def send(self) -> None: + webhook_url = current_app.config.get("MATTERMOST_WEBHOOK_URL") + if not webhook_url: + raise NotificationParamException( + "MATTERMOST_WEBHOOK_URL is not configured in superset_config" + ) + body = self._get_body(content=self._content) - channels = self._get_channels() - - has_screenshot = bool(self._content.screenshots) - has_csv = bool(self._content.csv) - has_pdf = bool(self._content.pdf) - - logger.info( - "[MATTERMOST STUB] Would send to channels=%s | " - "screenshot=%s | csv=%s | pdf=%s | body=%s", - channels, - has_screenshot, - has_csv, - has_pdf, - body[:200], - ) + image_uri = self._get_inline_image() + + if image_uri: + body += f"\n\n![report screenshot]({image_uri})" + + payload: dict = {"text": body} + + try: + resp = requests.post(webhook_url, json=payload, timeout=30) + resp.raise_for_status() + except requests.exceptions.RequestException as ex: + raise NotificationUnprocessableException( + f"Failed to send Mattermost notification: {ex}" + ) from ex + + logger.info("Report sent to Mattermost") From 117d5f663f6249d98cffedc96e6635c404669a1c Mon Sep 17 00:00:00 2001 From: James Date: Thu, 2 Apr 2026 15:19:02 -0400 Subject: [PATCH 05/12] feat: add inline base64 screenshot support to Mattermost notification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Encode PNG screenshots as data URIs and embed in webhook payload. Fails hard if screenshot encoding fails. Includes debug logging for content inspection. Note: large screenshots may exceed Mattermost's default max post size — Bot API file upload needed for production. Co-Authored-By: Claude Opus 4.6 (1M context) --- superset/reports/notifications/mattermost.py | 26 ++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/superset/reports/notifications/mattermost.py b/superset/reports/notifications/mattermost.py index 21e6aae549..81879905f7 100644 --- a/superset/reports/notifications/mattermost.py +++ b/superset/reports/notifications/mattermost.py @@ -53,9 +53,35 @@ def send(self) -> None: body = self._get_body(content=self._content) image_uri = self._get_inline_image() + if not image_uri and self._content.screenshots: + raise NotificationUnprocessableException( + "Failed to encode screenshot for Mattermost" + ) + if image_uri: body += f"\n\n![report screenshot]({image_uri})" + logger.info( + "[MATTERMOST DEBUG] screenshots=%s pdf=%s csv=%s text=%s body_len=%d", + len(self._content.screenshots) if self._content.screenshots else 0, + len(self._content.pdf) if self._content.pdf else 0, + len(self._content.csv) if self._content.csv else 0, + bool(self._content.text), + len(body), + ) + logger.info("[MATTERMOST DEBUG] body=%s", body[:500]) + + # Save files to /tmp + if self._content.screenshots: + for i, img in enumerate(self._content.screenshots): + with open(f"/tmp/mattermost_screenshot_{i}.png", "wb") as f: + f.write(img) + logger.info("[MATTERMOST DEBUG] Saved screenshot %d (%d bytes)", i, len(img)) + if self._content.pdf: + with open("/tmp/mattermost_report.pdf", "wb") as f: + f.write(self._content.pdf) + logger.info("[MATTERMOST DEBUG] Saved PDF (%d bytes)", len(self._content.pdf)) + payload: dict = {"text": body} try: From 13fd935e3888c93a9785aaffcd914bdc61a7340f Mon Sep 17 00:00:00 2001 From: James Date: Fri, 3 Apr 2026 16:24:02 -0400 Subject: [PATCH 06/12] refactor: move GeoSet config overrides out of upstream superset_config.py Create superset_config_docker_full.py as a committed override file for the full Docker Compose stack. This keeps upstream superset_config.py clean (zero diff against apache/superset) while preserving GeoSet-specific config: FixedExecutor fallback, cache warmup logging, Mattermost alerts, and webdriver settings. docker-compose.full.yml mounts it as superset_config_docker.py so it's auto-imported by the base config. Co-Authored-By: Claude Opus 4.6 (1M context) --- docker-compose.full.yml | 6 ++ docker/pythonpath_dev/.gitignore | 1 + docker/pythonpath_dev/superset_config.py | 9 +-- .../superset_config_docker_full.py | 74 +++++++++++++++++++ 4 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 docker/pythonpath_dev/superset_config_docker_full.py diff --git a/docker-compose.full.yml b/docker-compose.full.yml index 8a09c2740a..cc00ea757d 100644 --- a/docker-compose.full.yml +++ b/docker-compose.full.yml @@ -17,6 +17,8 @@ services: superset-init: environment: SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config.py + volumes: + - ./docker/pythonpath_dev/superset_config_docker_full.py:/app/docker/pythonpath_dev/superset_config_docker.py depends_on: db: condition: service_started @@ -28,6 +30,8 @@ services: superset: environment: SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config.py + volumes: + - ./docker/pythonpath_dev/superset_config_docker_full.py:/app/docker/pythonpath_dev/superset_config_docker.py depends_on: superset-init: condition: service_completed_successfully @@ -62,6 +66,7 @@ services: condition: service_started volumes: - ./docker:/app/docker + - ./docker/pythonpath_dev/superset_config_docker_full.py:/app/docker/pythonpath_dev/superset_config_docker.py - ./superset:/app/superset - ./superset-frontend:/app/superset-frontend - superset_home:/app/superset_home @@ -94,6 +99,7 @@ services: condition: service_started volumes: - ./docker:/app/docker + - ./docker/pythonpath_dev/superset_config_docker_full.py:/app/docker/pythonpath_dev/superset_config_docker.py - ./superset:/app/superset - ./superset-frontend:/app/superset-frontend - superset_home:/app/superset_home diff --git a/docker/pythonpath_dev/.gitignore b/docker/pythonpath_dev/.gitignore index 97cac2072f..dbb752bd13 100644 --- a/docker/pythonpath_dev/.gitignore +++ b/docker/pythonpath_dev/.gitignore @@ -21,4 +21,5 @@ !.gitignore !superset_config.py !superset_config_docker_light.py +!superset_config_docker_full.py !superset_config_local.example diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index b2c6497891..2de8f03794 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -100,12 +100,11 @@ class CeleryConfig: CELERY_CONFIG = CeleryConfig FEATURE_FLAGS = {"ALERT_REPORTS": True} -ALERT_REPORTS_NOTIFICATION_DRY_RUN = False -WEBDRIVER_BASEURL = os.environ.get("WEBDRIVER_BASEURL", f"http://superset_app{os.environ.get('SUPERSET_APP_ROOT', '/')}/") +ALERT_REPORTS_NOTIFICATION_DRY_RUN = True +WEBDRIVER_BASEURL = f"http://superset_app{os.environ.get('SUPERSET_APP_ROOT', '/')}/" # When using docker compose baseurl should be http://superset_nginx{ENV{BASEPATH}}/ # noqa: E501 # The base URL for the email report hyperlinks. -WEBDRIVER_BASEURL_USER_FRIENDLY = os.environ.get( - "WEBDRIVER_BASEURL_USER_FRIENDLY", - f"http://localhost:8888/{os.environ.get('SUPERSET_APP_ROOT', '/')}/", +WEBDRIVER_BASEURL_USER_FRIENDLY = ( + f"http://localhost:8888/{os.environ.get('SUPERSET_APP_ROOT', '/')}/" ) SQLLAB_CTAS_NO_LIMIT = True diff --git a/docker/pythonpath_dev/superset_config_docker_full.py b/docker/pythonpath_dev/superset_config_docker_full.py new file mode 100644 index 0000000000..c0d68904f3 --- /dev/null +++ b/docker/pythonpath_dev/superset_config_docker_full.py @@ -0,0 +1,74 @@ +"""GeoSet overrides for the full Docker Compose stack (Redis + Celery). + +Loaded automatically by superset_config.py via ``from superset_config_docker import *``. +docker-compose.full.yml mounts this file as superset_config_docker.py inside +the container so it is picked up without touching the upstream base config. +""" + +import logging +import os + +from celery.signals import task_failure, task_postrun, task_prerun +from superset.tasks.types import ExecutorType, FixedExecutor + +# Alerts & Reports +ALERT_REPORTS_NOTIFICATION_DRY_RUN = False +ALERT_REPORTS_NOTIFICATION_METHODS = ["Mattermost"] +WEBDRIVER_TYPE = "chrome" +FEATURE_FLAGS = {"ALERT_REPORTS": True, "PLAYWRIGHT_REPORTS_AND_THUMBNAILS": True} +MATTERMOST_WEBHOOK_URL = os.getenv( + "MATTERMOST_WEBHOOK_URL", + "https://mattermost.teamraft.com/hooks/q7co9uqot7g398cnzg63kfxaay", +) +WEBDRIVER_BASEURL = "http://superset:8088/" +WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL + +# Fallback executor for charts without owners (e.g. GeoSet example charts) +CACHE_WARMUP_EXECUTORS = [ExecutorType.OWNER, FixedExecutor("admin")] + +# --------------------------------------------------------------------------- +# Cache warmup logging via Celery signals +# --------------------------------------------------------------------------- +_warmup_logger = logging.getLogger("geoset.cache_warmup") + + +@task_prerun.connect(sender=None) +def _log_cache_warmup_start(sender=None, task_id=None, args=None, kwargs=None, **kw): + if sender and sender.name == "cache-warmup": + strategy = kwargs.get("strategy_name", "unknown") if kwargs else "unknown" + top_n = kwargs.get("top_n", "N/A") if kwargs else "N/A" + _warmup_logger.info( + "[CACHE-WARMUP] Starting | strategy=%s top_n=%s task_id=%s", + strategy, top_n, task_id, + ) + + +@task_postrun.connect(sender=None) +def _log_cache_warmup_done( + sender=None, task_id=None, retval=None, state=None, **kw +): + if sender and sender.name == "cache-warmup": + if isinstance(retval, dict): + scheduled = len(retval.get("scheduled", [])) + errors = len(retval.get("errors", [])) + _warmup_logger.info( + "[CACHE-WARMUP] Finished | scheduled=%d errors=%d state=%s task_id=%s", + scheduled, errors, state, task_id, + ) + else: + _warmup_logger.warning( + "[CACHE-WARMUP] Finished with non-dict result | result=%s state=%s task_id=%s", + retval, state, task_id, + ) + + +@task_failure.connect(sender=None) +def _log_cache_warmup_failure( + sender=None, task_id=None, exception=None, traceback=None, **kw +): + if sender and sender.name == "cache-warmup": + _warmup_logger.error( + "[CACHE-WARMUP] FAILED | exception=%s task_id=%s", + exception, task_id, + exc_info=True, + ) From ac30b799442234842e8885bee80c9e3ab1ef5055 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 3 Apr 2026 16:48:49 -0400 Subject: [PATCH 07/12] refactor: replace Dockerfile.chromium with Dockerfile.full, remove wildfire-alert Rename Dockerfile.chromium to Dockerfile.full and bake the GeoSet config override into the image instead of appending config lines at build time. Remove the wildfire-alert bootstrap job (POC cleanup). Update kustomize image tag from chromium to full. Co-Authored-By: Claude Opus 4.6 (1M context) --- kustomize/full/Dockerfile.chromium | 6 - kustomize/full/Dockerfile.full | 5 + kustomize/full/kustomization.yaml | 6 +- .../full/wildfire-alert/bootstrap-alert.sh | 122 ------------------ kustomize/full/wildfire-alert/job.yaml | 54 -------- 5 files changed, 6 insertions(+), 187 deletions(-) delete mode 100644 kustomize/full/Dockerfile.chromium create mode 100644 kustomize/full/Dockerfile.full delete mode 100755 kustomize/full/wildfire-alert/bootstrap-alert.sh delete mode 100644 kustomize/full/wildfire-alert/job.yaml diff --git a/kustomize/full/Dockerfile.chromium b/kustomize/full/Dockerfile.chromium deleted file mode 100644 index a0d36b2cec..0000000000 --- a/kustomize/full/Dockerfile.chromium +++ /dev/null @@ -1,6 +0,0 @@ -FROM jmeegan607/geoset -USER root -RUN uv pip install --python /app/.venv/bin/python playwright && playwright install-deps && /app/.venv/bin/playwright install chromium -RUN echo 'WEBDRIVER_TYPE = "chrome"' >> /app/docker/pythonpath_dev/superset_config_docker.py && \ - echo 'ALERT_REPORTS_NOTIFICATION_METHODS = ["Mattermost"]' >> /app/docker/pythonpath_dev/superset_config_docker.py -USER superset diff --git a/kustomize/full/Dockerfile.full b/kustomize/full/Dockerfile.full new file mode 100644 index 0000000000..8be0629008 --- /dev/null +++ b/kustomize/full/Dockerfile.full @@ -0,0 +1,5 @@ +FROM jmeegan607/geoset +USER root +RUN uv pip install --python /app/.venv/bin/python playwright && playwright install-deps && /app/.venv/bin/playwright install chromium +COPY docker/pythonpath_dev/superset_config_docker_full.py /app/docker/pythonpath_dev/superset_config_docker.py +USER superset diff --git a/kustomize/full/kustomization.yaml b/kustomize/full/kustomization.yaml index f3238e9b5f..b25b7162c6 100644 --- a/kustomize/full/kustomization.yaml +++ b/kustomize/full/kustomization.yaml @@ -10,7 +10,6 @@ resources: - redis/service.yaml - superset-worker/deployment.yaml - superset-beat/deployment.yaml - - wildfire-alert/job.yaml - mailhog/deployment.yaml - mailhog/service.yaml @@ -23,16 +22,13 @@ configMapGenerator: - name: superset-config-overrides files: - superset_config_docker.py=config/superset_config_docker.py - - name: wildfire-alert-bootstrap - files: - - bootstrap-alert.sh=wildfire-alert/bootstrap-alert.sh generatorOptions: disableNameSuffixHash: true images: - name: jmeegan607/geoset - newTag: chromium + newTag: full # Patch superset-web: bump replicas to 2 and wait for Redis patches: diff --git a/kustomize/full/wildfire-alert/bootstrap-alert.sh b/kustomize/full/wildfire-alert/bootstrap-alert.sh deleted file mode 100755 index 41c2872b15..0000000000 --- a/kustomize/full/wildfire-alert/bootstrap-alert.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Bootstrap the wildfire proximity alert in Superset Alerts & Reports. -# Runs once after superset-init to seed the SQL alert via the REST API. - -SUPERSET_URL="${SUPERSET_URL:-http://superset-web:8088}" -ADMIN_USER="${ADMIN_USERNAME:-admin}" -ADMIN_PASS="${ADMIN_PASSWORD}" -ALERT_RECIPIENT="${WILDFIRE_ALERT_RECIPIENT:-jmeegan@teamraft.com}" -COOKIE_JAR="/tmp/superset_cookies.txt" - -echo "Waiting for Superset API to be ready..." -until curl -sf "${SUPERSET_URL}/health" > /dev/null 2>&1; do - sleep 5 -done - -# Run the rest via Python to avoid bash quoting hell -python3 << PYEOF -import json, subprocess, sys - -SUPERSET_URL = "${SUPERSET_URL}" -ADMIN_USER = "${ADMIN_USER}" -ADMIN_PASS = "${ADMIN_PASS}" -ALERT_RECIPIENT = "${ALERT_RECIPIENT}" -COOKIE_JAR = "${COOKIE_JAR}" - -def curl(*args): - """Run curl and return stdout.""" - cmd = ["curl", "-s", "-b", COOKIE_JAR, "-c", COOKIE_JAR] + list(args) - result = subprocess.run(cmd, capture_output=True, text=True) - return result.stdout - -def curl_json(*args): - """Run curl and parse JSON response.""" - raw = curl(*args) - try: - return json.loads(raw) - except json.JSONDecodeError: - print(f"Failed to parse JSON: {raw[:200]}", file=sys.stderr) - sys.exit(1) - -# Login -print("Logging in...") -login_data = curl_json( - f"{SUPERSET_URL}/api/v1/security/login", - "-H", "Content-Type: application/json", - "-d", json.dumps({"username": ADMIN_USER, "password": ADMIN_PASS, "provider": "db"}) -) -token = login_data["access_token"] -auth = f"Bearer {token}" -print("Login OK") - -# CSRF - first call establishes session, second gets token -print("Fetching CSRF token...") -curl(f"{SUPERSET_URL}/api/v1/security/csrf_token/", "-H", f"Authorization: {auth}") -csrf_data = curl_json(f"{SUPERSET_URL}/api/v1/security/csrf_token/", "-H", f"Authorization: {auth}") -csrf_token = csrf_data["result"] -print("CSRF OK") - -# Check if alert already exists -reports = curl_json(f"{SUPERSET_URL}/api/v1/report/", "-H", f"Authorization: {auth}") -existing = [r for r in reports.get("result", []) if r.get("name") == "Wildfire Proximity Alert"] -if existing: - print("Wildfire Proximity Alert already exists, skipping.") - sys.exit(0) - -# Find DART database -databases = curl_json(f"{SUPERSET_URL}/api/v1/database/", "-H", f"Authorization: {auth}") -dart_dbs = [d for d in databases["result"] if "dart" in d["database_name"].lower()] -if not dart_dbs: - print("ERROR: No 'dart' database found in Superset", file=sys.stderr) - sys.exit(1) -database_id = dart_dbs[0]["id"] -print(f"Using database ID: {database_id}") - -# Find the wildfire proximity chart -charts = curl_json(f"{SUPERSET_URL}/api/v1/chart/", "-H", f"Authorization: {auth}") -chart_matches = [c for c in charts.get("result", []) if "active wildfire locations program office" in c.get("slice_name", "").lower()] -chart_id = chart_matches[0]["id"] if chart_matches else None -if chart_id: - print(f"Using chart ID: {chart_id}") -else: - print("WARNING: No 'active wildfire locations program office' chart found") - -# Build alert payload -alert = { - "type": "Alert", - "name": "Wildfire Proximity Alert", - "description": "Daily alert when active wildfires are within 25 miles of program offices (NIFC data)", - "active": True, - "crontab": "0 8 * * *", - "timezone": "America/New_York", - "database": database_id, - "sql": 'SELECT COUNT(*) FROM _program_locations_joined_with_active_nifc_fast WHERE "Name" IS NOT NULL AND is_fully_contained = false', - "validator_type": "operator", - "validator_config_json": {"op": ">", "threshold": 0}, - "recipients": [{"type": "Email", "recipient_config_json": {"target": ALERT_RECIPIENT}}], - "report_format": "TEXT", - "force_screenshot": False, -} -if chart_id: - alert["chart"] = chart_id - -# Create alert -print("Creating Wildfire Proximity Alert...") -result = curl_json( - f"{SUPERSET_URL}/api/v1/report/", - "-X", "POST", - "-H", f"Authorization: {auth}", - "-H", "Content-Type: application/json", - "-H", f"X-CSRFToken: {csrf_token}", - "-H", f"Referer: {SUPERSET_URL}/", - "-d", json.dumps(alert) -) - -if "id" in result: - print(f"Wildfire Proximity Alert created, id: {result['id']}") -else: - print(f"ERROR: {json.dumps(result, indent=2)}", file=sys.stderr) - sys.exit(1) -PYEOF diff --git a/kustomize/full/wildfire-alert/job.yaml b/kustomize/full/wildfire-alert/job.yaml deleted file mode 100644 index ecc0fcbd11..0000000000 --- a/kustomize/full/wildfire-alert/job.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: wildfire-alert-bootstrap -spec: - backoffLimit: 3 - template: - metadata: - labels: - app: wildfire-alert-bootstrap - component: superset - spec: - restartPolicy: OnFailure - initContainers: - - name: wait-for-superset - image: busybox:1.36 - command: - - sh - - -c - - | - until nc -z superset-web 8088; do - echo "waiting for superset-web..." - sleep 5 - done - containers: - - name: bootstrap - image: jmeegan607/geoset - command: ["/bin/bash", "/scripts/bootstrap-alert.sh"] - envFrom: - - configMapRef: - name: superset-env - env: - - name: ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: superset-secrets - key: ADMIN_PASSWORD - - name: WILDFIRE_ALERT_RECIPIENT - value: "jmeegan@teamraft.com" - volumeMounts: - - name: bootstrap-script - mountPath: /scripts - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 250m - memory: 128Mi - volumes: - - name: bootstrap-script - configMap: - name: wildfire-alert-bootstrap - defaultMode: 0755 From 0cdb9b1f4b86f6e64f4dd1213d6295d882b0055f Mon Sep 17 00:00:00 2001 From: James Date: Fri, 3 Apr 2026 16:52:34 -0400 Subject: [PATCH 08/12] chore: remove MailHog from kustomize full overlay Co-Authored-By: Claude Opus 4.6 (1M context) --- kustomize/full/config/superset-env-patch.env | 5 ++-- kustomize/full/kustomization.yaml | 2 -- kustomize/full/mailhog/deployment.yaml | 31 -------------------- kustomize/full/mailhog/service.yaml | 16 ---------- 4 files changed, 2 insertions(+), 52 deletions(-) delete mode 100644 kustomize/full/mailhog/deployment.yaml delete mode 100644 kustomize/full/mailhog/service.yaml diff --git a/kustomize/full/config/superset-env-patch.env b/kustomize/full/config/superset-env-patch.env index 7ffccef2cc..89e09f50c8 100644 --- a/kustomize/full/config/superset-env-patch.env +++ b/kustomize/full/config/superset-env-patch.env @@ -8,9 +8,8 @@ REDIS_CELERY_DB=0 REDIS_RESULTS_DB=1 # SMTP for Alerts & Reports -# Using MailHog (in-cluster fake SMTP) for dev/testing — swap to real SMTP for production -SMTP_HOST=mailhog -SMTP_PORT=1025 +SMTP_HOST=localhost +SMTP_PORT=25 SMTP_STARTTLS=false SMTP_SSL=false SMTP_USER= diff --git a/kustomize/full/kustomization.yaml b/kustomize/full/kustomization.yaml index b25b7162c6..672ce725b9 100644 --- a/kustomize/full/kustomization.yaml +++ b/kustomize/full/kustomization.yaml @@ -10,8 +10,6 @@ resources: - redis/service.yaml - superset-worker/deployment.yaml - superset-beat/deployment.yaml - - mailhog/deployment.yaml - - mailhog/service.yaml # Override the superset-env configmap to add Redis vars and switch to full config configMapGenerator: diff --git a/kustomize/full/mailhog/deployment.yaml b/kustomize/full/mailhog/deployment.yaml deleted file mode 100644 index fd54da4835..0000000000 --- a/kustomize/full/mailhog/deployment.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mailhog - labels: - app: mailhog -spec: - replicas: 1 - selector: - matchLabels: - app: mailhog - template: - metadata: - labels: - app: mailhog - spec: - containers: - - name: mailhog - image: mailhog/mailhog:latest - ports: - - name: smtp - containerPort: 1025 - - name: http - containerPort: 8025 - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 200m - memory: 128Mi diff --git a/kustomize/full/mailhog/service.yaml b/kustomize/full/mailhog/service.yaml deleted file mode 100644 index 88f9634b64..0000000000 --- a/kustomize/full/mailhog/service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: mailhog - labels: - app: mailhog -spec: - selector: - app: mailhog - ports: - - name: smtp - port: 1025 - targetPort: 1025 - - name: http - port: 8025 - targetPort: 8025 From cacb51710f64d3c4c74466be133c840a777c7c32 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 3 Apr 2026 16:54:27 -0400 Subject: [PATCH 09/12] chore: remove unused SMTP env vars from kustomize full overlay Co-Authored-By: Claude Opus 4.6 (1M context) --- kustomize/full/config/superset-env-patch.env | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/kustomize/full/config/superset-env-patch.env b/kustomize/full/config/superset-env-patch.env index 89e09f50c8..5417fcd8c8 100644 --- a/kustomize/full/config/superset-env-patch.env +++ b/kustomize/full/config/superset-env-patch.env @@ -7,21 +7,8 @@ REDIS_PORT=6379 REDIS_CELERY_DB=0 REDIS_RESULTS_DB=1 -# SMTP for Alerts & Reports -SMTP_HOST=localhost -SMTP_PORT=25 -SMTP_STARTTLS=false -SMTP_SSL=false -SMTP_USER= -SMTP_PASSWORD= -SMTP_MAIL_FROM=geoset-alerts@teamraft.com - # Alerts & Reports ALERT_REPORTS_DRY_RUN=false - -# Webdriver for chart screenshots (must resolve from within K8s cluster) WEBDRIVER_TYPE=chrome WEBDRIVER_BASEURL=http://superset-web:8088/ - -# Mattermost webhook for alerts & reports MATTERMOST_WEBHOOK_URL= From 76efa70944225c542d89be418e0bbc199f74a476 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 22 Jun 2026 12:08:06 -0400 Subject: [PATCH 10/12] save work so far --- kustomize/base/superset-web/deployment.yaml | 7 -- kustomize/full/Dockerfile.full | 1 - .../full/config/superset_config_docker.py | 70 ++++++++++++++++++- kustomize/full/redis/statefulset.yaml | 7 -- kustomize/full/superset-beat/deployment.yaml | 7 -- .../full/superset-worker/deployment.yaml | 7 -- 6 files changed, 68 insertions(+), 31 deletions(-) diff --git a/kustomize/base/superset-web/deployment.yaml b/kustomize/base/superset-web/deployment.yaml index b257083c1e..91c617e144 100644 --- a/kustomize/base/superset-web/deployment.yaml +++ b/kustomize/base/superset-web/deployment.yaml @@ -55,13 +55,6 @@ spec: value: "4" ports: - containerPort: 8088 - resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: "1" - memory: 1Gi readinessProbe: httpGet: path: /health diff --git a/kustomize/full/Dockerfile.full b/kustomize/full/Dockerfile.full index 8be0629008..7f1c0f07ba 100644 --- a/kustomize/full/Dockerfile.full +++ b/kustomize/full/Dockerfile.full @@ -1,5 +1,4 @@ FROM jmeegan607/geoset USER root RUN uv pip install --python /app/.venv/bin/python playwright && playwright install-deps && /app/.venv/bin/playwright install chromium -COPY docker/pythonpath_dev/superset_config_docker_full.py /app/docker/pythonpath_dev/superset_config_docker.py USER superset diff --git a/kustomize/full/config/superset_config_docker.py b/kustomize/full/config/superset_config_docker.py index c29e07dc7d..ee0638c45f 100644 --- a/kustomize/full/config/superset_config_docker.py +++ b/kustomize/full/config/superset_config_docker.py @@ -1,7 +1,73 @@ +"""GeoSet overrides for the full Kubernetes deployment (Redis + Celery). +Mounted as a ConfigMap into the container as superset_config_docker.py +so it is auto-imported by the upstream base config. +""" + +import logging import os -WEBDRIVER_TYPE = "chrome" +from celery.signals import task_failure, task_postrun, task_prerun +from superset.tasks.types import ExecutorType, FixedExecutor + +# Alerts & Reports +ALERT_REPORTS_NOTIFICATION_DRY_RUN = False ALERT_REPORTS_NOTIFICATION_METHODS = ["Mattermost"] +WEBDRIVER_TYPE = "chrome" FEATURE_FLAGS = {"ALERT_REPORTS": True, "PLAYWRIGHT_REPORTS_AND_THUMBNAILS": True} -MATTERMOST_WEBHOOK_URL = os.getenv("MATTERMOST_WEBHOOK_URL", "https://mattermost.teamraft.com/hooks/q7co9uqot7g398cnzg63kfxaay") +MATTERMOST_WEBHOOK_URL = os.getenv( + "MATTERMOST_WEBHOOK_URL", + "https://mattermost.teamraft.com/hooks/q7co9uqot7g398cnzg63kfxaay", +) +WEBDRIVER_BASEURL = "http://superset-web:8088/" +WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL + +# Fallback executor for charts without owners (e.g. GeoSet example charts) +CACHE_WARMUP_EXECUTORS = [ExecutorType.OWNER, FixedExecutor("admin")] + +# --------------------------------------------------------------------------- +# Cache warmup logging via Celery signals +# --------------------------------------------------------------------------- +_warmup_logger = logging.getLogger("geoset.cache_warmup") + + +@task_prerun.connect(sender=None) +def _log_cache_warmup_start(sender=None, task_id=None, args=None, kwargs=None, **kw): + if sender and sender.name == "cache-warmup": + strategy = kwargs.get("strategy_name", "unknown") if kwargs else "unknown" + top_n = kwargs.get("top_n", "N/A") if kwargs else "N/A" + _warmup_logger.info( + "[CACHE-WARMUP] Starting | strategy=%s top_n=%s task_id=%s", + strategy, top_n, task_id, + ) + + +@task_postrun.connect(sender=None) +def _log_cache_warmup_done( + sender=None, task_id=None, retval=None, state=None, **kw +): + if sender and sender.name == "cache-warmup": + if isinstance(retval, dict): + scheduled = len(retval.get("scheduled", [])) + errors = len(retval.get("errors", [])) + _warmup_logger.info( + "[CACHE-WARMUP] Finished | scheduled=%d errors=%d state=%s task_id=%s", + scheduled, errors, state, task_id, + ) + else: + _warmup_logger.warning( + "[CACHE-WARMUP] Finished with non-dict result | result=%s state=%s task_id=%s", + retval, state, task_id, + ) + + +@task_failure.connect(sender=None) +def _log_cache_warmup_failure( + sender=None, task_id=None, exception=None, traceback=None, **kw +): + if sender and sender.name == "cache-warmup": + _warmup_logger.error( + "[CACHE-WARMUP] FAILED | exception=%s task_id=%s", + exception, task_id, + exc_info=True, + ) diff --git a/kustomize/full/redis/statefulset.yaml b/kustomize/full/redis/statefulset.yaml index 8479f3401a..8d3a2a72a3 100644 --- a/kustomize/full/redis/statefulset.yaml +++ b/kustomize/full/redis/statefulset.yaml @@ -18,13 +18,6 @@ spec: image: redis:7-alpine ports: - containerPort: 6379 - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 256Mi volumeMounts: - name: redis-data mountPath: /data diff --git a/kustomize/full/superset-beat/deployment.yaml b/kustomize/full/superset-beat/deployment.yaml index 999d7cc642..dc789ba69c 100644 --- a/kustomize/full/superset-beat/deployment.yaml +++ b/kustomize/full/superset-beat/deployment.yaml @@ -61,13 +61,6 @@ spec: - name: config-overrides mountPath: /app/docker/pythonpath_dev/superset_config_docker.py subPath: superset_config_docker.py - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 250m - memory: 256Mi volumes: - name: config-overrides configMap: diff --git a/kustomize/full/superset-worker/deployment.yaml b/kustomize/full/superset-worker/deployment.yaml index 7afd509e69..4747cd1b02 100644 --- a/kustomize/full/superset-worker/deployment.yaml +++ b/kustomize/full/superset-worker/deployment.yaml @@ -55,13 +55,6 @@ spec: secretKeyRef: name: superset-secrets key: MAPBOX_API_KEY - resources: - requests: - cpu: 500m - memory: 512Mi - limits: - cpu: "2" - memory: 2Gi volumeMounts: - name: config-overrides mountPath: /app/docker/pythonpath_dev/superset_config_docker.py From dd3bd59e05efeae60e0b13940a41b1a9f6b9cd7c Mon Sep 17 00:00:00 2001 From: James Date: Tue, 23 Jun 2026 11:28:06 -0400 Subject: [PATCH 11/12] updating docs for kustomize and adding nameset --- kustomize/README.md | 30 ++++++++++++++++++++---------- kustomize/full/kustomization.yaml | 2 ++ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 7763202d08..420ec6e7f8 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -7,13 +7,13 @@ Kubernetes manifests for deploying GeoSet, organized as Kustomize overlays. | Overlay | Description | Use case | |---------|-------------|----------| | **base** | PostGIS, metadata DB, Superset web, sample data ingest. No Redis or Celery. | Local/dev clusters, quick demos | -| **full** | Everything in base + Redis, Celery workers, Celery beat (scheduled tasks, cache warmup, report generation), Flux GitOps | Staging/production | +| **full** | Everything in base + Redis, Celery workers, Celery beat, cache warmup/report configuration, Flux GitOps | Staging/production starting point | ## Prerequisites - A Kubernetes cluster (minikube, kind, EKS, etc.) - `kubectl` installed and configured -- Container images pushed (default: `jmeegan607/geoset:6.0.48`, `ebienstock/geoset:data-ingest-latest`) +- Container images pushed (default: `jmeegan607/geoset`, `jmeegan607/geoset:full`, `ebienstock/geoset:data-ingest-latest`) ## Setup @@ -53,15 +53,15 @@ You generally don't need to change these unless you're pointing at external data ### 3. Update container images (if needed) -The image tag is controlled in one place — the `images` block in `base/kustomization.yaml`: +The full overlay pins the Superset image tag in `full/kustomization.yaml`: ```yaml images: - name: jmeegan607/geoset - newTag: "6.0.48" + newTag: full ``` -Change `newTag` to use a different version. This applies to all manifests automatically. +Change `newTag` to use a different version. The base overlay leaves the image tag unset so it uses the image tag from the individual manifests. ## Deploy @@ -90,19 +90,29 @@ Superset web will be available on port `8088` via the `superset-web` service. To kubectl -n geoset port-forward svc/superset-web 8088:8088 ``` +### Validate manifests + +The deployment requires a local `secrets.yaml`, which is intentionally gitignored. To render the manifests for review or CI without creating local untracked files, validate against a temporary copy: + +```bash +tmp="$(mktemp -d)" +cp -R kustomize "$tmp/" +cp "$tmp/kustomize/base/secrets.yaml.example" "$tmp/kustomize/base/secrets.yaml" +kubectl kustomize "$tmp/kustomize/base" +kubectl kustomize "$tmp/kustomize/full" +``` + ## Full overlay extras The full overlay adds on top of base: - **Redis** — caching backend and Celery message broker - **Celery workers** (2 replicas) — async query execution -- **Celery beat** — scheduled tasks including cache warmup and automated report generation -- **MailHog** — in-cluster fake SMTP server for dev/testing alert email delivery (SMTP on port 1025, web UI on port 8025) -- **Wildfire Alert bootstrap** — one-time Job that seeds a daily "Wildfire Proximity Alert" SQL alert via the Superset REST API after init -- **SMTP / Alerts & Reports env vars** — pre-configured to use MailHog; swap env vars in `config/superset-env-patch.env` for production SMTP +- **Celery beat** — scheduled tasks including cache warmup and report generation +- **Alerts & Reports config** — enables Superset report generation and Mattermost notifications; set `MATTERMOST_WEBHOOK_URL` in `config/superset-env-patch.env` or your secret management flow - **Flux GitOps** — auto-syncs from `raft-tech/GeoSet` main branch -It also patches `superset-web` to 2 replicas and adds a Redis readiness check to its init container. +It also patches `superset-web` to 2 replicas, mounts the full deployment Superset config override, and adds a Redis readiness check to its init container. ## Teardown diff --git a/kustomize/full/kustomization.yaml b/kustomize/full/kustomization.yaml index 672ce725b9..ed5558d38f 100644 --- a/kustomize/full/kustomization.yaml +++ b/kustomize/full/kustomization.yaml @@ -1,6 +1,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: geoset + # Inherit everything from base resources: - ../base From 1408112dd9b124fe8855d7a21b5c7b92d6e09aa0 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 23 Jun 2026 16:12:44 -0400 Subject: [PATCH 12/12] chore: split runtime alert changes from kustomize --- docker-compose.full.yml | 110 ------------------ docker/pythonpath_dev/.gitignore | 1 - .../superset_config_docker_full.py | 74 ------------ kustomize/README.md | 17 +-- kustomize/full/Dockerfile.full | 4 - kustomize/full/config/superset-env-patch.env | 6 - .../full/config/superset_config_docker.py | 13 --- kustomize/full/kustomization.yaml | 4 - kustomize/full/superset-beat/deployment.yaml | 2 +- .../full/superset-worker/deployment.yaml | 2 +- .../alerts/components/NotificationMethod.tsx | 6 +- .../alerts/components/RecipientIcon.tsx | 6 - .../src/features/alerts/types.ts | 2 - superset/reports/models.py | 1 - superset/reports/notifications/__init__.py | 1 - superset/reports/notifications/mattermost.py | 95 --------------- superset/views/base.py | 6 +- 17 files changed, 8 insertions(+), 342 deletions(-) delete mode 100644 docker-compose.full.yml delete mode 100644 docker/pythonpath_dev/superset_config_docker_full.py delete mode 100644 kustomize/full/Dockerfile.full delete mode 100644 superset/reports/notifications/mattermost.py diff --git a/docker-compose.full.yml b/docker-compose.full.yml deleted file mode 100644 index cc00ea757d..0000000000 --- a/docker-compose.full.yml +++ /dev/null @@ -1,110 +0,0 @@ -# -# Full GeoSet development environment. -# Adds Redis, Celery worker, and Celery beat to the base compose. -# -# USAGE: -# docker compose -f docker-compose.yml -f docker-compose.full.yml up -# -services: - # ---------- Redis (Celery broker + cache) ---------- - redis: - image: redis:7-alpine - restart: unless-stopped - volumes: - - redis_data:/data - - # ---------- Override base superset to use full config ---------- - superset-init: - environment: - SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config.py - volumes: - - ./docker/pythonpath_dev/superset_config_docker_full.py:/app/docker/pythonpath_dev/superset_config_docker.py - depends_on: - db: - condition: service_started - postgis: - condition: service_healthy - redis: - condition: service_started - - superset: - environment: - SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config.py - volumes: - - ./docker/pythonpath_dev/superset_config_docker_full.py:/app/docker/pythonpath_dev/superset_config_docker.py - depends_on: - superset-init: - condition: service_completed_successfully - redis: - condition: service_started - - # ---------- Celery worker (executes alerts, takes screenshots) ---------- - superset-worker: - build: - context: . - dockerfile: ${DOCKERFILE:-Dockerfile} - target: dev - args: - DEV_MODE: "true" - INCLUDE_CHROMIUM: "true" - command: ["celery", "--app=superset.tasks.celery_app:app", - "worker", "--pool=prefork", "-O", "fair", - "-c", "2", "--loglevel=INFO"] - restart: unless-stopped - env_file: - - path: docker/.env - required: true - - path: docker/.env-local - required: false - environment: - SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config.py - user: root - depends_on: - superset-init: - condition: service_completed_successfully - redis: - condition: service_started - volumes: - - ./docker:/app/docker - - ./docker/pythonpath_dev/superset_config_docker_full.py:/app/docker/pythonpath_dev/superset_config_docker.py - - ./superset:/app/superset - - ./superset-frontend:/app/superset-frontend - - superset_home:/app/superset_home - - ./tests:/app/tests - - # ---------- Celery beat (schedules alerts) ---------- - superset-beat: - build: - context: . - dockerfile: ${DOCKERFILE:-Dockerfile} - target: dev - args: - DEV_MODE: "true" - command: ["celery", "--app=superset.tasks.celery_app:app", - "beat", "--loglevel=INFO", - "--schedule=/tmp/celerybeat-schedule"] - restart: unless-stopped - env_file: - - path: docker/.env - required: true - - path: docker/.env-local - required: false - environment: - SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config.py - user: root - depends_on: - superset-init: - condition: service_completed_successfully - redis: - condition: service_started - volumes: - - ./docker:/app/docker - - ./docker/pythonpath_dev/superset_config_docker_full.py:/app/docker/pythonpath_dev/superset_config_docker.py - - ./superset:/app/superset - - ./superset-frontend:/app/superset-frontend - - superset_home:/app/superset_home - - ./tests:/app/tests - -volumes: - redis_data: - external: false diff --git a/docker/pythonpath_dev/.gitignore b/docker/pythonpath_dev/.gitignore index dbb752bd13..97cac2072f 100644 --- a/docker/pythonpath_dev/.gitignore +++ b/docker/pythonpath_dev/.gitignore @@ -21,5 +21,4 @@ !.gitignore !superset_config.py !superset_config_docker_light.py -!superset_config_docker_full.py !superset_config_local.example diff --git a/docker/pythonpath_dev/superset_config_docker_full.py b/docker/pythonpath_dev/superset_config_docker_full.py deleted file mode 100644 index c0d68904f3..0000000000 --- a/docker/pythonpath_dev/superset_config_docker_full.py +++ /dev/null @@ -1,74 +0,0 @@ -"""GeoSet overrides for the full Docker Compose stack (Redis + Celery). - -Loaded automatically by superset_config.py via ``from superset_config_docker import *``. -docker-compose.full.yml mounts this file as superset_config_docker.py inside -the container so it is picked up without touching the upstream base config. -""" - -import logging -import os - -from celery.signals import task_failure, task_postrun, task_prerun -from superset.tasks.types import ExecutorType, FixedExecutor - -# Alerts & Reports -ALERT_REPORTS_NOTIFICATION_DRY_RUN = False -ALERT_REPORTS_NOTIFICATION_METHODS = ["Mattermost"] -WEBDRIVER_TYPE = "chrome" -FEATURE_FLAGS = {"ALERT_REPORTS": True, "PLAYWRIGHT_REPORTS_AND_THUMBNAILS": True} -MATTERMOST_WEBHOOK_URL = os.getenv( - "MATTERMOST_WEBHOOK_URL", - "https://mattermost.teamraft.com/hooks/q7co9uqot7g398cnzg63kfxaay", -) -WEBDRIVER_BASEURL = "http://superset:8088/" -WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL - -# Fallback executor for charts without owners (e.g. GeoSet example charts) -CACHE_WARMUP_EXECUTORS = [ExecutorType.OWNER, FixedExecutor("admin")] - -# --------------------------------------------------------------------------- -# Cache warmup logging via Celery signals -# --------------------------------------------------------------------------- -_warmup_logger = logging.getLogger("geoset.cache_warmup") - - -@task_prerun.connect(sender=None) -def _log_cache_warmup_start(sender=None, task_id=None, args=None, kwargs=None, **kw): - if sender and sender.name == "cache-warmup": - strategy = kwargs.get("strategy_name", "unknown") if kwargs else "unknown" - top_n = kwargs.get("top_n", "N/A") if kwargs else "N/A" - _warmup_logger.info( - "[CACHE-WARMUP] Starting | strategy=%s top_n=%s task_id=%s", - strategy, top_n, task_id, - ) - - -@task_postrun.connect(sender=None) -def _log_cache_warmup_done( - sender=None, task_id=None, retval=None, state=None, **kw -): - if sender and sender.name == "cache-warmup": - if isinstance(retval, dict): - scheduled = len(retval.get("scheduled", [])) - errors = len(retval.get("errors", [])) - _warmup_logger.info( - "[CACHE-WARMUP] Finished | scheduled=%d errors=%d state=%s task_id=%s", - scheduled, errors, state, task_id, - ) - else: - _warmup_logger.warning( - "[CACHE-WARMUP] Finished with non-dict result | result=%s state=%s task_id=%s", - retval, state, task_id, - ) - - -@task_failure.connect(sender=None) -def _log_cache_warmup_failure( - sender=None, task_id=None, exception=None, traceback=None, **kw -): - if sender and sender.name == "cache-warmup": - _warmup_logger.error( - "[CACHE-WARMUP] FAILED | exception=%s task_id=%s", - exception, task_id, - exc_info=True, - ) diff --git a/kustomize/README.md b/kustomize/README.md index 420ec6e7f8..4f71003bed 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -7,13 +7,13 @@ Kubernetes manifests for deploying GeoSet, organized as Kustomize overlays. | Overlay | Description | Use case | |---------|-------------|----------| | **base** | PostGIS, metadata DB, Superset web, sample data ingest. No Redis or Celery. | Local/dev clusters, quick demos | -| **full** | Everything in base + Redis, Celery workers, Celery beat, cache warmup/report configuration, Flux GitOps | Staging/production starting point | +| **full** | Everything in base + Redis, Celery workers, Celery beat, cache warmup configuration, Flux GitOps | Staging/production starting point | ## Prerequisites - A Kubernetes cluster (minikube, kind, EKS, etc.) - `kubectl` installed and configured -- Container images pushed (default: `jmeegan607/geoset`, `jmeegan607/geoset:full`, `ebienstock/geoset:data-ingest-latest`) +- Container images pushed (default: `jmeegan607/geoset`, `ebienstock/geoset:data-ingest-latest`) ## Setup @@ -53,15 +53,7 @@ You generally don't need to change these unless you're pointing at external data ### 3. Update container images (if needed) -The full overlay pins the Superset image tag in `full/kustomization.yaml`: - -```yaml -images: - - name: jmeegan607/geoset - newTag: full -``` - -Change `newTag` to use a different version. The base overlay leaves the image tag unset so it uses the image tag from the individual manifests. +The base and full overlays use the image tags from the individual manifests. Add an `images` block to the relevant `kustomization.yaml` if you need to pin a different tag for your environment. ## Deploy @@ -108,8 +100,7 @@ The full overlay adds on top of base: - **Redis** — caching backend and Celery message broker - **Celery workers** (2 replicas) — async query execution -- **Celery beat** — scheduled tasks including cache warmup and report generation -- **Alerts & Reports config** — enables Superset report generation and Mattermost notifications; set `MATTERMOST_WEBHOOK_URL` in `config/superset-env-patch.env` or your secret management flow +- **Celery beat** — scheduled tasks including cache warmup - **Flux GitOps** — auto-syncs from `raft-tech/GeoSet` main branch It also patches `superset-web` to 2 replicas, mounts the full deployment Superset config override, and adds a Redis readiness check to its init container. diff --git a/kustomize/full/Dockerfile.full b/kustomize/full/Dockerfile.full deleted file mode 100644 index 7f1c0f07ba..0000000000 --- a/kustomize/full/Dockerfile.full +++ /dev/null @@ -1,4 +0,0 @@ -FROM jmeegan607/geoset -USER root -RUN uv pip install --python /app/.venv/bin/python playwright && playwright install-deps && /app/.venv/bin/playwright install chromium -USER superset diff --git a/kustomize/full/config/superset-env-patch.env b/kustomize/full/config/superset-env-patch.env index 5417fcd8c8..401cbe4ff3 100644 --- a/kustomize/full/config/superset-env-patch.env +++ b/kustomize/full/config/superset-env-patch.env @@ -6,9 +6,3 @@ REDIS_HOST=redis REDIS_PORT=6379 REDIS_CELERY_DB=0 REDIS_RESULTS_DB=1 - -# Alerts & Reports -ALERT_REPORTS_DRY_RUN=false -WEBDRIVER_TYPE=chrome -WEBDRIVER_BASEURL=http://superset-web:8088/ -MATTERMOST_WEBHOOK_URL= diff --git a/kustomize/full/config/superset_config_docker.py b/kustomize/full/config/superset_config_docker.py index ee0638c45f..94df228a3d 100644 --- a/kustomize/full/config/superset_config_docker.py +++ b/kustomize/full/config/superset_config_docker.py @@ -5,23 +5,10 @@ """ import logging -import os from celery.signals import task_failure, task_postrun, task_prerun from superset.tasks.types import ExecutorType, FixedExecutor -# Alerts & Reports -ALERT_REPORTS_NOTIFICATION_DRY_RUN = False -ALERT_REPORTS_NOTIFICATION_METHODS = ["Mattermost"] -WEBDRIVER_TYPE = "chrome" -FEATURE_FLAGS = {"ALERT_REPORTS": True, "PLAYWRIGHT_REPORTS_AND_THUMBNAILS": True} -MATTERMOST_WEBHOOK_URL = os.getenv( - "MATTERMOST_WEBHOOK_URL", - "https://mattermost.teamraft.com/hooks/q7co9uqot7g398cnzg63kfxaay", -) -WEBDRIVER_BASEURL = "http://superset-web:8088/" -WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL - # Fallback executor for charts without owners (e.g. GeoSet example charts) CACHE_WARMUP_EXECUTORS = [ExecutorType.OWNER, FixedExecutor("admin")] diff --git a/kustomize/full/kustomization.yaml b/kustomize/full/kustomization.yaml index ed5558d38f..8654e9c29e 100644 --- a/kustomize/full/kustomization.yaml +++ b/kustomize/full/kustomization.yaml @@ -26,10 +26,6 @@ configMapGenerator: generatorOptions: disableNameSuffixHash: true -images: - - name: jmeegan607/geoset - newTag: full - # Patch superset-web: bump replicas to 2 and wait for Redis patches: - path: superset-web-patch.yaml diff --git a/kustomize/full/superset-beat/deployment.yaml b/kustomize/full/superset-beat/deployment.yaml index dc789ba69c..e65ca4d664 100644 --- a/kustomize/full/superset-beat/deployment.yaml +++ b/kustomize/full/superset-beat/deployment.yaml @@ -28,7 +28,7 @@ spec: done containers: - name: beat - image: jmeegan607/geoset:chromium + image: jmeegan607/geoset imagePullPolicy: Always command: ["celery", "--app=superset.tasks.celery_app:app", "beat", "--loglevel=INFO", diff --git a/kustomize/full/superset-worker/deployment.yaml b/kustomize/full/superset-worker/deployment.yaml index 4747cd1b02..f73c7c50b8 100644 --- a/kustomize/full/superset-worker/deployment.yaml +++ b/kustomize/full/superset-worker/deployment.yaml @@ -26,7 +26,7 @@ spec: done containers: - name: worker - image: jmeegan607/geoset:chromium + image: jmeegan607/geoset imagePullPolicy: Always command: ["celery", "--app=superset.tasks.celery_app:app", "worker", "--pool=prefork", "-O", "fair", diff --git a/superset-frontend/src/features/alerts/components/NotificationMethod.tsx b/superset-frontend/src/features/alerts/components/NotificationMethod.tsx index 9f5cd2910c..4c53cce544 100644 --- a/superset-frontend/src/features/alerts/components/NotificationMethod.tsx +++ b/superset-frontend/src/features/alerts/components/NotificationMethod.tsx @@ -325,8 +325,6 @@ export const NotificationMethod: FunctionComponent = ({ ); if (slackEnabled && !slackOptions[0]?.options.length) { updateSlackOptions(); - } else { - setMethodOptionsLoading(false); } }, []); @@ -341,8 +339,7 @@ export const NotificationMethod: FunctionComponent = ({ ((!isFeatureEnabled(FeatureFlag.AlertReportSlackV2) || useSlackV1) && method === NotificationMethodOption.Slack) || - method === NotificationMethodOption.Email || - method === NotificationMethodOption.Mattermost, + method === NotificationMethodOption.Email, ) .map(method => ({ label: @@ -525,7 +522,6 @@ export const NotificationMethod: FunctionComponent = ({ {[ NotificationMethodOption.Email, NotificationMethodOption.Slack, - NotificationMethodOption.Mattermost, ].includes(method) ? ( <>
diff --git a/superset-frontend/src/features/alerts/components/RecipientIcon.tsx b/superset-frontend/src/features/alerts/components/RecipientIcon.tsx index b04199e6da..229194f443 100644 --- a/superset-frontend/src/features/alerts/components/RecipientIcon.tsx +++ b/superset-frontend/src/features/alerts/components/RecipientIcon.tsx @@ -52,12 +52,6 @@ export default function RecipientIcon({ type }: { type: string }) { ); recipientIconConfig.label = NotificationMethodOption.Slack; break; - case NotificationMethodOption.Mattermost: - recipientIconConfig.icon = ( - - ); - recipientIconConfig.label = NotificationMethodOption.Mattermost; - break; default: recipientIconConfig.icon = null; recipientIconConfig.label = ''; diff --git a/superset-frontend/src/features/alerts/types.ts b/superset-frontend/src/features/alerts/types.ts index 9fc1dea847..06449697c3 100644 --- a/superset-frontend/src/features/alerts/types.ts +++ b/superset-frontend/src/features/alerts/types.ts @@ -45,7 +45,6 @@ export enum NotificationMethodOption { Email = 'Email', Slack = 'Slack', SlackV2 = 'SlackV2', - Mattermost = 'Mattermost', } export type SelectValue = { @@ -163,7 +162,6 @@ export enum RecipientIconName { Email = 'Email', Slack = 'Slack', SlackV2 = 'SlackV2', - Mattermost = 'Mattermost', } export interface AlertsReportsConfig { ALERT_REPORTS_DEFAULT_WORKING_TIMEOUT: number; diff --git a/superset/reports/models.py b/superset/reports/models.py index d53362af19..e4cdd7c9b4 100644 --- a/superset/reports/models.py +++ b/superset/reports/models.py @@ -63,7 +63,6 @@ class ReportRecipientType(StrEnum): EMAIL = "Email" SLACK = "Slack" SLACKV2 = "SlackV2" - MATTERMOST = "Mattermost" class ReportState(StrEnum): diff --git a/superset/reports/notifications/__init__.py b/superset/reports/notifications/__init__.py index b5d58b78ca..938e393447 100644 --- a/superset/reports/notifications/__init__.py +++ b/superset/reports/notifications/__init__.py @@ -17,7 +17,6 @@ from superset.reports.models import ReportRecipients from superset.reports.notifications.base import BaseNotification, NotificationContent from superset.reports.notifications.email import EmailNotification # noqa: F401 -from superset.reports.notifications.mattermost import MattermostNotification # noqa: F401 from superset.reports.notifications.slack import SlackNotification # noqa: F401 from superset.reports.notifications.slackv2 import SlackV2Notification # noqa: F401 diff --git a/superset/reports/notifications/mattermost.py b/superset/reports/notifications/mattermost.py deleted file mode 100644 index 81879905f7..0000000000 --- a/superset/reports/notifications/mattermost.py +++ /dev/null @@ -1,95 +0,0 @@ -import base64 -import logging - -import requests -from flask import current_app - -from superset.reports.models import ReportRecipientType -from superset.reports.notifications.base import BaseNotification -from superset.reports.notifications.exceptions import ( - NotificationParamException, - NotificationUnprocessableException, -) -from superset.reports.notifications.slack_mixin import SlackMixin -from superset.utils import json - -logger = logging.getLogger(__name__) - - -class MattermostNotification(SlackMixin, BaseNotification): - """ - Sends a notification to Mattermost via incoming webhook. - Embeds screenshots as inline base64 images. - """ - - type = ReportRecipientType.MATTERMOST - - def _get_inline_image(self) -> str | None: - """Get the first available image as a base64 data URI.""" - if self._content.screenshots: - b64 = base64.b64encode(self._content.screenshots[0]).decode() - return f"data:image/png;base64,{b64}" - if self._content.pdf: - # PDF is built from screenshots — encode it as-is - # Mattermost won't render PDF inline, but we can try - # converting via the first page if possible - try: - from superset.utils.pdf import build_pdf_from_screenshots # noqa - - # The PDF bytes are already built, no way to reverse easily. - # Just skip image for PDF format — text + link is sent. - return None - except ImportError: - return None - return None - - def send(self) -> None: - webhook_url = current_app.config.get("MATTERMOST_WEBHOOK_URL") - if not webhook_url: - raise NotificationParamException( - "MATTERMOST_WEBHOOK_URL is not configured in superset_config" - ) - - body = self._get_body(content=self._content) - image_uri = self._get_inline_image() - - if not image_uri and self._content.screenshots: - raise NotificationUnprocessableException( - "Failed to encode screenshot for Mattermost" - ) - - if image_uri: - body += f"\n\n![report screenshot]({image_uri})" - - logger.info( - "[MATTERMOST DEBUG] screenshots=%s pdf=%s csv=%s text=%s body_len=%d", - len(self._content.screenshots) if self._content.screenshots else 0, - len(self._content.pdf) if self._content.pdf else 0, - len(self._content.csv) if self._content.csv else 0, - bool(self._content.text), - len(body), - ) - logger.info("[MATTERMOST DEBUG] body=%s", body[:500]) - - # Save files to /tmp - if self._content.screenshots: - for i, img in enumerate(self._content.screenshots): - with open(f"/tmp/mattermost_screenshot_{i}.png", "wb") as f: - f.write(img) - logger.info("[MATTERMOST DEBUG] Saved screenshot %d (%d bytes)", i, len(img)) - if self._content.pdf: - with open("/tmp/mattermost_report.pdf", "wb") as f: - f.write(self._content.pdf) - logger.info("[MATTERMOST DEBUG] Saved PDF (%d bytes)", len(self._content.pdf)) - - payload: dict = {"text": body} - - try: - resp = requests.post(webhook_url, json=payload, timeout=30) - resp.raise_for_status() - except requests.exceptions.RequestException as ex: - raise NotificationUnprocessableException( - f"Failed to send Mattermost notification: {ex}" - ) from ex - - logger.info("Report sent to Mattermost") diff --git a/superset/views/base.py b/superset/views/base.py index 2a51655600..276ef56de8 100644 --- a/superset/views/base.py +++ b/superset/views/base.py @@ -385,11 +385,7 @@ def cached_common_bootstrap_data( # pylint: disable=unused-argument for k in FRONTEND_CONF_KEYS } - if app.config.get("ALERT_REPORTS_NOTIFICATION_METHODS"): - frontend_config["ALERT_REPORTS_NOTIFICATION_METHODS"] = app.config[ - "ALERT_REPORTS_NOTIFICATION_METHODS" - ] - elif app.config.get("SLACK_API_TOKEN"): + if app.config.get("SLACK_API_TOKEN"): frontend_config["ALERT_REPORTS_NOTIFICATION_METHODS"] = [ ReportRecipientType.EMAIL, ReportRecipientType.SLACK,