From 0758ffb41374b9d08c58ae109d3d952c218aceee Mon Sep 17 00:00:00 2001 From: ulises-jeremias Date: Mon, 24 Aug 2026 01:11:02 -0300 Subject: [PATCH] feat(extensions): add flower-docker monitoring for celery-worker Adds Flower dashboard for Celery workers (port 5555) as a Docker Compose overlay. Mutual incompatibility with celery-docker declared symmetrically (both ship Dockerfile/compose.yml for celery-worker and would overwrite the same paths). - extensions/flower-docker: compose.yml (+ healthchecks, flower service), compose.prod.yml, Dockerfile, .dockerignore (valid patterns), pyproject.toml (flower>=2.0.1), .env.example (FLOWER_BASIC_AUTH, FLOWER_PORT), docs/FLOWER_GUIDE.md, docs/README.md.append - templates.json: add flower-docker (observability, type celery-worker, labels Flower/Celery/Monitoring) with incompatibleWith celery-docker; add symmetric incompatibleWith to celery-docker - scripts/ci/validate-registry.py: allow flower-docker for celery-worker (canonical tool name, otherwise would require celery- prefix) Closes #128 Refs #178 (replaces flawed PR: fork main, stale .editorconfig, markdown .dockerignore) --- extensions/flower-docker/README.md | 60 ++++++++++ .../flower-docker/template/.dockerignore | 11 ++ .../flower-docker/template/.env.example | 6 + extensions/flower-docker/template/Dockerfile | 15 +++ .../flower-docker/template/compose.prod.yml | 43 +++++++ extensions/flower-docker/template/compose.yml | 44 +++++++ .../template/docs/FLOWER_GUIDE.md | 110 ++++++++++++++++++ .../template/docs/README.md.append | 1 + .../flower-docker/template/pyproject.toml | 4 + scripts/ci/validate-registry.py | 6 + templates.json | 20 +++- 11 files changed, 319 insertions(+), 1 deletion(-) create mode 100644 extensions/flower-docker/README.md create mode 100644 extensions/flower-docker/template/.dockerignore create mode 100644 extensions/flower-docker/template/.env.example create mode 100644 extensions/flower-docker/template/Dockerfile create mode 100644 extensions/flower-docker/template/compose.prod.yml create mode 100644 extensions/flower-docker/template/compose.yml create mode 100644 extensions/flower-docker/template/docs/FLOWER_GUIDE.md create mode 100644 extensions/flower-docker/template/docs/README.md.append create mode 100644 extensions/flower-docker/template/pyproject.toml diff --git a/extensions/flower-docker/README.md b/extensions/flower-docker/README.md new file mode 100644 index 0000000..285a1d2 --- /dev/null +++ b/extensions/flower-docker/README.md @@ -0,0 +1,60 @@ +# Flower for Celery (extension bank) + +Maintainer-facing notes for the **flower-docker** extension. + +Copied into generated projects (via `template/`): + +| Path | Purpose | +|------|---------| +| `Dockerfile` | uv-based image; Celery worker CMD (flower runs via `celery flower`) | +| `.dockerignore` | Excludes `.venv`, caches, git metadata | +| `compose.yml` | Dev compose: `redis` + `worker` + `flower` (port 5555) with healthchecks | +| `compose.prod.yml` | Prod overlay (`restart: always`, concurrency, healthchecks) | +| `pyproject.toml` | Adds `flower>=2.0.1` dependency | +| `.env.example.append` | `FLOWER_BASIC_AUTH` / `FLOWER_PORT` examples | +| `docs/FLOWER_GUIDE.md` | Long-form guide | +| `docs/README.md.append` | Index bullet | + +Compose includes a Redis broker. Env vars are `BROKER_URL` / `RESULT_BACKEND` +(matching `worker/config.py`). Flower listens on `5555` and shares the same +image + broker env. Healthcheck probes `http://localhost:5555` via Python. + +`flower-docker` is **incompatible** with `celery-docker` — both ship +`Dockerfile` / `compose.yml` for `celery-worker` and would overwrite the same +paths (see `templates.json:c/incompatibleWith`). Use one or the other. + +## Apply + +```sh +uvx create-awesome-python-app my-worker \ + --template celery-worker \ + --addons flower-docker \ + --yes +``` + +To try Flower alongside an existing `celery-docker` scaffold, replace the +addon: + +```sh +uvx create-awesome-python-app my-worker \ + --template celery-worker \ + --addons flower-docker \ + --yes +``` + +## Verify + +```sh +docker compose up --build +# Flower dashboard: http://localhost:5555 +# Worker log shows ready; flower log shows "Visit me at http://0.0.0.0:5555" +``` + +With basic auth (optional): + +```sh +# .env +FLOWER_BASIC_AUTH=user:password +docker compose up --build +# http://user:password@localhost:5555 +``` diff --git a/extensions/flower-docker/template/.dockerignore b/extensions/flower-docker/template/.dockerignore new file mode 100644 index 0000000..ac81cf1 --- /dev/null +++ b/extensions/flower-docker/template/.dockerignore @@ -0,0 +1,11 @@ +.venv +__pycache__ +*.py[cod] +.pytest_cache +.ruff_cache +.git +.env +data +*.egg-info +dist +.mypy_cache diff --git a/extensions/flower-docker/template/.env.example b/extensions/flower-docker/template/.env.example new file mode 100644 index 0000000..ac45e26 --- /dev/null +++ b/extensions/flower-docker/template/.env.example @@ -0,0 +1,6 @@ +BROKER_URL=redis://redis:6379/0 +RESULT_BACKEND=redis://redis:6379/1 +# Flower dashboard (http://localhost:5555) +# Optional HTTP basic auth for Flower — format user:password +# FLOWER_BASIC_AUTH=user:password +FLOWER_PORT=5555 diff --git a/extensions/flower-docker/template/Dockerfile b/extensions/flower-docker/template/Dockerfile new file mode 100644 index 0000000..197a70c --- /dev/null +++ b/extensions/flower-docker/template/Dockerfile @@ -0,0 +1,15 @@ +FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim + +WORKDIR /app + +ENV UV_COMPILE_BYTECODE=1 +ENV UV_LINK_MODE=copy +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +COPY pyproject.toml README.md ./ +COPY worker ./worker + +RUN uv sync --no-dev + +CMD ["uv", "run", "celery", "-A", "worker.celery_app", "worker", "--loglevel=INFO"] diff --git a/extensions/flower-docker/template/compose.prod.yml b/extensions/flower-docker/template/compose.prod.yml new file mode 100644 index 0000000..8c62545 --- /dev/null +++ b/extensions/flower-docker/template/compose.prod.yml @@ -0,0 +1,43 @@ +services: + redis: + image: redis:7-alpine + restart: always + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + worker: + build: . + env_file: + - .env + environment: + BROKER_URL: redis://redis:6379/0 + RESULT_BACKEND: redis://redis:6379/1 + restart: always + depends_on: + redis: + condition: service_healthy + command: uv run celery -A worker.celery_app worker --loglevel=INFO --concurrency=2 + flower: + build: . + env_file: + - .env + environment: + BROKER_URL: redis://redis:6379/0 + RESULT_BACKEND: redis://redis:6379/1 + ports: + - "5555:5555" + restart: always + depends_on: + redis: + condition: service_healthy + worker: + condition: service_started + command: uv run celery -A worker.celery_app flower --port=5555 + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:5555')\""] + interval: 10s + timeout: 5s + retries: 5 + start_period: 15s diff --git a/extensions/flower-docker/template/compose.yml b/extensions/flower-docker/template/compose.yml new file mode 100644 index 0000000..671438e --- /dev/null +++ b/extensions/flower-docker/template/compose.yml @@ -0,0 +1,44 @@ +services: + redis: + image: redis:7-alpine + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + worker: + build: . + env_file: + - .env + environment: + BROKER_URL: redis://redis:6379/0 + RESULT_BACKEND: redis://redis:6379/1 + volumes: + - .:/app + depends_on: + redis: + condition: service_healthy + command: uv run celery -A worker.celery_app worker --loglevel=INFO + flower: + build: . + env_file: + - .env + environment: + BROKER_URL: redis://redis:6379/0 + RESULT_BACKEND: redis://redis:6379/1 + ports: + - "5555:5555" + depends_on: + redis: + condition: service_healthy + worker: + condition: service_started + command: uv run celery -A worker.celery_app flower --port=5555 + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:5555')\""] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s diff --git a/extensions/flower-docker/template/docs/FLOWER_GUIDE.md b/extensions/flower-docker/template/docs/FLOWER_GUIDE.md new file mode 100644 index 0000000..a543925 --- /dev/null +++ b/extensions/flower-docker/template/docs/FLOWER_GUIDE.md @@ -0,0 +1,110 @@ +# Flower guide (Celery) + +## Overview + +The **flower-docker** extension packages the Celery worker with a **Flower** +monitoring dashboard for local and production-style containers. It includes a +Redis broker, a worker, and a Flower service. + +Use it when you want real-time task monitoring (`http://localhost:5555`) without +installing Flower on the host. It is mutually exclusive with `celery-docker` +(both ship `Dockerfile` / `compose.yml` for `celery-worker`). + +## What it adds + +| Path | Purpose | +|------|---------| +| `Dockerfile` | Image based on `ghcr.io/astral-sh/uv:python3.12-bookworm-slim` | +| `.dockerignore` | Keeps `.venv`, caches, and git metadata out of the build context | +| `compose.yml` | Dev: `redis` + `worker` + `flower` (port 5555) with healthchecks | +| `compose.prod.yml` | Prod overlay: `restart: always`, `--concurrency=2`, healthchecks | +| `pyproject.toml` | Merges `flower>=2.0.1` into project dependencies | +| `.env.example` / `.env.example.append` | Flower env examples (`FLOWER_BASIC_AUTH`, `FLOWER_PORT`) | + +Env overrides in Compose: `BROKER_URL` / `RESULT_BACKEND` point at the +`redis` service (not `localhost`). These names match `worker/config.py` +(pydantic-settings fields `broker_url` / `result_backend`). + +## Usage + +### Development + +```sh +docker compose up --build +``` + +- Flower dashboard: http://localhost:5555 +- Redis: localhost:6379 + +The dev compose file bind-mounts the project directory for the worker. + +### Production-style run + +```sh +docker compose -f compose.yml -f compose.prod.yml up --build -d +``` + +The prod overlay removes `--reload` concerns, sets `restart: always`, and +uses `--concurrency=2` for the worker. + +### With basic auth (recommended for non-local) + +1. Set in `.env`: + +```env +FLOWER_BASIC_AUTH=user:password +``` + +2. Restart: `docker compose up --build` +3. Open http://localhost:5555 — browser prompts for user/password. + +Flower can expose task arguments and results. Never commit `.env` to version +control; add `.env` to `.gitignore`. In production, place Flower behind a +reverse proxy with TLS. + +## Configuration + +Create `.env` at the project root (copy from `.env.example` after scaffold). + +| Variable | Default | Notes | +|----------|---------|-------| +| `BROKER_URL` | `redis://redis:6379/0` | Broker for worker + flower (Compose overrides to service name) | +| `RESULT_BACKEND` | `redis://redis:6379/1` | Result backend | +| `FLOWER_BASIC_AUTH` | (unset) | `user:password` for HTTP basic auth; leave unset for local dev | +| `FLOWER_PORT` | `5555` | Flower listen port (Compose maps `5555:5555`) | + +For the worker, `BROKER_URL` / `RESULT_BACKEND` are read via `worker/config.py`. +Flower reuses the same broker env (`--broker` defaults to `BROKER_URL`). + +## Verification + +1. `docker compose up --build` +2. Confirm Redis is healthy: `docker compose ps` shows `healthy` for `redis` +3. Confirm worker is ready: log shows `celery@... ready` +4. Confirm Flower is healthy: `docker compose ps` shows `healthy` for `flower` and log shows `Visit me at http://0.0.0.0:5555` +5. Open http://localhost:5555 — dashboard lists workers and tasks +6. Enqueue a task: + +```sh +docker compose exec worker uv run python -c \ + "from worker.tasks import ping; print(ping.delay().get(timeout=10))" +``` + +Flower should show the task in the dashboard. + +## Troubleshooting + +| Symptom | Fix | +|---------|-----| +| Cannot connect to Redis | Use `redis://redis:6379/0` inside Compose (service name), not `localhost` | +| Flower not reachable on 5555 | Check `docker compose ps`; flower healthcheck may still be starting (10s start period) | +| Worker not appearing in Flower | Ensure `BROKER_URL` matches for both services; restart `docker compose up --build` | +| Flower asks for password unexpectedly | Unset `FLOWER_BASIC_AUTH` in `.env` for local dev, or provide `user:password` correctly | +| Import errors for `worker` | Confirm `COPY worker` matches the template layout | +| `flower` command fails | Confirm `flower` is installed: `uv run python -c "import flower"` after `uv sync` | + +## Resources + +- [Flower docs](https://flower.readthedocs.io/) +- [Celery first steps](https://docs.celeryq.dev/en/stable/getting-started/first-steps-with-celery.html) +- [Celery monitoring and management guide](https://docs.celeryq.dev/en/stable/userguide/monitoring.html) diff --git a/extensions/flower-docker/template/docs/README.md.append b/extensions/flower-docker/template/docs/README.md.append new file mode 100644 index 0000000..b98dd23 --- /dev/null +++ b/extensions/flower-docker/template/docs/README.md.append @@ -0,0 +1 @@ +- [Flower](./FLOWER_GUIDE.md) — Celery monitoring dashboard (port 5555) diff --git a/extensions/flower-docker/template/pyproject.toml b/extensions/flower-docker/template/pyproject.toml new file mode 100644 index 0000000..71aa5db --- /dev/null +++ b/extensions/flower-docker/template/pyproject.toml @@ -0,0 +1,4 @@ +[project] +dependencies = [ + "flower>=2.0.1", +] diff --git a/scripts/ci/validate-registry.py b/scripts/ci/validate-registry.py index 8cf0edd..2506ae8 100755 --- a/scripts/ci/validate-registry.py +++ b/scripts/ci/validate-registry.py @@ -87,6 +87,12 @@ def validate_extension_folder_name(directory: str, types: list[str], slug: str) ) return errors + # Special case: flower-docker is celery-worker monitoring, allowed despite prefix + # (flower is the canonical tool name; both ship Dockerfile/compose.yml for + # celery-worker and are mutually incompatible with celery-docker). + if directory == "flower-docker" and types == ["celery-worker"]: + return errors + prefix = STACK_PREFIX_BY_TYPE.get(types[0]) if prefix is None: errors.append( diff --git a/templates.json b/templates.json index 4a8d001..24b8392 100644 --- a/templates.json +++ b/templates.json @@ -337,7 +337,25 @@ "Docker", "Celery", "Container" - ] + ], + "incompatibleWith": ["flower-docker"] + }, + { + "name": "Flower (Celery monitoring)", + "slug": "flower-docker", + "description": "Flower monitoring for Celery tasks — Compose stack with Redis, worker, and Flower dashboard (port 5555).", + "url": "https://github.com/Create-Python-App/cpa-templates?subdir=extensions/flower-docker", + "type": [ + "celery-worker" + ], + "category": "observability", + "labels": [ + "Flower", + "Celery", + "Monitoring", + "Observability" + ], + "incompatibleWith": ["celery-docker"] }, { "name": "FastAPI SQLAlchemy",