diff --git a/.copier-answers.resonant.yml b/.copier-answers.resonant.yml index 5479bef5..49467d29 100644 --- a/.copier-answers.resonant.yml +++ b/.copier-answers.resonant.yml @@ -1,4 +1,4 @@ -_commit: v0.48.1 +_commit: v0.50.5 _src_path: https://github.com/kitware-resonant/cookiecutter-resonant core_app_name: core include_example_code: false diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..0a485792 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,90 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose +{ + "name": "GeoDatalytics", + "dockerComposeFile": [ + "../docker-compose.yml", + "../docker-compose.override.yml", + "./docker-compose.devcontainer.yml" + ], + "service": "django", + "overrideCommand": true, + // The "vscode" user and remoteUser are set by the base image label (devcontainers/base). + "workspaceFolder": "/home/vscode/geodatalytics", + "features": { + "ghcr.io/devcontainers/features/git-lfs:1": {}, + "ghcr.io/devcontainers/features/node:1": { + "version": "24", + // Work around https://github.com/devcontainers/features/pull/1625 + "pnpmVersion": "none" + }, + "ghcr.io/rails/devcontainer/features/postgres-client:1": { + "version": 18 + }, + "ghcr.io/devcontainers/features/terraform:1": {}, + "ghcr.io/devcontainers/features/aws-cli:1": {}, + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/devcontainers-extra/features/heroku-cli:1": {} + }, + "customizations": { + "vscode": { + "extensions": [ + // Python + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.debugpy", + "ms-python.mypy-type-checker", + "charliermarsh.ruff", + // Django + "batisteo.vscode-django", + "augustocdias.tasks-shell-input", + // Other file formats + "editorconfig.editorconfig", + "mikestead.dotenv", + "tamasfe.even-better-toml", + "timonwong.shellcheck", + // Infrastructure + "ms-azuretools.vscode-containers", + "hashicorp.terraform", + "github.vscode-github-actions", + // Remove AWS extension, as only the CLI is wanted; see: https://github.com/devcontainers/features/issues/1228 + "-AmazonWebServices.aws-toolkit-vscode" + ], + "settings": { + "containers.containerClient": "com.microsoft.visualstudio.containers.docker", + // Container-specific Python paths + "python.defaultInterpreterPath": "/home/vscode/venv/bin/python", + // Disable automatic Python venv activation in new terminals. + "python-envs.terminal.autoActivationType": "off", + // Ensure that `envFile` from any user settings is ignored; Docker Compose provides it. + "python.envFile": "", + // Reduce file watcher overhead for generated/cache directories. + "files.watcherExclude": { + "**/__pycache__/**": true, + "**/.pytest_cache/**": true, + "**/node_modules/**": true + } + } + } + }, + // Prevent a prompt every time the debugger opens a port or Django auto-restarts. + "otherPortsAttributes": { + "onAutoForward": "silent" + }, + "portsAttributes": { + "8000": { + "label": "Django", + // Show a dialog if the port isn't free. + "requireLocalPort": true, + "onAutoForward": "silent" + } + }, + // Install a global Python and create a venv before VSCode extensions start, + // to prevent prompts and ensure test discovery works on first load. + "onCreateCommand": { + "python": ["uv", "python", "install", "--default"], + "venv": ["uv", "sync", "--all-extras", "--all-groups"] + }, + // Ensure it is re-synced on restarts. + "updateContentCommand": ["uv", "sync", "--all-extras", "--all-groups"] +} diff --git a/.devcontainer/docker-compose.devcontainer.yml b/.devcontainer/docker-compose.devcontainer.yml new file mode 100644 index 00000000..eb5cca88 --- /dev/null +++ b/.devcontainer/docker-compose.devcontainer.yml @@ -0,0 +1,10 @@ +services: + django: + # Don't expose ports, devcontainer forwarding is superior, since we can just bind to localhost. + ports: !reset [] + # Don't auto-run the default command, launch.json or the terminal will be used. + command: !reset [] + + celery: + # Celery will be started via launch.json or the terminal. + profiles: ["celery"] diff --git a/.editorconfig b/.editorconfig index b6494e09..e6c9576e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -27,6 +27,9 @@ indent_size = 2 indent_size = 4 max_line_length = 100 +[*.sh] +indent_size = 2 + [*.toml] indent_size = 2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68f48b7f..66507dcc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ on: permissions: contents: read jobs: +<<<<<<< before updating lint-client: runs-on: ubuntu-latest steps: @@ -24,6 +25,10 @@ jobs: test-server: runs-on: ubuntu-latest +======= + test: + runs-on: ubuntu-24.04 +>>>>>>> after updating services: postgres: image: postgis/postgis:18-3.6 @@ -37,7 +42,7 @@ jobs: ports: - 5432:5432 rabbitmq: - image: rabbitmq:management-alpine + image: rabbitmq:4.2-management-alpine options: >- --health-cmd "rabbitmq-diagnostics ping" --health-start-period 30s @@ -69,10 +74,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 - name: Install uv - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@v8.1.0 - name: Run tests run: | - uv run tox + uv run --locked tox env: DJANGO_DATABASE_URL: postgres://postgres:postgres@localhost:5432/django DJANGO_CELERY_BROKER_URL: amqp://localhost:5672/ diff --git a/README.md b/README.md index 1a04d336..79b7844f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # GeoDatalytics +<<<<<<< before updating [![License][apache-license-image]][license-link]

@@ -43,3 +44,37 @@ To run GeoDatalytics locally with `docker-compose`, follow the instructions in t [sds-lab-link]: https://sdslab.io [mass-mapper-link]: https://maps.massgis.digital.mass.gov/MassMapper/MassMapper.html [girder-4-cookiecutter-link]: https://github.com/girder/cookiecutter-girder-4 +======= +## Setup +1. Install [VS Code with dev container support](https://code.visualstudio.com/docs/devcontainers/containers#_installation). +1. Open the project in VS Code, then run `Dev Containers: Reopen in Container` + from the Command Palette (`Ctrl+Shift+P`). +1. Once the container is ready, open a terminal and run: + ```sh + ./manage.py migrate + ./manage.py createsuperuser + ``` + +## Run +Open the **Run and Debug** panel (`Ctrl+Shift+D`) and select a launch configuration: + +* **Django: Server** — Starts the development server at http://localhost:8000/ +* **Django: Server (eager Celery)** — Same, but Celery tasks run synchronously + in the web process (useful for debugging task code without a worker) +* **Celery: Worker** — Starts only the Celery worker +* **Django + Celery** — Starts both the server and a Celery worker +* **Django: Management Command** — Pick and run any management command + +## Test +Run the full test suite from a terminal: `tox` + +Auto-format code: `tox -e format` + +Run and debug individual tests from the **Testing** panel (`Ctrl+Shift+;`). + +## Rebuild +After changes to the Dockerfile, Docker Compose files, or `devcontainer.json`, +run `Dev Containers: Rebuild Container` from the Command Palette (`Ctrl+Shift+P`). + +For dependency changes in `pyproject.toml`, just run `uv sync --all-extras --all-groups`. +>>>>>>> after updating diff --git a/dev/django.Dockerfile b/dev/django.Dockerfile index d94c6b59..986065d0 100644 --- a/dev/django.Dockerfile +++ b/dev/django.Dockerfile @@ -1,13 +1,36 @@ -FROM ghcr.io/astral-sh/uv:debian - -# Make Python more friendly to running in containers -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 - -# Make uv install content in well-known locations -ENV UV_PROJECT_ENVIRONMENT=/var/lib/venv \ - UV_CACHE_DIR=/var/cache/uv/cache \ - UV_PYTHON_INSTALL_DIR=/var/cache/uv/bin \ - # The uv cache and environment are expected to be mounted on different volumes, - # so hardlinks won't work - UV_LINK_MODE=symlink +FROM mcr.microsoft.com/devcontainers/base:ubuntu-24.04 + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ + +# Ensure Python output appears immediately in container logs. +ENV PYTHONUNBUFFERED=1 + +# Override Node's default of attempting to bind to IPv6 interfaces over IPv4 +ENV NODE_OPTIONS=--dns-result-order=ipv4first + +# Put the uv and npm caches in a separate location, +# where they can persist and be shared across containers. +# The uv cache and virtual environment are on different volumes, so hardlinks won't work. +ENV UV_CACHE_DIR=/home/vscode/pkg-cache/uv \ + UV_PYTHON_INSTALL_DIR=/home/vscode/pkg-cache/uv-python \ + UV_LINK_MODE=symlink \ + NPM_CONFIG_CACHE=/home/vscode/pkg-cache/npm + +# Put the virtual environment outside the project directory, +# to improve performance on macOS and prevent accidental usage from the host machine. +# Activate it, so `uv run` doesn't need to be prefixed. +ENV UV_PROJECT_ENVIRONMENT=/home/vscode/venv \ + PATH="/home/vscode/venv/bin:$PATH" + +# Put tool scratch files outside the project directory too. +ENV TOX_WORK_DIR=/home/vscode/tox \ + RUFF_CACHE_DIR=/home/vscode/.cache/ruff \ + MYPY_CACHE_DIR=/home/vscode/.cache/mypy + +RUN ["chsh", "-s", "/usr/bin/zsh", "vscode"] + +USER vscode + +# Pre-create named volume mount points, so the new volume inherits `vscode` user ownership: +# https://docs.docker.com/engine/storage/volumes/#populate-a-volume-using-a-container +RUN ["mkdir", "/home/vscode/pkg-cache"] diff --git a/dev/docker-development.md b/dev/docker-development.md new file mode 100644 index 00000000..a5faba3c --- /dev/null +++ b/dev/docker-development.md @@ -0,0 +1,23 @@ +# Docker Compose Development (without VS Code) + +An alternative to the recommended [dev container](../README.md) workflow. + +## Setup +1. `docker compose run --rm django ./manage.py migrate` +1. `docker compose run --rm django ./manage.py createsuperuser` + +## Run +1. `docker compose up` +1. Access http://localhost:8000/ +1. `Ctrl+C` to stop + +To include the Celery worker: `docker compose --profile celery up` + +## Update +1. `docker compose down` +1. `docker compose pull` +1. `docker compose build --pull` +1. `docker compose run --rm django ./manage.py migrate` + +## Reset +Remove all data and volumes: `docker compose down -v` diff --git a/dev/export-env.sh b/dev/export-env.sh index ee0e3b0e..3e866bc2 100644 --- a/dev/export-env.sh +++ b/dev/export-env.sh @@ -1,3 +1,7 @@ +<<<<<<< before updating +======= +# shellcheck shell=bash +>>>>>>> after updating # Export environment variables from the .env file in the first argument. # If no argument is given, default to "dev/.env.docker-compose-native". # This file must be sourced, not run. @@ -21,6 +25,10 @@ fi # Using "set -a" allows .env files with spaces or comments to work seamlessly # https://stackoverflow.com/a/45971167 set -a +<<<<<<< before updating +======= +# shellcheck source=.env.docker-compose-native +>>>>>>> after updating . "$_dotenv_file" set +a diff --git a/dev/native-development.md b/dev/native-development.md new file mode 100644 index 00000000..9c410410 --- /dev/null +++ b/dev/native-development.md @@ -0,0 +1,19 @@ +# Native Development (advanced) + +Runs Python on the host while using Docker Compose for services. + +## Setup +1. [Install `uv`](https://docs.astral.sh/uv/getting-started/installation/) +1. Start services: `docker compose -f ./docker-compose.yml up -d` +1. Load environment: `source ./dev/export-env.sh` +1. `./manage.py migrate` +1. `./manage.py createsuperuser` + +## Run +1. Ensure services are running: `docker compose -f ./docker-compose.yml up -d` +1. `source ./dev/export-env.sh` +1. `./manage.py runserver` +1. In a separate terminal: `celery --app uvdat.celery worker --loglevel INFO --without-heartbeat` +1. Access http://localhost:8000/ + +Stop services when done: `docker compose stop` diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 73edc89f..1a32dd6a 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -9,12 +9,11 @@ services: ] # Log printing is enhanced by a TTY tty: true - environment: - UV_ENV_FILE: ./dev/.env.docker-compose - working_dir: /opt/django-project + working_dir: /home/vscode/geodatalytics + env_file: ./dev/.env.docker-compose volumes: - - .:/opt/django-project - - uv_cache:/var/cache/uv + - .:/home/vscode/geodatalytics + - pkg_cache:/home/vscode/pkg-cache ports: - 8000:8000 depends_on: @@ -41,14 +40,13 @@ services: "--loglevel", "INFO", "--without-heartbeat" ] - # Docker Compose does not set the TTY width, which causes Celery errors + # uv progress doesn't display properly with a Docker TTY tty: false - environment: - UV_ENV_FILE: ./dev/.env.docker-compose - working_dir: /opt/django-project + working_dir: /home/vscode/geodatalytics + env_file: ./dev/.env.docker-compose volumes: - - .:/opt/django-project - - uv_cache:/var/cache/uv + - .:/home/vscode/geodatalytics + - pkg_cache:/home/vscode/pkg-cache depends_on: postgres: condition: service_healthy @@ -113,5 +111,9 @@ services: - 8080:8080 volumes: +<<<<<<< before updating uv_cache: web_node_modules: +======= + pkg_cache: +>>>>>>> after updating diff --git a/docker-compose.yml b/docker-compose.yml index 09a3b47e..a8e8240b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: - postgres:/var/lib/postgresql rabbitmq: - image: rabbitmq:management-alpine + image: rabbitmq:4.2-management-alpine healthcheck: test: ["CMD", "rabbitmq-diagnostics", "ping"] start_period: 30s diff --git a/pyproject.toml b/pyproject.toml index 6d62347a..6c17e743 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -203,6 +203,10 @@ extend-immutable-calls = ["ninja.Query"] [tool.ruff.lint.flake8-self] extend-ignore-names = ["_base_manager", "_default_manager", "_meta"] +[tool.ruff.lint.flake8-type-checking] +runtime-evaluated-base-classes = ["pydantic.BaseModel"] +runtime-evaluated-decorators = ["pydantic.validate_call"] + [tool.ruff.lint.isort] # Sort by name, don't cluster "from" vs "import" force-sort-within-sections = true diff --git a/tox.ini b/tox.ini index ee053f3a..b322a949 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,8 @@ env_list = runner = uv-venv-lock-runner pass_env = DJANGO_* + RUFF_CACHE_DIR + MYPY_CACHE_DIR extras = development tasks diff --git a/uvdat/settings/heroku_production.py b/uvdat/settings/heroku_production.py index a2a6710b..59a0e758 100644 --- a/uvdat/settings/heroku_production.py +++ b/uvdat/settings/heroku_production.py @@ -16,5 +16,10 @@ # Heroku and Render automatically set this. SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") +<<<<<<< before updating # Heroku Redis uses self-signed certs CHANNEL_LAYERS["default"]["CONFIG"]["hosts"][0]["ssl_cert_reqs"] = ssl.CERT_NONE +======= +# Inform rate limiting that "X-Forwarded-For" should be trusted, as it's appended by Heroku. +ALLAUTH_TRUSTED_PROXY_COUNT = 1 +>>>>>>> after updating