diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ad21eadf..b13cdcfa 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,12 +5,15 @@ // over https. CPU only; a GPU gate (`just kokkos-cuda-test`) needs a host with a driver. The // first `nix develop` downloads the toolchain; later ones hit the store. Open a full clone, not a // git worktree: a worktree's .git points into a repository the container does not mount. +// cacheFrom reuses the published dev image (.github/workflows/docker-dev.yml, pushed by hand +// only), so Kokkos is not rebuilt; while none is pushed the build runs in full. { "name": "ww3-gpu", "build": { "dockerfile": "../docker/Dockerfile", "context": "..", - "target": "dev" + "target": "dev", + "cacheFrom": "ghcr.io/h0ffmann/ww3-gpu-dev:latest" }, "remoteUser": "ubuntu", "postCreateCommand": "bash scripts/agent_env.sh", diff --git a/.github/workflows/docker-dev.yml b/.github/workflows/docker-dev.yml new file mode 100644 index 00000000..d58cb53f --- /dev/null +++ b/.github/workflows/docker-dev.yml @@ -0,0 +1,60 @@ +# docker-dev — the development image: the `dev` stage of docker/Dockerfile (Ubuntu 26.04, Ubuntu's +# compilers and netCDF, Kokkos 5.2.2, the gate tools, Nix for picasso), the image +# .devcontainer/devcontainer.json builds. It holds no source: the checkout is mounted +# (scripts/docker.sh run-dev), so working on ww3-gpu needs only Docker. +# +# Manual only: no push, pull request or schedule builds it (the owner, 18 October 2026). A run +# started from Actions → docker-dev → Run workflow builds the image and checks its tools; with +# `push` ticked it also pushes ghcr.io//ww3-gpu-dev, and `latest` moves only when that run +# is on main. +name: docker-dev + +on: + workflow_dispatch: + inputs: + push: + description: Push the image to ghcr.io (unticked, the run only builds and checks it) + type: boolean + default: false + tag: + description: Image tag (empty, sha-) + type: string + default: "" + +permissions: + contents: read + +jobs: + # The environment `ghcr` is where the owner can require an approval before each run + # (Settings → Environments). + image: + runs-on: ubuntu-latest + timeout-minutes: 60 + environment: ghcr + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v7 + with: + submodules: false + - name: Build the dev stage (scripts/docker.sh build-dev) + run: bash scripts/docker.sh build-dev ci + - name: Its tools answer + run: docker run --rm ww3-gpu-dev:ci bash -c 'gfortran --version | head -1 && cmake --version | head -1 && just --version && shellcheck --version | head -2 && nix --version && test -d /opt/kokkos/include' + - name: Push to ghcr.io + if: inputs.push + env: + TAG: ${{ inputs.tag }} + OWNER: ${{ github.repository_owner }} + GH_TOKEN: ${{ github.token }} + run: | + tag="${TAG:-sha-${GITHUB_SHA::7}}" + [[ "$tag" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]] || { echo "bad tag: $tag" >&2; exit 64; } + [ "$tag" != latest ] || { echo "latest is set only by a run on main" >&2; exit 64; } + image="ghcr.io/${OWNER,,}/ww3-gpu-dev" + refs=("$image:$tag") + [ "$GITHUB_REF" = refs/heads/main ] && refs+=("$image:latest") + echo "$GH_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + bash scripts/docker.sh push ww3-gpu-dev:ci "${refs[@]}" + printf 'Pushed:\n' >> "$GITHUB_STEP_SUMMARY"; printf -- '- %s\n' "${refs[@]}" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/docker-dist.yml b/.github/workflows/docker-dist.yml new file mode 100644 index 00000000..55d3eb5c --- /dev/null +++ b/.github/workflows/docker-dist.yml @@ -0,0 +1,85 @@ +# docker-dist — the distribution images, binaries only: NOAA's WW3 at the WW3-noaa/ pin (--ww3) +# and ww3-gpu (--ww3-gpu, the fork with the Kokkos port) compiled side by side, so a user can run a +# case with both and compare the outputs pair to pair (scripts/ww3-gpu.sh is the entry point). +# cpu docker/Dockerfile ghcr.io//ww3-gpu Kokkos on OpenMP, plus WW4 +# cuda docker/Dockerfile.cuda ghcr.io//ww3-gpu-cuda Kokkos on CUDA, --ww3 on the CPU +# The final stage of each keeps the compiled programs, their runtime libraries and the case +# inputs; `scripts/docker.sh check` fails when a source file or either WW3 is missing. The labels +# io.github.h0ffmann.ww3-gpu.ww3-noaa and .ww3 name the two WW3 commits the binaries came from. +# +# Manual only: no push, pull request or schedule builds them (the owner, 18 October 2026). A run +# started from Actions → docker-dist → Run workflow builds and checks the chosen images (nothing +# runs on a GPU: the CUDA image is only compiled); with `push` ticked it also pushes them, and +# `latest` moves only when that run is on main. The CPU image's vulnerability scan is +# docker-scan.yml, manual too. +name: docker-dist + +on: + workflow_dispatch: + inputs: + flavour: + description: Which image + type: choice + options: [both, cpu, cuda] + default: both + push: + description: Push to ghcr.io (unticked, the run only builds and checks) + type: boolean + default: false + tag: + description: Image tag, e.g. a release 0.5.0 (empty, sha-) + type: string + default: "" + +permissions: + contents: read + +jobs: + # The environment `ghcr` is where the owner can require an approval before each run + # (Settings → Environments). + image: + runs-on: ubuntu-latest + timeout-minutes: 150 + environment: ghcr + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + flavour: ${{ fromJSON(inputs.flavour == 'both' && '["cpu","cuda"]' || format('["{0}"]', inputs.flavour)) }} + steps: + - uses: actions/checkout@v7 + with: + submodules: false + - name: Free disk space + run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL && df -h / + - name: Fetch WW3, WW3-noaa and WW4 at their pins + run: bash scripts/docker.sh submodules + - name: Build + env: + FLAVOUR: ${{ matrix.flavour }} + run: | + if [ "$FLAVOUR" = cuda ]; then bash scripts/docker.sh build-cuda ci; else bash scripts/docker.sh build ci; fi + - name: Binaries only, both WW3s present + env: + FLAVOUR: ${{ matrix.flavour }} + run: bash scripts/docker.sh check "$([ "$FLAVOUR" = cuda ] && echo ww3-gpu-cuda || echo ww3-gpu):ci" + - name: Push to ghcr.io + if: inputs.push + env: + FLAVOUR: ${{ matrix.flavour }} + TAG: ${{ inputs.tag }} + OWNER: ${{ github.repository_owner }} + GH_TOKEN: ${{ github.token }} + run: | + name=ww3-gpu; [ "$FLAVOUR" = cuda ] && name=ww3-gpu-cuda + tag="${TAG:-sha-${GITHUB_SHA::7}}" + [[ "$tag" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]] || { echo "bad tag: $tag" >&2; exit 64; } + [ "$tag" != latest ] || { echo "latest is set only by a run on main" >&2; exit 64; } + image="ghcr.io/${OWNER,,}/$name" + refs=("$image:$tag") + [ "$GITHUB_REF" = refs/heads/main ] && refs+=("$image:latest") + echo "$GH_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + bash scripts/docker.sh push "$name:ci" "${refs[@]}" + printf 'Pushed:\n' >> "$GITHUB_STEP_SUMMARY"; printf -- '- %s\n' "${refs[@]}" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/docker-scan.yml b/.github/workflows/docker-scan.yml index 97b0af19..22cbdcfa 100644 --- a/.github/workflows/docker-scan.yml +++ b/.github/workflows/docker-scan.yml @@ -1,17 +1,12 @@ # docker-scan — builds docker/Dockerfile, the image this repository distributes, and scans it with # Trivy (free, open source): the job fails on any HIGH or CRITICAL vulnerability that has a fix, # in an Ubuntu package or in a binary of the image. Unfixed ones are reported, not fatal: there -# is nothing to upgrade to. Weekly too, since new CVEs land without any change here. +# is nothing to upgrade to. Manual only (Actions → docker-scan → Run workflow): no Docker image +# is built by a push, a pull request or a schedule (the owner, 18 October 2026); `just dscan` +# runs the same scan locally. name: docker-scan on: - pull_request: - paths: [docker/**, .dockerignore, scripts/ww3-gpu.sh, scripts/docker.sh, kokkos/**, WW3, WW4, .github/workflows/docker-scan.yml] - push: - branches: [main] - paths: [docker/**, .dockerignore, scripts/ww3-gpu.sh, scripts/docker.sh, kokkos/**, WW3, WW4, .github/workflows/docker-scan.yml] - schedule: - - cron: "17 6 * * 1" workflow_dispatch: permissions: diff --git a/README.md b/README.md index 480e3af5..a87c85e9 100644 --- a/README.md +++ b/README.md @@ -472,11 +472,11 @@ become `dsmoke`, `dsmoke-ww3`, and so on. The image needs no Nix and no picasso: installs Ubuntu's compilers and libraries, compiles Kokkos 5.2.2 from its release tarball (Ubuntu packages an older one), and the final stage copies only the compiled programs. -CI scans the image with Trivy on every change to it and weekly (`.github/workflows/docker-scan.yml`): a HIGH or CRITICAL vulnerability with a fix fails the job. `just dscan` runs the same scan here. +No GitHub Actions workflow builds a Docker image on its own: every one starts by hand from Actions (Run workflow). `.github/workflows/docker-scan.yml` scans the image with Trivy when run: a HIGH or CRITICAL vulnerability with a fix fails the job. `just dscan` runs the same scan here. The Kokkos code on the GPU has its own image, `docker/Dockerfile.cuda`: NVIDIA's CUDA 13.4 on Ubuntu 26.04, Kokkos 5.2.2 built with CUDA for compute capability 8.9 (the RTX 4090; `KOKKOS_ARCH` -picks another), and only `--ww3-gpu --gpu` inside. The host needs the NVIDIA driver and the +picks another), with `--ww3-gpu --gpu` and, as its reference on the CPU, `--ww3`. Its kernels call CUDA's libdevice, not the device copies of glibc's libm (`WW_KOKKOS_LIBM=OFF`: Ubuntu ships libmvec stripped, and the copies need a symbol of it), so a GPU run in the image is judged by tolerance ([ADR-0014](docs/ADRs/ADR-0014-gpu-runs-judged-by-tolerance.md)). The host needs the NVIDIA driver and the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) for `docker run --gpus all`. As on the host, every `--gpu` run asks the supervising person for `yes` at the terminal ([Running on the GPU](AGENTS.md#running-on-the-gpu-hard-rule)). @@ -501,6 +501,26 @@ Ubuntu's: `--ww3-gpu` still compares the Kokkos and the Fortran DIA inside one e bit-for-bit verdict holds in the image; a number from the image is not a number from the pinned toolchain of [`docs/TOOLCHAIN.md`](docs/TOOLCHAIN.md), which uses another gfortran. +**Published images.** `.github/workflows/docker-dist.yml` runs only when a person starts it from +Actions (Run workflow); no push, pull request or schedule triggers it. It builds the chosen +runtime images and checks, without running a case, that each holds NOAA's WW3 (`--ww3`, compiled +from `WW3-noaa/` at its pin) and ww3-gpu (`--ww3-gpu`) and no source file (`just dcheck` runs the +same check). With `push` ticked it also pushes `ghcr.io/h0ffmann/ww3-gpu:` (CPU) and +`ghcr.io/h0ffmann/ww3-gpu-cuda:`, and `latest` too when the run is on main. The labels +`io.github.h0ffmann.ww3-gpu.ww3-noaa` and `io.github.h0ffmann.ww3-gpu.ww3` name the two WW3 commits +the binaries came from. With a published image, anyone can run NOAA's WW3 and ww3-gpu on the same +case and compare the outputs with only Docker: + +```console +img=ghcr.io/h0ffmann/ww3-gpu: +docker inspect "$img" --format '{{json .Config.Labels}}' # which WW3 commits +docker run --rm --user "$(id -u):$(id -g)" -v "$PWD/runs:/work/runs" "$img" --ww3 smoke +docker run --rm --user "$(id -u):$(id -g)" -v "$PWD/runs:/work/runs" "$img" --ww3-gpu smoke +``` + +(v) 2026-10-18: `scripts/docker.sh build` and `check` passed on the CPU image (WW3-noaa `03adda9`, WW3 `80644ba`); the CUDA image was not built here ⚠. No image has been pushed yet ⚠. GHCR may create the first push of a package as private; the owner +then makes it public in the package's settings. + ### 3 · the dev container `.devcontainer/devcontainer.json` builds the `dev` stage of `docker/Dockerfile`: Ubuntu 26.04 with @@ -521,6 +541,24 @@ npx --yes @devcontainers/cli@0.89.0 exec --workspace-folder . just smoke-ww3-gpu - `/nix` lives in the container, so a rebuilt container downloads picasso's toolchain again (about 3.1 GB, about a minute on the lab's link). +Without VS Code or the Dev Containers CLI, the dev image alone is enough. `just dbuild-dev` builds +it, or pull `ghcr.io/h0ffmann/ww3-gpu-dev`, which `.github/workflows/docker-dev.yml` builds and, +with `push` ticked, pushes, only when a person starts it from Actions. `just ddev` +then opens a shell with this checkout at `/workspaces/ww3-gpu`, and `just ddev ` runs one +command (`DOCKER_IMAGE=ghcr.io/h0ffmann/ww3-gpu-dev:latest` picks the published image). Without +just: + +```console +docker run --rm -it -v "$PWD:/workspaces/ww3-gpu" -w /workspaces/ww3-gpu ghcr.io/h0ffmann/ww3-gpu-dev:latest +``` + +The user inside is `ubuntu` (uid 1000), so files it writes belong to uid 1000 on the host. Run +`bash scripts/agent_env.sh` once inside when `picasso/` is not checked out. `devcontainer.json` +takes the published image as its build cache (`cacheFrom`), so Kokkos is not compiled again. +(v) 2026-10-18: `just dbuild-dev` built, and `docker-dev.yml`'s check found gfortran 15.2.0, +CMake 4.2.3, just 1.45.0, ShellCheck 0.11.0, Nix 2.34.3 and Kokkos in `/opt/kokkos`. ⚠ `just ddev +just kokkos-test serial-debug` and the cache from a pushed image are not run yet. + (v) 2026-10-15, CLI 0.89.0, Docker 29.9.1, `main` at `78e5a33`, full clone: `devcontainer up` passed, `just kokkos-test serial-debug` passed 10 of 10, `just smoke-ww3-gpu` gave byte-identical Fortran and Kokkos outputs, and every file the runs wrote belongs to the host user. diff --git a/README.pt-BR.md b/README.pt-BR.md index 105160e7..d3cde672 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -469,11 +469,11 @@ Cada receita que roda o `scripts/ww3-gpu.sh` no host tem seu par na imagem, com build instala os compiladores e as bibliotecas do Ubuntu, compila o Kokkos 5.2.2 a partir do tarball da versão (o Ubuntu empacota uma mais antiga), e o estágio final copia só os programas compilados. -O CI varre a imagem com o Trivy a cada mudança nela e toda semana (`.github/workflows/docker-scan.yml`): uma vulnerabilidade HIGH ou CRITICAL com correção reprova o job. `just dscan` roda a mesma varredura aqui. +Nenhum workflow do GitHub Actions constrói uma imagem Docker sozinho: todos começam à mão pelo Actions (Run workflow). O `.github/workflows/docker-scan.yml` varre a imagem com o Trivy quando disparado: uma vulnerabilidade HIGH ou CRITICAL com correção reprova o job. `just dscan` roda a mesma varredura aqui. O código Kokkos na GPU tem sua própria imagem, `docker/Dockerfile.cuda`: o CUDA 13.4 da NVIDIA sobre Ubuntu 26.04, o Kokkos 5.2.2 compilado com CUDA para a capacidade de computação 8.9 (a RTX -4090; `KOKKOS_ARCH` escolhe outra), e só o `--ww3-gpu --gpu` dentro. O host precisa do driver da +4090; `KOKKOS_ARCH` escolhe outra), com o `--ww3-gpu --gpu` e, como referência na CPU, o `--ww3`. Os kernels chamam a libdevice do CUDA, não as cópias da libm da glibc no dispositivo (`WW_KOKKOS_LIBM=OFF`: o Ubuntu distribui a libmvec sem a tabela de símbolos, e as cópias precisam de um símbolo dela), então uma rodada na GPU dentro da imagem é julgada por tolerância ([ADR-0014](docs/ADRs/ADR-0014-gpu-runs-judged-by-tolerance.md)). O host precisa do driver da NVIDIA e do [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) para `docker run --gpus all`. Como no host, toda rodada `--gpu` pede um `yes` no terminal a quem supervisiona ([Running on the GPU](AGENTS.md#running-on-the-gpu-hard-rule)). @@ -499,6 +499,27 @@ dentro de um mesmo executável, então o veredito bit a bit vale na imagem; um n imagem não é um número do toolchain fixado de [`docs/TOOLCHAIN.md`](docs/TOOLCHAIN.md), que usa outro gfortran. +**Imagens publicadas.** O `.github/workflows/docker-dist.yml` só roda quando uma pessoa o dispara +pelo Actions (Run workflow); nenhum push, pull request ou agendamento o aciona. Ele constrói as +imagens de execução escolhidas e confere, sem rodar caso nenhum, que cada uma traz o WW3 da NOAA +(`--ww3`, compilado do `WW3-noaa/` no seu pin) e o ww3-gpu (`--ww3-gpu`) e nenhum arquivo-fonte (o +`just dcheck` faz a mesma conferência). Com `push` marcado, ele também publica +`ghcr.io/h0ffmann/ww3-gpu:` (CPU) e `ghcr.io/h0ffmann/ww3-gpu-cuda:`, e também `latest` +quando a rodada é na main. Os rótulos +`io.github.h0ffmann.ww3-gpu.ww3-noaa` e `io.github.h0ffmann.ww3-gpu.ww3` dizem de quais commits do +WW3 vieram os binários. Com uma imagem publicada, qualquer pessoa roda o WW3 da NOAA e o ww3-gpu no +mesmo caso e compara as saídas só com o Docker: + +```console +img=ghcr.io/h0ffmann/ww3-gpu: +docker inspect "$img" --format '{{json .Config.Labels}}' # quais commits do WW3 +docker run --rm --user "$(id -u):$(id -g)" -v "$PWD/runs:/work/runs" "$img" --ww3 smoke +docker run --rm --user "$(id -u):$(id -g)" -v "$PWD/runs:/work/runs" "$img" --ww3-gpu smoke +``` + +(v) 18/10/2026: o `scripts/docker.sh build` e o `check` passaram na imagem de CPU (WW3-noaa `03adda9`, WW3 `80644ba`); a imagem CUDA não foi construída aqui ⚠. Nenhuma imagem foi publicada ainda ⚠. O GHCR pode criar como privada a primeira publicação de um +pacote; o dono então a torna pública nas configurações do pacote. + ### 3 · o dev container O `.devcontainer/devcontainer.json` constrói o estágio `dev` do `docker/Dockerfile`: Ubuntu 26.04 @@ -520,6 +541,25 @@ npx --yes @devcontainers/cli@0.89.0 exec --workspace-folder . just smoke-ww3-gpu - O `/nix` fica dentro do container, então um container reconstruído baixa de novo o toolchain do picasso (cerca de 3,1 GB, cerca de um minuto na rede do laboratório). +Sem o VS Code nem a Dev Containers CLI, a imagem de desenvolvimento basta. O `just dbuild-dev` a +constrói, ou baixe a `ghcr.io/h0ffmann/ww3-gpu-dev`, que o `.github/workflows/docker-dev.yml` +constrói e, com `push` marcado, publica, só quando uma pessoa o dispara pelo Actions. +O `just ddev` abre então um shell com este clone em `/workspaces/ww3-gpu`, e `just ddev ` +roda um comando (`DOCKER_IMAGE=ghcr.io/h0ffmann/ww3-gpu-dev:latest` escolhe a imagem publicada). +Sem o just: + +```console +docker run --rm -it -v "$PWD:/workspaces/ww3-gpu" -w /workspaces/ww3-gpu ghcr.io/h0ffmann/ww3-gpu-dev:latest +``` + +O usuário lá dentro é o `ubuntu` (uid 1000), então os arquivos que ele escreve pertencem ao uid 1000 +no host. Rode `bash scripts/agent_env.sh` uma vez lá dentro quando o `picasso/` não estiver no +clone. O `devcontainer.json` usa a imagem publicada como cache de build (`cacheFrom`), então o +Kokkos não é compilado de novo. (v) 18/10/2026: o `just dbuild-dev` construiu, e a conferência do +`docker-dev.yml` achou gfortran 15.2.0, CMake 4.2.3, just 1.45.0, ShellCheck 0.11.0, Nix 2.34.3 e o +Kokkos em `/opt/kokkos`. ⚠ O `just ddev just kokkos-test serial-debug` e o cache a partir de uma +imagem publicada ainda não rodaram. + (v) 15/10/2026, CLI 0.89.0, Docker 29.9.1, `main` em `78e5a33`, clone completo: o `devcontainer up` passou, o `just kokkos-test serial-debug` passou 10 de 10, o `just smoke-ww3-gpu` deu saídas em Fortran e em Kokkos idênticas byte a byte, e todo arquivo que as rodadas escreveram pertence ao diff --git a/docker/Dockerfile b/docker/Dockerfile index e39e1179..96804d21 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -148,6 +148,8 @@ RUN useradd --create-home --shell /bin/bash wave COPY --chmod=755 scripts/ww3-gpu.sh /opt/ww3-gpu/scripts/ww3-gpu.sh # Its list of Kokkos parts, one file per wiring unit. COPY kokkos/src/fortran_iface/parts.d/ /opt/ww3-gpu/kokkos/src/fortran_iface/parts.d/ +# The switch the programs were built with: ww3-gpu.sh reads it before any run (WW3GPU_SWITCH). +COPY switches/switch_lab_ompg /opt/ww3-gpu/switches/switch_lab_ompg ENV WW3GPU_PREBUILT=/opt/ww3-gpu/prebuilt \ WW3GPU_RUNS=/work/runs \ OMP_PROC_BIND=false diff --git a/docker/Dockerfile.cuda b/docker/Dockerfile.cuda index ca580249..a4f7bfb8 100644 --- a/docker/Dockerfile.cuda +++ b/docker/Dockerfile.cuda @@ -52,7 +52,10 @@ COPY WW3-noaa/ WW3-noaa/ COPY switches/switch_lab_ompg switches/ ENV P=/opt/ww3-gpu/prebuilt -RUN cmake -S kokkos --preset cuda-release -B /build/kokkos -DCMAKE_PREFIX_PATH=/opt/kokkos \ +# WW_KOKKOS_LIBM=OFF: the device's glibc libm (WFIP-0006) needs gen_tables to find the hidden +# __svml_spow_data in libmvec.so.1, and Ubuntu ships libmvec stripped (no .symtab), so the +# kernels call CUDA's libdevice here and a GPU run is judged by tolerance (ADR-0014). +RUN cmake -S kokkos --preset cuda-release -B /build/kokkos -DCMAKE_PREFIX_PATH=/opt/kokkos -DWW_KOKKOS_LIBM=OFF \ && cmake --build /build/kokkos \ && cmake --install /build/kokkos --prefix /opt/ww_kokkos \ && install -D -m 755 /build/kokkos/tools/nccmp-tol/nccmp-tol "$P/bin/nccmp-tol" \ @@ -87,7 +90,7 @@ ARG WW3_REV=? ARG WW3_NOAA_REV=? RUN { echo "ww3-gpu ${WW3_GPU_REV}"; echo "WW3 ${WW3_REV} (h0ffmann/WW3)"; \ echo "WW3-noaa ${WW3_NOAA_REV} (NOAA-EMC/WW3)"; \ - echo "kokkos ${KOKKOS_VERSION} CUDA ${KOKKOS_ARCH}"; nvcc --version | tail -1; \ + echo "kokkos ${KOKKOS_VERSION} CUDA ${KOKKOS_ARCH} WW_KOKKOS_LIBM=OFF (libdevice)"; nvcc --version | tail -1; \ gfortran --version | head -1; cmake --version | head -1; } > /opt/ww3-gpu/BUILD_INFO FROM ${CUDA_RUNTIME} @@ -100,6 +103,8 @@ RUN useradd --create-home --shell /bin/bash wave COPY --chmod=755 scripts/ww3-gpu.sh /opt/ww3-gpu/scripts/ww3-gpu.sh # Its list of Kokkos parts, one file per wiring unit. COPY kokkos/src/fortran_iface/parts.d/ /opt/ww3-gpu/kokkos/src/fortran_iface/parts.d/ +# The switch the programs were built with: ww3-gpu.sh reads it before any run (WW3GPU_SWITCH). +COPY switches/switch_lab_ompg /opt/ww3-gpu/switches/switch_lab_ompg ENV WW3GPU_PREBUILT=/opt/ww3-gpu/prebuilt \ WW3GPU_RUNS=/work/runs WORKDIR /work diff --git a/justfile b/justfile index b80cd3ea..9a9b5e85 100644 --- a/justfile +++ b/justfile @@ -209,6 +209,19 @@ dbuild-cuda tag="local": drun-cuda *args: bash scripts/docker.sh run-cuda {{args}} +# Build the dev image ww3-gpu-dev:, the dev stage of docker/Dockerfile (no sources inside) +dbuild-dev tag="local": + bash scripts/docker.sh build-dev {{tag}} + +# DOCKER_IMAGE=ghcr.io/h0ffmann/ww3-gpu-dev:latest uses the published image. +# A shell (or a command) in the dev image with this checkout mounted: `just ddev just kokkos-test serial-debug` +ddev *args: + bash scripts/docker.sh run-dev {{args}} + +# A runtime image holds NOAA's WW3 and ww3-gpu compiled, and no source file (docker-dist.yml) +dcheck image="ww3-gpu:local": + bash scripts/docker.sh check {{image}} + # NOAA's WW3 (--ww3, the Fortran on the CPU) against --ww3-gpu on CUDA (--kokkos gpu: W3SRCE on # Kokkos, the rest of WW3 in Fortran): parity first, then runs of each and their medians. # Asks the supervising human for "yes" (AGENTS.md). `dperf-ww3-gpu` is the same in the GPU image. diff --git a/scripts/docker.sh b/scripts/docker.sh index 49fec376..386110f2 100755 --- a/scripts/docker.sh +++ b/scripts/docker.sh @@ -9,10 +9,16 @@ # what it was built from. # scripts/docker.sh build-cuda [tag] build ww3-gpu-cuda: from docker/Dockerfile.cuda # scripts/docker.sh run-cuda [args] run ww3-gpu.sh in the GPU image with --gpus all +# scripts/docker.sh build-dev [tag] build ww3-gpu-dev:, the dev stage of docker/Dockerfile +# scripts/docker.sh run-dev [cmd] a shell (or cmd) in the dev image, this checkout mounted +# scripts/docker.sh submodules fetch WW3, WW3-noaa and WW4 at their pins over https +# scripts/docker.sh check a runtime image holds both WW3s compiled and no source file, +# and its entry point starts (--build-only, no run) +# scripts/docker.sh push ... tag a local image as each ref and push it (by hand only) # scripts/docker.sh --dry-run ... print the docker command, run nothing # scripts/docker.sh --self-test # -# DOCKER_IMAGE overrides the image name (ww3-gpu:local, ww3-gpu-cuda:local). The CPU image has no +# DOCKER_IMAGE overrides the image name (ww3-gpu:local, ww3-gpu-cuda:local, ww3-gpu-dev:local). The CPU image has no # CUDA build, and ww3-gpu.sh refuses --gpu in it (exit 64); the GPU image runs only --ww3-gpu --gpu. set -euo pipefail @@ -24,6 +30,20 @@ usage() { sed -n '2,/^set -euo/{/^set -euo/!p}' "${BASH_SOURCE[0]}" | sed 's/^# say() { printf '%q ' "$@"; echo; } do_or_say() { if [ -n "$dry" ]; then say "$@"; else "$@"; fi; } rev() { git -C "$1" rev-parse --short HEAD 2>/dev/null || echo "?"; } +# A submodule's pin as the superproject records it, checked out or not. +pin() { git -C "$root" rev-parse --short "HEAD:$1" 2>/dev/null || echo "?"; } +# OCI labels: the source label links a GHCR package to this repository; the pins say which WW3 +# (NOAA's and the fork) the binaries were compiled from without the sources being in the image. +# $1 the image's title, $2 its description (else the base image's "ubuntu" shows on GHCR). +labels() { + printf -- '--label\n%s\n' \ + "org.opencontainers.image.title=$1" "org.opencontainers.image.description=$2" \ + "org.opencontainers.image.source=https://github.com/h0ffmann/ww3-gpu" \ + "org.opencontainers.image.revision=$(git -C "$root" rev-parse HEAD 2>/dev/null || echo "?")" \ + "org.opencontainers.image.licenses=MIT" \ + "io.github.h0ffmann.ww3-gpu.ww3-noaa=$(pin WW3-noaa)" \ + "io.github.h0ffmann.ww3-gpu.ww3=$(pin WW3)" +} if [ "${1:-}" = --self-test ]; then out="$(bash "${BASH_SOURCE[0]}" --dry-run run --ww3-gpu smoke)" @@ -40,6 +60,23 @@ if [ "${1:-}" = --self-test ]; then done out="$(bash "${BASH_SOURCE[0]}" --dry-run build-cuda t1)" grep -q -- "docker/Dockerfile.cuda" <<<"$out" || { echo "docker: build-cuda lost its Dockerfile: $out" >&2; exit 1; } + out="$(bash "${BASH_SOURCE[0]}" --dry-run build-dev t1)" + for want in "--target dev" ww3-gpu-dev:t1 BUILDKIT_INLINE_CACHE=1 org.opencontainers.image.source=; do + grep -q -- "$want" <<<"$out" || { echo "docker: build-dev lost '$want': $out" >&2; exit 1; } + done + out="$(bash "${BASH_SOURCE[0]}" --dry-run run-dev just --list &2; exit 1; } + done + out="$(bash "${BASH_SOURCE[0]}" --dry-run push ww3-gpu:t1 ghcr.io/x/ww3-gpu:a ghcr.io/x/ww3-gpu:b)" + for want in "tag ww3-gpu:t1 ghcr.io/x/ww3-gpu:a" "push ghcr.io/x/ww3-gpu:b"; do + grep -q -- "$want" <<<"$out" || { echo "docker: push lost '$want': $out" >&2; exit 1; } + done + out="$(bash "${BASH_SOURCE[0]}" --dry-run check ww3-gpu:t1)" + for want in "--entrypoint sh ww3-gpu:t1" BUILD_INFO "binaries only" "--ww3 smoke --build-only" "--ww3-gpu smoke --build-only"; do + grep -q -- "$want" <<<"$out" || { echo "docker: check lost '$want': $out" >&2; exit 1; } + done + bash "${BASH_SOURCE[0]}" --dry-run push ww3-gpu:t1 >/dev/null 2>&1 && { echo "docker: push without a ref" >&2; exit 1; } bash "${BASH_SOURCE[0]}" --dry-run bogus >/dev/null 2>&1 && { echo "docker: accepted a bad command" >&2; exit 1; } echo "docker self-test: ok" exit 0 @@ -58,7 +95,8 @@ case "$cmd" in git -C "$root" submodule update --init --depth 1 WW4 git -C "$root/WW4" submodule update --init --depth 1 externals/yaml-cpp externals/googletest } - do_or_say docker build -f "$root/docker/Dockerfile" -t "ww3-gpu:$tag" \ + mapfile -t lab < <(labels ww3-gpu "WW3 (NOAA, --ww3) and ww3-gpu (Kokkos OpenMP, --ww3-gpu) compiled side by side, and WW4; binaries only") + do_or_say docker build -f "$root/docker/Dockerfile" -t "ww3-gpu:$tag" "${lab[@]}" \ --build-arg "WW3_GPU_REV=$(rev "$root")" --build-arg "WW3_REV=$(rev "$root/WW3")" \ --build-arg "WW3_NOAA_REV=$(rev "$root/WW3-noaa")" \ --build-arg "WW4_REV=$(rev "$root/WW4")" "$root" ;; @@ -74,7 +112,8 @@ case "$cmd" in # pure NOAA WW3 (WW3-noaa/) on the CPU as its reference. [ -n "$dry" ] || [ -f "$root/WW3-noaa/model/src/CMakeLists.txt" ] \ || git -C "$root" submodule update --init --depth 1 WW3-noaa - do_or_say docker build -f "$root/docker/Dockerfile.cuda" -t "ww3-gpu-cuda:${1:-local}" \ + mapfile -t lab < <(labels ww3-gpu-cuda "ww3-gpu on Kokkos CUDA (--ww3-gpu --gpu) and WW3 (NOAA, --ww3) on the CPU; binaries only") + do_or_say docker build -f "$root/docker/Dockerfile.cuda" -t "ww3-gpu-cuda:${1:-local}" "${lab[@]}" \ --build-arg "WW3_GPU_REV=$(rev "$root")" --build-arg "WW3_REV=$(rev "$root/WW3")" \ --build-arg "WW3_NOAA_REV=$(rev "$root/WW3-noaa")" "$root" ;; run-cuda) @@ -85,5 +124,54 @@ case "$cmd" in do_or_say docker run --rm "${tty[@]}" --gpus all --user "$(id -u):$(id -g)" \ -e "WW3GPU_ROUTINES=${WW3GPU_ROUTINES:-}" -v "$root/build/docker-runs:/work/runs" \ "${DOCKER_IMAGE:-ww3-gpu-cuda:local}" "$@" ;; + build-dev) + # The dev stage copies no source, so no submodule is needed. The inline cache lets + # .devcontainer/devcontainer.json (cacheFrom) reuse a pushed image instead of rebuilding Kokkos. + mapfile -t lab < <(labels ww3-gpu-dev "Development image of ww3-gpu: compilers, netCDF, Kokkos, gate tools, Nix; mount the checkout") + do_or_say docker build -f "$root/docker/Dockerfile" --target dev -t "ww3-gpu-dev:${1:-local}" \ + "${lab[@]}" --build-arg BUILDKIT_INLINE_CACHE=1 "$root" ;; + run-dev) + # The checkout is mounted where the dev container mounts it; the user is the image's ubuntu + # (uid 1000), who owns /nix, so files written there belong to uid 1000 on the host. + tty=(); [ -t 0 ] && tty=(-it) + do_or_say docker run --rm "${tty[@]}" -v "$root:/workspaces/ww3-gpu" -w /workspaces/ww3-gpu \ + "${DOCKER_IMAGE:-ww3-gpu-dev:local}" "${@:-bash}" ;; + submodules) + # .gitmodules uses ssh URLs, hence https; a pin on a topic branch may not be fetched by + # `submodule update`, so the fallback fetches the SHA itself. + export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=url.https://github.com/.insteadOf GIT_CONFIG_VALUE_0=git@github.com: + for sm in WW3 WW3-noaa WW4; do + if [ -n "$dry" ]; then say git submodule update --init --depth 1 "$sm"; continue; fi + git -C "$root" submodule update --init --depth 1 "$sm" || { + sha=$(git -C "$root" ls-tree HEAD "$sm" | awk '{print $3}') + git -C "$root/$sm" fetch --depth 1 origin "$sha" && git -C "$root/$sm" checkout -q "$sha" + } + done + do_or_say git -C "$root/WW4" submodule update --init --depth 1 externals/yaml-cpp externals/googletest ;; + check) + # Runs no case: --ww3 (NOAA) and --ww3-gpu must both be compiled in, and nothing in the + # image's tree may be a Fortran or C/C++ source, so the image is a binary distribution. + img="${1:-}"; [ -n "$img" ] || { echo "docker: check " >&2; exit 64; } + # shellcheck disable=SC2016 # expanded by the container's sh + do_or_say docker run --rm --entrypoint sh "$img" -c ' + set -e; cat /opt/ww3-gpu/BUILD_INFO; P=/opt/ww3-gpu/prebuilt + test -x "$P/ww3/bin/ww3_shel" || { echo "no NOAA ww3_shel" >&2; exit 1; } + ls -d "$P"/ww3-gpu-*/bin/ww3_shel || { echo "no ww3-gpu ww3_shel" >&2; exit 1; } + src=$(find /opt/ww3-gpu /work -type f \( -name "*.[fF]90" -o -name "*.ftn" -o -name "*.[ch]" \ + -o -name "*.[ch]pp" -o -name "*.cu" -o -name CMakeLists.txt \) | head) + [ -z "$src" ] || { echo "sources in the image:" >&2; echo "$src" >&2; exit 1; } + echo "binaries only: ok"' + # The entry point must get past its argument and file checks in both modes; --build-only + # stops before any run (and --gpu --build-only needs neither a GPU nor a person). + gpu=() + if [ -z "$dry" ] && docker run --rm --entrypoint test "$img" -d /opt/ww3-gpu/prebuilt/ww3-gpu-cuda; then gpu=(--gpu); fi + do_or_say docker run --rm "$img" --ww3 smoke --build-only + do_or_say docker run --rm "$img" --ww3-gpu smoke "${gpu[@]}" --build-only ;; + push) + # Never called by CI on its own: .github/workflows/docker-*.yml run it only from a manual + # workflow_dispatch with push set. + src="${1:-}"; shift || true + [ -n "$src" ] && [ "$#" -gt 0 ] || { echo "docker: push ..." >&2; exit 64; } + for ref in "$@"; do do_or_say docker tag "$src" "$ref"; do_or_say docker push "$ref"; done ;; *) echo "docker: unknown command: $cmd" >&2; usage ;; esac