Port the translators to the moveapps R package

.dockerignore

@@ -14,5 +14,7 @@ x-archive
 # dev-only bulk, copied by neither the App image nor the test image.
 # `python/sample/**` is deliberately NOT excluded - the tests build their fixtures from it.
 r/data/
+# ...except the samples the round trip reads in CI
+!r/data/raw
 notebook/
 python/notebooks/

.github/workflows/test.yml

@@ -11,12 +11,12 @@ on:
   # so a fresh resolution can be checked after a suspected break, instead of waiting for Monday
   workflow_dispatch:
 
-# `true` exactly for the runs that must resolve the conda environment from scratch.
-# `test/Dockerfile` resolves the environment in its own layer - correctly, it is what keeps a
-# source-only change cheap - but since v2.2.0 that environment is deliberately open: only `pandas`
-# carries a floor and a ceiling. So a cached layer lets this suite report green against a
-# resolution nobody has re-checked, and nothing evicts it on its own, because every push's
-# `cache-from` resets GitHub's seven-day idle timer.
+# `true` exactly for the runs that must resolve from scratch. The App and the python tests install
+# `python/conda-linux-64.lock`, so an ordinary run is reproducible. But `environment.yml` is deliberately
+# open since v2.2.0 - only `pandas` carries a floor and a ceiling - and the lock is refreshed from it
+# before a release. These runs resolve it as it stands today and test that: the next lock. They skip
+# every cache, which nothing would evict on its own, because every push's `cache-from` resets
+# GitHub's seven-day idle timer.
 env:
   FRESH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
 
@@ -57,3 +57,66 @@ jobs:
           # after the tests have already passed. Import is unaffected - a PR may read the cache of
           # its own branch, the base branch and the default branch - so fork PRs stay warm.
           cache-to: type=gha,mode=max,ignore-error=true
+          # fresh runs test environment.yml resolved today, ordinary runs the lock the App installs
+          build-args: ENVIRONMENT=${{ env.FRESH == 'true' && 'fresh' || 'lock' }}
+          # the comparison below reads the image
+          load: ${{ env.FRESH == 'true' }}
+          tags: link-r-python-python-tests
+
+      # A lock behind today's resolution is no failure - it is refreshed before a release - but it must
+      # not go unnoticed.
+      - name: Report whether the lock is behind
+        if: env.FRESH == 'true'
+        run: |
+          docker run --rm link-r-python-python-tests cat resolved.txt | sort > fresh.txt
+          grep '^https' python/conda-linux-64.lock | sort > lock.txt
+          if diff lock.txt fresh.txt > drift.txt; then
+            echo 'python/conda-linux-64.lock is current' >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo '::warning::python/conda-linux-64.lock is behind environment.yml resolved today - run python/lock.sh'
+            { echo '### python/conda-linux-64.lock is behind'; echo '```diff'; cat drift.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+  app:
+    name: App images
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    # about twice a cold run (8m36s measured, no cache): cuts off a hang, not a slow runner
+    timeout-minutes: 20
+    env:
+      CACHE: gha
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build both Apps and test them inside the images
+        uses: docker/bake-action@v7
+        with:
+          source: .
+          files: test/docker-bake.hcl
+          targets: test
+
+      # The platform passes no build arguments, so a Dockerfile copy without its DIRECTION edit must be
+      # refused. The builder still holds every layer from the step above, so this costs the last ones only.
+      # The grep makes sure the build failed at the check, not somewhere else: it matches the message
+      # the check prints, anchored at the line end, because the log also quotes the command that prints it.
+      - name: A build without a valid DIRECTION must fail
+        env:
+          CACHE: ''
+        run: |
+          for target in app-without-direction app-with-invalid-direction; do
+            if docker buildx bake -f test/docker-bake.hcl "$target" > "$target.log" 2>&1; then
+              echo "::error::$target was built"
+              exit 1
+            fi
+            if ! grep -q -E 'DIRECTION must be r2python or python2r$' "$target.log"; then
+              cat "$target.log"
+              echo "::error::$target failed for another reason"
+              exit 1
+            fi
+          done

Dockerfile

@@ -1,63 +1,61 @@
-FROM registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-v3-r:sdk-v3.2.0_geospatial-4.3.2_3649
+# Builds either translator App - `DIRECTION` below says which. The platform passes no build
+# arguments: BASE and CONDA fall back to their defaults, and DIRECTION must be set in the copy of this
+# file that goes onto the App version (see README).
+ARG BASE=registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-r:v4.0.0_sdk-v1.0.3_geospatial-4.6.1_4674
+# The python co-pilot, for its conda: FROM condaforge/miniforge3:26.7.2-0, /opt/conda untouched.
+# Taken from the platform's registry, not Docker Hub: no App version has ever pulled from elsewhere,
+# and the platform's builder holds no Docker Hub credentials.
+ARG CONDA=registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-v1-python:v3.1.0
 
-# install miniconda aka conda-forge
-ENV MINICONDA_VERSION=latest
-ENV CONDA_DIR=$HOME/miniconda3
-RUN wget --quiet -O ~/Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \
-    chmod +x ~/Miniforge3.sh && \
-    bash ~/Miniforge3.sh -b -p $CONDA_DIR && \
-    rm ~/Miniforge3.sh
-# make non-activate conda commands available
-ENV PATH=$CONDA_DIR/bin:$PATH
-# make conda activate command available from /bin/bash --login shells
-RUN echo ". $CONDA_DIR/etc/profile.d/conda.sh" >> ~/.profile
-# make conda activate command available from /bin/bash --interative shells
-RUN conda init bash
+FROM ${CONDA} AS conda
 
+FROM ${BASE}
+COPY --from=conda /opt/conda /opt/conda
+ENV PATH=/opt/conda/bin:$PATH
+# conda caches in ~/.cache/conda, but the co-pilot created ~/.cache as root (for ~/.cache/R)
+USER root
+RUN install -d -o $UID -g $GID $HOME/.cache/conda
 USER $USER
 
 # the app
 ENV PROJECT_DIR=$HOME/co-pilot-r
 WORKDIR $PROJECT_DIR
 
-# the python part
+# the python part: the explicit lock python/lock.sh resolves from python/environment.yml, so the App gets
+# exactly what CI tested
 WORKDIR $PROJECT_DIR/python
-COPY --chown=$UID:$GID python/environment.yml /tmp/
-# build the conda environment
+COPY --chown=$UID:$GID python/conda-linux-64.lock /tmp/
 # keep in mind that anaconda aka channel `conda` is blocked in production env
 # https://gitlab.mpcdf.mpg.de/mpcdf-hpc-cloud/mvpr-moveapps/-/issues/29
-# miniconda uses channel `conda-forge` as default - so nothing to do here
+# miniforge uses channel `conda-forge` as default - so nothing to do here
 ENV ENV_PREFIX=$PROJECT_DIR/python-env
-RUN conda env create --prefix $ENV_PREFIX --file /tmp/environment.yml && \
+RUN conda create --yes --prefix $ENV_PREFIX --file /tmp/conda-linux-64.lock && \
     conda clean --all --yes
 
-# the r part
+# the r part: the translator's own renv project in r/
 WORKDIR $PROJECT_DIR/r
-# move the co-pilot-r sdk into this sub-directory
-USER root:root
-RUN mv ../src .
-# cleanup: remove unusuded files form co-pilot-r
-RUN rm -rf ../renv ../renv.lock ../.Rprofile ../RFunction.R src/io/shiny_bookmark_handler.R
-# patch the co-pilot-r file `src/io/rds.R` in order to not dependend on `move1`
-COPY r/src/io/rds.R src/io/rds.R
-USER $USER
+# cleanup: the co-pilot's own renv project and entry point; the translator brings both
+RUN rm -rf ../renv ../renv.lock ../.Rprofile ../RFunction.R ../app.R
 # renv: restore the current snapshot
 COPY --chown=$UID:$GID r/renv.lock r/.Rprofile ./
 COPY --chown=$UID:$GID r/renv/activate.R r/renv/settings.dcf ./renv/
 RUN R -e 'renv::restore()'
 # be prepared for the hangar inspection
 RUN cp renv.lock ../
 
-# the project
+# the project: both directions, so both Apps share every layer up to here
 WORKDIR $PROJECT_DIR
-# r -> python
-COPY --chown=$UID:$GID python/csv_2_pickle.py python/transform_to_pickle.py ./python/
-COPY --chown=$UID:$GID r/rds_2_csv.R ./r/
-# python -> r
-COPY --chown=$UID:$GID python/pickle_2_csv.py python/transform_to_csv.py ./python/
-COPY --chown=$UID:$GID r/csv_2_rds.R ./r/
+COPY --chown=$UID:$GID python/csv_2_pickle.py python/transform_to_pickle.py python/pickle_2_csv.py python/transform_to_csv.py ./python/
+COPY --chown=$UID:$GID r/link.R r/rds_2_csv.R r/csv_2_rds.R ./r/
+COPY --chown=$UID:$GID r2python.sh python2r.sh ./
 
-# r -> python
-#COPY --chown=$UID:$GID r2python.sh start-process.sh
-# python -> r
-#COPY --chown=$UID:$GID python2r.sh start-process.sh
+# The one line in which the two Apps differ: r2python (move2_loc to MovingPandas) or python2r
+# (MovingPandas to move2_loc). No default on purpose - a forgotten edit must fail the build, not
+# silently build the other App. Declared last: a changed ARG invalidates every RUN after it.
+# The start script is picked here, not by `COPY ${DIRECTION}.sh`: BuildKit resolves a COPY source
+# before this check has run, and fails on "/.sh: not found" instead of the message.
+ARG DIRECTION
+RUN case "$DIRECTION" in \
+      r2python|python2r) mv "$DIRECTION.sh" start-process.sh && rm -f r2python.sh python2r.sh ;; \
+      *) echo "DIRECTION must be r2python or python2r" >&2; exit 1 ;; \
+    esac

README.md

@@ -17,6 +17,35 @@ Both Apps create out of the input data one .csv file `meta.csv`. It contains tra
 
 **Python to R**: This App reads as input a MovingPandas TrajectoryCollection in Python, there transfers the data to csv files (see above), reads the information from those files into R and creates a move2_loc as output. This output can then be used as input in R Apps (of that IO type).
 
+## Build
+One `Dockerfile` builds both Apps. The direction is the build argument `DIRECTION`, deliberately without a default:
+
+    docker build --platform linux/amd64 --build-arg DIRECTION=r2python .   # move2_loc to MovingPandas
+    docker build --platform linux/amd64 --build-arg DIRECTION=python2r .   # MovingPandas to move2_loc
+
+The platform passes no build arguments. The Dockerfile of an App version is therefore this `Dockerfile` with exactly one
+line changed: `ARG DIRECTION` becomes `ARG DIRECTION=r2python` or `ARG DIRECTION=python2r`. A copy without that edit
+fails the build.
+
+## Python environment
+`python/environment.yml` says what the python half needs; `python/conda-linux-64.lock` is what the App and the python
+tests install. Refresh the lock before every release and after any change to `environment.yml`, then commit it:
+
+    python/lock.sh
+
+It resolves for linux-64 with the Miniforge of `test/Dockerfile`. CI fails while the lock was not made from the current
+`environment.yml`; the weekly run tests a fresh resolution and warns when the lock is behind it.
+
+## Tests
+- python: `docker build -f test/Dockerfile .` (linux/amd64, emulated on an arm64 host)
+- both Apps, on public images only - the R tests, the round trip R → python → R and a check that the pinned versions
+  agree: `docker buildx bake -f test/docker-bake.hcl test`
+- the golden CSV pairs (`test/contract/`) and the round-trip expectations (`test/roundtrip/*.csv`) were produced by the
+  v2.2.0 production images; `test/contract/generate.sh` and `test/roundtrip/measure.sh` reproduce them against any two
+  App images (registry access needed for the defaults).
+
 ## Null or error handling:
-Empty MovingPandas TrajectoryCollections will be transferred to a NULL object. An appropriate error will be shown in any following R App.
+Empty MovingPandas TrajectoryCollections are transferred to a NULL object. An appropriate error will be shown in any following R App.
 Null objects from the R world will cause an error in the R to Python App.
+A track with a single location cannot become a MovingPandas trajectory and is dropped by the R to Python App; the App log names every dropped track.
+Movebank data are UTC. Data in another timezone are transferred as UTC instants, the timezone travels along.