diff --git a/.dockerignore b/.dockerignore index 0b7048f..eda0ed7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,5 +14,7 @@ x-archive # dev-only bulk, copied by neither the App image nor the test image. # `python/sample/**` is deliberately NOT excluded - the tests build their fixtures from it. r/data/ +# ...except the samples the round trip reads in CI +!r/data/raw notebook/ python/notebooks/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 85c6d36..82bf1e4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,12 +11,12 @@ on: # so a fresh resolution can be checked after a suspected break, instead of waiting for Monday workflow_dispatch: -# `true` exactly for the runs that must resolve the conda environment from scratch. -# `test/Dockerfile` resolves the environment in its own layer - correctly, it is what keeps a -# source-only change cheap - but since v2.2.0 that environment is deliberately open: only `pandas` -# carries a floor and a ceiling. So a cached layer lets this suite report green against a -# resolution nobody has re-checked, and nothing evicts it on its own, because every push's -# `cache-from` resets GitHub's seven-day idle timer. +# `true` exactly for the runs that must resolve from scratch. The App and the python tests install +# `python/conda-linux-64.lock`, so an ordinary run is reproducible. But `environment.yml` is deliberately +# open since v2.2.0 - only `pandas` carries a floor and a ceiling - and the lock is refreshed from it +# before a release. These runs resolve it as it stands today and test that: the next lock. They skip +# every cache, which nothing would evict on its own, because every push's `cache-from` resets +# GitHub's seven-day idle timer. env: FRESH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} @@ -57,3 +57,66 @@ jobs: # after the tests have already passed. Import is unaffected - a PR may read the cache of # its own branch, the base branch and the default branch - so fork PRs stay warm. cache-to: type=gha,mode=max,ignore-error=true + # fresh runs test environment.yml resolved today, ordinary runs the lock the App installs + build-args: ENVIRONMENT=${{ env.FRESH == 'true' && 'fresh' || 'lock' }} + # the comparison below reads the image + load: ${{ env.FRESH == 'true' }} + tags: link-r-python-python-tests + + # A lock behind today's resolution is no failure - it is refreshed before a release - but it must + # not go unnoticed. + - name: Report whether the lock is behind + if: env.FRESH == 'true' + run: | + docker run --rm link-r-python-python-tests cat resolved.txt | sort > fresh.txt + grep '^https' python/conda-linux-64.lock | sort > lock.txt + if diff lock.txt fresh.txt > drift.txt; then + echo 'python/conda-linux-64.lock is current' >> "$GITHUB_STEP_SUMMARY" + else + echo '::warning::python/conda-linux-64.lock is behind environment.yml resolved today - run python/lock.sh' + { echo '### python/conda-linux-64.lock is behind'; echo '```diff'; cat drift.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + fi + + app: + name: App images + runs-on: ubuntu-latest + permissions: + contents: read + # about twice a cold run (8m36s measured, no cache): cuts off a hang, not a slow runner + timeout-minutes: 20 + env: + CACHE: gha + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v4 + + - name: Build both Apps and test them inside the images + uses: docker/bake-action@v7 + with: + source: . + files: test/docker-bake.hcl + targets: test + + # The platform passes no build arguments, so a Dockerfile copy without its DIRECTION edit must be + # refused. The builder still holds every layer from the step above, so this costs the last ones only. + # The grep makes sure the build failed at the check, not somewhere else: it matches the message + # the check prints, anchored at the line end, because the log also quotes the command that prints it. + - name: A build without a valid DIRECTION must fail + env: + CACHE: '' + run: | + for target in app-without-direction app-with-invalid-direction; do + if docker buildx bake -f test/docker-bake.hcl "$target" > "$target.log" 2>&1; then + echo "::error::$target was built" + exit 1 + fi + if ! grep -q -E 'DIRECTION must be r2python or python2r$' "$target.log"; then + cat "$target.log" + echo "::error::$target failed for another reason" + exit 1 + fi + done diff --git a/Dockerfile b/Dockerfile index 31ba624..e4ed521 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,46 +1,41 @@ -FROM registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-v3-r:sdk-v3.2.0_geospatial-4.3.2_3649 +# Builds either translator App - `DIRECTION` below says which. The platform passes no build +# arguments: BASE and CONDA fall back to their defaults, and DIRECTION must be set in the copy of this +# file that goes onto the App version (see README). +ARG BASE=registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-r:v4.0.0_sdk-v1.0.3_geospatial-4.6.1_4674 +# The python co-pilot, for its conda: FROM condaforge/miniforge3:26.7.2-0, /opt/conda untouched. +# Taken from the platform's registry, not Docker Hub: no App version has ever pulled from elsewhere, +# and the platform's builder holds no Docker Hub credentials. +ARG CONDA=registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-v1-python:v3.1.0 -# install miniconda aka conda-forge -ENV MINICONDA_VERSION=latest -ENV CONDA_DIR=$HOME/miniconda3 -RUN wget --quiet -O ~/Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \ - chmod +x ~/Miniforge3.sh && \ - bash ~/Miniforge3.sh -b -p $CONDA_DIR && \ - rm ~/Miniforge3.sh -# make non-activate conda commands available -ENV PATH=$CONDA_DIR/bin:$PATH -# make conda activate command available from /bin/bash --login shells -RUN echo ". $CONDA_DIR/etc/profile.d/conda.sh" >> ~/.profile -# make conda activate command available from /bin/bash --interative shells -RUN conda init bash +FROM ${CONDA} AS conda +FROM ${BASE} +COPY --from=conda /opt/conda /opt/conda +ENV PATH=/opt/conda/bin:$PATH +# conda caches in ~/.cache/conda, but the co-pilot created ~/.cache as root (for ~/.cache/R) +USER root +RUN install -d -o $UID -g $GID $HOME/.cache/conda USER $USER # the app ENV PROJECT_DIR=$HOME/co-pilot-r WORKDIR $PROJECT_DIR -# the python part +# the python part: the explicit lock python/lock.sh resolves from python/environment.yml, so the App gets +# exactly what CI tested WORKDIR $PROJECT_DIR/python -COPY --chown=$UID:$GID python/environment.yml /tmp/ -# build the conda environment +COPY --chown=$UID:$GID python/conda-linux-64.lock /tmp/ # keep in mind that anaconda aka channel `conda` is blocked in production env # https://gitlab.mpcdf.mpg.de/mpcdf-hpc-cloud/mvpr-moveapps/-/issues/29 -# miniconda uses channel `conda-forge` as default - so nothing to do here +# miniforge uses channel `conda-forge` as default - so nothing to do here ENV ENV_PREFIX=$PROJECT_DIR/python-env -RUN conda env create --prefix $ENV_PREFIX --file /tmp/environment.yml && \ +RUN conda create --yes --prefix $ENV_PREFIX --file /tmp/conda-linux-64.lock && \ conda clean --all --yes -# the r part +# the r part: the translator's own renv project in r/ WORKDIR $PROJECT_DIR/r -# move the co-pilot-r sdk into this sub-directory -USER root:root -RUN mv ../src . -# cleanup: remove unusuded files form co-pilot-r -RUN rm -rf ../renv ../renv.lock ../.Rprofile ../RFunction.R src/io/shiny_bookmark_handler.R -# patch the co-pilot-r file `src/io/rds.R` in order to not dependend on `move1` -COPY r/src/io/rds.R src/io/rds.R -USER $USER +# cleanup: the co-pilot's own renv project and entry point; the translator brings both +RUN rm -rf ../renv ../renv.lock ../.Rprofile ../RFunction.R ../app.R # renv: restore the current snapshot COPY --chown=$UID:$GID r/renv.lock r/.Rprofile ./ COPY --chown=$UID:$GID r/renv/activate.R r/renv/settings.dcf ./renv/ @@ -48,16 +43,19 @@ RUN R -e 'renv::restore()' # be prepared for the hangar inspection RUN cp renv.lock ../ -# the project +# the project: both directions, so both Apps share every layer up to here WORKDIR $PROJECT_DIR -# r -> python -COPY --chown=$UID:$GID python/csv_2_pickle.py python/transform_to_pickle.py ./python/ -COPY --chown=$UID:$GID r/rds_2_csv.R ./r/ -# python -> r -COPY --chown=$UID:$GID python/pickle_2_csv.py python/transform_to_csv.py ./python/ -COPY --chown=$UID:$GID r/csv_2_rds.R ./r/ +COPY --chown=$UID:$GID python/csv_2_pickle.py python/transform_to_pickle.py python/pickle_2_csv.py python/transform_to_csv.py ./python/ +COPY --chown=$UID:$GID r/link.R r/rds_2_csv.R r/csv_2_rds.R ./r/ +COPY --chown=$UID:$GID r2python.sh python2r.sh ./ -# r -> python -#COPY --chown=$UID:$GID r2python.sh start-process.sh -# python -> r -#COPY --chown=$UID:$GID python2r.sh start-process.sh +# The one line in which the two Apps differ: r2python (move2_loc to MovingPandas) or python2r +# (MovingPandas to move2_loc). No default on purpose - a forgotten edit must fail the build, not +# silently build the other App. Declared last: a changed ARG invalidates every RUN after it. +# The start script is picked here, not by `COPY ${DIRECTION}.sh`: BuildKit resolves a COPY source +# before this check has run, and fails on "/.sh: not found" instead of the message. +ARG DIRECTION +RUN case "$DIRECTION" in \ + r2python|python2r) mv "$DIRECTION.sh" start-process.sh && rm -f r2python.sh python2r.sh ;; \ + *) echo "DIRECTION must be r2python or python2r" >&2; exit 1 ;; \ + esac diff --git a/README.md b/README.md index 8b74a23..b2e0ba5 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,35 @@ Both Apps create out of the input data one .csv file `meta.csv`. It contains tra **Python to R**: This App reads as input a MovingPandas TrajectoryCollection in Python, there transfers the data to csv files (see above), reads the information from those files into R and creates a move2_loc as output. This output can then be used as input in R Apps (of that IO type). +## Build +One `Dockerfile` builds both Apps. The direction is the build argument `DIRECTION`, deliberately without a default: + + docker build --platform linux/amd64 --build-arg DIRECTION=r2python . # move2_loc to MovingPandas + docker build --platform linux/amd64 --build-arg DIRECTION=python2r . # MovingPandas to move2_loc + +The platform passes no build arguments. The Dockerfile of an App version is therefore this `Dockerfile` with exactly one +line changed: `ARG DIRECTION` becomes `ARG DIRECTION=r2python` or `ARG DIRECTION=python2r`. A copy without that edit +fails the build. + +## Python environment +`python/environment.yml` says what the python half needs; `python/conda-linux-64.lock` is what the App and the python +tests install. Refresh the lock before every release and after any change to `environment.yml`, then commit it: + + python/lock.sh + +It resolves for linux-64 with the Miniforge of `test/Dockerfile`. CI fails while the lock was not made from the current +`environment.yml`; the weekly run tests a fresh resolution and warns when the lock is behind it. + +## Tests +- python: `docker build -f test/Dockerfile .` (linux/amd64, emulated on an arm64 host) +- both Apps, on public images only - the R tests, the round trip R → python → R and a check that the pinned versions + agree: `docker buildx bake -f test/docker-bake.hcl test` +- the golden CSV pairs (`test/contract/`) and the round-trip expectations (`test/roundtrip/*.csv`) were produced by the + v2.2.0 production images; `test/contract/generate.sh` and `test/roundtrip/measure.sh` reproduce them against any two + App images (registry access needed for the defaults). + ## Null or error handling: -Empty MovingPandas TrajectoryCollections will be transferred to a NULL object. An appropriate error will be shown in any following R App. +Empty MovingPandas TrajectoryCollections are transferred to a NULL object. An appropriate error will be shown in any following R App. Null objects from the R world will cause an error in the R to Python App. +A track with a single location cannot become a MovingPandas trajectory and is dropped by the R to Python App; the App log names every dropped track. +Movebank data are UTC. Data in another timezone are transferred as UTC instants, the timezone travels along. diff --git a/docs/superpowers/plans/2026-09-30-port-to-moveapps-package.md b/docs/superpowers/plans/2026-09-30-port-to-moveapps-package.md new file mode 100644 index 0000000..722d1d2 --- /dev/null +++ b/docs/superpowers/plans/2026-09-30-port-to-moveapps-package.md @@ -0,0 +1,2383 @@ +# Port to the `moveapps` R package — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Both translator Apps build `FROM` the current R co-pilot and run on the `moveapps` package with output equal to `v2.2.0`, one `Dockerfile` builds both, and R side, python side and the CSV contract between them are tested in CI inside the real App images. + +**Architecture:** The R logic moves from two top-level scripts into pure functions in `r/link.R`; the entry scripts shrink to SDK glue. Golden CSV pairs and round-trip expectations are produced **once by the v2.2.0 production images** and pinned in the repository; every later step proves itself by reproducing them (`git diff --exit-code`). CI builds the real `Dockerfile` on a public stand-in for the private co-pilot via `docker buildx bake`. + +**Tech Stack:** R 4.6.1, renv 1.2.4, move2, sf, testthat 3.3.2, `moveapps` 1.0.3; python (conda-forge, pandas <3, geopandas, movingpandas), unittest; Docker BuildKit / buildx bake; GitHub Actions. + +**Spec:** `docs/superpowers/specs/2026-09-30-port-to-moveapps-package-design.md` — read it before any task. **The plan may be wrong.** Where a step's code or expectation contradicts what you measure, stop and report the measurement; do not bend the test or the code to make the plan true. + +## Global Constraints + +- Repository: `/opt/couchbits/projects/max-planck-gesellschaft/moveapps/apps/link-r-python/.claude/worktrees/port-to-moveapps-package`, branch `feature/port-to-moveapps-package` (no upstream on purpose). Every command below runs from this directory unless it says otherwise. Commit only here — never in the main checkout `/opt/…/apps/link-r-python`. +- **Never** push, open a pull request, tag, or create App versions without an explicit go from Clemens in chat. That is Task 14, and it waits. +- Code, comments, commit messages: English. Chat: German. +- New tests mark their blocks `# arrange` / `# act` / `# assert`. Existing tests keep their words (`# prepare` / `# execute` / `# verify`), also where this plan edits them. +- R: camelCase names like the SDK (`linkMeta`, `readLink`); call functions with named arguments where the original code did or where you write new calls. +- Commit messages: subject ≤ 50 chars, imperative, capitalised, no period; blank line; body wrapped at 72, explains what and why; last line `Co-Authored-By: Claude Opus 5.5 `. No "Generated with" line anywhere. +- Images (exact): + - R co-pilot: `registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-r:v4.0.0_sdk-v1.0.3_geospatial-4.6.1_4674` + - python co-pilot (conda source): `registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-v1-python:v3.1.0` + - rocker for the stand-in: `rocker/geospatial@sha256:4cee92a576e26c31546860eb252290f8f259e8236d49600d6bee695dfb96cae5` (tag `4.6.1`) + - Miniforge for CI: `condaforge/miniforge3:26.7.2-0` + - v2.2.0 production, R → python: `gitlab-registry.mpcdf.mpg.de/moveapps/apps/production-a81e3046-bc48-4fcb-8d28-4291000c77f9-move2loc-to-movingpandas:7` + - v2.2.0 production, python → R: `gitlab-registry.mpcdf.mpg.de/moveapps/apps/production-ed490356-9d55-4cb3-90d1-b8a2fc18ce10-movingpandas-to-move2loc:6` +- Template test data: `movestore/Template_R_Function_App` at `e00af0d779f9756c40161bffd8d6f0a5ebc77ced`; local clone at `/opt/couchbits/projects/max-planck-gesellschaft/moveapps/apps/Template_R_Function_App`. +- Apple Silicon: all App images are `linux/amd64` and run emulated. Pass `--platform linux/amd64` where a command below does. **Never** set `DOCKER_DEFAULT_PLATFORM`. +- Docker Desktop cannot bind-mount the macOS temp directories (`/private/tmp/…`). The scripts below copy in and out with `docker create` / `docker cp` / `docker start --attach` instead of `-v`. +- A long build (> 2 min) runs in the background; confirm its end by its output (`naming to …`, `exporting`, or an `ERROR` line), never by a notification alone. +- Subagents: do the work yourself; do not dispatch further agents. + +## File Structure + +| Path | Responsibility | Task | +|---|---|---| +| `r/data/raw/*.rds`, `r/data/raw/README.txt` | sample inputs: Template set + the List file | 1 | +| `test/images.sh` | host helper: run a command in an App image, copy results out | 2 | +| `test/roundtrip/r2python.sh`, `test/roundtrip/python2r.sh` | run every case through one entry point, record exit codes | 2 | +| `test/roundtrip/synthetic.R` | derive the two non-UTC cases from input1 | 2 | +| `test/roundtrip/empty.sh` | write the python-born empty case | 2 | +| `test/roundtrip/stage-r2python.sh`, `test/roundtrip/stage-python2r.sh` | one round-trip half inside one image | 2 | +| `test/roundtrip/describe.R` | describe a round trip; as a script, pin it | 2 | +| `test/roundtrip/measure.sh` | host: measure two images, write the pins | 2 | +| `test/roundtrip/expected.csv`, `test/roundtrip/classes.csv` | the pins | 2, 7–11 | +| `test/contract/cut.R`, `test/contract/prepare.sh`, `test/contract/generate.sh` | cut the contract cases, produce the golden CSV pairs | 2 | +| `test/contract//{input.rds,meta.csv,link.csv,py/meta.csv,py/link.csv}` | the goldens, 6 cases | 2, 8 | +| `python/tests/test_contract.py` | python half of the contract | 3 | +| `python/tests/test_transform_to_{csv,pickle}.py` | python unit gaps, later the fixes | 3, 7, 8, 10 | +| `r/link.R` | the R logic: `linkMeta`, `linkTable`, `writeLink`, `readLink` | 4, 7–9, 11 | +| `r/rds_2_csv.R`, `r/csv_2_rds.R` | entry points: SDK glue only | 4, 5, 7 | +| `r/.renvignore` | keep `tests/` out of the App's lock | 4 | +| `r/tests/testthat/*.R` | R unit and contract tests | 4, 7–9, 11 | +| `r/renv.lock`, `r/renv/activate.R` | R 4.6.1 lock | 5 | +| `r/src/**` | deleted | 5 | +| `Dockerfile`, `r2python.sh`, `python2r.sh` | App build and entry, both directions | 5 | +| `test/Dockerfile` | python unit tests; Miniforge bump | 3, 5 | +| `test/co-pilot-r.Dockerfile` | public stand-in for the R co-pilot | 6 | +| `test/docker-bake.hcl` | build graph for CI and local runs | 6 | +| `test/app-test.Dockerfile` | R unit tests, round trip, coherence inside the App images | 6 | +| `test/roundtrip/run.sh`, `test/roundtrip/compare.R` | in-image round trip and its check | 6 | +| `test/coherence.R` | cross-file version pins agree | 6 | +| `.dockerignore`, `.github/workflows/test.yml` | build context, CI job `app` | 6 | +| `README.md` | build and test instructions, behaviour | 12 | + +--- + +### Task 1: Replace the sample inputs with the Template set + +**Files:** +- Delete: `r/data/raw/input{1,2,3,4}_move2loc_{LatLon,Mollweide}.rds` (8 files) +- Create: the 8 files of the same names from the Template commit +- Modify: `r/data/raw/README.txt` +- Keep: `r/data/raw/input_move2loc_List.rds` + +**Interfaces:** +- Produces: `r/data/raw/input1_move2loc_LatLon.rds` … `input4_move2loc_Mollweide.rds` (current Movebank cut), `r/data/raw/input_move2loc_List.rds` (unchanged). Later tasks read `r/data/raw/*move2loc*.rds`. + +- [ ] **Step 1: Replace the files** + +```bash +TEMPLATE=/opt/couchbits/projects/max-planck-gesellschaft/moveapps/apps/Template_R_Function_App +SHA=e00af0d779f9756c40161bffd8d6f0a5ebc77ced +for f in $(git -C "$TEMPLATE" ls-tree --name-only "$SHA" data/raw/ | grep move2loc); do + git -C "$TEMPLATE" show "$SHA:$f" > "r/data/raw/$(basename "$f")" +done +``` + +- [ ] **Step 2: Verify they are the Template's blobs, byte for byte** + +```bash +for f in $(git -C "$TEMPLATE" ls-tree --name-only "$SHA" data/raw/ | grep move2loc); do + test "$(git hash-object "r/data/raw/$(basename "$f")")" = "$(git -C "$TEMPLATE" rev-parse "$SHA:$f")" && echo "ok $f" || echo "MISMATCH $f" +done +``` +Expected: 8 lines `ok data/raw/…`, no `MISMATCH`. `git status --short r/data/raw` shows 8 `M` lines and nothing else. + +- [ ] **Step 3: Rewrite `r/data/raw/README.txt`** + +```text +Set of input data to test apps. + +Source: github.com/movestore/Template_R_Function_App, data/raw at commit +e00af0d779f9756c40161bffd8d6f0a5ebc77ced (2026-09-28), the move2_loc files only - the +telemetry.list files are no input type of the translators. +input_move2loc_List.rds is this repository's own: the only input with list columns in its +track data. + +*Content* +- input1: 1 goat, median fix rate = 30mins, tracking duration 7.5 month, gps, local movement +- input2: 3 storks, median fix rate = 1sec, tracking duration 2 weeks, gps, local movement +- input3: 1 stork, one track per deployment, median fix rate = 1h | 1day | 1 week, tracking duration 11.5 years, argos, includes migration +- input4: 3 geese, median fix rate = 1h | 4h, tracking duration 1.5 years, gps, includes migration +- input_move2loc_List: 5 locations in 4 tracks, three of them with a single fix + +*Projection* +- data are provided in "lat/long" (EPSG:4326) and projected to "Mollweide" (ESRI:54009) in order to test your app accordingly for not projected and projected data. + +*File names* +input1_move2loc_LatLon.rds +input1_move2loc_Mollweide.rds + +input2_move2loc_LatLon.rds +input2_move2loc_Mollweide.rds + +input3_move2loc_LatLon.rds +input3_move2loc_Mollweide.rds + +input4_move2loc_LatLon.rds +input4_move2loc_Mollweide.rds + +input_move2loc_List.rds +``` + +- [ ] **Step 4: Commit** + +```bash +git add r/data/raw/ +git commit -F - <<'EOF' +Use the current Template sample data + +The eight move2_loc inputs came from an older cut of the Template's +data: same observations, but more event columns and another track id +column than Movebank delivers today. Taken byte for byte from +Template_R_Function_App e00af0d. The List file stays, it is the only +input with list columns. + +Co-Authored-By: Claude Opus 5.5 +EOF +``` + +--- + +### Task 2: Pin v2.2.0 — goldens and round-trip expectations from the production images + +The production images are the reference. Everything here is tooling plus the files it writes; the tooling is reused unchanged by Tasks 4, 5 and 7–11 and by CI. + +**Files:** +- Create: `test/images.sh`, `test/roundtrip/{r2python.sh,python2r.sh,synthetic.R,empty.sh,stage-r2python.sh,stage-python2r.sh,describe.R,measure.sh}`, `test/contract/{cut.R,prepare.sh,generate.sh}` +- Create (generated): `test/roundtrip/expected.csv`, `test/roundtrip/classes.csv`, `test/contract//{input.rds,meta.csv,link.csv,py/meta.csv,py/link.csv}` for the six cases + +**Interfaces:** +- Produces: + - `runInImage ` (bash function in `test/images.sh`); env `OVERLAY=1` copies this checkout's `r/link.R r/rds_2_csv.R r/csv_2_rds.R` into the image first. + - `test/roundtrip/r2python.sh ` → `//{input.rds,output.pickle,meta.csv,link.csv,r2python.exit,r2python.log}` + - `test/roundtrip/python2r.sh ` → `/{output.rds,py/meta.csv,py/link.csv,python2r.exit,python2r.log}` for each case holding `output.pickle` + - `test/roundtrip/stage-r2python.sh `, `test/roundtrip/stage-python2r.sh ` (env `START`, default `./start-process.sh`) + - `test/roundtrip/describe.R`: `describeCase(dir)`, `describeAll(cases)`, `writeDescription(described, expectedFile, classesFile)`; as a script `Rscript describe.R ` + - `test/roundtrip/measure.sh`, `test/contract/generate.sh` — env `R2PY_IMAGE`, `PY2R_IMAGE` (default: v2.2.0 production), `OVERLAY`, `KEEP=` (measure only), `CUT=1` (generate only) + - Case names: `latlon-three-tracks`, `mollweide-midnight`, `subsecond`, `argos-sfc`, `list-columns`, `berlin`; round-trip cases additionally `input1_move2loc_LatLon_berlin`, `dst_ambiguous`, `empty_trajectory_collection`. + +- [ ] **Step 1: Pull the production images** + +```bash +docker pull --platform linux/amd64 gitlab-registry.mpcdf.mpg.de/moveapps/apps/production-a81e3046-bc48-4fcb-8d28-4291000c77f9-move2loc-to-movingpandas:7 +docker pull --platform linux/amd64 gitlab-registry.mpcdf.mpg.de/moveapps/apps/production-ed490356-9d55-4cb3-90d1-b8a2fc18ce10-movingpandas-to-move2loc:6 +``` +Expected: both end with the image name. A `denied` means registry login — stop and ask Clemens. + +- [ ] **Step 2: Write `test/images.sh`** + +```bash +# Runs a command inside an App image and copies its results out. No bind mounts: Docker Desktop on +# macOS cannot mount the per-user temp directories, and copying leaves the image's own files alone. +# +# runInImage +# -> /tmp/in read only: owned by root inside the container +# runs as the image's user in /home/moveapps/co-pilot-r and writes to /tmp/out +# /tmp/out -> +# +# OVERLAY=1 first replaces the image's r/link.R, r/rds_2_csv.R and r/csv_2_rds.R with this +# checkout's - how a change to the R side is proved against an older image. +runInImage() { + local image="$1" input="$2" command="$3" output="$4" + local container status + + container="$(docker create --platform linux/amd64 --entrypoint bash -w /home/moveapps/co-pilot-r "$image" -c "$command")" + docker cp "$input" "$container:/tmp/in" > /dev/null + + if [ "${OVERLAY:-0}" = 1 ]; then + for file in r/link.R r/rds_2_csv.R r/csv_2_rds.R; do + docker cp "$file" "$container:/home/moveapps/co-pilot-r/$file" > /dev/null + done + fi + + docker start --attach "$container" && status=0 || status=$? + + rm -rf "$output" + docker cp "$container:/tmp/out" "$output" > /dev/null || true + docker rm "$container" > /dev/null + return "$status" +} +``` + +- [ ] **Step 3: Write the per-entry-point runners** + +`test/roundtrip/r2python.sh`: +```bash +#!/bin/bash +# Runs every /.rds through the R -> python entry point, as the platform would: +# //{input.rds, meta.csv, link.csv, output.pickle, r2python.log, r2python.exit} +# Never fails itself: a failing App is a measurement, recorded in r2python.exit. +set -o nounset + +start="$1" inputs="$2" cases="$3" + +for input in "$inputs"/*.rds; do + case="$(basename "$input" .rds)" + dir="$cases/$case" + mkdir -p "$dir" + cp "$input" "$dir/input.rds" + + SOURCE_FILE="$dir/input.rds" OUTPUT_FILE="$dir/output.pickle" ERROR_FILE="$dir/error.log" \ + LINK_R_PYTHON_BUFFER="$dir/link.csv" LINK_R_PYTHON_META="$dir/meta.csv" \ + "$start" > "$dir/r2python.log" 2>&1 + echo $? > "$dir/r2python.exit" +done +``` + +`test/roundtrip/python2r.sh`: +```bash +#!/bin/bash +# Runs every //output.pickle through the python -> R entry point: +# //{py/meta.csv, py/link.csv, output.rds, python2r.log, python2r.exit} +# A case without a pickle (its R -> python step failed) is skipped. +set -o nounset + +start="$1" cases="$2" + +for dir in "$cases"/*/; do + dir="${dir%/}" + [ -f "$dir/output.pickle" ] || continue + mkdir -p "$dir/py" + + SOURCE_FILE="$dir/output.pickle" OUTPUT_FILE="$dir/output.rds" ERROR_FILE="$dir/py/error.log" \ + LINK_R_PYTHON_BUFFER="$dir/py/link.csv" LINK_R_PYTHON_META="$dir/py/meta.csv" \ + "$start" > "$dir/python2r.log" 2>&1 + echo $? > "$dir/python2r.exit" +done +``` + +- [ ] **Step 4: Write the case generators** + +`test/roundtrip/synthetic.R`: +```r +# Derives the non-UTC round-trip cases from input1. Movebank data are UTC; these are the edge cases. +# input1_move2loc_LatLon_berlin.rds - all of input1, displayed in Europe/Berlin +# dst_ambiguous.rds - two fixes at 00:30 and 01:30 UTC on 2021-10-31, both 02:30 local +# Rscript synthetic.R +suppressMessages(library("move2")) + +args <- commandArgs(trailingOnly = TRUE) +input1 <- readRDS(args[1]) + +berlin <- input1 +attr(berlin[[mt_time_column(berlin)]], "tzone") <- "Europe/Berlin" +saveRDS(berlin, file.path(args[2], "input1_move2loc_LatLon_berlin.rds")) + +ambiguous <- input1[1:2, ] +ambiguous[[mt_time_column(ambiguous)]] <- as.POSIXct(c("2021-10-31 00:30:00", "2021-10-31 01:30:00"), tz = "UTC") +attr(ambiguous[[mt_time_column(ambiguous)]], "tzone") <- "Europe/Berlin" +saveRDS(ambiguous, file.path(args[2], "dst_ambiguous.rds")) +``` + +`test/roundtrip/empty.sh`: +```bash +#!/bin/bash +# The python-born edge case: an empty TrajectoryCollection, as a python App may hand one on. +# /empty_trajectory_collection/output.pickle +set -o errexit -o nounset + +dir="$1/empty_trajectory_collection" +mkdir -p "$dir" + +conda run --no-capture-output --prefix "$ENV_PREFIX" python -c ' +import sys +import movingpandas as mpd +import pandas as pd +pd.to_pickle(mpd.TrajectoryCollection([]), sys.argv[1], compression="gzip") +' "$dir/output.pickle" +``` + +- [ ] **Step 5: Write the two stages** + +`test/roundtrip/stage-r2python.sh`: +```bash +#!/bin/bash +# One half of the round trip, inside an R -> python App image: the sample inputs of plus the +# synthetic cases through the entry point $START (default ./start-process.sh) -> /cases/ +set -o errexit -o nounset + +here="$(cd "$(dirname "$0")" && pwd)" +raw="$1" out="$2" + +mkdir -p "$out/inputs" +cp "$raw"/*move2loc*.rds "$out/inputs/" +(cd r && Rscript "$here/synthetic.R" "$out/inputs/input1_move2loc_LatLon.rds" "$out/inputs") + +"$here/r2python.sh" "${START:-./start-process.sh}" "$out/inputs" "$out/cases" +``` + +`test/roundtrip/stage-python2r.sh`: +```bash +#!/bin/bash +# The other half, inside a python -> R App image: every pickle of back through the entry +# point $START (default ./start-process.sh), plus the python-born empty case -> /cases/ +set -o errexit -o nounset + +here="$(cd "$(dirname "$0")" && pwd)" +cases="$1" out="$2" + +mkdir -p "$out" +cp -R "$cases" "$out/cases" +"$here/empty.sh" "$out/cases" + +"$here/python2r.sh" "${START:-./start-process.sh}" "$out/cases" +``` + +`chmod +x test/roundtrip/*.sh`. + +- [ ] **Step 6: Write `test/roundtrip/describe.R`** + +```r +# Describes what the round trip R -> python -> R did to every case directory in : +# /input.rds what went in (absent for a python-born case) +# /r2python.exit exit code of the R -> python App; /r2python.log its log +# /python2r.exit exit code of the python -> R App +# /output.rds what came back (absent if an App failed, empty for the NULL result) +# +# Sourced by compare.R. Run as a script it pins the description: +# Rscript describe.R + +suppressMessages({ + library("move2") + library("sf") +}) + +readExit <- function(file) { + if (file.exists(file)) as.integer(readLines(file, n = 1)) else NA_integer_ +} + +classOf <- function(x) paste(class(x), collapse = "/") + +# the input rows of the tracks the output kept, and the output, both ordered by track id and time; +# radix ordering is locale-free, so two images with different locales order alike +alignRows <- function(input, output) { + input <- input[as.character(mt_track_id(input)) %in% as.character(mt_track_id(output)), ] + list( + input = input[order(as.character(mt_track_id(input)), as.numeric(mt_time(input)), method = "radix"), ], + output = output[order(as.character(mt_track_id(output)), as.numeric(mt_time(output)), method = "radix"), ] + ) +} + +describeCase <- function(dir) { + case <- basename(dir) + log <- file.path(dir, "r2python.log") + summary <- data.frame( + case = case, + r2python_exit = readExit(file.path(dir, "r2python.exit")), + python2r_exit = readExit(file.path(dir, "python2r.exit")), + rows_out = NA_integer_, + tracks_out = NA_integer_, + max_time_deviation_s = NA_character_, + time_column_out = NA_character_, + tzone_out = NA_character_, + dropped_tracks_warned = if (file.exists(log)) sum(grepl("^\\[WARN\\] track ", readLines(log, warn = FALSE))) else NA_integer_ + ) + described <- list(summary = summary, classes = NULL, aligned = NULL) + + outputFile <- file.path(dir, "output.rds") + if (!file.exists(outputFile)) { + return(described) + } + if (file.size(outputFile) == 0) { + described$summary$rows_out <- 0L + return(described) + } + + output <- readRDS(outputFile) + described$summary$rows_out <- nrow(output) + described$summary$tracks_out <- mt_n_tracks(output) + described$summary$time_column_out <- mt_time_column(output) + described$summary$tzone_out <- attr(mt_time(output), "tzone") + + inputFile <- file.path(dir, "input.rds") + if (!file.exists(inputFile)) { + return(described) + } + input <- readRDS(inputFile) + aligned <- alignRows(input = input, output = output) + if (nrow(aligned$input) == nrow(aligned$output) && nrow(aligned$output) > 0) { + deviation <- abs(as.numeric(mt_time(aligned$input)) - as.numeric(mt_time(aligned$output))) + described$summary$max_time_deviation_s <- sprintf("%.3f", max(deviation)) + } + + # the input as it travels: track attributes become event attributes on the way out + events <- mt_as_event_attribute(input, names(mt_track_data(input))) + namesIn <- setdiff(names(events), attr(events, "sf_column")) + namesOut <- setdiff(names(output), attr(output, "sf_column")) + columns <- union(namesIn, namesOut) + described$classes <- data.frame( + case = case, + column = columns, + class_in = vapply(columns, function(column) if (column %in% namesIn) classOf(events[[column]]) else NA_character_, character(1)), + class_out = vapply(columns, function(column) if (column %in% namesOut) classOf(output[[column]]) else NA_character_, character(1)), + row.names = NULL + ) + + described$aligned <- aligned + described$input <- input + described$output <- output + described$namesIn <- namesIn + described$namesOut <- namesOut + described +} + +describeAll <- function(cases) { + lapply(sort(list.dirs(cases, recursive = FALSE), method = "radix"), describeCase) +} + +# one CSV line per case, and one per case and column +writeDescription <- function(described, expectedFile, classesFile) { + write.csv(do.call(rbind, lapply(described, `[[`, "summary")), expectedFile, row.names = FALSE, na = "") + write.csv(do.call(rbind, lapply(described, `[[`, "classes")), classesFile, row.names = FALSE, na = "") +} + +if (sys.nframe() == 0) { + args <- commandArgs(trailingOnly = TRUE) + writeDescription(described = describeAll(cases = args[1]), expectedFile = args[2], classesFile = args[3]) +} +``` + +- [ ] **Step 7: Write `test/roundtrip/measure.sh`** + +```bash +#!/bin/bash +# Measures what a round trip R -> python -> R does to every sample input, and pins it: +# test/roundtrip/expected.csv - per case: exit codes, rows and tracks out, time deviation, time column and tzone out +# test/roundtrip/classes.csv - per case and column: class in, class out +# +# [R2PY_IMAGE=…] [PY2R_IMAGE=…] [OVERLAY=1] [KEEP=] test/roundtrip/measure.sh +# +# Defaults to the v2.2.0 production images (registry access needed). KEEP copies the cases out, +# outputs included. OVERLAY: see test/images.sh. +set -o errexit -o nounset -o pipefail + +cd "$(dirname "$0")/../.." +source test/images.sh + +R2PY_IMAGE="${R2PY_IMAGE:-gitlab-registry.mpcdf.mpg.de/moveapps/apps/production-a81e3046-bc48-4fcb-8d28-4291000c77f9-move2loc-to-movingpandas:7}" +PY2R_IMAGE="${PY2R_IMAGE:-gitlab-registry.mpcdf.mpg.de/moveapps/apps/production-ed490356-9d55-4cb3-90d1-b8a2fc18ce10-movingpandas-to-move2loc:6}" + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# R -> python: sample inputs and synthetic cases +mkdir -p "$work/in/raw" +cp r/data/raw/*move2loc*.rds "$work/in/raw/" +cp -R test "$work/in/test" +runInImage "$R2PY_IMAGE" "$work/in" '/tmp/in/test/roundtrip/stage-r2python.sh /tmp/in/raw /tmp/out' "$work/stage1" + +# python -> R, then describe with that image's R +cp -R test "$work/stage1/test" +runInImage "$PY2R_IMAGE" "$work/stage1" \ + '/tmp/in/test/roundtrip/stage-python2r.sh /tmp/in/cases /tmp/out && cd r && Rscript /tmp/in/test/roundtrip/describe.R /tmp/out/cases /tmp/out/expected.csv /tmp/out/classes.csv' \ + "$work/stage2" + +cp "$work/stage2/expected.csv" "$work/stage2/classes.csv" test/roundtrip/ +if [ -n "${KEEP:-}" ]; then + rm -rf "$KEEP" + cp -R "$work/stage2/cases" "$KEEP" +fi +``` + +- [ ] **Step 8: Write the contract generator** + +`test/contract/cut.R`: +```r +# Cuts the contract cases out of the sample inputs: a few rows each, by row number of the full input. +# Rscript cut.R +suppressMessages(library("move2")) + +cases <- list( + "latlon-three-tracks" = list(file = "input4_move2loc_LatLon.rds", rows = c(1:4, 3011:3014, 4506:4509)), + "mollweide-midnight" = list(file = "input1_move2loc_Mollweide.rds", rows = 2728:2731), + "subsecond" = list(file = "input2_move2loc_LatLon.rds", rows = 3443:3446), + "argos-sfc" = list(file = "input3_move2loc_LatLon.rds", rows = c(1, 2, 25, 26, 63, 64, 365, 366, 938, 939)), + "list-columns" = list(file = "input_move2loc_List.rds", rows = 1:5), + "berlin" = list(file = "input1_move2loc_LatLon.rds", rows = 1:4, tzone = "Europe/Berlin") +) + +args <- commandArgs(trailingOnly = TRUE) +for (name in names(cases)) { + spec <- cases[[name]] + data <- readRDS(file.path(args[1], spec$file))[spec$rows, ] + if (!is.null(spec$tzone)) { + attr(data[[mt_time_column(data)]], "tzone") <- spec$tzone + } + dir.create(file.path(args[2], name), recursive = TRUE, showWarnings = FALSE) + saveRDS(data, file.path(args[2], name, "input.rds")) +} +``` + +`test/contract/prepare.sh`: +```bash +#!/bin/bash +# Inside the R -> python image: collects every //input.rds as /inputs/.rds, +# after cutting them from first when is 1. +set -o errexit -o nounset + +here="$(cd "$(dirname "$0")" && pwd)" +raw="$1" out="$2" cut="$3" + +mkdir -p "$out/contract" "$out/inputs" +cp -R "$here/." "$out/contract/" +if [ "$cut" = 1 ]; then + (cd r && Rscript "$here/cut.R" "$raw" "$out/contract") +fi + +for dir in "$out"/contract/*/; do + [ -f "$dir/input.rds" ] || continue + cp "$dir/input.rds" "$out/inputs/$(basename "$dir").rds" +done +``` + +`test/contract/generate.sh`: +```bash +#!/bin/bash +# Produces the golden CSV pairs of test/contract// from each /input.rds, through two Apps: +# input.rds --R->python--> meta.csv, link.csv, pickle --python->R--> py/meta.csv, py/link.csv +# +# [R2PY_IMAGE=…] [PY2R_IMAGE=…] [CUT=1] [OVERLAY=1] test/contract/generate.sh +# +# Defaults to the v2.2.0 production images (registry access needed). CUT=1 first cuts every +# input.rds out of r/data/raw (see cut.R). OVERLAY: see test/images.sh. +set -o errexit -o nounset -o pipefail + +cd "$(dirname "$0")/../.." +source test/images.sh + +R2PY_IMAGE="${R2PY_IMAGE:-gitlab-registry.mpcdf.mpg.de/moveapps/apps/production-a81e3046-bc48-4fcb-8d28-4291000c77f9-move2loc-to-movingpandas:7}" +PY2R_IMAGE="${PY2R_IMAGE:-gitlab-registry.mpcdf.mpg.de/moveapps/apps/production-ed490356-9d55-4cb3-90d1-b8a2fc18ce10-movingpandas-to-move2loc:6}" +CUT="${CUT:-0}" + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +mkdir -p "$work/in" +cp -R test "$work/in/test" +cp -R r/data/raw "$work/in/raw" +runInImage "$R2PY_IMAGE" "$work/in" \ + "/tmp/in/test/contract/prepare.sh /tmp/in/raw /tmp/out $CUT && /tmp/in/test/roundtrip/r2python.sh ./start-process.sh /tmp/out/inputs /tmp/out/cases" \ + "$work/stage1" + +cp -R test "$work/stage1/test" +runInImage "$PY2R_IMAGE" "$work/stage1" '/tmp/in/test/roundtrip/stage-python2r.sh /tmp/in/cases /tmp/out' "$work/stage2" + +for dir in "$work"/stage1/contract/*/; do + [ -f "$dir/input.rds" ] || continue + case="$(basename "$dir")" + for exit in "$work/stage1/cases/$case/r2python.exit" "$work/stage2/cases/$case/python2r.exit"; do + if [ "$(cat "$exit")" != 0 ]; then + echo "$case: $(basename "$exit" .exit) failed" >&2 + cat "${exit%.exit}.log" >&2 + exit 1 + fi + done + + target="test/contract/$case" + mkdir -p "$target/py" + if [ "$CUT" = 1 ]; then + cp "$dir/input.rds" "$target/input.rds" + fi + for file in meta.csv link.csv; do + cp "$work/stage1/cases/$case/$file" "$target/$file" + cp "$work/stage2/cases/$case/py/$file" "$target/py/$file" + done +done +``` + +`chmod +x test/contract/*.sh`. + +- [ ] **Step 9: Produce the goldens and the pins** + +```bash +CUT=1 test/contract/generate.sh +test/roundtrip/measure.sh +``` +Both take a few minutes each (emulated). Expected: exit 0; `ls test/contract` shows the six case directories, each with `input.rds meta.csv link.csv py/`. + +- [ ] **Step 10: Check the pins against the measurement taken while planning** + +```bash +cat test/roundtrip/expected.csv +grep -c '' test/contract/list-columns/py/link.csv +grep -o '"2021-07-01 0[0-9]:40:00.000"' test/contract/berlin/link.csv | head -1 +``` +Expected, from the spec's "round trip of v2.2.0, measured": +- every `r/data/raw` case: `r2python_exit` 0, `python2r_exit` 0, `time_column_out` `timestamp_utc`, `tzone_out` `UTC`, `dropped_tracks_warned` 0; +- `input2_move2loc_LatLon` and `input2_move2loc_Mollweide`: `max_time_deviation_s` `0.999`; every other case with an output `0.000`; +- `input_move2loc_List`: `rows_out` 2, `tracks_out` 1; +- `dst_ambiguous`: `r2python_exit` 1, no `python2r_exit`, no rows; +- `empty_trajectory_collection`: empty `r2python_exit`, `python2r_exit` 1; +- `list-columns/py/link.csv`: 3 lines (header + 2); +- berlin: `"2021-07-01 08:40:00.000"` (Berlin wall-clock time). + +Any other value: stop and report it — the pins must describe production, not this plan. + +- [ ] **Step 11: Prove the tooling deterministic** + +```bash +git add test/ +test/contract/generate.sh && test/roundtrip/measure.sh +git diff --exit-code test/ +``` +Expected: exit 0 — the second run writes the very bytes of the first. + +- [ ] **Step 12: Commit** + +```bash +git add test/ +git commit -F - <<'EOF' +Pin v2.2.0: golden CSV pairs and round-trip data + +The v2.2.0 production images, pulled from the platform registry, +produced every golden and every expectation here - so what later +steps must reproduce is literally what runs in production. The +scripts that did it are committed alongside and are reused to prove +the refactoring, the port and each fix. + +Co-Authored-By: Claude Opus 5.5 +EOF +``` + +--- + +### Task 3: Python — close the unit gaps, pin the contract + +**Files:** +- Create: `python/tests/test_contract.py` +- Modify: `python/tests/test_transform_to_csv.py`, `python/tests/test_transform_to_pickle.py`, `test/Dockerfile` + +**Interfaces:** +- Consumes: `test/contract//{meta.csv,link.csv,py/meta.csv,py/link.csv}` (Task 2). +- Produces: test module `python.tests.test_contract`, listed in `test/Dockerfile`. + +- [ ] **Step 1: Write `python/tests/test_contract.py`** + +```python +import os +import tempfile +import unittest + +from ..transform_to_csv import TransformToCsv +from ..transform_to_pickle import TransformToPickle + +CONTRACT_DIR = './test/contract' + + +class ContractTestCase(unittest.TestCase): + """ + The CSV pair between R and python, pinned by golden files the v2.2.0 production images produced: + R's link.csv and meta.csv go in, and what python hands back to R must equal py/link.csv and + py/meta.csv byte for byte. + """ + + def test_it_should_hand_back_the_python_csv_pair_of_every_case(self): + cases = sorted(case for case in os.listdir(CONTRACT_DIR) if os.path.isdir(os.path.join(CONTRACT_DIR, case))) + self.assertEqual(6, len(cases)) + + for case in cases: + with self.subTest(case=case), tempfile.TemporaryDirectory() as tmp: + # arrange + case_dir = os.path.join(CONTRACT_DIR, case) + pickle = os.path.join(tmp, 'output_file') + link = os.path.join(tmp, 'link.csv') + meta = os.path.join(tmp, 'meta.csv') + TransformToPickle().convert( + input_data_file_name=os.path.join(case_dir, 'link.csv'), + input_meta_file_name=os.path.join(case_dir, 'meta.csv'), + output_file_name=pickle + ) + + # act + TransformToCsv().convert(input_data_file_name=pickle, output_file_name=link, output_meta_file_name=meta) + + # assert + self.assertEqual(self.__read(os.path.join(case_dir, 'py', 'meta.csv')), self.__read(meta)) + self.assertEqual(self.__read(os.path.join(case_dir, 'py', 'link.csv')), self.__read(link)) + + @staticmethod + def __read(file_path) -> bytes: + with open(file_path, 'rb') as file: + return file.read() +``` + +- [ ] **Step 2: Register it in `test/Dockerfile`** + +Add after `COPY python/ ./python/`: +```dockerfile +# the golden CSV pairs the contract test reads, shared with the R tests +COPY test/contract/ ./test/contract/ +``` +and extend the test command's module list: +```dockerfile +RUN conda run --no-capture-output --prefix ${ENV_PREFIX} python -m unittest \ + python.tests.test_transform_to_pickle python.tests.test_transform_to_csv python.tests.test_contract +``` + +- [ ] **Step 3: Add the unit gaps to `python/tests/test_transform_to_pickle.py`** + +Change the import to `from ..transform_to_pickle import Meta, TransformToPickle` and add to `TransformToPickleTestCase`: +```python + def test_it_should_read_the_meta_csv(self): + # arrange + file_path = './python/sample/input4/meta.csv' + + # act + actual = self.sut.read_meta_csv(file_path=file_path) + + # assert + self.assertEqual( + Meta(projection='EPSG:4326', timezone='UTC', time_col_name='timestamp', track_id_col_name='individual_name_deployment_id'), + actual + ) + + def test_it_should_create_one_trajectory_per_track_in_the_crs_of_the_meta_csv(self): + # arrange + meta = self.sut.read_meta_csv(file_path='./python/sample/input4/meta.csv') + data = self.sut.read_data_csv(file_path='./python/sample/input4/link.csv', time_col_name=meta.time_col_name) + self.sut.adjust_timestamps(data=data, timezone=meta.timezone, time_col_name=meta.time_col_name) + + # act + actual = self.sut.create_moving_pandas(data=data, projection=meta.projection, track_id_col_name=meta.track_id_col_name) + + # assert: the sample carries three distinct tracks + self.assertEqual(3, len(actual.trajectories)) + self.assertEqual('EPSG:4326', actual.trajectories[0].df.crs.to_string()) +``` + +- [ ] **Step 4: Add the unit gaps to `python/tests/test_transform_to_csv.py`** + +```python + def test_it_should_write_crs_timezone_and_column_names_to_the_meta_csv(self): + # arrange + data = self.sut.read_data_pickle(file_path=self.compressed) + geopandas = self.sut.create_geopandas(data=data) + meta = os.path.join(self.tmp.name, 'meta.csv') + + # act + self.sut.write_meta_csv(geopanda=geopandas, movingpanda=data, file_path=meta) + + # assert: the time column is the index movingpandas built from `timestamp_utc` + with open(meta) as written: + actual = written.read().splitlines() + self.assertEqual( + ['crs,tzone,timeColName,trackIdColName', 'EPSG:4326,UTC,timestamp_utc,individual_name_deployment_id'], + actual + ) + + def test_it_should_write_the_time_index_and_the_coordinates_to_the_link_csv(self): + # arrange + data = self.sut.read_data_pickle(file_path=self.compressed) + geopandas = self.sut.create_geopandas(data=data) + link = os.path.join(self.tmp.name, 'link.csv') + + # act + self.sut.write_result(file_name=link, data=geopandas) + + # assert + with open(link) as written: + actual = written.readline().rstrip('\n').split(',') + self.assertEqual('timestamp_utc', actual[0]) + self.assertIn('coords_x', actual) + self.assertIn('coords_y', actual) +``` + +- [ ] **Step 5: Run the python suite** + +Run: `docker build -f test/Dockerfile -t link-r-python-python-tests . 2>&1 | tail -20` +Expected: `Ran 15 tests` and `OK` (10 existing + 2 + 2 + 1). + +If `test_contract` fails, **do not regenerate the goldens.** Diff the two files the assertion names (`cmp` then `diff`) and report the difference: the test environment resolves conda afresh, the goldens came from the production images, and a difference is a real contract drift that Clemens decides on. + +- [ ] **Step 6: Mutate once** + +Change one character in `test/contract/subsecond/py/link.csv`, rerun Step 5: `test_contract` must fail naming `subsecond`. Revert with `git checkout -- test/contract/subsecond/py/link.csv`. + +- [ ] **Step 7: Commit** + +```bash +git add python/tests/ test/Dockerfile +git commit -F - <<'EOF' +Test the python half of the CSV contract + +The goldens from v2.2.0 now bind python: R's CSV pair goes in, and +what python hands back must match production byte for byte. Adds the +first tests of the meta.csv and link.csv python writes, and of reading +meta.csv and building the collection. + +Co-Authored-By: Claude Opus 5.5 +EOF +``` + +--- + +### Task 4: R — extract `r/link.R`, test it, prove the refactoring on v2.2.0 + +**Files:** +- Create: `r/link.R`, `r/.renvignore`, `r/tests/testthat/helper-link.R`, `r/tests/testthat/test-link-meta.R`, `r/tests/testthat/test-link-table.R`, `r/tests/testthat/test-read-link.R`, `r/tests/testthat/test-contract.R` +- Modify: `r/rds_2_csv.R`, `r/csv_2_rds.R` (they still `source()` the old SDK — the port is Task 5) + +**Interfaces:** +- Consumes: goldens and pins (Task 2), `OVERLAY=1` of `test/images.sh`. +- Produces (R, in `r/link.R`): + - `linkMeta(data)` → data.frame(crs, tzone, timeColName, trackIdColName) + - `linkTable(data)` → data.frame, the rows of `link.csv` + - `writeLink(data, bufferFile, metaFile)` → writes `metaFile`, then `bufferFile` + - `readLink(bufferFile, metaFile)` → move2, or `NULL` for a buffer without rows + +- [ ] **Step 1: Write the tests first** + +`r/tests/testthat/helper-link.R`: +```r +# the code under test, and the contract cases shared with the python tests +source(file.path("..", "..", "link.R")) + +contractDir <- file.path("..", "..", "..", "test", "contract") +contractCases <- basename(list.dirs(contractDir, recursive = FALSE)) + +readCase <- function(case) readRDS(file.path(contractDir, case, "input.rds")) + +writeTempLines <- function(lines) { + file <- tempfile(fileext = ".csv") + writeLines(lines, file) + file +} + +metaFileFor <- function(tzone = "UTC") { + writeTempLines(c( + "crs,tzone,timeColName,trackIdColName", + sprintf("EPSG:4326,%s,timestamp_utc,track", tzone) + )) +} +``` + +`r/tests/testthat/test-link-meta.R`: +```r +test_that("linkMeta names crs, tzone, time and track column", { + # arrange + data <- readCase("latlon-three-tracks") + + # act + actual <- linkMeta(data = data) + + # assert + expect_equal(actual, data.frame(crs = "EPSG:4326", tzone = "UTC", timeColName = "timestamp", trackIdColName = "individual_local_identifier")) +}) + +test_that("linkMeta keeps the crs of a projected input", { + # arrange + data <- readCase("mollweide-midnight") + + # act + actual <- linkMeta(data = data) + + # assert + expect_equal(actual$crs, "ESRI:54009") +}) + +test_that("linkMeta keeps a timezone other than UTC", { + # arrange + data <- readCase("berlin") + + # act + actual <- linkMeta(data = data) + + # assert + expect_equal(actual$tzone, "Europe/Berlin") +}) +``` + +`r/tests/testthat/test-link-table.R`: +```r +test_that("linkTable carries one row per location with its coordinates, without geometry", { + # arrange + data <- readCase("latlon-three-tracks") + + # act + actual <- linkTable(data = data) + + # assert + expect_equal(nrow(actual), 12) + expect_equal(actual$coords_x, unname(sf::st_coordinates(data)[, 1])) + expect_equal(actual$coords_y, unname(sf::st_coordinates(data)[, 2])) + expect_false("geometry" %in% names(actual)) +}) + +test_that("linkTable repeats the track attributes on every row", { + # arrange + data <- readCase("latlon-three-tracks") + + # act + actual <- linkTable(data = data) + + # assert: data.frame() makes the names syntactic, as v2.2.0 did + expect_true(all(make.names(names(mt_track_data(data))) %in% names(actual))) +}) + +test_that("linkTable writes sfc attributes as WKT", { + # arrange + data <- readCase("argos-sfc") + + # act + actual <- linkTable(data = data) + + # assert + expect_type(actual$argos_location_1, "character") + expect_match(actual$argos_location_1, "^POINT \\(") +}) + +test_that("linkTable leaves no list column behind", { + # arrange + data <- readCase("list-columns") + + # act + actual <- linkTable(data = data) + + # assert + expect_false(any(vapply(actual, is.list, logical(1)))) +}) + +test_that("linkTable keeps the milliseconds of a midnight fix", { + # arrange + data <- readCase("mollweide-midnight") + + # act + actual <- linkTable(data = data) + + # assert + expect_true("2021-12-10 00:00:00.000" %in% actual$timestamp) +}) + +test_that("linkTable keeps milliseconds below the second", { + # arrange + data <- readCase("subsecond") + + # act + actual <- linkTable(data = data) + + # assert + expect_equal(actual$timestamp, c("2014-08-06 09:19:35.000", "2014-08-06 09:19:35.999", "2014-08-06 09:19:36.999", "2014-08-06 09:19:38.000")) +}) +``` + +`r/tests/testthat/test-read-link.R`: +```r +test_that("readLink returns NULL for a buffer without rows", { + # arrange + buffer <- writeTempLines("timestamp_utc,track,coords_x,coords_y") + + # act + actual <- readLink(bufferFile = buffer, metaFile = metaFileFor()) + + # assert + expect_null(actual) +}) + +test_that("readLink builds a move2 in the crs of meta.csv, ordered by track and time", { + # arrange + buffer <- writeTempLines(c( + "timestamp_utc,track,coords_x,coords_y", + "2021-07-01 06:46:00,b,1,2", + "2021-07-01 06:40:00,b,3,4", + "2021-07-01 06:40:00,a,5,6" + )) + + # act + actual <- readLink(bufferFile = buffer, metaFile = metaFileFor()) + + # assert + expect_true(mt_is_move2(actual)) + expect_equal(sf::st_crs(actual), sf::st_crs("EPSG:4326")) + expect_equal(as.character(mt_track_id(actual)), c("a", "b", "b")) + expect_equal(format(mt_time(actual), "%H:%M"), c("06:40", "06:40", "06:46")) +}) + +test_that("readLink reads the times in the tzone of meta.csv", { + # arrange + buffer <- writeTempLines(c( + "timestamp_utc,track,coords_x,coords_y", + "2021-07-01 06:40:00,a,1,2", + "2021-07-01 06:46:00,a,1,2" + )) + + # act + actual <- readLink(bufferFile = buffer, metaFile = metaFileFor(tzone = "Asia/Kolkata")) + + # assert: 06:40 in Kolkata (UTC+05:30) is 01:10 UTC + expect_equal(attr(mt_time(actual), "tzone"), "Asia/Kolkata") + expect_equal(format(mt_time(actual)[1], "%Y-%m-%d %H:%M:%S", tz = "UTC"), "2021-07-01 01:10:00") +}) + +test_that("readLink truncates sub-second times (v2.2.0, pinned)", { + # arrange + buffer <- writeTempLines(c( + "timestamp_utc,track,coords_x,coords_y", + "2021-07-01 06:40:00.123,a,1,2", + "2021-07-01 06:40:01.000,a,1,2" + )) + + # act + actual <- readLink(bufferFile = buffer, metaFile = metaFileFor()) + + # assert + expect_equal(as.numeric(mt_time(actual))[1] %% 1, 0) +}) + +test_that("readLink leaves python's True and False as text (v2.2.0, pinned)", { + # arrange + buffer <- writeTempLines(c( + "timestamp_utc,track,coords_x,coords_y,visible", + "2021-07-01 06:40:00,a,1,2,True", + "2021-07-01 06:46:00,a,1,2,False" + )) + + # act + actual <- readLink(bufferFile = buffer, metaFile = metaFileFor()) + + # assert + expect_equal(actual$visible, c("True", "False")) +}) +``` + +`r/tests/testthat/test-contract.R`: +```r +# the CSV pair between R and python, pinned by the goldens the v2.2.0 production images produced +bytesOf <- function(file) readBin(file, what = "raw", n = file.size(file)) + +for (case in contractCases) { + test_that(sprintf("writeLink reproduces the v2.2.0 CSV pair of '%s' byte for byte", case), { + # arrange + buffer <- tempfile(fileext = ".csv") + meta <- tempfile(fileext = ".csv") + + # act + writeLink(data = readCase(case), bufferFile = buffer, metaFile = meta) + + # assert + expect_identical(bytesOf(meta), bytesOf(file.path(contractDir, case, "meta.csv"))) + expect_identical(bytesOf(buffer), bytesOf(file.path(contractDir, case, "link.csv"))) + }) + + test_that(sprintf("readLink turns python's CSV pair of '%s' back into its locations", case), { + # arrange: python keeps only the tracks with at least two fixes + input <- readCase(case) + + # act + actual <- readLink(bufferFile = file.path(contractDir, case, "py", "link.csv"), metaFile = file.path(contractDir, case, "py", "meta.csv")) + + # assert: v2.2.0 truncates sub-second times + kept <- input[as.character(mt_track_id(input)) %in% as.character(mt_track_id(actual)), ] + expect_equal(nrow(actual), nrow(kept)) + expect_equal(sf::st_crs(actual), sf::st_crs(input)) + expect_equal(sort(as.numeric(mt_time(actual))), floor(sort(as.numeric(mt_time(kept))))) + }) +} +``` + +- [ ] **Step 2: Run them — they must fail** + +Dev loop, native and fast; `--no-init-file` keeps the old renv project out: +```bash +COPYFILE_DISABLE=1 tar --no-xattrs -cf - r/tests test/contract | docker run --rm -i rocker/geospatial:4.6.1 bash -c ' + mkdir /w && tar -C /w -xf - && cd /w/r && + R -q -s -e "install.packages(c(\"move2\", \"testthat\"), quiet = TRUE)" > /dev/null 2>&1 && + Rscript --no-init-file -e "testthat::test_dir(\"tests/testthat\")"' 2>&1 | tail -15 +``` +Expected: FAIL at `helper-link.R` — `cannot open file '../../link.R'`. + +This dev loop runs on R 4.6.1 with the newest CRAN packages, not on either lock. It is for fast feedback only; the binding runs are Step 6 below (v2.2.0) and Tasks 5 and 6 (the new lock). + +- [ ] **Step 3: Write `r/link.R`** — the v2.2.0 logic moved, not changed + +```r +# Converts between a move2 object and the CSV pair python reads and writes: +# +# move2 --writeLink()--> meta.csv + link.csv --(python)--> pickle +# pickle --(python)--> meta.csv + link.csv --readLink()--> move2 +# +# meta.csv carries what a plain CSV cannot: crs, tzone, and which columns hold time and track id. + +library("move2") +library("sf") +library("dplyr") +library("vctrs") +library("purrr") +library("rlang") + +# meta.csv: one row with what link.csv cannot say itself +linkMeta <- function(data) { + prj <- st_crs(data)[[1]] + tz <- attr(mt_time(data),'tzone') + data.frame(crs=c(prj), tzone=c(tz), timeColName=mt_time_column(data), trackIdColName=mt_track_id_column(data)) +} + +# link.csv: one row per location, the track attributes repeated on every row, geometry as coordinates +linkTable <- function(data) { + ## checking if there are columns in the track data that are a list. If yes, check if the content is the same, if yes remove list. If list columns are left over because content is different transform these into a character string (could be done as well as json, but think that average user will be more comfortable with text?. Easy to change in the future. See Issue #78 on move2) + if(any(sapply(mt_track_data(data), is_bare_list))){ + ## reduce all columns were entry is the same to one (so no list anymore) + data <- data |> mutate_track_data(across( + where( ~is_bare_list(.x) && all(purrr::map_lgl(.x, function(y) 1==length(unique(y)) ))), + ~do.call(vctrs::vec_c,purrr::map(.x, head,1)))) + if(any(sapply(mt_track_data(data), is_bare_list))){ + ## transform those that are still a list into a character string + data <- data |> mutate_track_data(across( + where( ~is_bare_list(.x) && any(purrr::map_lgl(.x, function(y) 1!=length(unique(y)) ))), + ~unlist(purrr::map(.x, paste, collapse=",")))) + } + } + data <- mt_as_event_attribute(data, names(mt_track_data(data))) + data <- dplyr::mutate(data, coords_x=sf::st_coordinates(data)[,1], + coords_y=sf::st_coordinates(data)[,2]) + data <- sf::st_drop_geometry(data) ## removes the sf geometry column from the table + sfc_cols <- names(data)[unlist(lapply(data, inherits, 'sfc'))] ## get the col names that are spacial + + for(x in sfc_cols){ ## converting the "point" columns into characters, ie into WKT (Well-known text) + data[[x]] <- st_as_text(data[[x]]) + } ## st_as_sfc() can be used to convert these columns back to spacial + + data.csv <- data.frame(data) + data.csv[,mt_time_column(data)] <- format(data.csv[,mt_time_column(data)],format="%Y-%m-%d %H:%M:%OS3") ## if time is 00:00:00 it gets rounded just to the date, and if miliseconds are .000 it gets rounded to seconds when saved as csv. This ensures this does not happen. All timestamps will always have miliseconds. + data.csv +} + +# meta.csv first, then link.csv - the order of v2.2.0 +writeLink <- function(data, bufferFile, metaFile) { + write.csv(linkMeta(data = data), metaFile, row.names=FALSE) + write.csv(linkTable(data = data), bufferFile, row.names=FALSE) +} + +# NULL for a buffer without rows +readLink <- function(bufferFile, metaFile) { + # always includes "coords_x", "coords_y" + datapy <- read.csv(bufferFile, header=TRUE) + # always includes crs, tzone, timeColName, trackIdColName + meta <- read.csv(metaFile, header=TRUE) + + if (dim(datapy)[1]==0) { + return(NULL) + } + + datapy[meta$timeColName] <- as.POSIXct(datapy %>% select(meta$timeColName) %>% sapply(as.character) %>% as.vector,format="%Y-%m-%d %H:%M:%S", tz=meta$tzone) + result <- mt_as_move2(datapy, + coords = c("coords_x", "coords_y"), + time_column= meta$timeColName, + track_id_column= meta$trackIdColName, + crs= meta$crs + ) + result |> dplyr::arrange(mt_track_id(result),mt_time(result)) +} +``` + +- [ ] **Step 4: Reduce the entry scripts to glue** + +`r/rds_2_csv.R`: +```r +source("src/common/logger.R") +source("src/common/runtime_configuration.R") +source("src/io/app_files.R") +source("src/io/io_handler.R") +source("src/io/rds.R") +source("link.R") + +tryCatch( + { + Sys.setenv(tz="UTC") + + writeLink(data = readInput(sourceFile()), bufferFile = Sys.getenv(x = "LINK_R_PYTHON_BUFFER"), metaFile = Sys.getenv(x = "LINK_R_PYTHON_META")) + }, + error = function(e) + { + # error handler picks up where error was generated + print(paste("ERROR: ", e)) + storeToFile(e, errorFile()) + stop(e) # re-throw the exception + } +) +``` + +`r/csv_2_rds.R`: +```r +source("src/common/logger.R") +source("src/common/runtime_configuration.R") +source("src/io/app_files.R") +source("src/io/io_handler.R") +source("src/io/rds.R") +source("link.R") + +tryCatch( + { + Sys.setenv(tz="UTC") + + result <- readLink(bufferFile = Sys.getenv(x = "LINK_R_PYTHON_BUFFER"), metaFile = Sys.getenv(x = "LINK_R_PYTHON_META")) + # v2.2.0 stores no output at all for a buffer without rows + if (!is.null(result)) storeResult(result = result, outputFile = outputFile()) + }, + error = function(e) + { + # error handler picks up where error was generated + print(paste("ERROR: ", e)) + storeToFile(e, errorFile()) + stop(e) # re-throw the exception + } +) +``` + +`r/.renvignore`: +```text +# test-only code: keeps testthat out of the App's lock, and so out of the hangar inspection +tests/ +``` + +- [ ] **Step 5: Run the tests (dev loop)** — add `r/link.R` to the tar list of Step 2: + +```bash +COPYFILE_DISABLE=1 tar --no-xattrs -cf - r/link.R r/tests test/contract | docker run --rm -i rocker/geospatial:4.6.1 bash -c ' + mkdir /w && tar -C /w -xf - && cd /w/r && + R -q -s -e "install.packages(c(\"move2\", \"testthat\"), quiet = TRUE)" > /dev/null 2>&1 && + Rscript --no-init-file -e "testthat::test_dir(\"tests/testthat\")"' 2>&1 | tail -15 +``` +Expected: `[ FAIL 0 | … | PASS n ]`. A failure in `test-contract.R` alone means the newest CRAN packages already differ from v2.2.0 — note which case and field, and continue: Step 6 is the binding proof for this task, Task 5 the one for the new stack. + +- [ ] **Step 6: Prove the refactoring changes nothing — on v2.2.0 itself** + +The overlay runs this checkout's `link.R`, `rds_2_csv.R` and `csv_2_rds.R` inside the production images; their old `src/` is still there. +```bash +OVERLAY=1 test/contract/generate.sh && OVERLAY=1 test/roundtrip/measure.sh && git diff --exit-code test/ +``` +Expected: exit 0, no diff. Any diff: the refactoring changed behaviour — fix `link.R`, not the goldens. + +- [ ] **Step 7: Mutate once** + +In `linkTable` change `"%Y-%m-%d %H:%M:%OS3"` to `"%Y-%m-%d %H:%M:%OS2"`, rerun Step 6: `git diff` must show `link.csv` changes in all six cases. Revert the edit, rerun, expect no diff again. + +- [ ] **Step 8: Commit** + +```bash +git add r/link.R r/.renvignore r/rds_2_csv.R r/csv_2_rds.R r/tests/ +git commit -F - <<'EOF' +Move the R logic into testable functions + +Both entry scripts were a single top-level tryCatch, so nothing on the +R side could be tested. link.R holds the conversion as functions, the +entry scripts keep only the SDK glue. Running this checkout's R files +inside the v2.2.0 production images reproduces every golden and every +round-trip expectation, so the move changes nothing. Includes the +first R tests: unit tests per function and the contract goldens. + +Co-Authored-By: Claude Opus 5.5 +EOF +``` + +--- + +### Task 5: The port — `moveapps`, R 4.6.1, conda from the python co-pilot, one Dockerfile + +**Files:** +- Modify: `r/rds_2_csv.R`, `r/csv_2_rds.R`, `r/renv.lock`, `r/renv/activate.R`, `Dockerfile`, `r2python.sh`, `python2r.sh`, `test/Dockerfile` +- Delete: `r/src/` (all six files) + +**Interfaces:** +- Consumes: `r/link.R` (Task 4), the pins (Task 2). +- Produces: `Dockerfile` with `ARG BASE`, `ARG CONDA`, `ARG DIRECTION`; images `link-r-python:r2python`, `link-r-python:python2r` built locally on the real co-pilots (used by Tasks 5 and 13). + +- [ ] **Step 1: Port the entry scripts to `moveapps`** + +`r/rds_2_csv.R`: +```r +library("moveapps") +moveapps::logger.init() +source("link.R") + +tryCatch( + { + Sys.setenv(tz="UTC") + + writeLink(data = moveapps::readInput(moveapps::sourceFile()), bufferFile = Sys.getenv(x = "LINK_R_PYTHON_BUFFER"), metaFile = Sys.getenv(x = "LINK_R_PYTHON_META")) + }, + error = function(e) + { + # error handler picks up where error was generated + print(paste("ERROR: ", e)) + moveapps::storeToFile(e, moveapps::errorFile()) + stop(e) # re-throw the exception + } +) +``` + +`r/csv_2_rds.R`: +```r +library("moveapps") +moveapps::logger.init() +source("link.R") + +tryCatch( + { + Sys.setenv(tz="UTC") + + result <- readLink(bufferFile = Sys.getenv(x = "LINK_R_PYTHON_BUFFER"), metaFile = Sys.getenv(x = "LINK_R_PYTHON_META")) + # v2.2.0 stores no output at all for a buffer without rows + if (!is.null(result)) moveapps::storeResult(result = result, outputFile = moveapps::outputFile()) + }, + error = function(e) + { + # error handler picks up where error was generated + print(paste("ERROR: ", e)) + moveapps::storeToFile(e, moveapps::errorFile()) + stop(e) # re-throw the exception + } +) +``` + +```bash +git rm -r -q r/src +``` + +- [ ] **Step 2: Regenerate the lock inside the R co-pilot** + +P3M binaries make the emulated install minutes instead of hours; the versions are CRAN's today, and the lock records CRAN as its repository. +```bash +CO_PILOT=registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-r:v4.0.0_sdk-v1.0.3_geospatial-4.6.1_4674 +container="$(docker create --platform linux/amd64 --entrypoint bash -w /home/moveapps \ + -e RENV_CONFIG_REPOS_OVERRIDE=https://p3m.dev/cran/__linux__/noble/latest "$CO_PILOT" -c ' + set -o errexit + cp -R /tmp/in /tmp/r && cd /tmp/r + cp /home/moveapps/co-pilot-r/renv/activate.R renv/activate.R + rm -f renv.lock + R -q -e "renv::install(c(\"move2\", \"sf\", \"dplyr\", \"vctrs\", \"purrr\", \"rlang\", \"movestore/moveapps-sdk-r-package@v1.0.3\"), prompt = FALSE); renv::snapshot(prompt = FALSE, repos = c(CRAN = \"https://cloud.r-project.org\"))"')" +docker cp r "$container:/tmp/in" +docker start --attach "$container" 2>&1 | tail -20 +docker cp "$container:/tmp/r/renv.lock" r/renv.lock +docker cp "$container:/tmp/r/renv/activate.R" r/renv/activate.R +docker rm "$container" > /dev/null +``` + +- [ ] **Step 3: Check the lock** + +```bash +jq -r '.R.Version, .Packages.renv.Version, .Packages.moveapps.Version, .Packages.moveapps.RemoteRef' r/renv.lock +jq -r '.Packages | keys[]' r/renv.lock | grep -x -E 'move2|sf|dplyr|vctrs|purrr|rlang|moveapps|testthat|httr' +grep -m1 'version <-' r/renv/activate.R +``` +Expected: `4.6.1`, `1.2.4`, `1.0.3`, `v1.0.3`; the second list shows exactly the seven packages `dplyr moveapps move2 purrr rlang sf vctrs` — **no** `testthat`, **no** `httr`; `version <- "1.2.4"`. + +- [ ] **Step 4: Rewrite `Dockerfile`** + +```dockerfile +# Builds either translator App - `DIRECTION` below says which. The platform passes no build +# arguments: BASE and CONDA fall back to their defaults, and DIRECTION must be set in the copy of this +# file that goes onto the App version (see README). +ARG BASE=registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-r:v4.0.0_sdk-v1.0.3_geospatial-4.6.1_4674 +# The python co-pilot, for its conda: FROM condaforge/miniforge3:26.7.2-0, /opt/conda untouched. +# Taken from the platform's registry, not Docker Hub: no App version has ever pulled from elsewhere, +# and the platform's builder holds no Docker Hub credentials. +ARG CONDA=registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-v1-python:v3.1.0 + +FROM ${CONDA} AS conda + +FROM ${BASE} +COPY --from=conda /opt/conda /opt/conda +ENV PATH=/opt/conda/bin:$PATH + +# the app +ENV PROJECT_DIR=$HOME/co-pilot-r +WORKDIR $PROJECT_DIR + +# the python part, built like every python App +WORKDIR $PROJECT_DIR/python +COPY --chown=$UID:$GID python/environment.yml /tmp/ +# keep in mind that anaconda aka channel `conda` is blocked in production env +# https://gitlab.mpcdf.mpg.de/mpcdf-hpc-cloud/mvpr-moveapps/-/issues/29 +# miniforge uses channel `conda-forge` as default - so nothing to do here +ENV ENV_PREFIX=$PROJECT_DIR/python-env +RUN conda env create --prefix $ENV_PREFIX --file /tmp/environment.yml && \ + conda clean --all --yes + +# the r part: the translator's own renv project in r/ +WORKDIR $PROJECT_DIR/r +# cleanup: the co-pilot's own renv project and entry point; the translator brings both +RUN rm -rf ../renv ../renv.lock ../.Rprofile ../RFunction.R ../app.R +# renv: restore the current snapshot +COPY --chown=$UID:$GID r/renv.lock r/.Rprofile ./ +COPY --chown=$UID:$GID r/renv/activate.R r/renv/settings.dcf ./renv/ +RUN R -e 'renv::restore()' +# be prepared for the hangar inspection +RUN cp renv.lock ../ + +# the project: both directions, so both Apps share every layer up to here +WORKDIR $PROJECT_DIR +COPY --chown=$UID:$GID python/csv_2_pickle.py python/transform_to_pickle.py python/pickle_2_csv.py python/transform_to_csv.py ./python/ +COPY --chown=$UID:$GID r/link.R r/rds_2_csv.R r/csv_2_rds.R ./r/ + +# The one line in which the two Apps differ: r2python (move2_loc to MovingPandas) or python2r +# (MovingPandas to move2_loc). No default on purpose - a forgotten edit must fail the build, not +# silently build the other App. Declared last: a changed ARG invalidates every RUN after it. +ARG DIRECTION +RUN case "$DIRECTION" in r2python|python2r) ;; *) echo "DIRECTION must be r2python or python2r" >&2; exit 1 ;; esac +COPY --chown=$UID:$GID ${DIRECTION}.sh start-process.sh +``` + +- [ ] **Step 5: Start python via `conda run`** + +`r2python.sh` — replace the shebang with `#!/bin/bash` and the python line with: +```bash +(cd python && conda run --no-capture-output --prefix "$ENV_PREFIX" python csv_2_pickle.py) +``` +`python2r.sh` — the same shebang, and: +```bash +(cd python && conda run --no-capture-output --prefix "$ENV_PREFIX" python pickle_2_csv.py) +``` +Leave the rest of both files unchanged. (`--no-capture-output`: without it conda swallows the output, a failure included. The login shell existed only for `conda activate`.) + +- [ ] **Step 6: Align `test/Dockerfile` with the python co-pilot's conda** + +`FROM condaforge/miniforge3:26.3.2-3` → `FROM condaforge/miniforge3:26.7.2-0`. Update its comment: the tag is the python co-pilot's own base, and `test/coherence.R` (Task 6) holds CI to it. + +- [ ] **Step 7: Prove `/opt/conda` of the python co-pilot is Miniforge's** + +```bash +for image in registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-v1-python:v3.1.0 condaforge/miniforge3:26.7.2-0; do + docker run --rm --platform linux/amd64 --entrypoint sh "$image" -c 'cd /opt/conda && find . -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum' +done +``` +Expected: two identical sums. Different: stop and report — the `CONDA` override in CI would then not mirror production. + +- [ ] **Step 8: Build both Apps on the real co-pilots** (background; the first build compiles every R package from source, emulated — expect well over 30 minutes) + +```bash +docker build --platform linux/amd64 --build-arg DIRECTION=r2python -t link-r-python:r2python . > /tmp/link-r2python-build.log 2>&1 +docker build --platform linux/amd64 --build-arg DIRECTION=python2r -t link-r-python:python2r . > /tmp/link-python2r-build.log 2>&1 +``` +Expected: both logs end with `naming to docker.io/library/link-r-python:…`; the second build shows every step up to `ARG DIRECTION` as `CACHED`. + +- [ ] **Step 9: A missing or wrong `DIRECTION` must fail** + +```bash +docker build --platform linux/amd64 . 2>&1 | grep -E 'DIRECTION must be|ERROR' | head -3 +docker build --platform linux/amd64 --build-arg DIRECTION=sideways . 2>&1 | grep -E 'DIRECTION must be|ERROR' | head -3 +``` +Expected: both print `DIRECTION must be r2python or python2r` and an `ERROR` line. + +- [ ] **Step 10: The port reproduces v2.2.0** + +```bash +R2PY_IMAGE=link-r-python:r2python PY2R_IMAGE=link-r-python:python2r test/contract/generate.sh +R2PY_IMAGE=link-r-python:r2python PY2R_IMAGE=link-r-python:python2r test/roundtrip/measure.sh +git diff --exit-code test/ +``` +Expected: exit 0. Any diff is a behaviour change by R 4.6.1 or the new packages: stop, report the diff, do not update the pins. + +- [ ] **Step 11: The R tests pass in the App image** + +```bash +container="$(docker create --platform linux/amd64 --entrypoint bash -w /home/moveapps/co-pilot-r/r link-r-python:r2python -c \ + 'R -q -e "renv::install(\"testthat@3.3.2\")" > /dev/null && Rscript -e "testthat::test_dir(\"tests/testthat\")"')" +docker cp r/tests "$container:/home/moveapps/co-pilot-r/r/tests" +docker cp test "$container:/home/moveapps/co-pilot-r/test" +docker start --attach "$container" 2>&1 | tail -8 +docker rm "$container" > /dev/null +``` +Expected: `[ FAIL 0 | … ]`. + +- [ ] **Step 12: Commit** + +```bash +git add -A r/ Dockerfile r2python.sh python2r.sh test/Dockerfile +git commit -F - <<'EOF' +Port the translators to the moveapps R package + +Since R co-pilot v4 the SDK is the moveapps package and the image has +no src/, so the old build failed at mv ../src. The entry scripts now +call the package; its readRdsInput needs no move1, so the patched copy +goes too. The lock moves to R 4.6.1. + +conda comes from the python co-pilot's image instead of a Miniforge +installer fetched from "latest", so the python half resolves like every +python App. One Dockerfile builds both Apps: DIRECTION has no default, +so a forgotten edit fails the build instead of building the other App. + +Built on the real co-pilots, both Apps reproduce every v2.2.0 golden +and round-trip expectation. + +Co-Authored-By: Claude Opus 5.5 +EOF +``` + +--- + +### Task 6: CI — the real Dockerfile on a public stand-in, tested inside the images + +**Files:** +- Create: `test/co-pilot-r.Dockerfile`, `test/docker-bake.hcl`, `test/app-test.Dockerfile`, `test/coherence.R`, `test/roundtrip/run.sh`, `test/roundtrip/compare.R` +- Modify: `.dockerignore`, `.github/workflows/test.yml` + +**Interfaces:** +- Consumes: `describe.R` functions (Task 2), `stage-*.sh` (Task 2), the Dockerfile ARGs (Task 5). +- Produces: bake targets `co-pilot-r`, `app-r2python`, `app-python2r`, `test`, `app-without-direction`, `app-with-invalid-direction`; bake variables `CACHE` (`gha` or empty), `FRESH` (`true`/`false`), `CONDA`. + +- [ ] **Step 1: Write `test/co-pilot-r.Dockerfile`** + +```dockerfile +######################################################################################################################## +# A public stand-in for the private R co-pilot, so CI can build the real App Dockerfile. +# Mirrors co-pilot-v1/co-pilot-r/r/Dockerfile of movestore-groundcontrol: same rocker digest, user, directories and +# renv setup. Left out: the Java part, and the co-pilot's own renv project - the translator brings `moveapps` through +# r/renv.lock. Keep the digest in step with the co-pilot; test/coherence.R can check the R version, not the digest, +# because the co-pilot's registry is private. +######################################################################################################################## +# rocker/geospatial:4.6.1 +FROM rocker/geospatial@sha256:4cee92a576e26c31546860eb252290f8f259e8236d49600d6bee695dfb96cae5 + +RUN apt-get update && apt-get install -y \ + libcurl4-openssl-dev \ +# packages that build a vendored C++ dependency need it, e.g. s2 (Abseil) + cmake \ + && apt-get clean + +ARG username=moveapps +ARG uid=1001 +ARG gid=staff +ENV USER=$username +ENV UID=$uid +ENV GID=$gid +ENV HOME=/home/$USER + +RUN adduser --disabled-password \ + --gecos "Non-root user" \ + --uid $UID \ + --ingroup $GID \ + --home $HOME \ + $USER +RUN install -d -o moveapps -g staff $HOME/co-pilot-r +RUN install -d -o moveapps -g staff $HOME/.cache/R +USER $USER +WORKDIR $HOME/co-pilot-r + +ENV RENV_PATHS_CACHE=$HOME/.cache/R/renv +# The one deviation from the co-pilot, which compiles from https://cloud.r-project.org: P3M binaries of the very +# versions the lock pins, so a cold CI run takes minutes. A version P3M lacks falls back to source (hence cmake). +ENV RENV_CONFIG_REPOS_OVERRIDE=https://p3m.dev/cran/__linux__/noble/latest +ENV RENV_CONFIG_SANDBOX_ENABLED=FALSE + +RUN R -e "if (!requireNamespace('renv', quietly = TRUE)) install.packages('renv')" +``` + +- [ ] **Step 2: Write `test/docker-bake.hcl`** + +```hcl +# Builds and tests both translator Apps from the real Dockerfile, on public images only: +# +# co-pilot-r (stand-in) --BASE--+--> app-r2python --+ +# miniforge -----------CONDA----+--> app-python2r --+--> test: R unit tests, round trip, coherence +# +# docker buildx bake -f test/docker-bake.hcl test + +# "gha" inside GitHub Actions; empty elsewhere, where no Actions cache exists +variable "CACHE" { + default = "" +} + +# "true" for the runs that must resolve everything from scratch (see .github/workflows/test.yml) +variable "FRESH" { + default = "false" +} + +# the python co-pilot's own base; test/coherence.R holds test/Dockerfile to the same tag +variable "CONDA" { + default = "condaforge/miniforge3:26.7.2-0" +} + +function "cacheFrom" { + params = [scope] + result = CACHE == "gha" && FRESH != "true" ? ["type=gha,scope=${scope}"] : [] +} + +function "cacheTo" { + params = [scope] + result = CACHE == "gha" ? ["type=gha,mode=max,scope=${scope},ignore-error=true"] : [] +} + +target "co-pilot-r" { + context = "." + dockerfile = "test/co-pilot-r.Dockerfile" + platforms = ["linux/amd64"] + no-cache = FRESH == "true" + cache-from = cacheFrom("co-pilot-r") + cache-to = cacheTo("co-pilot-r") +} + +target "app" { + name = "app-${direction}" + matrix = { + direction = ["r2python", "python2r"] + } + context = "." + dockerfile = "Dockerfile" + platforms = ["linux/amd64"] + tags = ["link-r-python:${direction}"] + contexts = { + co-pilot-r = "target:co-pilot-r" + } + args = { + BASE = "co-pilot-r" + CONDA = CONDA + DIRECTION = direction + } + no-cache = FRESH == "true" + cache-from = cacheFrom("app-${direction}") + cache-to = cacheTo("app-${direction}") +} + +target "test" { + context = "." + dockerfile = "test/app-test.Dockerfile" + platforms = ["linux/amd64"] + contexts = { + app-r2python = "target:app-r2python" + app-python2r = "target:app-python2r" + } + no-cache = FRESH == "true" + cache-from = cacheFrom("test") + cache-to = cacheTo("test") +} + +# Both must fail. The platform passes no build arguments, so a Dockerfile copy without its DIRECTION +# edit has to be refused rather than built. +target "app-without-direction" { + context = "." + dockerfile = "Dockerfile" + platforms = ["linux/amd64"] + contexts = { + co-pilot-r = "target:co-pilot-r" + } + args = { + BASE = "co-pilot-r" + CONDA = CONDA + } +} + +target "app-with-invalid-direction" { + inherits = ["app-without-direction"] + args = { + DIRECTION = "sideways" + } +} +``` + +- [ ] **Step 3: Write `test/roundtrip/compare.R`** + +```r +# Checks a round trip against its pins (see describe.R), then the invariants no pin may relax: the +# track of every row, the coordinates, the crs, and every input column present on the way back. +# Rscript compare.R +here <- dirname(sub("^--file=", "", grep("^--file=", commandArgs(trailingOnly = FALSE), value = TRUE))) +source(file.path(here, "describe.R")) + +args <- commandArgs(trailingOnly = TRUE) +described <- describeAll(cases = args[1]) +failures <- character() + +# the pins, line by line +actualExpected <- tempfile(fileext = ".csv") +actualClasses <- tempfile(fileext = ".csv") +writeDescription(described = described, expectedFile = actualExpected, classesFile = actualClasses) +for (files in list(c(args[2], actualExpected), c(args[3], actualClasses))) { + pinned <- readLines(files[1]) + measured <- readLines(files[2]) + failures <- c(failures, + paste0(basename(files[1]), ", pinned but not measured: ", setdiff(pinned, measured)), + paste0(basename(files[1]), ", measured but not pinned: ", setdiff(measured, pinned))) +} + +# the invariants +for (case in described) { + if (is.null(case$aligned)) next + name <- case$summary$case + input <- case$aligned$input + output <- case$aligned$output + + if (nrow(input) != nrow(output)) { + failures <- c(failures, sprintf("%s: %d input rows in the kept tracks, %d rows back", name, nrow(input), nrow(output))) + next + } + if (!identical(as.character(mt_track_id(input)), as.character(mt_track_id(output)))) { + failures <- c(failures, sprintf("%s: the track of a row changed", name)) + } + coordinatesIn <- st_coordinates(input) + coordinatesOut <- st_coordinates(output) + if (any(abs(coordinatesIn - coordinatesOut) > 1e-12 * pmax(1, abs(coordinatesIn)))) { + failures <- c(failures, sprintf("%s: coordinates changed", name)) + } + if (st_crs(case$input) != st_crs(case$output)) { + failures <- c(failures, sprintf("%s: crs changed", name)) + } + lost <- setdiff(case$namesIn, case$namesOut) + if (length(lost) > 0) { + failures <- c(failures, sprintf("%s: columns lost: %s", name, paste(lost, collapse = ", "))) + } +} + +if (length(failures) > 0) { + cat(failures, sep = "\n") + quit(status = 1) +} +cat(sprintf("round trip: %d cases as pinned\n", length(described))) +``` + +- [ ] **Step 4: Write `test/roundtrip/run.sh`** and `chmod +x` it + +```bash +#!/bin/bash +# Inside the test image, which holds both entry points: the round trip of every case, checked against +# expected.csv and classes.csv. +# r/data/raw/*.rds --start-process.sh--> pickle --start-process-python2r.sh--> rds +set -o errexit -o nounset + +here="$(cd "$(dirname "$0")" && pwd)" +work="$(mktemp -d)" + +"$here/stage-r2python.sh" r/data/raw "$work/stage1" +START=./start-process-python2r.sh "$here/stage-python2r.sh" "$work/stage1/cases" "$work/stage2" + +(cd r && Rscript "$here/compare.R" "$work/stage2/cases" "$here/expected.csv" "$here/classes.csv") +``` + +- [ ] **Step 5: Write `test/coherence.R`** + +```r +# The versions pinned in separate files must agree - nothing else compares them. +# Rscript coherence.R (inside the test image: R is the stand-in's) +root <- commandArgs(trailingOnly = TRUE)[1] + +field <- function(text, pattern) { + match <- regmatches(text, regexec(pattern, text))[[1]] + if (length(match) == 2) match[2] else NA_character_ +} + +base <- grep("^ARG BASE=", readLines(file.path(root, "Dockerfile")), value = TRUE) +lock <- jsonlite::fromJSON(file.path(root, "r", "renv.lock")) +bake <- paste(readLines(file.path(root, "test", "docker-bake.hcl")), collapse = "\n") +testDockerfile <- paste(readLines(file.path(root, "test", "Dockerfile")), collapse = "\n") + +check <- function(label, left, right) { + ok <- !is.na(left) && !is.na(right) && identical(left, right) + cat(sprintf("%s %s: %s / %s\n", if (ok) "ok " else "FAIL", label, left, right)) + ok +} + +results <- c( + check("R of this image / R in r/renv.lock", as.character(getRversion()), lock$R$Version), + check("R in r/renv.lock / geospatial-X of BASE", lock$R$Version, field(base, "_geospatial-([0-9.]+)_")), + check("moveapps in r/renv.lock / sdk-vX of BASE", lock$Packages$moveapps$Version, field(base, "_sdk-v([0-9.]+)_")), + check("Miniforge CI passes as CONDA / Miniforge of test/Dockerfile", + field(bake, 'default = "condaforge/miniforge3:([0-9A-Za-z._-]+)"'), + field(testDockerfile, "FROM condaforge/miniforge3:([0-9A-Za-z._-]+)")) +) +if (!all(results)) quit(status = 1) +``` + +- [ ] **Step 6: Write `test/app-test.Dockerfile`** + +```dockerfile +# Runs the coherence check, the R unit tests and the round trip inside the real App images. +FROM app-r2python + +# the python -> R entry point; everything else in the two images is identical +COPY --from=app-python2r --chown=$UID:$GID /home/moveapps/co-pilot-r/start-process.sh /home/moveapps/co-pilot-r/start-process-python2r.sh + +# testthat is a test dependency only: installed here, never part of the App's lock +WORKDIR /home/moveapps/co-pilot-r/r +RUN R -q -e 'renv::install("testthat@3.3.2")' + +WORKDIR /home/moveapps/co-pilot-r +COPY --chown=$UID:$GID Dockerfile ./Dockerfile +COPY --chown=$UID:$GID test/ ./test/ +COPY --chown=$UID:$GID r/tests/ ./r/tests/ +COPY --chown=$UID:$GID r/data/raw/ ./r/data/raw/ + +RUN cd r && Rscript ../test/coherence.R .. +RUN cd r && Rscript -e 'testthat::test_dir("tests/testthat")' +RUN test/roundtrip/run.sh +``` + +- [ ] **Step 7: Let the samples into the build context** — `.dockerignore`, directly after `r/data/`: + +```text +# ...except the samples the round trip reads in CI +!r/data/raw +``` + +- [ ] **Step 8: Run the whole suite locally** (background; cold: rocker pull + emulated build) + +Run: `docker buildx bake -f test/docker-bake.hcl test > /tmp/link-bake.log 2>&1` +Expected in the log: four `ok` lines from `coherence.R`, testthat `[ FAIL 0 | …`, `round trip: 12 cases as pinned` (9 sample inputs, 2 synthetic, 1 empty — the data lines of `expected.csv`), and the build finishing without `ERROR`. + +- [ ] **Step 9: The negative targets fail, for the right reason** + +```bash +for target in app-without-direction app-with-invalid-direction; do + if docker buildx bake -f test/docker-bake.hcl "$target" > "/tmp/$target.log" 2>&1; then echo "BUILT $target"; fi + grep -c 'DIRECTION must be r2python or python2r' "/tmp/$target.log" +done +``` +Expected: no `BUILT` line; two counts ≥ 1. + +- [ ] **Step 10: Mutate — every guard must bite** + +One at a time, each followed by Step 8, each reverted with `git checkout -- `: +1. `test/roundtrip/expected.csv`: change the `rows_out` of `input1_move2loc_LatLon` → the build fails with `pinned but not measured`. +2. `test/Dockerfile`: tag `26.7.2-0` → `26.7.2-1` → `FAIL Miniforge CI passes as CONDA …`. +3. `r/tests/testthat/test-link-meta.R`: expected crs `"EPSG:4326"` → `"EPSG:4327"` → testthat `FAIL 1`. + +- [ ] **Step 11: Add the CI job** — `.github/workflows/test.yml` + +Extend the comment above `env: FRESH` with one sentence: +```yaml +# The `app` job's images resolve the same open environment inside the App Dockerfile. +``` +Append the job (the existing `test` job keeps its id and name): +```yaml + app: + name: App images + runs-on: ubuntu-latest + permissions: + contents: read + # a first guess with headroom; set from the first measured cold run + timeout-minutes: 90 + env: + CACHE: gha + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v4 + + - name: Build both Apps and test them inside the images + uses: docker/bake-action@v7 + with: + source: . + files: test/docker-bake.hcl + targets: test + + # The platform passes no build arguments, so a Dockerfile copy without its DIRECTION edit must be + # refused. The builder still holds every layer from the step above, so this costs the last ones only; + # the grep makes sure the build failed at the check, not somewhere else. + - name: A build without a valid DIRECTION must fail + env: + CACHE: '' + run: | + for target in app-without-direction app-with-invalid-direction; do + if docker buildx bake -f test/docker-bake.hcl "$target" > "$target.log" 2>&1; then + echo "::error::$target was built" + exit 1 + fi + if ! grep -q 'DIRECTION must be r2python or python2r' "$target.log"; then + cat "$target.log" + echo "::error::$target failed for another reason" + exit 1 + fi + done +``` + +- [ ] **Step 12: Commit** + +```bash +git add test/ .dockerignore .github/workflows/test.yml +git commit -F - <<'EOF' +Build and test the App images in CI + +The App Dockerfile was built by nothing but the platform, which is how +the mv ../src break stayed invisible. CI now builds the real Dockerfile +for both directions, on a public stand-in for the private co-pilot, and +runs the R unit tests, the round trip and a coherence check of the +version pins inside the images. A build without a valid DIRECTION must +fail, at the check. + +Co-Authored-By: Claude Opus 5.5 +EOF +``` + +--- + +The five defect fixes follow, one task and one commit each. Each fix changes the pins it corrects — the diff of `expected.csv`, `classes.csv` or a golden is part of its evidence. Rebuild and re-measure after each fix with the local bake images (P3M; same pinned versions as production): +```bash +docker buildx bake -f test/docker-bake.hcl --load app-r2python app-python2r +R2PY_IMAGE=link-r-python:r2python PY2R_IMAGE=link-r-python:python2r test/roundtrip/measure.sh +``` + +### Task 7: Defect 1 — an empty collection yields the NULL output + +**Files:** +- Modify: `python/transform_to_csv.py`, `python/tests/test_transform_to_csv.py`, `r/link.R`, `r/csv_2_rds.R`, `r/tests/testthat/test-read-link.R`, `test/roundtrip/expected.csv` + +- [ ] **Step 1: Failing python test** — add to `TransformToCsvTestCase`: + +```python + def test_it_should_write_empty_files_for_an_empty_trajectory_collection(self): + # arrange + empty = os.path.join(self.tmp.name, 'empty_output_file') + pd.to_pickle(mpd.TrajectoryCollection([]), empty, compression='gzip') + link = os.path.join(self.tmp.name, 'link.csv') + meta = os.path.join(self.tmp.name, 'meta.csv') + + # act + self.sut.convert(input_data_file_name=empty, output_file_name=link, output_meta_file_name=meta) + + # assert: empty files tell R to hand on the NULL result + self.assertEqual(0, os.path.getsize(link)) + self.assertEqual(0, os.path.getsize(meta)) +``` +with `import movingpandas as mpd` and `import pandas as pd` at the top. Run `docker build -f test/Dockerfile .` → FAIL, `ValueError: No objects to concatenate`. + +- [ ] **Step 2: Failing R test** — add to `test-read-link.R`: + +```r +test_that("readLink returns NULL for an empty buffer file", { + # arrange: python writes one for an empty TrajectoryCollection + buffer <- tempfile(fileext = ".csv") + file.create(buffer) + meta <- tempfile(fileext = ".csv") + file.create(meta) + + # act + actual <- readLink(bufferFile = buffer, metaFile = meta) + + # assert + expect_null(actual) +}) +``` +Run the dev loop (Task 4, Step 5) → FAIL, `no lines available in input`. + +- [ ] **Step 3: Fix python** — in `TransformToCsv.convert`, directly after reading the pickle: + +```python + if len(data.trajectories) == 0: + # nothing to hand over, not even a crs: empty files tell R to store the NULL result + open(output_file_name, 'w').close() + open(output_meta_file_name, 'w').close() + return +``` + +- [ ] **Step 4: Fix R** — at the top of `readLink`: + +```r + # python writes an empty buffer for an empty TrajectoryCollection + if (file.size(bufferFile) == 0) { + return(NULL) + } +``` +and in `r/csv_2_rds.R` replace the two lines from `result <- readLink(` to the `if (!is.null(result))` line by: +```r + # NULL for an empty buffer: storeResult then writes the empty file the next App reads as NULL input + moveapps::storeResult(result = readLink(bufferFile = Sys.getenv(x = "LINK_R_PYTHON_BUFFER"), metaFile = Sys.getenv(x = "LINK_R_PYTHON_META")), outputFile = moveapps::outputFile()) +``` + +- [ ] **Step 5: Tests green, pins moved** — python suite, dev loop, then rebuild + re-measure (see above). Expected diff in `expected.csv`: only the `empty_trajectory_collection` line, `python2r_exit` `1` → `0` and `rows_out` empty → `0`. `docker buildx bake -f test/docker-bake.hcl test` green. + +- [ ] **Step 6: Commit** + +```bash +git add python/ r/ test/roundtrip/expected.csv +git commit -F - <<'EOF' +Hand on NULL for an empty TrajectoryCollection + +The README promises a NULL object, but the python half failed with +"No objects to concatenate", and the R half would not have stored an +output for zero rows either. Python now writes empty files, and R +stores the NULL result: the empty file the next R App rejects with +code 10. + +Co-Authored-By: Claude Opus 5.5 +EOF +``` + +--- + +### Task 8: Defect 2 — transmit UTC instants, not wall-clock times + +**Files:** +- Modify: `r/link.R`, `r/tests/testthat/test-link-table.R`, `python/transform_to_pickle.py`, `python/tests/test_transform_to_pickle.py`, `python/environment.yml` (one comment sentence), `test/contract/berlin/{link.csv,py/link.csv}`, `test/roundtrip/{expected.csv,classes.csv}` + +- [ ] **Step 1: Failing R test** — add to `test-link-table.R`: + +```r +test_that("linkTable writes UTC instants, whatever the tzone", { + # arrange + data <- readCase("berlin") + + # act + actual <- linkTable(data = data) + + # assert: 06:40 UTC is 08:40 in Berlin; link.csv carries the instant, meta.csv the zone + expect_equal(actual$timestamp[1], "2021-07-01 06:40:00.000") +}) +``` +Dev loop → FAIL: `"2021-07-01 08:40:00.000"`. + +- [ ] **Step 2: Rewrite the python timezone tests** — they pin wall-clock semantics, which is the defect. In `test_transform_to_pickle.py` (keep their `# prepare / # execute / # verify` words): +- `test_apply_timezone_plusOffset`: expected `timestamp_tz` `datetime.datetime(2013, 8, 8, 8, 47, 31, tzinfo=ZoneInfo('Europe/Berlin'))`, `timestamp_utc` `datetime.datetime(2013, 8, 8, 6, 47, 31)`; comment `# csv value: 2013-08-08 06:47:31, a UTC instant`. +- `test_apply_timezone_name_kolkata`: expected `timestamp_tz` `datetime.datetime(2013, 8, 8, 12, 17, 31, tzinfo=ZoneInfo('Asia/Kolkata'))`, `timestamp_utc` `datetime.datetime(2013, 8, 8, 6, 47, 31)`. +- `test_apply_timezone_name_utc`: unchanged. +- `test_ambiguous_dst_timestamp_should_raise_error` → rename `test_an_instant_in_the_repeated_dst_hour_should_convert`: + ```python + # verify: 01:16:03 UTC is 01:16:03 GMT, just after British Summer Time ended at 01:00 UTC + actual = self.sut.adjust_timestamps(data, timezone='Europe/London', time_col_name='timestamp') + self.assertEqual(datetime.datetime(2013, 10, 27, 1, 16, 3, tzinfo=datetime.timezone.utc), actual['timestamp_tz'][0].to_pydatetime()) + self.assertEqual(datetime.datetime(2013, 10, 27, 1, 16, 3), actual['timestamp_utc'][0].to_pydatetime()) + ``` +- `test_non_existent_timestamp_should_raise_error` → rename `test_an_instant_in_the_skipped_dst_hour_should_convert`: + ```python + # verify: 01:24:20 UTC is 02:24:20 BST, British Summer Time began at 01:00 UTC + actual = self.sut.adjust_timestamps(data, timezone='Europe/London', time_col_name='timestamp') + self.assertEqual(datetime.datetime(2014, 3, 30, 1, 24, 20, tzinfo=datetime.timezone.utc), actual['timestamp_tz'][0].to_pydatetime()) + self.assertEqual(datetime.datetime(2014, 3, 30, 1, 24, 20), actual['timestamp_utc'][0].to_pydatetime()) + ``` +- remove `import pytz`, now unused. + +Python suite → the four changed tests FAIL. + +- [ ] **Step 3: Fix R** — in `linkTable`, the format line gains `tz="UTC"` and a comment: + +```r + ## UTC instants: a wall-clock time repeats in the autumn DST hour and python could not tell the two apart; meta.csv keeps the tzone + data.csv[,mt_time_column(data)] <- format(data.csv[,mt_time_column(data)],format="%Y-%m-%d %H:%M:%OS3", tz="UTC") ## if time is 00:00:00 it gets rounded just to the date, and if miliseconds are .000 it gets rounded to seconds when saved as csv. This ensures this does not happen. All timestamps will always have miliseconds. +``` + +- [ ] **Step 4: Fix python** — in `adjust_timestamps`, the `timestamp_tz` line: + +```python + # link.csv carries UTC instants; `timezone` is how R displayed them + # kudos: https://stackoverflow.com/a/18912631/810944 + data['timestamp_tz'] = data[time_col_name].apply(lambda x: x.tz_localize('UTC').tz_convert(timezone)) +``` +In `python/environment.yml` delete the sentence `It also drops the pytz dependency that` … `borrows transitively.` — the tests no longer borrow it. + +- [ ] **Step 5: Regenerate the goldens with the fixed images, check the diff** + +```bash +docker buildx bake -f test/docker-bake.hcl --load app-r2python app-python2r +R2PY_IMAGE=link-r-python:r2python PY2R_IMAGE=link-r-python:python2r test/contract/generate.sh +R2PY_IMAGE=link-r-python:r2python PY2R_IMAGE=link-r-python:python2r test/roundtrip/measure.sh +git diff --stat test/ +``` +Expected: `test/contract/berlin/link.csv` (time column 08:40 → 06:40 etc.), `test/contract/berlin/py/link.csv` (the `timestamp` column, now UTC); `expected.csv`: only the `dst_ambiguous` line (`r2python_exit` 0, `python2r_exit` 0, `rows_out` 2, `tracks_out` 1, `0.000`, `timestamp_utc`, `UTC`); `classes.csv`: new `dst_ambiguous` lines only. No other golden changes — UTC data are byte-identical. + +- [ ] **Step 6: All suites green** — python suite, dev loop, `docker buildx bake -f test/docker-bake.hcl test`. + +- [ ] **Step 7: Commit** + +```bash +git add r/ python/ test/ +git commit -F - <<'EOF' +Hand timestamps to python as UTC instants + +R wrote wall-clock times and python re-localised them, so two fixes an +hour apart in the repeated autumn hour became the same local time and +the App failed with AmbiguousTimeError. link.csv now carries UTC; +meta.csv still names the zone and python converts. UTC data - all real +Movebank data - stay byte-identical. + +Co-Authored-By: Claude Opus 5.5 +EOF +``` + +--- + +### Task 9: Defect 3 — keep sub-second times on the way back + +**Files:** +- Modify: `r/link.R`, `r/tests/testthat/test-read-link.R`, `r/tests/testthat/test-contract.R`, `test/roundtrip/expected.csv` + +- [ ] **Step 1: Turn the pinned tests into the corrected behaviour** + +In `test-read-link.R` replace `readLink truncates sub-second times (v2.2.0, pinned)` by: +```r +test_that("readLink keeps sub-second times", { + # arrange + buffer <- writeTempLines(c( + "timestamp_utc,track,coords_x,coords_y", + "2021-07-01 06:40:00.123,a,1,2", + "2021-07-01 06:40:01.000,a,1,2" + )) + + # act + actual <- readLink(bufferFile = buffer, metaFile = metaFileFor()) + + # assert + expect_equal(as.numeric(mt_time(actual))[1] %% 1, 0.123, tolerance = 1e-6) +}) +``` +In `test-contract.R` the last assertion becomes exact: +```r + # assert + kept <- input[as.character(mt_track_id(input)) %in% as.character(mt_track_id(actual)), ] + expect_equal(nrow(actual), nrow(kept)) + expect_equal(sf::st_crs(actual), sf::st_crs(input)) + expect_equal(sort(as.numeric(mt_time(actual))), sort(as.numeric(mt_time(kept))), tolerance = 1e-6) +``` +Dev loop → FAIL for the unit test and for the `subsecond` contract case. + +- [ ] **Step 2: Fix** — in `readLink`, `format="%Y-%m-%d %H:%M:%S"` → `format="%Y-%m-%d %H:%M:%OS"`, with the comment `# %OS: python writes fractions of a second, %S would drop them`. + +- [ ] **Step 3: Rebuild, re-measure** (see above). Expected diff: `expected.csv` lines `input2_move2loc_LatLon` and `input2_move2loc_Mollweide`, `max_time_deviation_s` `0.999` → `0.000`, nothing else. All suites green. + +- [ ] **Step 4: Commit** + +```bash +git add r/ test/roundtrip/expected.csv +git commit -F - <<'EOF' +Keep sub-second times from python + +Python writes fractions of a second, R parsed them with %S and dropped +them: 75 fixes of the stork sample came back up to 0.999 s early. + +Co-Authored-By: Claude Opus 5.5 +EOF +``` + +--- + +### Task 10: Defect 4 — warn about the tracks movingpandas drops + +**Files:** +- Modify: `python/transform_to_pickle.py`, `python/tests/test_transform_to_pickle.py`, `test/roundtrip/expected.csv` + +- [ ] **Step 1: Failing test** — add to `TransformToPickleTestCase` (imports `contextlib`, `io`, `pandas as pd`): + +```python + def test_it_should_warn_about_every_track_movingpandas_drops(self): + # arrange: track `a` has one fix, movingpandas needs two for a trajectory + data = pd.DataFrame({ + 'track': ['a', 'b', 'b'], + 'timestamp_utc': pd.to_datetime(['2021-07-01 06:40', '2021-07-01 06:40', '2021-07-01 06:46']), + 'coords_x': [1.0, 2.0, 2.1], + 'coords_y': [1.0, 2.0, 2.1], + }) + output = io.StringIO() + + # act + with contextlib.redirect_stdout(output): + self.sut.create_moving_pandas(data=data, projection='EPSG:4326', track_id_col_name='track') + + # assert + actual = [line for line in output.getvalue().splitlines() if line.startswith('[WARN]')] + self.assertEqual(['[WARN] track a dropped: a trajectory needs at least two fixes, it has 1'], actual) +``` +Python suite → FAIL, `[] != [...]`. + +- [ ] **Step 2: Fix** — in `TransformToPickle.create_moving_pandas`, after building `move`: + +```python + self.warn_about_dropped_tracks(data=data, move=move, track_id_col_name=track_id_col_name) +``` +and the new method: +```python + def warn_about_dropped_tracks(self, data, move, track_id_col_name): + # movingpandas needs two fixes for a trajectory and drops a track with fewer without a word; + # the loss cannot be avoided in this format, but it must not stay silent + kept = {trajectory.id for trajectory in move.trajectories} + rows_per_track = data[track_id_col_name].value_counts() + for track_id, rows in sorted(rows_per_track.items(), key=lambda item: str(item[0])): + if track_id not in kept: + print(f'[WARN] track {track_id} dropped: a trajectory needs at least two fixes, it has {rows}') +``` + +- [ ] **Step 3: Rebuild, re-measure.** Expected diff: `expected.csv` line `input_move2loc_List`, `dropped_tracks_warned` `0` → `3`, nothing else. All suites green. + +- [ ] **Step 4: Commit** + +```bash +git add python/ test/roundtrip/expected.csv +git commit -F - <<'EOF' +Warn about the tracks movingpandas drops + +A track with a single fix cannot become a trajectory, and movingpandas +drops it without a word: the List sample goes in with 4 tracks and +comes back with 1. The format cannot keep them, so the App log now +names every dropped track and its number of fixes. + +Co-Authored-By: Claude Opus 5.5 +EOF +``` + +--- + +### Task 11: Defect 5 — logical comes back as logical + +**Files:** +- Modify: `r/link.R`, `r/tests/testthat/test-read-link.R`, `test/roundtrip/classes.csv` + +- [ ] **Step 1: Turn the pinned test into the corrected behaviour** — replace `readLink leaves python's True and False as text (v2.2.0, pinned)` by: + +```r +test_that("readLink turns python's True, False and missing value back into logical", { + # arrange: pandas writes a missing value as an empty field + buffer <- writeTempLines(c( + "timestamp_utc,track,coords_x,coords_y,visible,comment", + "2021-07-01 06:40:00,a,1,2,True,True story", + "2021-07-01 06:46:00,a,1,2,False,", + "2021-07-01 06:52:00,a,1,2,,x" + )) + + # act + actual <- readLink(bufferFile = buffer, metaFile = metaFileFor()) + + # assert: a column with other text stays text + expect_equal(actual$visible, c(TRUE, FALSE, NA)) + expect_type(actual$comment, "character") +}) +``` +Dev loop → FAIL. + +- [ ] **Step 2: Fix** — in `r/link.R`, a helper above `readLink`: + +```r +# python writes logical values as True/False and a missing one as an empty field; read.csv keeps all +# three as text. A column holding nothing else goes back to TRUE/FALSE/NA; any other column is untouched. +logicalFromPython <- function(x) { + if (!is.character(x)) { + return(x) + } + values <- x[!is.na(x) & x != ""] + if (length(values) == 0 || !all(values %in% c("True", "False"))) { + return(x) + } + ifelse(x == "True", TRUE, ifelse(x == "False", FALSE, NA)) +} +``` +and in `readLink`, after the empty-rows check: +```r + datapy[] <- lapply(datapy, logicalFromPython) +``` + +- [ ] **Step 3: Rebuild, re-measure.** Expected diff: `classes.csv` lines whose `class_in` is `logical` and `class_out` was `character` now end in `logical`; `expected.csv` unchanged. Any other class change: stop and report. All suites green. + +- [ ] **Step 4: Commit** + +```bash +git add r/ test/roundtrip/classes.csv +git commit -F - <<'EOF' +Read python's True and False back as logical + +Python writes booleans as True/False, which read.csv leaves as text, so +every logical column came back as character. The other class changes +of the CSV bridge stay as they are and stay pinned. + +Co-Authored-By: Claude Opus 5.5 +EOF +``` + +--- + +### Task 12: README + +**Files:** Modify: `README.md` + +- [ ] **Step 1: Add, after `## Documentation`:** + +```markdown +## Build +One `Dockerfile` builds both Apps. The direction is the build argument `DIRECTION`, deliberately without a default: + + docker build --platform linux/amd64 --build-arg DIRECTION=r2python . # move2_loc to MovingPandas + docker build --platform linux/amd64 --build-arg DIRECTION=python2r . # MovingPandas to move2_loc + +The platform passes no build arguments. The Dockerfile of an App version is therefore this `Dockerfile` with exactly one +line changed: `ARG DIRECTION` becomes `ARG DIRECTION=r2python` or `ARG DIRECTION=python2r`. A copy without that edit +fails the build. + +## Tests +- python: `docker build -f test/Dockerfile .` +- both Apps, on public images only - the R tests, the round trip R → python → R and a check that the pinned versions + agree: `docker buildx bake -f test/docker-bake.hcl test` +- the golden CSV pairs (`test/contract/`) and the round-trip expectations (`test/roundtrip/*.csv`) were produced by the + v2.2.0 production images; `test/contract/generate.sh` and `test/roundtrip/measure.sh` reproduce them against any two + App images (registry access needed for the defaults). +``` + +- [ ] **Step 2: Replace `## Null or error handling:` by:** + +```markdown +## Null or error handling: +Empty MovingPandas TrajectoryCollections are transferred to a NULL object. An appropriate error will be shown in any following R App. +Null objects from the R world will cause an error in the R to Python App. +A track with a single location cannot become a MovingPandas trajectory and is dropped by the R to Python App; the App log names every dropped track. +Movebank data are UTC. Data in another timezone are transferred as UTC instants, the timezone travels along. +``` + +- [ ] **Step 3: Commit** + +```bash +git add README.md +git commit -F - <<'EOF' +Document the build argument and the tests + +Co-Authored-By: Claude Opus 5.5 +EOF +``` + +--- + +### Task 13: Local proofs for the pull request + +Nothing here is committed; the results go into the pull request description (Task 14). + +- [ ] **Step 1: Rebuild both Apps on the real co-pilots** (Task 5, Step 8, now with every fix; background). + +- [ ] **Step 2: Everything on the real co-pilots** + +```bash +R2PY_IMAGE=link-r-python:r2python PY2R_IMAGE=link-r-python:python2r KEEP=/tmp/link-new-cases test/roundtrip/measure.sh +R2PY_IMAGE=link-r-python:r2python PY2R_IMAGE=link-r-python:python2r test/contract/generate.sh +git diff --exit-code test/ +``` +Expected: exit 0 — the stand-in (CI) and the real co-pilots measure alike. + +- [ ] **Step 3: The new rds, read by v2.2.0** — the downstream R Apps that have not been rebuilt. Feed every new output through the v2.2.0 R → python App: + +```bash +source test/images.sh +mkdir -p /tmp/link-old-read/raw +for f in /tmp/link-new-cases/*/output.rds; do + [ -s "$f" ] && cp "$f" "/tmp/link-old-read/raw/$(basename "$(dirname "$f")")_move2loc.rds" +done +cp -R test /tmp/link-old-read/test +runInImage gitlab-registry.mpcdf.mpg.de/moveapps/apps/production-a81e3046-bc48-4fcb-8d28-4291000c77f9-move2loc-to-movingpandas:7 \ + /tmp/link-old-read '/tmp/in/test/roundtrip/r2python.sh ./start-process.sh /tmp/in/raw /tmp/out/cases' /tmp/link-old-read-out +grep -H . /tmp/link-old-read-out/cases/*/r2python.exit +``` +Expected: every case `0`. + +- [ ] **Step 4: DIRECTION on the real co-pilot** — Task 5, Step 9 again. Expected: both refused with the message. + +- [ ] **Step 5: Draft the pull request description** in the scratchpad (not in the repository), in English: what and why (spec link), the measured v2.2.0 defects and their fixes, the proofs of Steps 2–4 and of Tasks 4 and 5, the App-version instructions (the one `ARG DIRECTION` line), and the test counts before/after (python: `Ran N tests`; R: testthat `PASS n`; round trip: cases). No "Generated with" line: the team's guidelines forbid generation signatures in pull request descriptions. + +--- + +### Task 14: Push, pull request, first CI run — **only after Clemens says go** + +- [ ] **Step 1:** Show Clemens the branch log (`git log --oneline origin/main..HEAD`) and the drafted description; wait for an explicit go. +- [ ] **Step 2:** `git push -u origin feature/port-to-moveapps-package`; open the pull request against `main` with the approved description (`gh pr create --base main`). +- [ ] **Step 3:** Watch the first run of the `app` job. If it fails with `no space left on device`, add as its first step and push again: + ```yaml + - name: Free disk space for the App images + run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + ``` +- [ ] **Step 4:** Read the cold run's duration from the job summary; set `timeout-minutes` of `app` to about twice that, rounded up to 10 minutes; commit (`Set the App job timeout from its first run`), push. +- [ ] **Step 5:** Stop. Merge, tag `v2.3.0` and the two App versions in the admin UI happen after review, each on its own go. diff --git a/docs/superpowers/specs/2026-09-30-port-to-moveapps-package-design.md b/docs/superpowers/specs/2026-09-30-port-to-moveapps-package-design.md new file mode 100644 index 0000000..d3f644b --- /dev/null +++ b/docs/superpowers/specs/2026-09-30-port-to-moveapps-package-design.md @@ -0,0 +1,383 @@ +# Port the translator Apps to the `moveapps` R package, and put both halves under test + +Date: 2026-09-30 +Ticket: [Trello 1402](https://trello.com/c/9sVeyETS) — follow-up of the python co-pilot v3.1.0 +runbook ([Trello 1401](https://trello.com/c/Wm20TyGx)) +Touches: `Dockerfile`, `r2python.sh`, `python2r.sh`, `r/`, `python/tests/`, `test/`, +`.github/workflows/test.yml`, `README.md` + +## Problem + +The two translator Apps — `move2_loc to MovingPandas` (v7) and `MovingPandas to move2_loc` (v6), +both on tag `v2.2.0` — are pinned to the **old** R co-pilot, +`co-pilot-v3-r:sdk-v3.2.0_geospatial-4.3.2_3649` (R 4.3.2), and cannot follow the R upgrade: + +- `r/csv_2_rds.R` and `r/rds_2_csv.R` `source()` the old SDK's files + (`src/common/logger.R`, `src/common/runtime_configuration.R`, `src/io/app_files.R`, + `src/io/io_handler.R`, `src/io/rds.R`). +- The `Dockerfile` moves the co-pilot's `src/` into `r/` and overwrites `src/io/rds.R` with a copy + that drops the `move1` dependency. +- Since R co-pilot `v4.0.0` the SDK is the R package `moveapps` and there is no `src/`. With + `co-pilot-r:v4.0.0_sdk-v1.0.3_geospatial-4.6.1_4674` the build fails at `RUN mv ../src .`. + +Three further problems surfaced while scoping this: + +- **The R side has no tests at all**, and the CSV pair that carries the data between the two + languages (`meta.csv`, `link.csv`) is specified nowhere. The python side has ten tests, mostly + timezones and gzip. +- **The App `Dockerfile` is never built by anything but the platform.** The repository's copy has + both `start-process.sh` lines commented out, and each App version carries a hand-edited copy. The + break above would have been invisible until someone built an App version. +- **Its conda is unpinned and unlike every python App.** It installs Miniforge from + `releases/latest` at build time and activates via `conda init`, `.profile` and a login shell. The + python co-pilot takes conda from `condaforge/miniforge3:26.7.2-0` and starts with `conda run`. + `test/Dockerfile` uses a third version, `26.3.2-3`. + +## Context, measured + +**Production Dockerfiles** (prod snapshot, `app_versions.dockerfile`). `move2_loc to MovingPandas` +v7 equals v6, `MovingPandas to move2_loc` v6 equals v5. Against the repository `Dockerfile` they +differ only in legacy `ENV key value` syntax, one comment, and the direction-specific lines +(`r2python.sh` + `csv_2_pickle.py` + `transform_to_pickle.py` + `rds_2_csv.R`, or the mirror +image). Nothing lives in production that the repository lacks. + +**A current python App** (`Stationary Tags` v6, the generated Dockerfile shared by most v3.1.0 +rebuilds): + +```dockerfile +FROM registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-v1-python:v3.1.0 +COPY --chown=$UID:$GID environment.yml $PROJECT_DIR/env.yml +RUN conda env create --prefix $ENV_PREFIX --file $PROJECT_DIR/env.yml && \ + conda clean --all --yes +``` + +The co-pilot is `FROM condaforge/miniforge3:26.7.2-0` (Ubuntu 24.04, `/opt/conda` owned by root, +`PATH=/opt/conda/bin:…`, conda 26.7.2, user `moveapps`) and starts the App with +`conda run --no-capture-output --prefix $ENV_PREFIX python3 sdk.py`. + +**A current R App** installs its dependencies unpinned into the co-pilot's own renv project +(`RUN R -e 'remotes::install_version("move2")'` …) and finishes with `RUN R -e 'renv::snapshot()'`. +The translators deliberately do **not** follow this: platform R Apps float because renv is not +imposed on R developers. That reason does not apply here, and a pinned lock means the tests below +exercise exactly the versions that ship. + +**The R co-pilot** (`co-pilot-v1/co-pilot-r/r/Dockerfile` in groundcontrol) is +`rocker/geospatial@sha256:4cee92a576e26c31546860eb252290f8f259e8236d49600d6bee695dfb96cae5` +(4.6.1) plus JRE, `libcurl4-openssl-dev`, `cmake`, user `moveapps` (uid 1001, group `staff`), +`HOME=/home/moveapps`, workdir `/home/moveapps/co-pilot-r`, `RENV_PATHS_CACHE=$HOME/.cache/R/renv`, +`RENV_CONFIG_REPOS_OVERRIDE=https://cloud.r-project.org`, `RENV_CONFIG_SANDBOX_ENABLED=FALSE`, and a +renv project holding `moveapps` 1.0.3 (GitHub `movestore/moveapps-sdk-r-package`). The hangar +inspection reads `/home/moveapps/co-pilot-r/renv.lock`. The image lives in a **private** GitLab +registry: GitHub Actions cannot pull it. + +**`moveapps` 1.0.3** exports every function the translators use under the same name +(`readInput`, `storeResult`, `storeToFile`, `sourceFile`, `outputFile`, `errorFile`, `logger.*`). +Its `readRdsInput` is line for line the repository's patched `r/src/io/rds.R` — no `move1`. The +patch has no reason to exist any more. + +**The platform build** runs +`/kaniko/executor --cache=true --dockerfile=/build/Dockerfile --context=dir:///build --destination=…` +— no `--build-arg`. groundcontrol does not parse the `FROM` line of stored Dockerfiles. + +**Test data.** Profiled in R 4.6.1: + +| | `r/data/raw` today | `Template_R_Function_App@e00af0d` (2026-09-28) | +|---|---|---| +| observations | input1–4: 3243 / 98285 / 3169 / 7770 rows | identical | +| event columns | 23–37 | 11–21 — the current Movebank cut | +| track id | `individual_name_deployment_id` | mostly `individual_local_identifier` | +| column types | character, factor, ordered, integer, integer64, logical, numeric, POSIXct, Date, units, sfc | the same families | +| list columns | only in `input_move2loc_List.rds` | none | +| timezone | UTC throughout | UTC throughout | +| edge cases | a midnight fix (input1), 75 sub-second fixes (input2), extra sfc event columns (input3) | the same | + +Movebank's principle is "everything in UTC", so a non-UTC `tzone` is an edge case — but a +reachable one: nothing stops a user from converting. + +**The round trip of `v2.2.0`, measured** on the production images themselves +(`…-move2loc-to-movingpandas:7`, `…-movingpandas-to-move2loc:6`), every Template input plus the +List file, R → python → R through each image's own `start-process.sh`: + +- intact: row and track count (except below), every instant (except below), coordinates within + 5e-9, crs, track ids; +- the time column comes back as `timestamp_utc` in UTC, next to an added `timestamp_tz`; the + original `timestamp` survives as a character column — by design of the python side; +- attribute classes change in ways the CSV bridge implies: factor → character/integer, + integer64 → integer/numeric, units → integer/numeric, Date and POSIXct attributes → + character, sfc → WKT character; +- the defects listed under [Defects](#defects). + +## Goals + +1. Both translators build `FROM` the current R co-pilot and run on `moveapps`, R 4.6.1, with + output equivalent to `v2.2.0`. +2. One `Dockerfile` builds both Apps; the two App versions differ in exactly one line. +3. The Python side uses the same conda as the python Apps. +4. The R side, the python side and the CSV contract between them are tested; the real App images + are built and exercised in CI on every change. +5. Defects the new tests uncover are fixed in this change, one commit each. + +## Non-goals + +- The pickle format and the python dependency resolution (`python/environment.yml`). Both were + settled by the gzip rollout (`v2.2.0`). A conda lock would be the python counterpart of + `r/renv.lock`; it stays a follow-up question. +- The move1 translators (`legacy/move1`). +- groundcontrol. The only groundcontrol-side follow-up is documentation, see the end. + +## Design + +### R code + +- **New `r/link.R`** — pure functions, camelCase like the SDK: + - `linkMeta(data)` → the `meta.csv` data.frame (`crs`, `tzone`, `timeColName`, + `trackIdColName`). + - `linkTable(data)` → the `link.csv` data.frame: collapse list columns, track attributes to + event attributes, `coords_x`/`coords_y`, geometry dropped, sfc columns as WKT, the time + column formatted `%Y-%m-%d %H:%M:%OS3`. + - `writeLink(data, bufferFile, metaFile)` → writes `meta.csv`, then `link.csv`, in the order and + with the `write.csv` calls of `v2.2.0`. + - `readLink(bufferFile, metaFile)` → a `move2` object, or `NULL` for an empty buffer. +- **`r/rds_2_csv.R` and `r/csv_2_rds.R`** stay the entry points `start-process.sh` calls, reduced + to: `library("moveapps")`, `moveapps::logger.init()`, `source("link.R")`, and the `tryCatch` + around `moveapps::readInput` / `moveapps::storeResult` / `moveapps::storeToFile`. The logic moves, + it does not change (proved by the goldens, below). +- **`r/src/` is deleted.** +- **`r/renv.lock` and `r/renv/activate.R`** are regenerated inside the new co-pilot image: + R 4.6.1, renv 1.2.4, `moveapps` 1.0.3 from GitHub, plus `move2`, `sf`, `dplyr`, `vctrs`, + `purrr`, `rlang`. The translator keeps its own renv project in `r/`. +- **`r/.renvignore`** excludes `tests/`, so testthat never enters the App's lock — and therefore + never the hangar inspection. + +### App `Dockerfile` + +```dockerfile +ARG BASE=registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-r:v4.0.0_sdk-v1.0.3_geospatial-4.6.1_4674 +# the very conda the python co-pilot ships +ARG CONDA=registry.gitlab.com/couchbits/movestore/movestore-groundcontrol/co-pilot-v1-python:v3.1.0 +FROM ${CONDA} AS conda +FROM ${BASE} +COPY --from=conda /opt/conda /opt/conda +ENV PATH=/opt/conda/bin:$PATH + +# python: exactly as the python Apps +# conda env create --prefix $ENV_PREFIX --file … && conda clean --all --yes +# r: the translator's own renv project in r/ +# restore the lock; drop the co-pilot's unused project and entry point +# (../renv ../renv.lock ../.Rprofile ../RFunction.R ../app.R); cp renv.lock ../ for the hangar +# inspection +# the scripts of both directions, as today + +ARG DIRECTION +RUN case "$DIRECTION" in r2python|python2r) ;; *) echo "DIRECTION must be r2python or python2r" >&2; exit 1 ;; esac +COPY --chown=$UID:$GID ${DIRECTION}.sh start-process.sh +``` + +- **`BASE`** exists for CI only. Its default is the private co-pilot, so the platform, which + passes no build arguments, builds exactly what it builds today. +- **`CONDA`** likewise. Its default is the python co-pilot `v3.1.0`, which is + `FROM condaforge/miniforge3:26.7.2-0` and leaves `/opt/conda` untouched. conda is taken from + there rather than from Docker Hub because no App version on the platform has ever pulled from + outside the couchbits GitLab registry: kaniko holds credentials for gitlab.com and the MPCDF + GitLab only, and no mirror. CI, which cannot pull the python co-pilot, passes + `CONDA=condaforge/miniforge3:26.7.2-0`. +- **`DIRECTION` has no default, on purpose.** With one, forgetting to change it would silently + build the wrong App; without one, the build fails and says why. +- **`DIRECTION` is declared last, on purpose.** A changed ARG value is a cache miss for every + following `RUN`; declared at the top, the two directions would not share the conda and renv + layers. The price: a missing `DIRECTION` fails after the long steps rather than before. +- **Removed:** the Miniforge download from `releases/latest`, `conda init`, the `.profile` line, + `MINICONDA_VERSION`, `CONDA_DIR`, and the `mv ../src` / `rm` / `COPY r/src/io/rds.R` block + together with its `USER root:root` switch. +- **On the platform** the App version's Dockerfile is the repository's `Dockerfile` with one line + changed: `ARG DIRECTION=r2python` or `ARG DIRECTION=python2r`. + +### Start scripts + +`r2python.sh` and `python2r.sh` call python via +`conda run --no-capture-output --prefix "$ENV_PREFIX" python …` — the python co-pilot's own form; +without `--no-capture-output` conda swallows the output, including a failure. The shebang drops +`--login`, which existed only for `conda activate`. + +### Test data and fixtures + +- **`r/data/raw/`** is replaced by the eight `move2loc` files of + `movestore/Template_R_Function_App@e00af0d779f9756c40161bffd8d6f0a5ebc77ced`. + `input_move2loc_List.rds` stays — the only source of list columns. The `telemetrylist` files are + not a translator input type and are left out. The old files remain in history. +- **The non-UTC cases are derived at run time**, not stored as further binaries: input1 with + `tzone` set to `Europe/Berlin` (it has no fix in the repeated hour of 2021-10-31), and a + two-fix track at 00:30 and 01:30 UTC on 2021-10-31 in `Europe/Berlin` — both `02:30` local. +- **Golden files** in `test/contract//`: + - `input.rds` — a few rows cut from `r/data/raw` at fixed indices; + - `meta.csv`, `link.csv` — the R → python direction; + - `py/meta.csv`, `py/link.csv` — the python → R direction. + + Cases: LatLon with three tracks, Mollweide including the midnight fix, sub-second fixes, Argos + sfc event columns, list columns, non-UTC outside a DST switch. + + **The goldens are produced once, by `v2.2.0`** — by the production images themselves, pulled + from the platform registry, so the reference is literally what runs in production. That makes + them the proof that the port changes nothing, carried into CI permanently. From then on a change + to a golden is a deliberate contract change and shows as such in the diff. The generator is + committed alongside. +- **The round-trip expectations** are measured the same way, on all inputs, and pinned in two + files the round-trip test reads: `test/roundtrip/expected.csv` (per input: rows and tracks out, + largest time deviation, output time column and `tzone`) and `test/roundtrip/classes.csv` (per + input and column: class in, class out). A defect fix changes the lines it corrects, visibly. + +### Tests + +New tests mark their blocks `# arrange` / `# act` / `# assert`; existing tests keep their words. + +**R** — testthat, `r/tests/testthat/`: +- `linkMeta`, `linkTable` and `readLink` on their own: crs (`EPSG:4326`, `ESRI:54009`), tzone, + column names; list columns collapsed when equal per track and pasted when not; track attributes + present per event; sfc as WKT; milliseconds kept at midnight, at `.000` and below the second; + times parsed in `tzone`; rows ordered by track and time; an empty buffer. +- Contract: the new code on `input.rds` reproduces `meta.csv` and `link.csv` **byte for byte**; + `readLink` on `py/*.csv` yields the expected `move2` (rows, tracks, times, crs, columns). + +**Python** — `python/tests/`: +- New coverage: `write_meta_csv`, the columns of the written `link.csv`, `read_meta_csv`, + `create_moving_pandas` (crs, number of trajectories). +- `test_contract.py`: golden `*.csv` → pickle → `*.csv` equals `py/*.csv`. + +**Round trip** — CI, inside the real App images: for every `r/data/raw/*move2loc*.rds` plus the +non-UTC cases, the r2python image's `start-process.sh` writes a pickle, the python2r image's +`start-process.sh` turns it back into an rds. Asserted against `expected.csv`: rows and tracks out, +the largest time deviation, output time column and `tzone`; strictly: track per row, coordinates +within 1e-8, crs, every input column name present. Classes are asserted against `classes.csv`. +Plus one python-born case: an empty `TrajectoryCollection` through the python2r image. + +**Local only, once** — the old image is private, so these land in the pull request description: +- old against new on every input, outputs compared; +- the new rds read by the old image (R 4.3.2, old `move2`) — the downstream R Apps that have not + been rebuilt; +- both App images build; a build without, and one with an invalid, `DIRECTION` fails. + +### CI + +`.github/workflows/test.yml`: + +- **`test` / "Unit tests"** — the existing job, same id and name so a required status check keeps + matching. `test/Dockerfile` moves to `condaforge/miniforge3:26.7.2-0` and runs all python tests, + the contract tests included. +- **`app`** — new, via `docker buildx bake` (`test/docker-bake.hcl`): + 1. `co-pilot-r` — `test/co-pilot-r.Dockerfile`: a public stand-in for the private co-pilot. Same + rocker digest, `cmake`, user `moveapps`, `HOME`/`UID`/`GID`, `/home/moveapps/co-pilot-r`, the + renv environment variables, renv. No Java part, no co-pilot renv project — the translator + brings `moveapps` through its own lock. + 2. `app-r2python`, `app-python2r` — the **real** `Dockerfile`, `BASE=co-pilot-r` through a bake + context `target:co-pilot-r`, and the respective `DIRECTION`. + 3. `test` — `test/app-test.Dockerfile` on top of both App images: the R unit tests (testthat + installed at a pinned version, outside the lock), the round trip, the coherence check. A + failing test fails the build — the pattern the python job already uses. + 4. The build without and with an invalid `DIRECTION` **must** fail. With the cache warm this + costs the last layers only. +- **Coherence check** — the pins nobody else compares. All must agree: + - the stand-in's R version = `r/renv.lock` `.R.Version` = `geospatial-X` in the `BASE` default; + - `moveapps` in `r/renv.lock` = `sdk-vX` in the `BASE` default; + - the Miniforge tag CI passes as `CONDA` = the one in `test/Dockerfile`. + + Two pairs cannot be checked, because the co-pilots are private: the stand-in's rocker digest + against the R co-pilot's, and the Miniforge tag against the python co-pilot's base. Each is + stated where it is pinned. +- **P3M binaries, in the stand-in only**: + `RENV_CONFIG_REPOS_OVERRIDE=https://p3m.dev/cran/__linux__/noble/latest`. The versions stay the + lock's; only the build path differs from the platform, which compiles from source. Where P3M + has no binary for a pinned version renv falls back to source, which is why `cmake` is there. +- **Cache** `type=gha`, one scope per target. **Fresh runs** (weekly, `workflow_dispatch`) drop the + cache for both jobs: the App image resolves the same open conda environment. R is pinned by + its lock either way. +- **Timeout** for `app`: measured on the first cold run, then set with headroom. `test` keeps 30 + minutes. +- **Dependabot** stays on `github-actions`. The rocker digest and the Miniforge tag are pinned by + hand and guarded by the coherence check. + +### Defects + +All measured on the `v2.2.0` production images. Fixed in this change, each in its own commit, each +with the test that pins the corrected behaviour. + +1. **An empty `TrajectoryCollection` fails the python → R App.** `TransformToCsv.create_geopandas` + raises `ValueError: No objects to concatenate`, while the README promises a NULL object. The + same happens when every track has a single fix. Behind it, `csv_2_rds.R` would not have written + an output for zero rows either: it calls `storeResult` only in the non-empty branch. Corrected + behaviour: python writes an empty `link.csv` and `meta.csv`, R stores the NULL output — an + empty file, the platform's convention, which the next R App rejects with code 10. +2. **DST ambiguity, R → python.** R writes local wall-clock times without an offset; python + re-localises them with `tz_localize(tzone)`. Two fixes at 00:30 and 01:30 UTC on 2021-10-31 in + `Europe/Berlin` both become `2021-10-31 02:30:00.000`, and the App fails with + `AmbiguousTimeError` — for instants R held unambiguously. The fix transmits UTC in `link.csv` + (`tzone` stays in `meta.csv`) and converts on the python side. The existing tests that pin the + raise and localise wall-clock times are changed deliberately. For UTC data — all real data — + `link.csv` stays byte-identical. +3. **Sub-second precision is lost, python → R.** python writes `…:31.123`, `readLink` parses with + `%S` and truncates: 75 fixes of input2 come back up to 0.999 s early. The fix parses with `%OS`. +4. **Single-fix tracks vanish silently, R → python.** movingpandas cannot form a trajectory from + one point; `input_move2loc_List` goes in with 5 rows in 4 tracks and comes back with 2 rows in + 1 track. Not fixable within the format. The fix: `TransformToPickle` logs a warning naming + every dropped track and its row count, so the App log shows the loss. The output stays as is. +5. **logical comes back as character, python → R.** python writes `True`/`False`, which + `read.csv` does not recognise. The fix: `readLink` turns a column holding only `True`, `False` + and `NA` back into logical. The other class changes are inherent to the CSV bridge and stay + pinned in `classes.csv`. + +Measured and **not** a defect: a tz-aware index. movingpandas drops the timezone itself, so +`TransformToCsv.get_timezone` never sees one; its tz-aware branch is dead code and stays untouched. + +The CSV pair is created and consumed inside one App, so a contract fix creates no ordering +constraint between App versions. + +## Sequence and proofs + +One commit per step, each green on its own: + +1. Replace `r/data/raw/` with the Template set. +2. Goldens and type mapping, produced by `v2.2.0`; generator committed. +3. Python gaps and contract tests, pinning today's behaviour and green against today's code. A + test for a defect arrives with its fix in step 7, not here. +4. R refactoring into `link.R` plus testthat — **still on the old image**: the refactoring alone + changes nothing. +5. The port: `moveapps`, the new lock, the `Dockerfile` (`BASE`, conda from the image, + `DIRECTION`), `conda run` in the scripts, `r/src/` removed: R 4.6.1 and the new packages change + nothing. Steps 4 and 5 are separate proofs, so a deviation has exactly one cause. +6. CI: stand-in, bake, test target, coherence check, the `app` job. +7. Defect fixes, one commit each. Until its fix, the DST case runs in the round trip as an expected + failure of the r2python step; the empty case joins with fix 1. +8. README: building with `--build-arg DIRECTION=…`, and the one line that differs per App. + +## Delivery + +Each step after an explicit go: + +1. Push, pull request against `main`, in English, with the local proofs. +2. After the merge: tag `v2.3.0`. The runtime moves, the external formats do not. +3. The App versions are created by hand in the admin UI. Dockerfile: the repository's, with + `ARG DIRECTION=r2python` or `ARG DIRECTION=python2r`. Their order does not matter: the rds and + pickle formats are unchanged — the rds side proved by the old-image read above. +4. An A3-style workflow run on mtest1 (R → python → R). + +## Risks + +- **bake resolving `FROM ${BASE}` through a `target:` context** — measured locally (buildx 0.37.1, + BuildKit 0.25.1) with stand-in images: the matrix over both directions, `COPY --from` between the + App targets, and the refusal of an invalid `DIRECTION` all work. Not yet measured on the GitHub + runner's builder. Fallback: a local registry as a service container. +- **The stand-in can drift from the co-pilot.** The coherence check catches a version mismatch; + a republished rocker tag under the same R version it cannot. The digest comment and the + groundcontrol follow-up below are the mitigation. +- **Runner disk and the Actions cache.** rocker/geospatial alone is 4.3 GB, and the Actions cache + is capped at 10 GB per repository. Measured on the first cold run; eviction costs time, not + correctness. +- **P3M may not carry binaries for every pinned version** — then those compile, and the cold run + takes longer. + +## Follow-ups + +Not part of this change; tickets only after Clemens approves each: + +- groundcontrol: list `link-r-python` in `upgrade-co-pilot-runtime/references/r.md` §5 — an R + upgrade needs its lock re-snapshotted and the stand-in's digest moved. +- A conda lock for the python side. diff --git a/python/conda-linux-64.lock b/python/conda-linux-64.lock new file mode 100644 index 0000000..b94c21f --- /dev/null +++ b/python/conda-linux-64.lock @@ -0,0 +1,167 @@ +# environment.yml md5: 2c86d26fc89448aaabee7a897d2ffed8 +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: linux-64 +# created-by: conda 26.7.2 +@EXPLICIT +https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda#0f51e2391ade309db462a55611263e9c +https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2#0c96522c6bdaed4b1566d11387caaf45 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2#34893075a5c9e55cdafac56607368fc6 +https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2#4d59c254e01d9cde7957100457e2d5fb +https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda#49023d73832ef61042f6a237cb2687e7 +https://conda.anaconda.org/conda-forge/linux-64/libgomp-16.2.0-he0feb66_7.conda#d904c7ddd95e026254b4b80808ee9e12 +https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.2-h25fd6f3_3.conda#0de0122d9570a8ab637c6b73db268389 +https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-9_cp314.conda#350d89b50d7de805abf2e63e29dee451 +https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda#fcb489df604d100968b737f2cb6076c6 +https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda#a9f577daf3de00bca7c3c76c0ecbd1de +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-hc364b38_1.conda#a7970cd949a077b7cb9696379d338681 +https://conda.anaconda.org/conda-forge/linux-64/zlib-1.3.2-h25fd6f3_3.conda#6acb86426229f96f93e5468d1df3a5e8 +https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_7.conda#aa459086047c0e5e27023ab19f8cb86a +https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.46.1-default_hbd61a6d_102.conda#449500f2c089da11c40f5c21312e3e07 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-16.2.0-ha9f2e26_7.conda#735629b400b70eeab21a2ba8f5b1c2ce +https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_10.conda#e675fabcf81499adc7edf58124fb1e01 +https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.8-hebe6cf0_2.conda#3ad2b403be8fc5612b9159e2ce621f69 +https://conda.anaconda.org/conda-forge/linux-64/fribidi-1.0.17-h7cc23a3_0.conda#f9d5302bca196ee87e62f393f2626562 +https://conda.anaconda.org/conda-forge/linux-64/giflib-6.1.3-hebe6cf0_2.conda#07bdd65afe03df3fb5dbcaa8257c64a3 +https://conda.anaconda.org/conda-forge/linux-64/json-c-0.18-hebe6cf0_2.conda#9aa77c4e7f9a988df94da18a7b9b90fe +https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.3-h7cc23a3_1.conda#ba55d1b89fd7775e67de8291029b4059 +https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.2.0-h39a168f_4.conda#df210655e30a05e3b069c91b7aaddd2d +https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.25-hd45a770_1.conda#40f9b31aa9cf007789867df0decd0492 +https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h280c20c_3.conda#7bc31538d6e3fb349e897353969237ec +https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.8.5-hd2095e1_0.conda#72cfd2b95b0540247220b42f58a148d7 +https://conda.anaconda.org/conda-forge/linux-64/libffi-3.7.0-h81df57d_1.conda#6525a0b06aa4fd390795f0740636a9dd +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-16.2.0-h69a702a_7.conda#2005515ac9340f816fb2ac7e83afe553 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-16.2.0-h6b99dfc_7.conda#7719ed7be38ad8c492c4bd5cc2bce0c4 +https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h0cb94f2_3.conda#f92233bf33e24a25668bb2119e2c51f9 +https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.2.0-hb03c661_1.conda#898d1c9793eaa52efc4727bd84d2e39a +https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.3-hb03c661_1.conda#1390b7c5ac0b1d8e447bc5efa6d3c8c2 +https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb03c661_2.conda#fcfed1dc5053eb1901b66e7b1fc32588 +https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hb9d3cd8_1.conda#d864d34357c3b65a4b731f78c0801dc4 +https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.59-h922cc85_0.conda#8c9caf742a485ac688ffa88bbadbf0be +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-16.2.0-h934c35e_7.conda#9b6707e0caca917fa6ea9ee473f37a5d +https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.42.4-hcfc3c73_0.conda#54a948337e66706380ff24d6b5fde63f +https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.6.0-hd42ef1d_1.conda#9332b53d0ea93c5d39e33be03a0c611a +https://conda.anaconda.org/conda-forge/linux-64/lzo-2.10-hebe6cf0_1003.conda#10e0aa58ed390ba598b728badf9eb1b7 +https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.6-hdb14827_1.conda#ee6c0cd80a60961a1f48aa3e0b91f986 +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.4-h781a0a9_0.conda#16e3034a330cc625f2c685edec60cf4e +https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h7cc23a3_1004.conda#bb66b610707b811e3c65181a6431e7a8 +https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h1df4ec4_4.conda#676a2f9b1c4fcf57b70aa5c65f6c60d0 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.2-h280c20c_0.conda#85c9442aec283b4e464fa9ecc484a2f3 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.12-h7cc23a3_2.conda#da33efee1a93603d92e06d4a2b68def6 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.5-h7cc23a3_2.conda#eabfc45786244de252090cf8d55128a8 +https://conda.anaconda.org/conda-forge/linux-64/geos-3.14.1-h55b0958_0.conda#5f856b72ca9cd5b5b4a1c43551804622 +https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.15-h54a6638_1.conda#f9fe2984587fa8235a6af6004760cd18 +https://conda.anaconda.org/conda-forge/linux-64/icu-78.3-py310h44b86e0_2.conda#72a381cbad04f24b1c2a43ef707f45b4 +https://conda.anaconda.org/conda-forge/linux-64/lerc-4.2.0-hdb68285_0.conda#fb9d356b1a57d6d54768be7ebd5fce09 +https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.2.0-ha411449_4.conda#0cfd601eb03cca403dcc248844787486 +https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.2.0-h018ffa1_4.conda#4136f6e4ef4835cc7df39a707cbd511c +https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h373387f_1.conda#50708d3b951d0f8e2d7f2df5b5edc040 +https://conda.anaconda.org/conda-forge/linux-64/libfreetype6-2.14.3-h5e6c136_2.conda#b72a266a9317036fd0464cb98b027cd0 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-16.2.0-h69a702a_7.conda#3a8d285f5f22138ef826ad54cf267e73 +https://conda.anaconda.org/conda-forge/linux-64/libhwy-1.4.0-h57c4cff_1.conda#d58bfbaaf51ed85771eae92c2e7f5816 +https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.68.1-h74cf4be_1.conda#75d211f04ac87506f1f43420712a69fe +https://conda.anaconda.org/conda-forge/linux-64/libpython-3.14.7-hdc7f604_106_cp314.conda#28af2158bb8dbb62e55b0af3b44f9d4a +https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.11.1-h6154650_1.conda#5c526561e1d2a2e071941174eae84557 +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-16.2.0-hdf11a46_7.conda#25a23bde5e26d548374f3e02b4669670 +https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.17.0-hb83e432_2.conda#61f0ffba9161cbb3a77722869b21e4bb +https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.10.0-hee9eb32_2.conda#e38a3253d72ab07d471483f24947e2a2 +https://conda.anaconda.org/conda-forge/linux-64/minizip-4.2.2-haac24f3_1.conda#3fee8d11b0d8a54fb899749052accb4f +https://conda.anaconda.org/conda-forge/linux-64/muparser-2.3.5-hee9eb32_1.conda#7de84fd76ec2bdb7ac617507de3d3052 +https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.47-h8b3dc9c_1.conda#06f6113cf1ff4a54b65f87ead132a5f1 +https://conda.anaconda.org/conda-forge/linux-64/pixman-0.46.4-h54a6638_3.conda#0ee5bb30034b081a1386c1e2c98ab0a7 +https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-hd6e31c0_1.conda#69c01c781e7c8190bbdaf79e3848c5ca +https://conda.anaconda.org/conda-forge/linux-64/snappy-1.2.2-h34e00bb_2.conda#2b7e31ffbd476eecf33b3b833692339c +https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.6-h0d788c3_1.conda#aa7459ed9ad086ba11dda843d764b33d +https://conda.anaconda.org/conda-forge/linux-64/zlib-ng-2.3.3-hce19668_1.conda#1726acec19beeed1c764385cd8622405 +https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.6-he440d0b_1.conda#2c2fae981fd2afd00812c92ac47d023d +https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.2.0-h9908984_4.conda#c48c09444f3736800ea0b9550c365baf +https://conda.anaconda.org/conda-forge/linux-64/freexl-2.0.0-h1f12c17_3.conda#4a0a8fd87ee4767db791047546f56501 +https://conda.anaconda.org/conda-forge/linux-64/krb5-1.22.2-hbc21106_2.conda#53318d715316929a574f83591308b1f8 +https://conda.anaconda.org/conda-forge/linux-64/libfreetype-2.14.3-ha770c72_2.conda#f8054e759d0ddddaf34a5c8fedc900a1 +https://conda.anaconda.org/conda-forge/linux-64/libglib-2.90.0-h569388d_0.conda#c85e9539551f1fd21104e518d940c244 +https://conda.anaconda.org/conda-forge/linux-64/libjxl-0.12.0-heb2dce7_2.conda#485788d339785bac87fe86315b4a0627 +https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.34-pthreads_hf13c14d_2.conda#7e19fcd73d7f0997d3cb3326d2586a86 +https://conda.anaconda.org/conda-forge/linux-64/libpsl-0.23.1-hd9e3e90_1.conda#77be60417aa572afcb48cbe0f967401d +https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-h55b0958_21.conda#6c639697369e9669032ca08d0a566282 +https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.53.4-h13e7031_1.conda#e72bbec309c2b0f37823ee7d4fabfcd3 +https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.7.2-hcc2c06a_1.conda#0907d876f460ebc941ae590338b6102f +https://conda.anaconda.org/conda-forge/linux-64/libxml2-16-2.15.4-hf3af7cc_0.conda#b7545c57a73a51d72aa6eee5695cf051 +https://conda.anaconda.org/conda-forge/linux-64/qhull-2020.2-h434a139_5.conda#353823361b1d27eb3960efb076dfcaf6 +https://conda.anaconda.org/conda-forge/linux-64/uriparser-0.9.8-hac33072_0.conda#d71d3a66528853c0a1ac2c02d79a0284 +https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.3.0-h78fed68_2.conda#f2ff0de6c209e90c53b3a195c2708f5f +https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.13-he1eb515_1.conda#8c282bbe4808a3cc80a5c98e9aec1cfc +https://conda.anaconda.org/conda-forge/linux-64/brotli-1.2.0-h505cf86_4.conda#746aa619a1bcaf4da541101179d7c60e +https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.18.3-h4db4eae_1.conda#922776b528a470ab5afa81fd42abfa1d +https://conda.anaconda.org/conda-forge/linux-64/freetype-2.14.3-ha770c72_2.conda#2d0ea23b23603e07ca47f23f949f67b6 +https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.19.1-h9073bf1_3.conda#596464faa06e0b656b35416188ef9bba +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.11.0-11_h4a7cf45_openblas.conda#b8cce1486f3c62f33291d6318c193d0d +https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.22.0-ha042cf0_0.conda#e617deee7af8eb0c04f6296d12b54157 +https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-haa4a5bd_1023.conda#953b7cca897e21215302dbfe2af5cd0c +https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.15.4-h7df9aa5_0.conda#ea192d1ef556bf1d55de4acb8c0f5b65 +https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.4-heb1ab33_2.conda#a3f3ac7a68d01c198e276dbb99e62032 +https://conda.anaconda.org/conda-forge/linux-64/python-3.14.7-hcd007b5_106_cp314.conda#56ee91e118243e46bfc0c41e4030d354 +https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.53.4-h9ffa6c4_1.conda#d42b24574925bfa3167efb3571c905ad +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.7-h7cc23a3_1.conda#e5b6b28536b81b3f4cb20db4668a4642 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.12-hb03c661_1.conda#e470d224a7a5be1b1d021bded7abb536 +https://conda.anaconda.org/conda-forge/noarch/backports.zstd-1.7.0-py314h680f03e_1.conda#67924260a851b5273e39f9b28b6b5e24 +https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.2.0-py314hcd2bdb6_4.conda#d9aabceecc99b3b8b0eb7090c5776af4 +https://conda.anaconda.org/conda-forge/linux-64/cairo-1.18.6-h3c89d7e_0.conda#7920701f9abcb488411495742cf365a7 +https://conda.anaconda.org/conda-forge/noarch/certifi-2026.7.22-pyhd8ed1ab_0.conda#37e13edbe3b48f1095a9d085ef9cd83b +https://conda.anaconda.org/conda-forge/noarch/cloudpickle-3.1.2-pyhcf101f3_1.conda#61b8078a0905b12529abc622406cb62c +https://conda.anaconda.org/conda-forge/noarch/cpython-3.14.7-py314hd8ed1ab_106.conda#3b52b830ac00367685c08f98f50963bf +https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhcf101f3_2.conda#4c2a8fef270f6c69591889b93f9f55c1 +https://conda.anaconda.org/conda-forge/noarch/geographiclib-2.1-pyhd8ed1ab_0.conda#43dd16b113cc7b244d923b630026ff4f +https://conda.anaconda.org/conda-forge/noarch/hpack-4.2.0-pyhd8ed1ab_0.conda#b395909221b9bd1df066e5930e18855b +https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda#8e6923fc12f1fe8f8c4e5c9f343256ac +https://conda.anaconda.org/conda-forge/noarch/idna-3.20-pyh5ded981_0.conda#ee73be987f9de9ea5c2fdaeefa7b6491 +https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.5.1-py314h5383ef5_3.conda#f8b25504f794b11a671868f7262aeab0 +https://conda.anaconda.org/conda-forge/linux-64/libarchive-3.8.9-gpl_h3152399_101.conda#680520cff78a974d564cd99cdf5ec0b5 +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.11.0-11_h0358290_openblas.conda#73a248c30811075059b5aeda766c2624 +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.11.0-11_h47877c9_openblas.conda#5622a13917855ce4b953b9e655cf9262 +https://conda.anaconda.org/conda-forge/linux-64/libxml2-devel-2.15.4-h7df9aa5_0.conda#9a414200ffe436e47d72c9f4c11c3658 +https://conda.anaconda.org/conda-forge/linux-64/markupsafe-3.0.3-py314h67df5f8_1.conda#9a17c4307d23318476d7fbf0fedc0cde +https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyhd8ed1ab_1.conda#37293a85a0f4f77bbd9cf7aaefc62609 +https://conda.anaconda.org/conda-forge/noarch/narwhals-2.26.0-pyh5ded981_0.conda#afb7146667be7c8121f5f8337be60680 +https://conda.anaconda.org/conda-forge/noarch/networkx-3.7-pyhc364b38_0.conda#ddc679d85e7c9c3738efd303b6580c69 +https://conda.anaconda.org/conda-forge/noarch/packaging-26.3-pyhc364b38_0.conda#936687ed80f295a1f5dbcf8bd34c252c +https://conda.anaconda.org/conda-forge/linux-64/pillow-12.3.0-py314h50bfbbb_4.conda#2180323126daa9971a80cc679cf12f46 +https://conda.anaconda.org/conda-forge/noarch/pip-26.2.1-pyh145f28c_0.conda#573cb3ed111004230ed3de650653cd4d +https://conda.anaconda.org/conda-forge/linux-64/proj-9.9.0-h3767062_0.conda#29e9221e3a2122d57234d424b94649b6 +https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.3.3-pyh5ded981_0.conda#8bf43ae1312e9b966063459e04c20fa7 +https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda#461219d1a5bd61342293efa2c0c90eac +https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2026.4-pyh5ded981_0.conda#6fc085ba8fa446b73bff580281b21ce3 +https://conda.anaconda.org/conda-forge/noarch/pytz-2026.4-pyh5ded981_0.conda#88f9c01fa949cb51dcd33d36e547a30e +https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda#3339e3b65d58accf4ca4fb8748ab16b3 +https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.7.0-pyhc455866_0.conda#8c05910e780502e362d658f00f5ec10f +https://conda.anaconda.org/conda-forge/noarch/xyzservices-2026.9.1-pyhd8ed1ab_0.conda#5d7c149ba9c728d4951f3fe6d3ce0f84 +https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.66.1-py314hd42368d_0.conda#3265ea810d2c81a2b37f0d5047179418 +https://conda.anaconda.org/conda-forge/noarch/geopy-2.5.0-pyhd8ed1ab_0.conda#43372c0cbf9e460f218de5ed1543a011 +https://conda.anaconda.org/conda-forge/noarch/h2-4.4.1-pyhcf101f3_0.conda#e652ac7756069c456d0da2a922cd7df5 +https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.6-pyhcf101f3_1.conda#04558c96691bed63104678757beb4f8d +https://conda.anaconda.org/conda-forge/noarch/joblib-1.6.0-pyhcf101f3_0.conda#4ebc70ef1af2e21cdc8edb1bfa1ec28d +https://conda.anaconda.org/conda-forge/linux-64/libharfbuzz-14.5.0-h23af247_0.conda#4c0d9662a054c51e69b66a427e7ead03 +https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.1.0-gpl_h2988700_122.conda#2f88d147c5cf80d3f4e8f019fa6cbfed +https://conda.anaconda.org/conda-forge/linux-64/numpy-2.5.3-py314hd3a7d6b_0.conda#d1ae35d69e57534c8713ff0368b9fdb9 +https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.8.0-py314h08533f4_2.conda#1a8b87c5df1bef0ea42136b513843d05 +https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda#5b8d21249ff20967101ffa321cab24e8 +https://conda.anaconda.org/conda-forge/noarch/python-gil-3.14.7-h4df99d1_106.conda#b672ecb44198ade19ef249bf99f4b1ff +https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_3.conda#3845f3d75991bae0fb90884662f4327c +https://conda.anaconda.org/conda-forge/noarch/branca-0.8.2-pyhd8ed1ab_0.conda#1fcdf88e7a8c296d3df8409bf0690db4 +https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.4.0-py314h5383ef5_1.conda#2848b95cbf026062b30f3f1f504a2f61 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-core-3.13.3-h7fa801c_2.conda#0fcbb3a961299633e8f825899471b3e8 +https://conda.anaconda.org/conda-forge/linux-64/libraqm-0.11.0-h6406941_0.conda#3ac89a48d224409739dbf6200e524373 +https://conda.anaconda.org/conda-forge/linux-64/pandas-2.3.3-py314ha0b5721_2.conda#fe3a5c8be07a7b82058bdeb39d33d93b +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.18.1-py314h36f0660_1.conda#2c5a3b238546edf4e3ea3bd9c1b9fa03 +https://conda.anaconda.org/conda-forge/linux-64/shapely-2.1.2-py314he501186_3.conda#047df7fa5fee2bfc538e185f6268a61b +https://conda.anaconda.org/conda-forge/noarch/urllib3-2.8.0-pyhd8ed1ab_0.conda#407a3d3778570bae8e8fcf554e803e43 +https://conda.anaconda.org/conda-forge/linux-64/charset-normalizer-3.5.2-h1114479_0.conda#78d5c6112c9320a1a797935cecfec7c2 +https://conda.anaconda.org/conda-forge/noarch/geopandas-base-1.2.0-pyha770c72_0.conda#9b99d8df91a5d477372858fa9dd94a25 +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.11.2-py314h9124352_1.conda#55d364956c0d3a01f6e39db3f6f6cc9b +https://conda.anaconda.org/conda-forge/linux-64/pyogrio-0.13.0-py314hab130c5_0.conda#95fcdefe34572c4db193d0d09c88a74e +https://conda.anaconda.org/conda-forge/linux-64/scikit-learn-1.9.1-np2py314h9e1d7c2_1.conda#b0dd28a54d731a0ea172b28d9bc36e35 +https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.11.0-pyhd8ed1ab_0.conda#0323c95182d828bb1bcf708c4077db3b +https://conda.anaconda.org/conda-forge/noarch/movingpandas-0.23.0-pyhd8ed1ab_0.conda#43b9b2c8a351bdca95e9a0dd80ad358d +https://conda.anaconda.org/conda-forge/noarch/requests-2.34.2-pyhcf101f3_0.conda#4a85203c1d80c1059086ae860836ffb9 +https://conda.anaconda.org/conda-forge/noarch/folium-0.20.0-pyhd8ed1ab_0.conda#a6997a7dcd6673c0692c61dfeaea14ab +https://conda.anaconda.org/conda-forge/noarch/geopandas-1.2.0-pyhd8ed1ab_0.conda#676ee32e43cfbac783986c54ae489949 diff --git a/python/environment.yml b/python/environment.yml index 970b1c9..723d08d 100644 --- a/python/environment.yml +++ b/python/environment.yml @@ -1,3 +1,4 @@ +# The App installs python/conda-linux-64.lock, resolved from this file: run python/lock.sh after any change here. name: link-r-python channels: - conda-forge @@ -12,8 +13,7 @@ dependencies: - python # The ceiling is deliberate and belongs in its own ticket: pandas 3 switches the default string # dtype and datetime resolution (`str`, `datetime64[us]`), which reaches the R side through the CSV - # this App writes, and no test here covers that. It also drops the pytz dependency that - # `tests/test_transform_to_pickle.py` borrows transitively. + # this App writes, and no test here covers that. - pandas>=2.3.3,<3 - geopandas - movingpandas \ No newline at end of file diff --git a/python/lock.sh b/python/lock.sh new file mode 100755 index 0000000..c6f3d63 --- /dev/null +++ b/python/lock.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Resolves python/environment.yml into python/conda-linux-64.lock - the explicit package list the App and +# the python tests install, so an App version gets exactly what CI tested. Run before every release, +# and after any change to environment.yml: +# python/lock.sh +# Resolves for linux-64, the platform's architecture, with the Miniforge of test/Dockerfile (the python +# co-pilot's base). The first line holds the md5 of environment.yml; test/coherence.R holds them together. +set -o errexit -o nounset -o pipefail + +cd "$(dirname "$0")/.." +miniforge="$(grep -m1 -o 'condaforge/miniforge3:[0-9A-Za-z._-]*' test/Dockerfile)" + +docker run --rm -i --platform linux/amd64 "$miniforge" bash -c ' + set -o errexit + cat > /tmp/environment.yml + conda env create --quiet --prefix /tmp/env --file /tmp/environment.yml > /dev/null + echo "# environment.yml md5: $(md5sum /tmp/environment.yml | cut -d " " -f 1)" + conda list --prefix /tmp/env --explicit --md5 +' < python/environment.yml > python/conda-linux-64.lock.new +mv python/conda-linux-64.lock.new python/conda-linux-64.lock diff --git a/python/tests/test_contract.py b/python/tests/test_contract.py new file mode 100644 index 0000000..15a4ba7 --- /dev/null +++ b/python/tests/test_contract.py @@ -0,0 +1,45 @@ +import os +import tempfile +import unittest + +from ..transform_to_csv import TransformToCsv +from ..transform_to_pickle import TransformToPickle + +CONTRACT_DIR = './test/contract' + + +class ContractTestCase(unittest.TestCase): + """ + The CSV pair between R and python, pinned by golden files the v2.2.0 production images produced: + R's link.csv and meta.csv go in, and what python hands back to R must equal py/link.csv and + py/meta.csv byte for byte. + """ + + def test_it_should_hand_back_the_python_csv_pair_of_every_case(self): + cases = sorted(case for case in os.listdir(CONTRACT_DIR) if os.path.isdir(os.path.join(CONTRACT_DIR, case))) + self.assertEqual(6, len(cases)) + + for case in cases: + with self.subTest(case=case), tempfile.TemporaryDirectory() as tmp: + # arrange + case_dir = os.path.join(CONTRACT_DIR, case) + pickle = os.path.join(tmp, 'output_file') + link = os.path.join(tmp, 'link.csv') + meta = os.path.join(tmp, 'meta.csv') + TransformToPickle().convert( + input_data_file_name=os.path.join(case_dir, 'link.csv'), + input_meta_file_name=os.path.join(case_dir, 'meta.csv'), + output_file_name=pickle + ) + + # act + TransformToCsv().convert(input_data_file_name=pickle, output_file_name=link, output_meta_file_name=meta) + + # assert + self.assertEqual(self.__read(os.path.join(case_dir, 'py', 'meta.csv')), self.__read(meta)) + self.assertEqual(self.__read(os.path.join(case_dir, 'py', 'link.csv')), self.__read(link)) + + @staticmethod + def __read(file_path) -> bytes: + with open(file_path, 'rb') as file: + return file.read() diff --git a/python/tests/test_transform_to_csv.py b/python/tests/test_transform_to_csv.py index 46e63cf..ba5b3e0 100644 --- a/python/tests/test_transform_to_csv.py +++ b/python/tests/test_transform_to_csv.py @@ -4,6 +4,9 @@ import tempfile import unittest +import movingpandas as mpd +import pandas as pd + from ..transform_to_csv import TransformToCsv from ..transform_to_pickle import TransformToPickle @@ -66,6 +69,53 @@ def test_it_should_detect_the_absence_of_compression(self): # verify: `None` is how pandas is told not to decompress at all self.assertIsNone(actual) + def test_it_should_write_crs_timezone_and_column_names_to_the_meta_csv(self): + # arrange + data = self.sut.read_data_pickle(file_path=self.compressed) + geopandas = self.sut.create_geopandas(data=data) + meta = os.path.join(self.tmp.name, 'meta.csv') + + # act + self.sut.write_meta_csv(geopanda=geopandas, movingpanda=data, file_path=meta) + + # assert: the time column is the index movingpandas built from `timestamp_utc` + with open(meta) as written: + actual = written.read().splitlines() + self.assertEqual( + ['crs,tzone,timeColName,trackIdColName', 'EPSG:4326,UTC,timestamp_utc,individual_name_deployment_id'], + actual + ) + + def test_it_should_write_the_time_index_and_the_coordinates_to_the_link_csv(self): + # arrange + data = self.sut.read_data_pickle(file_path=self.compressed) + geopandas = self.sut.create_geopandas(data=data) + link = os.path.join(self.tmp.name, 'link.csv') + + # act + self.sut.write_result(file_name=link, data=geopandas) + + # assert + with open(link) as written: + actual = written.readline().rstrip('\n').split(',') + self.assertEqual('timestamp_utc', actual[0]) + self.assertIn('coords_x', actual) + self.assertIn('coords_y', actual) + + def test_it_should_write_empty_files_for_an_empty_trajectory_collection(self): + # arrange + empty = os.path.join(self.tmp.name, 'empty_output_file') + pd.to_pickle(mpd.TrajectoryCollection([]), empty, compression='gzip') + link = os.path.join(self.tmp.name, 'link.csv') + meta = os.path.join(self.tmp.name, 'meta.csv') + + # act + self.sut.convert(input_data_file_name=empty, output_file_name=link, output_meta_file_name=meta) + + # assert: empty files tell R to hand on the NULL result + self.assertEqual(0, os.path.getsize(link)) + self.assertEqual(0, os.path.getsize(meta)) + def __uncompressed_copy(self) -> str: """ Unpacks the compressed output into what an App on the previous release would have written. diff --git a/python/tests/test_transform_to_pickle.py b/python/tests/test_transform_to_pickle.py index aa5de89..4725450 100644 --- a/python/tests/test_transform_to_pickle.py +++ b/python/tests/test_transform_to_pickle.py @@ -1,13 +1,14 @@ +import contextlib import datetime +import io import os import tempfile import unittest from zoneinfo import ZoneInfo import pandas as pd -import pytz -from ..transform_to_pickle import TransformToPickle +from ..transform_to_pickle import Meta, TransformToPickle class TransformToPickleTestCase(unittest.TestCase): @@ -19,10 +20,10 @@ def test_apply_timezone_plusOffset(self): # execute actual = self.sut.adjust_timestamps(data, timezone='+02:00', time_col_name='timestamp') # verify - # csv value: 2013-08-08 06:47:31 - expected = datetime.datetime(2013, 8, 8, 6, 47, 31, tzinfo=ZoneInfo('Europe/Berlin')) + # csv value: 2013-08-08 06:47:31, a UTC instant + expected = datetime.datetime(2013, 8, 8, 8, 47, 31, tzinfo=ZoneInfo('Europe/Berlin')) self.assertEqual(expected, actual['timestamp_tz'][0].to_pydatetime()) - self.assertEqual(datetime.datetime(2013, 8, 8, 4, 47, 31, tzinfo=None), actual['timestamp_utc'][0].to_pydatetime()) + self.assertEqual(datetime.datetime(2013, 8, 8, 6, 47, 31, tzinfo=None), actual['timestamp_utc'][0].to_pydatetime()) def test_apply_timezone_name_kolkata(self): # prepare @@ -31,11 +32,11 @@ def test_apply_timezone_name_kolkata(self): # Asia/Kolkata aka UTC+05:30 actual = self.sut.adjust_timestamps(data, timezone='Asia/Kolkata', time_col_name='timestamp') # verify - # csv value: 2013-08-08 06:47:31 - expected = datetime.datetime(2013, 8, 8, 6, 47, 31, tzinfo=ZoneInfo('Asia/Kolkata')) + # csv value: 2013-08-08 06:47:31, a UTC instant + # 06:47:31 UTC is 12:17:31 at UTC+05:30 on the same day - India has no DST + expected = datetime.datetime(2013, 8, 8, 12, 17, 31, tzinfo=ZoneInfo('Asia/Kolkata')) self.assertEqual(expected, actual['timestamp_tz'][0].to_pydatetime()) - # 06:47:31 at UTC+05:30 is 01:17:31 UTC on the same day - India has no DST - self.assertEqual(datetime.datetime(2013, 8, 8, 1, 17, 31, tzinfo=None), actual['timestamp_utc'][0].to_pydatetime()) + self.assertEqual(datetime.datetime(2013, 8, 8, 6, 47, 31, tzinfo=None), actual['timestamp_utc'][0].to_pydatetime()) def test_apply_timezone_name_utc(self): # prepare @@ -48,19 +49,23 @@ def test_apply_timezone_name_utc(self): self.assertEqual(expected, actual['timestamp_tz'][0].to_pydatetime()) self.assertEqual(datetime.datetime(2013, 8, 8, 6, 47, 31, tzinfo=None), actual['timestamp_utc'][0].to_pydatetime()) - def test_ambiguous_dst_timestamp_should_raise_error(self): + def test_an_instant_in_the_repeated_dst_hour_should_convert(self): # prepare data = self.sut.read_data_csv(file_path='./python/tests/data/link-dst-ambiguous.csv', time_col_name='timestamp') - # execute & verify - # pytz.exceptions.AmbiguousTimeError: Cannot infer dst time from 2013-10-27 01:16:03, try using the 'ambiguous' argument - self.assertRaises(pytz.exceptions.AmbiguousTimeError, self.sut.adjust_timestamps, data, timezone='Europe/London', time_col_name='timestamp') + # execute + actual = self.sut.adjust_timestamps(data, timezone='Europe/London', time_col_name='timestamp') + # verify: 01:16:03 UTC is 01:16:03 GMT, just after British Summer Time ended at 01:00 UTC + self.assertEqual(datetime.datetime(2013, 10, 27, 1, 16, 3, tzinfo=datetime.timezone.utc), actual['timestamp_tz'][0].to_pydatetime()) + self.assertEqual(datetime.datetime(2013, 10, 27, 1, 16, 3), actual['timestamp_utc'][0].to_pydatetime()) - def test_non_existent_timestamp_should_raise_error(self): + def test_an_instant_in_the_skipped_dst_hour_should_convert(self): # prepare data = self.sut.read_data_csv(file_path='./python/tests/data/link-dst-nonexistent.csv', time_col_name='timestamp') - # execute & verify - # pytz.exceptions.NonExistentTimeError: 2014-03-30 01:24:20 - self.assertRaises(pytz.exceptions.NonExistentTimeError, self.sut.adjust_timestamps, data, timezone='Europe/London', time_col_name='timestamp') + # execute + actual = self.sut.adjust_timestamps(data, timezone='Europe/London', time_col_name='timestamp') + # verify: 01:24:20 UTC is 02:24:20 BST, British Summer Time began at 01:00 UTC + self.assertEqual(datetime.datetime(2014, 3, 30, 1, 24, 20, tzinfo=datetime.timezone.utc), actual['timestamp_tz'][0].to_pydatetime()) + self.assertEqual(datetime.datetime(2014, 3, 30, 1, 24, 20), actual['timestamp_utc'][0].to_pydatetime()) def test_it_should_write_a_gzip_compressed_pickle(self): # prepare @@ -79,6 +84,50 @@ def test_it_should_write_a_gzip_compressed_pickle(self): actual = written.read(2) self.assertEqual(b'\x1f\x8b', actual) + def test_it_should_read_the_meta_csv(self): + # arrange + file_path = './python/sample/input4/meta.csv' + + # act + actual = self.sut.read_meta_csv(file_path=file_path) + + # assert + self.assertEqual( + Meta(projection='EPSG:4326', timezone='UTC', time_col_name='timestamp', track_id_col_name='individual_name_deployment_id'), + actual + ) + + def test_it_should_create_one_trajectory_per_track_in_the_crs_of_the_meta_csv(self): + # arrange + meta = self.sut.read_meta_csv(file_path='./python/sample/input4/meta.csv') + data = self.sut.read_data_csv(file_path='./python/sample/input4/link.csv', time_col_name=meta.time_col_name) + self.sut.adjust_timestamps(data=data, timezone=meta.timezone, time_col_name=meta.time_col_name) + + # act + actual = self.sut.create_moving_pandas(data=data, projection=meta.projection, track_id_col_name=meta.track_id_col_name) + + # assert: the sample carries three distinct tracks + self.assertEqual(3, len(actual.trajectories)) + self.assertEqual('EPSG:4326', actual.trajectories[0].df.crs.to_string()) + + def test_it_should_warn_about_every_track_movingpandas_drops(self): + # arrange: track `a` has one fix, movingpandas needs two for a trajectory + data = pd.DataFrame({ + 'track': ['a', 'b', 'b'], + 'timestamp_utc': pd.to_datetime(['2021-07-01 06:40', '2021-07-01 06:40', '2021-07-01 06:46']), + 'coords_x': [1.0, 2.0, 2.1], + 'coords_y': [1.0, 2.0, 2.1], + }) + output = io.StringIO() + + # act + with contextlib.redirect_stdout(output): + self.sut.create_moving_pandas(data=data, projection='EPSG:4326', track_id_col_name='track') + + # assert + actual = [line for line in output.getvalue().splitlines() if line.startswith('[WARN]')] + self.assertEqual(['[WARN] track a dropped: a trajectory needs at least two fixes, it has 1'], actual) + def test_it_should_write_trajectories_with_point_locations(self): # Readers access `trajectory.df.geometry` directly: movingpandas < 0.22 does it in # `TrajectoryStopDetector`, Apps do it in their own code. A geometry movingpandas >= 0.22 diff --git a/python/transform_to_csv.py b/python/transform_to_csv.py index 21451cf..c659ebd 100644 --- a/python/transform_to_csv.py +++ b/python/transform_to_csv.py @@ -75,6 +75,11 @@ def write_result(self, file_name, data: gpd.GeoDataFrame): def convert(self, input_data_file_name, output_file_name, output_meta_file_name): data = self.read_data_pickle(file_path=input_data_file_name) + if len(data.trajectories) == 0: + # nothing to hand over, not even a crs: empty files tell R to store the NULL result + open(output_file_name, 'w').close() + open(output_meta_file_name, 'w').close() + return geopandas = self.create_geopandas(data=data) self.write_result(file_name=output_file_name, data=geopandas) self.write_meta_csv(geopanda=geopandas, movingpanda=data, file_path=output_meta_file_name) diff --git a/python/transform_to_pickle.py b/python/transform_to_pickle.py index 12eeb5c..04723b4 100644 --- a/python/transform_to_pickle.py +++ b/python/transform_to_pickle.py @@ -36,8 +36,9 @@ def read_data_csv(self, file_path, time_col_name): return deserialized def adjust_timestamps(self, data, timezone, time_col_name): + # link.csv carries UTC instants; `timezone` is how R displayed them # kudos: https://stackoverflow.com/a/18912631/810944 - data['timestamp_tz'] = data[time_col_name].apply(lambda x: x.tz_localize(timezone)) + data['timestamp_tz'] = data[time_col_name].apply(lambda x: x.tz_localize('UTC').tz_convert(timezone)) # prepare data for movingpandas b/c of https://github.com/movingpandas/movingpandas/issues/303 # we need our timestamps in timezone 'UTC'; movingpandas can not work w/ timezone-info @@ -63,8 +64,18 @@ def create_moving_pandas(self, data, projection, track_id_col_name): t='timestamp_utc' # use our converted timezone column (UTC) ) print(move) + self.warn_about_dropped_tracks(data=data, move=move, track_id_col_name=track_id_col_name) return move + def warn_about_dropped_tracks(self, data, move, track_id_col_name): + # movingpandas needs two fixes for a trajectory and drops a track with fewer without a word; + # the loss cannot be avoided in this format, but it must not stay silent + kept = {trajectory.id for trajectory in move.trajectories} + rows_per_track = data[track_id_col_name].value_counts() + for track_id, rows in sorted(rows_per_track.items(), key=lambda item: str(item[0])): + if track_id not in kept: + print(f'[WARN] track {track_id} dropped: a trajectory needs at least two fixes, it has {rows}') + def write_result(self, file_name, data): print(type(data)) # MoveApps dictates the output path and it carries no file extension, so the compression has diff --git a/python2r.sh b/python2r.sh index 873c651..466517e 100755 --- a/python2r.sh +++ b/python2r.sh @@ -1,4 +1,4 @@ -#!/bin/bash --login +#!/bin/bash ######################################################################################################################## # converts from Python .pickle to R .rds ######################################################################################################################## @@ -13,5 +13,5 @@ set -a : ${LINK_R_PYTHON_META:=/tmp/artifacts/meta.csv} set +a -(cd python && conda activate "$HOME"/co-pilot-r/python-env && python pickle_2_csv.py) +(cd python && conda run --no-capture-output --prefix "$ENV_PREFIX" python pickle_2_csv.py) (cd r && Rscript csv_2_rds.R) diff --git a/r/.renvignore b/r/.renvignore new file mode 100644 index 0000000..14ecd58 --- /dev/null +++ b/r/.renvignore @@ -0,0 +1,2 @@ +# test-only code: keeps testthat out of the App's lock, and so out of the hangar inspection +tests/ diff --git a/r/csv_2_rds.R b/r/csv_2_rds.R index bfd88ef..0776be3 100644 --- a/r/csv_2_rds.R +++ b/r/csv_2_rds.R @@ -1,42 +1,19 @@ -source("src/common/logger.R") -source("src/common/runtime_configuration.R") -source("src/io/app_files.R") -source("src/io/io_handler.R") -source("src/io/rds.R") - -library(move2) -library(dplyr) +library("moveapps") +moveapps::logger.init() +source("link.R") tryCatch( { Sys.setenv(tz="UTC") - - # always includes "coords_x", "coords_y" - datapy <- read.csv(Sys.getenv(x = "LINK_R_PYTHON_BUFFER"),header=TRUE) - # always includes crs, tzone, timeColName, trackIdColName - meta <- read.csv(Sys.getenv(x = "LINK_R_PYTHON_META"),header=TRUE) - - if (dim(datapy)[1]==0) result <- NULL else - { - datapy[meta$timeColName] <- as.POSIXct(datapy %>% select(meta$timeColName) %>% sapply(as.character) %>% as.vector,format="%Y-%m-%d %H:%M:%S", tz=meta$tzone) - result <- mt_as_move2(datapy, - coords = c("coords_x", "coords_y"), - time_column= meta$timeColName, - track_id_column= meta$trackIdColName, - # track_attributes=c(), - crs= meta$crs - ) - ## add remove empty locs?? - result <- result |> dplyr::arrange(mt_track_id(result),mt_time(result)) - - storeResult(result = result, outputFile = outputFile()) - } + + # NULL for an empty buffer: storeResult then writes the empty file the next App reads as NULL input + moveapps::storeResult(result = readLink(bufferFile = Sys.getenv(x = "LINK_R_PYTHON_BUFFER"), metaFile = Sys.getenv(x = "LINK_R_PYTHON_META")), outputFile = moveapps::outputFile()) }, error = function(e) { # error handler picks up where error was generated print(paste("ERROR: ", e)) - storeToFile(e, errorFile()) + moveapps::storeToFile(e, moveapps::errorFile()) stop(e) # re-throw the exception } ) diff --git a/r/data/raw/README.txt b/r/data/raw/README.txt index 4329082..b18e0e4 100644 --- a/r/data/raw/README.txt +++ b/r/data/raw/README.txt @@ -1,20 +1,20 @@ Set of input data to test apps. +Source: github.com/movestore/Template_R_Function_App, data/raw at commit +e00af0d779f9756c40161bffd8d6f0a5ebc77ced (2026-09-28), the move2_loc files only - the +telemetry.list files are no input type of the translators. +input_move2loc_List.rds is this repository's own: the only input with list columns in its +track data. + *Content* - input1: 1 goat, median fix rate = 30mins, tracking duration 7.5 month, gps, local movement - input2: 3 storks, median fix rate = 1sec, tracking duration 2 weeks, gps, local movement -- input3: 1 stork, median fix rate = 1h | 1day | 1 week, tracking duration 11.5 years, argos, includes migration +- input3: 1 stork, one track per deployment, median fix rate = 1h | 1day | 1 week, tracking duration 11.5 years, argos, includes migration - input4: 3 geese, median fix rate = 1h | 4h, tracking duration 1.5 years, gps, includes migration - -*I/O types* -- all data sets are provided as 'move2_loc' -- input1 & input2 are also provided as 'telemetry.list' +- input_move2loc_List: 5 locations in 4 tracks, three of them with a single fix *Projection* -- for 'move2_loc', data are provided in "lat/long" and projected to "Mollweide" in order to test your app accordingly for not projected and projected data. If your app does not allow projected data or only can deal with projected data, document and either build a automatic transformation in the app or make it fail with an informative error message. The app "Change projection" can be refered to for the user to change the projection of the data acordingly previous to your app. - -- the 'telemetry.list' examples are in a "aeqd" projection with 0,0 in the center of the track, as this is a common projection used within the ctmm library - +- data are provided in "lat/long" (EPSG:4326) and projected to "Mollweide" (ESRI:54009) in order to test your app accordingly for not projected and projected data. *File names* input1_move2loc_LatLon.rds @@ -29,9 +29,4 @@ input3_move2loc_Mollweide.rds input4_move2loc_LatLon.rds input4_move2loc_Mollweide.rds - -input1_telemetrylist_aeqd.rds -input2_telemetrylist_aeqd.rds - - - +input_move2loc_List.rds diff --git a/r/data/raw/input1_move2loc_LatLon.rds b/r/data/raw/input1_move2loc_LatLon.rds index 8f52791..9e87a45 100644 Binary files a/r/data/raw/input1_move2loc_LatLon.rds and b/r/data/raw/input1_move2loc_LatLon.rds differ diff --git a/r/data/raw/input1_move2loc_Mollweide.rds b/r/data/raw/input1_move2loc_Mollweide.rds index 3312cea..4c501dc 100644 Binary files a/r/data/raw/input1_move2loc_Mollweide.rds and b/r/data/raw/input1_move2loc_Mollweide.rds differ diff --git a/r/data/raw/input2_move2loc_LatLon.rds b/r/data/raw/input2_move2loc_LatLon.rds index 428cec9..47b701a 100644 Binary files a/r/data/raw/input2_move2loc_LatLon.rds and b/r/data/raw/input2_move2loc_LatLon.rds differ diff --git a/r/data/raw/input2_move2loc_Mollweide.rds b/r/data/raw/input2_move2loc_Mollweide.rds index 57398eb..7aabdfb 100644 Binary files a/r/data/raw/input2_move2loc_Mollweide.rds and b/r/data/raw/input2_move2loc_Mollweide.rds differ diff --git a/r/data/raw/input3_move2loc_LatLon.rds b/r/data/raw/input3_move2loc_LatLon.rds index a77ceae..5d571a8 100644 Binary files a/r/data/raw/input3_move2loc_LatLon.rds and b/r/data/raw/input3_move2loc_LatLon.rds differ diff --git a/r/data/raw/input3_move2loc_Mollweide.rds b/r/data/raw/input3_move2loc_Mollweide.rds index 8e0e52f..2a48cef 100644 Binary files a/r/data/raw/input3_move2loc_Mollweide.rds and b/r/data/raw/input3_move2loc_Mollweide.rds differ diff --git a/r/data/raw/input4_move2loc_LatLon.rds b/r/data/raw/input4_move2loc_LatLon.rds index 8d7825b..fac3cb4 100644 Binary files a/r/data/raw/input4_move2loc_LatLon.rds and b/r/data/raw/input4_move2loc_LatLon.rds differ diff --git a/r/data/raw/input4_move2loc_Mollweide.rds b/r/data/raw/input4_move2loc_Mollweide.rds index 0fc0fac..f6a8ac0 100644 Binary files a/r/data/raw/input4_move2loc_Mollweide.rds and b/r/data/raw/input4_move2loc_Mollweide.rds differ diff --git a/r/link.R b/r/link.R new file mode 100644 index 0000000..b234f00 --- /dev/null +++ b/r/link.R @@ -0,0 +1,99 @@ +# Converts between a move2 object and the CSV pair python reads and writes: +# +# move2 --writeLink()--> meta.csv + link.csv --(python)--> pickle +# pickle --(python)--> meta.csv + link.csv --readLink()--> move2 +# +# meta.csv carries what a plain CSV cannot: crs, tzone, and which columns hold time and track id. + +library("move2") +library("sf") +library("dplyr") +library("vctrs") +library("purrr") +library("rlang") + +# meta.csv: one row with what link.csv cannot say itself +linkMeta <- function(data) { + prj <- st_crs(data)[[1]] + tz <- attr(mt_time(data),'tzone') + data.frame(crs=c(prj), tzone=c(tz), timeColName=mt_time_column(data), trackIdColName=mt_track_id_column(data)) +} + +# link.csv: one row per location, the track attributes repeated on every row, geometry as coordinates +linkTable <- function(data) { + ## checking if there are columns in the track data that are a list. If yes, check if the content is the same, if yes remove list. If list columns are left over because content is different transform these into a character string (could be done as well as json, but think that average user will be more comfortable with text?. Easy to change in the future. See Issue #78 on move2) + if(any(sapply(mt_track_data(data), is_bare_list))){ + ## reduce all columns were entry is the same to one (so no list anymore) + data <- data |> mutate_track_data(across( + where( ~is_bare_list(.x) && all(purrr::map_lgl(.x, function(y) 1==length(unique(y)) ))), + ~do.call(vctrs::vec_c,purrr::map(.x, head,1)))) + if(any(sapply(mt_track_data(data), is_bare_list))){ + ## transform those that are still a list into a character string + data <- data |> mutate_track_data(across( + where( ~is_bare_list(.x) && any(purrr::map_lgl(.x, function(y) 1!=length(unique(y)) ))), + ~unlist(purrr::map(.x, paste, collapse=",")))) + } + } + data <- mt_as_event_attribute(data, names(mt_track_data(data))) + data <- dplyr::mutate(data, coords_x=sf::st_coordinates(data)[,1], + coords_y=sf::st_coordinates(data)[,2]) + data <- sf::st_drop_geometry(data) ## removes the sf geometry column from the table + sfc_cols <- names(data)[unlist(lapply(data, inherits, 'sfc'))] ## get the col names that are spacial + + for(x in sfc_cols){ ## converting the "point" columns into characters, ie into WKT (Well-known text) + data[[x]] <- st_as_text(data[[x]]) + } ## st_as_sfc() can be used to convert these columns back to spacial + + data.csv <- data.frame(data) + ## UTC instants: a wall-clock time repeats in the autumn DST hour and python could not tell the two apart; meta.csv keeps the tzone + ## + half a millisecond: %OS3 truncates, so 35.999 stored as 35.998999... would lose its millisecond + data.csv[,mt_time_column(data)] <- format(data.csv[,mt_time_column(data)] + 0.0005,format="%Y-%m-%d %H:%M:%OS3", tz="UTC") ## if time is 00:00:00 it gets rounded just to the date, and if miliseconds are .000 it gets rounded to seconds when saved as csv. This ensures this does not happen. All timestamps will always have miliseconds. + data.csv +} + +# meta.csv first, then link.csv - the order of v2.2.0 +writeLink <- function(data, bufferFile, metaFile) { + write.csv(linkMeta(data = data), metaFile, row.names=FALSE) + write.csv(linkTable(data = data), bufferFile, row.names=FALSE) +} + +# python writes logical values as True/False and a missing one as an empty field; read.csv keeps all +# three as text. A column holding nothing else goes back to TRUE/FALSE/NA; any other column is untouched. +logicalFromPython <- function(x) { + if (!is.character(x)) { + return(x) + } + values <- x[!is.na(x) & x != ""] + if (length(values) == 0 || !all(values %in% c("True", "False"))) { + return(x) + } + ifelse(x == "True", TRUE, ifelse(x == "False", FALSE, NA)) +} + +# NULL for an empty buffer or a buffer without rows +readLink <- function(bufferFile, metaFile) { + # python writes an empty buffer for an empty TrajectoryCollection + if (file.size(bufferFile) == 0) { + return(NULL) + } + + # always includes "coords_x", "coords_y" + datapy <- read.csv(bufferFile, header=TRUE) + # always includes crs, tzone, timeColName, trackIdColName + meta <- read.csv(metaFile, header=TRUE) + + if (dim(datapy)[1]==0) { + return(NULL) + } + + datapy[] <- lapply(datapy, logicalFromPython) + # %OS: python writes fractions of a second, %S would drop them + datapy[meta$timeColName] <- as.POSIXct(datapy %>% select(meta$timeColName) %>% sapply(as.character) %>% as.vector,format="%Y-%m-%d %H:%M:%OS", tz=meta$tzone) + result <- mt_as_move2(datapy, + coords = c("coords_x", "coords_y"), + time_column= meta$timeColName, + track_id_column= meta$trackIdColName, + crs= meta$crs + ) + result |> dplyr::arrange(mt_track_id(result),mt_time(result)) +} diff --git a/r/rds_2_csv.R b/r/rds_2_csv.R index 5b0d2d0..47b4005 100644 --- a/r/rds_2_csv.R +++ b/r/rds_2_csv.R @@ -1,63 +1,18 @@ -source("src/common/logger.R") -source("src/common/runtime_configuration.R") -source("src/io/app_files.R") -source("src/io/io_handler.R") -source("src/io/rds.R") - -library("move2") -library("sf") -library("dplyr") -library("vctrs") -library("purrr") -library("rlang") +library("moveapps") +moveapps::logger.init() +source("link.R") tryCatch( { Sys.setenv(tz="UTC") - - data <- readInput(sourceFile()) # .rds - - ## get meta.csv - prj <- st_crs(data)[[1]] - tz <- attr(mt_time(data),'tzone') - meta <- data.frame(crs=c(prj), tzone=c(tz), timeColName=mt_time_column(data), trackIdColName=mt_track_id_column(data)) - write.csv(meta,Sys.getenv(x = "LINK_R_PYTHON_META"),row.names=FALSE) - - ## get link.csv - ## checking if there are columns in the track data that are a list. If yes, check if the content is the same, if yes remove list. If list columns are left over because content is different transform these into a character string (could be done as well as json, but think that average user will be more comfortable with text?. Easy to change in the future. See Issue #78 on move2) - if(any(sapply(mt_track_data(data), is_bare_list))){ - ## reduce all columns were entry is the same to one (so no list anymore) - data <- data |> mutate_track_data(across( - where( ~is_bare_list(.x) && all(purrr::map_lgl(.x, function(y) 1==length(unique(y)) ))), - ~do.call(vctrs::vec_c,purrr::map(.x, head,1)))) - if(any(sapply(mt_track_data(data), is_bare_list))){ - ## transform those that are still a list into a character string - data <- data |> mutate_track_data(across( - where( ~is_bare_list(.x) && any(purrr::map_lgl(.x, function(y) 1!=length(unique(y)) ))), - ~unlist(purrr::map(.x, paste, collapse=",")))) - } - } - data <- mt_as_event_attribute(data, names(mt_track_data(data))) - data <- dplyr::mutate(data, coords_x=sf::st_coordinates(data)[,1], - coords_y=sf::st_coordinates(data)[,2]) - data <- sf::st_drop_geometry(data) ## removes the sf geometry column from the table - sfc_cols <- names(data)[unlist(lapply(data, inherits, 'sfc'))] ## get the col names that are spacial - - for(x in sfc_cols){ ## converting the "point" columns into characters, ie into WKT (Well-known text) - data[[x]] <- st_as_text(data[[x]]) - } ## st_as_sfc() can be used to convert these columns back to spacial - - data.csv <- data.frame(data) - data.csv[,mt_time_column(data)] <- format(data.csv[,mt_time_column(data)],format="%Y-%m-%d %H:%M:%OS3") ## if time is 00:00:00 it gets rounded just to the date, and if miliseconds are .000 it gets rounded to seconds when saved as csv. This ensures this does not happen. All timestamps will always have miliseconds. - write.csv(data.csv,Sys.getenv(x = "LINK_R_PYTHON_BUFFER"),row.names=FALSE) - + writeLink(data = moveapps::readInput(moveapps::sourceFile()), bufferFile = Sys.getenv(x = "LINK_R_PYTHON_BUFFER"), metaFile = Sys.getenv(x = "LINK_R_PYTHON_META")) }, error = function(e) { # error handler picks up where error was generated print(paste("ERROR: ", e)) - storeToFile(e, errorFile()) + moveapps::storeToFile(e, moveapps::errorFile()) stop(e) # re-throw the exception } ) diff --git a/r/renv.lock b/r/renv.lock index 6b63033..021af8d 100644 --- a/r/renv.lock +++ b/r/renv.lock @@ -1,635 +1,1742 @@ { "R": { - "Version": "4.3.2", + "Version": "4.6.1", "Repositories": [ { "Name": "CRAN", - "URL": "https://ftp.gwdg.de/pub/misc/cran" + "URL": "https://cloud.r-project.org" } ] }, "Packages": { "DBI": { "Package": "DBI", - "Version": "1.2.0", + "Version": "1.3.0", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "methods" + "Title": "R Database Interface", + "Date": "2026-02-11", + "Authors@R": "c( person(\"R Special Interest Group on Databases (R-SIG-DB)\", role = \"aut\"), person(\"Hadley\", \"Wickham\", role = \"aut\"), person(\"Kirill\", \"Müller\", , \"kirill@cynkra.com\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-1416-3412\")), person(\"R Consortium\", role = \"fnd\") )", + "Description": "A database interface definition for communication between R and relational database management systems. All classes in this package are virtual and need to be extended by the various R/DBMS implementations.", + "License": "LGPL (>= 2.1)", + "URL": "https://dbi.r-dbi.org, https://github.com/r-dbi/DBI", + "BugReports": "https://github.com/r-dbi/DBI/issues", + "Depends": [ + "methods", + "R (>= 3.0.0)" ], - "Hash": "3e0051431dff9acfe66c23765e55c556" + "Suggests": [ + "arrow", + "blob", + "callr", + "covr", + "DBItest (>= 1.8.2)", + "dbplyr", + "downlit", + "dplyr", + "glue", + "hms", + "knitr", + "magrittr", + "nanoarrow (>= 0.3.0.1)", + "otel", + "otelsdk", + "RMariaDB", + "rmarkdown", + "rprojroot", + "RSQLite (>= 1.1-2)", + "testthat (>= 3.0.0)", + "vctrs", + "xml2" + ], + "VignetteBuilder": "knitr", + "Config/autostyle/scope": "line_breaks", + "Config/autostyle/strict": "false", + "Config/Needs/check": "r-dbi/DBItest", + "Config/Needs/website": "r-dbi/DBItest, r-dbi/dbitemplate, adbi, AzureKusto, bigrquery, DatabaseConnector, dittodb, duckdb, implyr, lazysf, odbc, pool, RAthena, IMSMWU/RClickhouse, RH2, RJDBC, RMariaDB, RMySQL, RPostgres, RPostgreSQL, RPresto, RSQLite, sergeant, sparklyr, withr", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.3.9000", + "NeedsCompilation": "no", + "Author": "R Special Interest Group on Databases (R-SIG-DB) [aut], Hadley Wickham [aut], Kirill Müller [aut, cre] (ORCID: ), R Consortium [fnd]", + "Maintainer": "Kirill Müller ", + "Repository": "CRAN" }, "KernSmooth": { "Package": "KernSmooth", - "Version": "2.23-22", + "Version": "2.23-26", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", + "Priority": "recommended", + "Date": "2024-12-10", + "Title": "Functions for Kernel Smoothing Supporting Wand & Jones (1995)", + "Authors@R": "c(person(\"Matt\", \"Wand\", role = \"aut\", email = \"Matt.Wand@uts.edu.au\"), person(\"Cleve\", \"Moler\", role = \"ctb\", comment = \"LINPACK routines in src/d*\"), person(\"Brian\", \"Ripley\", role = c(\"trl\", \"cre\", \"ctb\"), email = \"Brian.Ripley@R-project.org\", comment = \"R port and updates\"))", + "Note": "Maintainers are not available to give advice on using a package they did not author.", + "Depends": [ + "R (>= 2.5.0)", "stats" ], - "Hash": "2fecebc3047322fa5930f74fae5de70f" + "Suggests": [ + "MASS", + "carData" + ], + "Description": "Functions for kernel smoothing (and density estimation) corresponding to the book: Wand, M.P. and Jones, M.C. (1995) \"Kernel Smoothing\".", + "License": "Unlimited", + "ByteCompile": "yes", + "NeedsCompilation": "yes", + "Author": "Matt Wand [aut], Cleve Moler [ctb] (LINPACK routines in src/d*), Brian Ripley [trl, cre, ctb] (R port and updates)", + "Maintainer": "Brian Ripley ", + "Repository": "CRAN" }, "MASS": { "Package": "MASS", - "Version": "7.3-60", + "Version": "7.3-65", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", + "Priority": "recommended", + "Date": "2025-02-19", + "Revision": "$Rev: 3681 $", + "Depends": [ + "R (>= 4.4.0)", "grDevices", "graphics", - "methods", "stats", "utils" ], - "Hash": "a56a6365b3fa73293ea8d084be0d9bb0" + "Imports": [ + "methods" + ], + "Suggests": [ + "lattice", + "nlme", + "nnet", + "survival" + ], + "Authors@R": "c(person(\"Brian\", \"Ripley\", role = c(\"aut\", \"cre\", \"cph\"), email = \"Brian.Ripley@R-project.org\"), person(\"Bill\", \"Venables\", role = c(\"aut\", \"cph\")), person(c(\"Douglas\", \"M.\"), \"Bates\", role = \"ctb\"), person(\"Kurt\", \"Hornik\", role = \"trl\", comment = \"partial port ca 1998\"), person(\"Albrecht\", \"Gebhardt\", role = \"trl\", comment = \"partial port ca 1998\"), person(\"David\", \"Firth\", role = \"ctb\", comment = \"support functions for polr\"))", + "Description": "Functions and datasets to support Venables and Ripley, \"Modern Applied Statistics with S\" (4th edition, 2002).", + "Title": "Support Functions and Datasets for Venables and Ripley's MASS", + "LazyData": "yes", + "ByteCompile": "yes", + "License": "GPL-2 | GPL-3", + "URL": "http://www.stats.ox.ac.uk/pub/MASS4/", + "Contact": "", + "NeedsCompilation": "yes", + "Author": "Brian Ripley [aut, cre, cph], Bill Venables [aut, cph], Douglas M. Bates [ctb], Kurt Hornik [trl] (partial port ca 1998), Albrecht Gebhardt [trl] (partial port ca 1998), David Firth [ctb] (support functions for polr)", + "Maintainer": "Brian Ripley ", + "Repository": "CRAN" }, "R6": { "Package": "R6", - "Version": "2.5.1", + "Version": "2.6.1", "Source": "Repository", - "Repository": "RSPM", - "Requirements": [ - "R" + "Title": "Encapsulated Classes with Reference Semantics", + "Authors@R": "c( person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Creates classes with reference semantics, similar to R's built-in reference classes. Compared to reference classes, R6 classes are simpler and lighter-weight, and they are not built on S4 classes so they do not require the methods package. These classes allow public and private members, and they support inheritance, even when the classes are defined in different packages.", + "License": "MIT + file LICENSE", + "URL": "https://r6.r-lib.org, https://github.com/r-lib/R6", + "BugReports": "https://github.com/r-lib/R6/issues", + "Depends": [ + "R (>= 3.6)" + ], + "Suggests": [ + "lobstr", + "testthat (>= 3.0.0)" ], - "Hash": "470851b6d5d0ac559e9d01bb352b4021" + "Config/Needs/website": "tidyverse/tidytemplate, ggplot2, microbenchmark, scales", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "NeedsCompilation": "no", + "Author": "Winston Chang [aut, cre], Posit Software, PBC [cph, fnd]", + "Maintainer": "Winston Chang ", + "Repository": "CRAN" }, "Rcpp": { "Package": "Rcpp", - "Version": "1.0.12", + "Version": "1.1.2", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ + "Title": "Seamless R and C++ Integration", + "Date": "2026-07-01", + "Authors@R": "c(person(\"Dirk\", \"Eddelbuettel\", role = c(\"aut\", \"cre\"), email = \"edd@debian.org\", comment = c(ORCID = \"0000-0001-6419-907X\")), person(\"Romain\", \"Francois\", role = \"aut\", comment = c(ORCID = \"0000-0002-2444-4226\")), person(\"JJ\", \"Allaire\", role = \"aut\", comment = c(ORCID = \"0000-0003-0174-9868\")), person(\"Kevin\", \"Ushey\", role = \"aut\", comment = c(ORCID = \"0000-0003-2880-7407\")), person(\"Qiang\", \"Kou\", role = \"aut\", comment = c(ORCID = \"0000-0001-6786-5453\")), person(\"Nathan\", \"Russell\", role = \"aut\"), person(\"Iñaki\", \"Ucar\", role = \"aut\", comment = c(ORCID = \"0000-0001-6403-5550\")), person(\"Doug\", \"Bates\", role = \"aut\", comment = c(ORCID = \"0000-0001-8316-9503\")), person(\"John\", \"Chambers\", role = \"aut\"))", + "Description": "The 'Rcpp' package provides R functions as well as C++ classes which offer a seamless integration of R and C++. Many R data types and objects can be mapped back and forth to C++ equivalents which facilitates both writing of new code as well as easier integration of third-party libraries. Documentation about 'Rcpp' is provided by several vignettes included in this package, via the 'Rcpp Gallery' site at , the paper by Eddelbuettel and Francois (2011, ), the book by Eddelbuettel (2013, ) and the paper by Eddelbuettel and Balamuta (2018, ); see 'citation(\"Rcpp\")' for details.", + "Depends": [ + "R (>= 3.5.0)" + ], + "Imports": [ "methods", "utils" ], - "Hash": "5ea2700d21e038ace58269ecdbeb9ec0" - }, - "askpass": { - "Package": "askpass", - "Version": "1.2.0", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "sys" + "Suggests": [ + "tinytest", + "inline", + "rbenchmark", + "pkgKitten (>= 0.1.2)" ], - "Hash": "cad6cf7f1d5f6e906700b9d3e718c796" + "URL": "https://www.rcpp.org, https://dirk.eddelbuettel.com/code/rcpp.html, https://github.com/RcppCore/Rcpp", + "License": "GPL (>= 2)", + "BugReports": "https://github.com/RcppCore/Rcpp/issues", + "RoxygenNote": "6.1.1", + "Encoding": "UTF-8", + "VignetteBuilder": "Rcpp", + "NeedsCompilation": "yes", + "Author": "Dirk Eddelbuettel [aut, cre] (ORCID: ), Romain Francois [aut] (ORCID: ), JJ Allaire [aut] (ORCID: ), Kevin Ushey [aut] (ORCID: ), Qiang Kou [aut] (ORCID: ), Nathan Russell [aut], Iñaki Ucar [aut] (ORCID: ), Doug Bates [aut] (ORCID: ), John Chambers [aut]", + "Maintainer": "Dirk Eddelbuettel ", + "Repository": "CRAN" }, "assertthat": { "Package": "assertthat", "Version": "0.2.1", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ + "Title": "Easy Pre and Post Assertions", + "Authors@R": "person(\"Hadley\", \"Wickham\", , \"hadley@rstudio.com\", c(\"aut\", \"cre\"))", + "Description": "An extension to stopifnot() that makes it easy to declare the pre and post conditions that you code should satisfy, while also producing friendly error messages so that your users know what's gone wrong.", + "License": "GPL-3", + "Imports": [ "tools" ], - "Hash": "50c838a310445e954bc13f26f26a6ecf" + "Suggests": [ + "testthat", + "covr" + ], + "RoxygenNote": "6.0.1", + "Collate": "'assert-that.r' 'on-failure.r' 'assertions-file.r' 'assertions-scalar.R' 'assertions.r' 'base.r' 'base-comparison.r' 'base-is.r' 'base-logical.r' 'base-misc.r' 'utils.r' 'validate-that.R'", + "NeedsCompilation": "no", + "Author": "Hadley Wickham [aut, cre]", + "Maintainer": "Hadley Wickham ", + "Repository": "CRAN", + "Encoding": "UTF-8" }, "bit": { "Package": "bit", - "Version": "4.0.5", + "Version": "4.6.0", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R" + "Title": "Classes and Methods for Fast Memory-Efficient Boolean Selections", + "Authors@R": "c( person(\"Michael\", \"Chirico\", email = \"MichaelChirico4@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Jens\", \"Oehlschlägel\", role = \"aut\"), person(\"Brian\", \"Ripley\", role = \"ctb\") )", + "Depends": [ + "R (>= 3.4.0)" ], - "Hash": "d242abec29412ce988848d0294b208fd" + "Suggests": [ + "testthat (>= 3.0.0)", + "roxygen2", + "knitr", + "markdown", + "rmarkdown", + "microbenchmark", + "bit64 (>= 4.0.0)", + "ff (>= 4.0.0)" + ], + "Description": "Provided are classes for boolean and skewed boolean vectors, fast boolean methods, fast unique and non-unique integer sorting, fast set operations on sorted and unsorted sets of integers, and foundations for ff (range index, compression, chunked processing).", + "License": "GPL-2 | GPL-3", + "LazyLoad": "yes", + "ByteCompile": "yes", + "Encoding": "UTF-8", + "URL": "https://github.com/r-lib/bit", + "VignetteBuilder": "knitr, rmarkdown", + "RoxygenNote": "7.3.2", + "Config/testthat/edition": "3", + "NeedsCompilation": "yes", + "Author": "Michael Chirico [aut, cre], Jens Oehlschlägel [aut], Brian Ripley [ctb]", + "Maintainer": "Michael Chirico ", + "Repository": "CRAN" }, "bit64": { "Package": "bit64", - "Version": "4.0.5", + "Version": "4.8.6", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "bit", + "Title": "A S3 Class for Vectors of 64bit Integers", + "Authors@R": "c( person(\"Michael\", \"Chirico\", email=\"michaelchirico4@gmail.com\", role=c(\"aut\", \"cre\")), person(\"Jens\", \"Oehlschlägel\", role=\"aut\"), person(\"Leonardo\", \"Silvestri\", role=\"ctb\"), person(\"Ofek\", \"Shilon\", role=\"ctb\"), person(\"Christian\", \"Ullerich\", role=\"ctb\") )", + "Depends": [ + "R (>= 3.5.0)" + ], + "Description": "Package 'bit64' provides serializable S3 atomic 64bit (signed) integers. These are useful for handling database keys and exact counting in +-2^63. WARNING: do not use them as replacement for 32bit integers, integer64 are not supported for subscripting by R-core and they have different semantics when combined with double, e.g. integer64 + double => integer64. Class integer64 can be used in vectors, matrices, arrays and data.frames. Methods are available for coercion from and to logicals, integers, doubles, characters and factors as well as many elementwise and summary functions. Many fast algorithmic operations such as 'match' and 'order' support inter- active data exploration and manipulation and optionally leverage caching.", + "License": "GPL-2 | GPL-3", + "LazyLoad": "yes", + "ByteCompile": "yes", + "URL": "https://github.com/r-lib/bit64, https://bit64.r-lib.org", + "Encoding": "UTF-8", + "Imports": [ + "bit (>= 4.0.0)", + "graphics", "methods", "stats", "utils" ], - "Hash": "9fe98599ca456d6552421db0d6772d8f" + "Suggests": [ + "patrick (>= 0.3.0)", + "testthat (>= 3.3.0)", + "withr" + ], + "Config/testthat/edition": "3", + "Config/Needs/development": "patrick, testthat", + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/roxygen2/version": "8.0.0", + "NeedsCompilation": "yes", + "Author": "Michael Chirico [aut, cre], Jens Oehlschlägel [aut], Leonardo Silvestri [ctb], Ofek Shilon [ctb], Christian Ullerich [ctb]", + "Maintainer": "Michael Chirico ", + "Repository": "CRAN" }, "class": { "Package": "class", - "Version": "7.3-22", + "Version": "7.3-23", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "MASS", - "R", + "Priority": "recommended", + "Date": "2025-01-01", + "Depends": [ + "R (>= 3.0.0)", "stats", "utils" ], - "Hash": "f91f6b29f38b8c280f2b9477787d4bb2" + "Imports": [ + "MASS" + ], + "Authors@R": "c(person(\"Brian\", \"Ripley\", role = c(\"aut\", \"cre\", \"cph\"), email = \"Brian.Ripley@R-project.org\"), person(\"William\", \"Venables\", role = \"cph\"))", + "Description": "Various functions for classification, including k-nearest neighbour, Learning Vector Quantization and Self-Organizing Maps.", + "Title": "Functions for Classification", + "ByteCompile": "yes", + "License": "GPL-2 | GPL-3", + "URL": "http://www.stats.ox.ac.uk/pub/MASS4/", + "NeedsCompilation": "yes", + "Author": "Brian Ripley [aut, cre, cph], William Venables [cph]", + "Maintainer": "Brian Ripley ", + "Repository": "CRAN" }, "classInt": { "Package": "classInt", - "Version": "0.4-10", + "Version": "0.4-11", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "KernSmooth", - "R", - "class", - "e1071", + "Date": "2025-01-06", + "Title": "Choose Univariate Class Intervals", + "Authors@R": "c( person(\"Roger\", \"Bivand\", role=c(\"aut\", \"cre\"), email=\"Roger.Bivand@nhh.no\", comment=c(ORCID=\"0000-0003-2392-6140\")), person(\"Bill\", \"Denney\", role=\"ctb\", comment=c(ORCID=\"0000-0002-5759-428X\")), person(\"Richard\", \"Dunlap\", role=\"ctb\"), person(\"Diego\", \"Hernangómez\", role=\"ctb\", comment=c(ORCID=\"0000-0001-8457-4658\")), person(\"Hisaji\", \"Ono\", role=\"ctb\"), person(\"Josiah\", \"Parry\", role = \"ctb\", comment = c(ORCID = \"0000-0001-9910-865X\")), person(\"Matthieu\", \"Stigler\", role=\"ctb\", comment =c(ORCID=\"0000-0002-6802-4290\")))", + "Depends": [ + "R (>= 2.2)" + ], + "Imports": [ "grDevices", + "stats", "graphics", - "stats" + "e1071", + "class", + "KernSmooth" ], - "Hash": "f5a40793b1ae463a7ffb3902a95bf864" + "Suggests": [ + "spData (>= 0.2.6.2)", + "units", + "knitr", + "rmarkdown", + "tinytest" + ], + "NeedsCompilation": "yes", + "Description": "Selected commonly used methods for choosing univariate class intervals for mapping or other graphics purposes.", + "License": "GPL (>= 2)", + "URL": "https://r-spatial.github.io/classInt/, https://github.com/r-spatial/classInt/", + "BugReports": "https://github.com/r-spatial/classInt/issues/", + "RoxygenNote": "6.1.1", + "Encoding": "UTF-8", + "VignetteBuilder": "knitr", + "Author": "Roger Bivand [aut, cre] (), Bill Denney [ctb] (), Richard Dunlap [ctb], Diego Hernangómez [ctb] (), Hisaji Ono [ctb], Josiah Parry [ctb] (), Matthieu Stigler [ctb] ()", + "Maintainer": "Roger Bivand ", + "Repository": "CRAN" }, "cli": { "Package": "cli", - "Version": "3.6.2", + "Version": "3.6.6", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", + "Title": "Helpers for Developing Command Line Interfaces", + "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"gabor@posit.co\", role = c(\"aut\", \"cre\")), person(\"Hadley\", \"Wickham\", role = \"ctb\"), person(\"Kirill\", \"Müller\", role = \"ctb\"), person(\"Salim\", \"Brüggemann\", , \"salim-b@pm.me\", role = \"ctb\", comment = c(ORCID = \"0000-0002-5329-5987\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", + "Description": "A suite of tools to build attractive command line interfaces ('CLIs'), from semantic elements: headings, lists, alerts, paragraphs, etc. Supports custom themes via a 'CSS'-like language. It also contains a number of lower level 'CLI' elements: rules, boxes, trees, and 'Unicode' symbols with 'ASCII' alternatives. It support ANSI colors and text styles as well.", + "License": "MIT + file LICENSE", + "URL": "https://cli.r-lib.org, https://github.com/r-lib/cli", + "BugReports": "https://github.com/r-lib/cli/issues", + "Depends": [ + "R (>= 3.4)" + ], + "Imports": [ "utils" ], - "Hash": "1216ac65ac55ec0058a6f75d7ca0fd52" + "Suggests": [ + "callr", + "covr", + "crayon", + "digest", + "glue (>= 1.6.0)", + "grDevices", + "htmltools", + "htmlwidgets", + "knitr", + "methods", + "processx", + "ps (>= 1.3.4.9000)", + "rlang (>= 1.0.2.9003)", + "rmarkdown", + "rprojroot", + "rstudioapi", + "testthat (>= 3.2.0)", + "tibble", + "whoami", + "withr" + ], + "Config/Needs/website": "r-lib/asciicast, bench, brio, cpp11, decor, desc, fansi, prettyunits, sessioninfo, tidyverse/tidytemplate, usethis, vctrs", + "Config/testthat/edition": "3", + "Config/usethis/last-upkeep": "2025-04-25", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2.9000", + "NeedsCompilation": "yes", + "Author": "Gábor Csárdi [aut, cre], Hadley Wickham [ctb], Kirill Müller [ctb], Salim Brüggemann [ctb] (ORCID: ), Posit Software, PBC [cph, fnd] (ROR: )", + "Maintainer": "Gábor Csárdi ", + "Repository": "CRAN" }, "cpp11": { "Package": "cpp11", - "Version": "0.4.7", + "Version": "0.5.5", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R" + "Title": "A C++11 Interface for R's C Interface", + "Authors@R": "c( person(\"Davis\", \"Vaughan\", email = \"davis@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-4777-038X\")), person(\"Jim\",\"Hester\", role = \"aut\", comment = c(ORCID = \"0000-0002-2739-7082\")), person(\"Romain\", \"François\", role = \"aut\", comment = c(ORCID = \"0000-0002-2444-4226\")), person(\"Benjamin\", \"Kietzman\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Provides a header only, C++11 interface to R's C interface. Compared to other approaches 'cpp11' strives to be safe against long jumps from the C API as well as C++ exceptions, conform to normal R function semantics and supports interaction with 'ALTREP' vectors.", + "License": "MIT + file LICENSE", + "URL": "https://cpp11.r-lib.org, https://github.com/r-lib/cpp11", + "BugReports": "https://github.com/r-lib/cpp11/issues", + "Depends": [ + "R (>= 4.0.0)" + ], + "Suggests": [ + "bench", + "brio", + "callr", + "cli", + "covr", + "decor", + "desc", + "ggplot2", + "glue", + "knitr", + "lobstr", + "mockery", + "progress", + "rmarkdown", + "scales", + "Rcpp", + "testthat (>= 3.2.0)", + "tibble", + "utils", + "vctrs", + "withr" ], - "Hash": "5a295d7d963cc5035284dcdbaf334f4e" + "VignetteBuilder": "knitr", + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Config/Needs/cpp11/cpp_register": "brio, cli, decor, desc, glue, tibble, vctrs", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.3", + "NeedsCompilation": "no", + "Author": "Davis Vaughan [aut, cre] (ORCID: ), Jim Hester [aut] (ORCID: ), Romain François [aut] (ORCID: ), Benjamin Kietzman [ctb], Posit Software, PBC [cph, fnd]", + "Maintainer": "Davis Vaughan ", + "Repository": "CRAN" }, "crayon": { "Package": "crayon", - "Version": "1.5.2", + "Version": "1.5.3", "Source": "Repository", - "Repository": "RSPM", - "Requirements": [ + "Title": "Colored Terminal Output", + "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Brodie\", \"Gaslam\", , \"brodie.gaslam@yahoo.com\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "The crayon package is now superseded. Please use the 'cli' package for new projects. Colored terminal output on terminals that support 'ANSI' color and highlight codes. It also works in 'Emacs' 'ESS'. 'ANSI' color support is automatically detected. Colors and highlighting can be combined and nested. New styles can also be created easily. This package was inspired by the 'chalk' 'JavaScript' project.", + "License": "MIT + file LICENSE", + "URL": "https://r-lib.github.io/crayon/, https://github.com/r-lib/crayon", + "BugReports": "https://github.com/r-lib/crayon/issues", + "Imports": [ "grDevices", "methods", "utils" ], - "Hash": "e8a1e41acf02548751f45c718d55aa6a" - }, - "curl": { - "Package": "curl", - "Version": "5.2.0", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R" + "Suggests": [ + "mockery", + "rstudioapi", + "testthat", + "withr" ], - "Hash": "ce88d13c0b10fe88a37d9c59dba2d7f9" + "Config/Needs/website": "tidyverse/tidytemplate", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.1", + "Collate": "'aaa-rstudio-detect.R' 'aaaa-rematch2.R' 'aab-num-ansi-colors.R' 'aac-num-ansi-colors.R' 'ansi-256.R' 'ansi-palette.R' 'combine.R' 'string.R' 'utils.R' 'crayon-package.R' 'disposable.R' 'enc-utils.R' 'has_ansi.R' 'has_color.R' 'link.R' 'styles.R' 'machinery.R' 'parts.R' 'print.R' 'style-var.R' 'show.R' 'string_operations.R'", + "NeedsCompilation": "no", + "Author": "Gábor Csárdi [aut, cre], Brodie Gaslam [ctb], Posit Software, PBC [cph, fnd]", + "Maintainer": "Gábor Csárdi ", + "Repository": "CRAN" }, "dplyr": { "Package": "dplyr", - "Version": "1.1.4", + "Version": "1.2.1", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "R6", - "cli", + "Type": "Package", + "Title": "A Grammar of Data Manipulation", + "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Romain\", \"François\", role = \"aut\", comment = c(ORCID = \"0000-0002-2444-4226\")), person(\"Lionel\", \"Henry\", role = \"aut\"), person(\"Kirill\", \"Müller\", role = \"aut\", comment = c(ORCID = \"0000-0002-1416-3412\")), person(\"Davis\", \"Vaughan\", , \"davis@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4777-038X\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "A fast, consistent tool for working with data frame like objects, both in memory and out of memory.", + "License": "MIT + file LICENSE", + "URL": "https://dplyr.tidyverse.org, https://github.com/tidyverse/dplyr", + "BugReports": "https://github.com/tidyverse/dplyr/issues", + "Depends": [ + "R (>= 4.1.0)" + ], + "Imports": [ + "cli (>= 3.6.2)", "generics", - "glue", - "lifecycle", - "magrittr", + "glue (>= 1.3.2)", + "lifecycle (>= 1.0.5)", + "magrittr (>= 1.5)", "methods", - "pillar", - "rlang", - "tibble", - "tidyselect", + "pillar (>= 1.9.0)", + "R6", + "rlang (>= 1.1.7)", + "tibble (>= 3.2.0)", + "tidyselect (>= 1.2.0)", "utils", - "vctrs" + "vctrs (>= 0.7.1)" + ], + "Suggests": [ + "broom", + "covr", + "DBI", + "dbplyr (>= 2.2.1)", + "ggplot2", + "knitr", + "Lahman", + "lobstr", + "nycflights13", + "purrr", + "rmarkdown", + "RSQLite", + "stringi (>= 1.7.6)", + "testthat (>= 3.1.5)", + "tidyr (>= 1.3.0)", + "withr" ], - "Hash": "fedd9d00c2944ff00a0e2696ccf048ec" + "VignetteBuilder": "knitr", + "Config/build/compilation-database": "true", + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "LazyData": "true", + "RoxygenNote": "7.3.3", + "NeedsCompilation": "yes", + "Author": "Hadley Wickham [aut, cre] (ORCID: ), Romain François [aut] (ORCID: ), Lionel Henry [aut], Kirill Müller [aut] (ORCID: ), Davis Vaughan [aut] (ORCID: ), Posit Software, PBC [cph, fnd]", + "Maintainer": "Hadley Wickham ", + "Repository": "CRAN" }, "e1071": { "Package": "e1071", - "Version": "1.7-14", + "Version": "1.7-17", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "class", - "grDevices", + "Title": "Misc Functions of the Department of Statistics, Probability Theory Group (Formerly: E1071), TU Wien", + "Imports": [ "graphics", - "methods", - "proxy", + "grDevices", + "class", "stats", - "utils" + "methods", + "utils", + "proxy" + ], + "Suggests": [ + "cluster", + "mlbench", + "nnet", + "randomForest", + "rpart", + "SparseM", + "xtable", + "Matrix", + "MASS", + "slam" ], - "Hash": "4ef372b716824753719a8a38b258442d" + "Authors@R": "c(person(given = \"David\", family = \"Meyer\", role = c(\"aut\", \"cre\"), email = \"David.Meyer@R-project.org\", comment = c(ORCID = \"0000-0002-5196-3048\")), person(given = \"Evgenia\", family = \"Dimitriadou\", role = c(\"aut\",\"cph\")), person(given = \"Kurt\", family = \"Hornik\", role = \"aut\", email = \"Kurt.Hornik@R-project.org\", comment = c(ORCID = \"0000-0003-4198-9911\")), person(given = \"Andreas\", family = \"Weingessel\", role = \"aut\"), person(given = \"Friedrich\", family = \"Leisch\", role = \"aut\"), person(given = \"Chih-Chung\", family = \"Chang\", role = c(\"ctb\",\"cph\"), comment = \"libsvm C++-code\"), person(given = \"Chih-Chen\", family = \"Lin\", role = c(\"ctb\",\"cph\"), comment = \"libsvm C++-code\"))", + "Description": "Functions for latent class analysis, short time Fourier transform, fuzzy clustering, support vector machines, shortest path computation, bagged clustering, naive Bayes classifier, generalized k-nearest neighbour ...", + "License": "GPL-2 | GPL-3", + "LazyLoad": "yes", + "NeedsCompilation": "yes", + "Author": "David Meyer [aut, cre] (ORCID: ), Evgenia Dimitriadou [aut, cph], Kurt Hornik [aut] (ORCID: ), Andreas Weingessel [aut], Friedrich Leisch [aut], Chih-Chung Chang [ctb, cph] (libsvm C++-code), Chih-Chen Lin [ctb, cph] (libsvm C++-code)", + "Maintainer": "David Meyer ", + "Repository": "CRAN", + "Encoding": "UTF-8" }, - "fansi": { - "Package": "fansi", - "Version": "1.0.6", + "fs": { + "Package": "fs", + "Version": "2.1.0", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "grDevices", - "utils" + "Title": "Cross-Platform File System Operations Based on 'libuv'", + "Authors@R": "c( person(\"Jim\", \"Hester\", role = \"aut\"), person(\"Hadley\", \"Wickham\", role = \"aut\"), person(\"Gábor\", \"Csárdi\", role = \"aut\"), person(\"Jeroen\", \"Ooms\", , \"jeroenooms@gmail.com\", role = \"cre\"), person(\"libuv project contributors\", role = \"cph\", comment = \"libuv library\"), person(\"Joyent, Inc. and other Node contributors\", role = \"cph\", comment = \"libuv library\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", + "Description": "A cross-platform interface to file system operations, built on top of the 'libuv' C library.", + "License": "MIT + file LICENSE", + "URL": "https://fs.r-lib.org, https://github.com/r-lib/fs", + "BugReports": "https://github.com/r-lib/fs/issues", + "Depends": [ + "R (>= 4.1)" ], - "Hash": "962174cf2aeb5b9eea581522286a911f" + "Imports": [ + "methods" + ], + "Suggests": [ + "covr", + "crayon", + "knitr", + "pillar (>= 1.0.0)", + "rmarkdown", + "spelling", + "testthat (>= 3.0.0)", + "tibble (>= 1.1.0)", + "vctrs (>= 0.3.0)", + "withr" + ], + "VignetteBuilder": "knitr", + "SystemRequirements": "libuv: libuv-devel (rpm) or libuv1-dev (deb). Alternatively to build the vendored libuv 'cmake' is required. GNU make.", + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Config/usethis/last-upkeep": "2025-04-23", + "Copyright": "file COPYRIGHTS", + "Encoding": "UTF-8", + "Language": "en-US", + "RoxygenNote": "7.3.3", + "NeedsCompilation": "yes", + "Author": "Jim Hester [aut], Hadley Wickham [aut], Gábor Csárdi [aut], Jeroen Ooms [cre], libuv project contributors [cph] (libuv library), Joyent, Inc. and other Node contributors [cph] (libuv library), Posit Software, PBC [cph, fnd] (ROR: )", + "Maintainer": "Jeroen Ooms ", + "Repository": "CRAN" }, "generics": { "Package": "generics", - "Version": "0.1.3", + "Version": "0.1.4", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", + "Title": "Common S3 Generics not Provided by Base R Methods Related to Model Fitting", + "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Max\", \"Kuhn\", , \"max@posit.co\", role = \"aut\"), person(\"Davis\", \"Vaughan\", , \"davis@posit.co\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"https://ror.org/03wc8by49\")) )", + "Description": "In order to reduce potential package dependencies and conflicts, generics provides a number of commonly used S3 generics.", + "License": "MIT + file LICENSE", + "URL": "https://generics.r-lib.org, https://github.com/r-lib/generics", + "BugReports": "https://github.com/r-lib/generics/issues", + "Depends": [ + "R (>= 3.6)" + ], + "Imports": [ "methods" ], - "Hash": "15e9634c0fcd294799e9b2e929ed1b86" + "Suggests": [ + "covr", + "pkgload", + "testthat (>= 3.0.0)", + "tibble", + "withr" + ], + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "NeedsCompilation": "no", + "Author": "Hadley Wickham [aut, cre] (ORCID: ), Max Kuhn [aut], Davis Vaughan [aut], Posit Software, PBC [cph, fnd] (ROR: )", + "Maintainer": "Hadley Wickham ", + "Repository": "CRAN" }, "glue": { "Package": "glue", - "Version": "1.7.0", + "Version": "1.8.1", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", + "Title": "Interpreted String Literals", + "Authors@R": "c( person(\"Jim\", \"Hester\", role = \"aut\", comment = c(ORCID = \"0000-0002-2739-7082\")), person(\"Jennifer\", \"Bryan\", , \"jenny@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-6983-2759\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", + "Description": "An implementation of interpreted string literals, inspired by Python's Literal String Interpolation and Docstrings and Julia's Triple-Quoted String Literals .", + "License": "MIT + file LICENSE", + "URL": "https://glue.tidyverse.org/, https://github.com/tidyverse/glue", + "BugReports": "https://github.com/tidyverse/glue/issues", + "Depends": [ + "R (>= 4.1)" + ], + "Imports": [ "methods" ], - "Hash": "e0b3a53876554bd45879e596cdb10a52" + "Suggests": [ + "crayon", + "DBI (>= 1.2.0)", + "dplyr", + "knitr", + "rlang", + "rmarkdown", + "RSQLite", + "testthat (>= 3.2.0)", + "vctrs (>= 0.3.0)", + "waldo (>= 0.5.3)", + "withr" + ], + "VignetteBuilder": "knitr", + "ByteCompile": "true", + "Config/Needs/website": "bench, forcats, ggbeeswarm, ggplot2, R.utils, rprintf, tidyr, tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Config/usethis/last-upkeep": "2026-04-16", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.3", + "NeedsCompilation": "yes", + "Author": "Jim Hester [aut] (ORCID: ), Jennifer Bryan [aut, cre] (ORCID: ), Posit Software, PBC [cph, fnd] (ROR: )", + "Maintainer": "Jennifer Bryan ", + "Repository": "CRAN" }, "hms": { "Package": "hms", - "Version": "1.1.3", + "Version": "1.1.4", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ + "Title": "Pretty Time of Day", + "Date": "2025-10-11", + "Authors@R": "c( person(\"Kirill\", \"Müller\", , \"kirill@cynkra.com\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-1416-3412\")), person(\"R Consortium\", role = \"fnd\"), person(\"Posit Software, PBC\", role = \"fnd\", comment = c(ROR = \"03wc8by49\")) )", + "Description": "Implements an S3 class for storing and formatting time-of-day values, based on the 'difftime' class.", + "License": "MIT + file LICENSE", + "URL": "https://hms.tidyverse.org/, https://github.com/tidyverse/hms", + "BugReports": "https://github.com/tidyverse/hms/issues", + "Imports": [ + "cli", "lifecycle", "methods", "pkgconfig", - "rlang", - "vctrs" + "rlang (>= 1.0.2)", + "vctrs (>= 0.3.8)" ], - "Hash": "b59377caa7ed00fa41808342002138f9" - }, - "httr": { - "Package": "httr", - "Version": "1.4.7", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "R6", - "curl", - "jsonlite", - "mime", - "openssl" + "Suggests": [ + "crayon", + "lubridate", + "pillar (>= 1.1.0)", + "testthat (>= 3.0.0)" ], - "Hash": "ac107251d9d9fd72f0ca8049988f1d7f" + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.3.9000", + "NeedsCompilation": "no", + "Author": "Kirill Müller [aut, cre] (ORCID: ), R Consortium [fnd], Posit Software, PBC [fnd] (ROR: )", + "Maintainer": "Kirill Müller ", + "Repository": "CRAN" }, "jsonlite": { "Package": "jsonlite", - "Version": "1.8.8", + "Version": "2.0.0", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ + "Title": "A Simple and Robust JSON Parser and Generator for R", + "License": "MIT + file LICENSE", + "Depends": [ "methods" ], - "Hash": "e1b9c55281c5adc4dd113652d9e26768" + "Authors@R": "c( person(\"Jeroen\", \"Ooms\", role = c(\"aut\", \"cre\"), email = \"jeroenooms@gmail.com\", comment = c(ORCID = \"0000-0002-4035-0289\")), person(\"Duncan\", \"Temple Lang\", role = \"ctb\"), person(\"Lloyd\", \"Hilaiel\", role = \"cph\", comment=\"author of bundled libyajl\"))", + "URL": "https://jeroen.r-universe.dev/jsonlite https://arxiv.org/abs/1403.2805", + "BugReports": "https://github.com/jeroen/jsonlite/issues", + "Maintainer": "Jeroen Ooms ", + "VignetteBuilder": "knitr, R.rsp", + "Description": "A reasonably fast JSON parser and generator, optimized for statistical data and the web. Offers simple, flexible tools for working with JSON in R, and is particularly powerful for building pipelines and interacting with a web API. The implementation is based on the mapping described in the vignette (Ooms, 2014). In addition to converting JSON data from/to R objects, 'jsonlite' contains functions to stream, validate, and prettify JSON data. The unit tests included with the package verify that all edge cases are encoded and decoded consistently for use with dynamic data in systems and applications.", + "Suggests": [ + "httr", + "vctrs", + "testthat", + "knitr", + "rmarkdown", + "R.rsp", + "sf" + ], + "RoxygenNote": "7.3.2", + "Encoding": "UTF-8", + "NeedsCompilation": "yes", + "Author": "Jeroen Ooms [aut, cre] (), Duncan Temple Lang [ctb], Lloyd Hilaiel [cph] (author of bundled libyajl)", + "Repository": "CRAN" }, "lifecycle": { "Package": "lifecycle", - "Version": "1.0.4", + "Version": "1.0.5", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "cli", - "glue", - "rlang" + "Title": "Manage the Life Cycle of your Package Functions", + "Authors@R": "c( person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = c(\"aut\", \"cre\")), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Manage the life cycle of your exported functions with shared conventions, documentation badges, and user-friendly deprecation warnings.", + "License": "MIT + file LICENSE", + "URL": "https://lifecycle.r-lib.org/, https://github.com/r-lib/lifecycle", + "BugReports": "https://github.com/r-lib/lifecycle/issues", + "Depends": [ + "R (>= 3.6)" + ], + "Imports": [ + "cli (>= 3.4.0)", + "rlang (>= 1.1.0)" + ], + "Suggests": [ + "covr", + "knitr", + "lintr (>= 3.1.0)", + "rmarkdown", + "testthat (>= 3.0.1)", + "tibble", + "tidyverse", + "tools", + "vctrs", + "withr", + "xml2" ], - "Hash": "b8552d117e1b808b09a832f589b79035" + "VignetteBuilder": "knitr", + "Config/Needs/website": "tidyverse/tidytemplate, usethis", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.3", + "NeedsCompilation": "no", + "Author": "Lionel Henry [aut, cre], Hadley Wickham [aut] (ORCID: ), Posit Software, PBC [cph, fnd]", + "Maintainer": "Lionel Henry ", + "Repository": "CRAN" }, "magrittr": { "Package": "magrittr", - "Version": "2.0.3", + "Version": "2.0.5", "Source": "Repository", - "Repository": "RSPM", - "Requirements": [ - "R" + "Type": "Package", + "Title": "A Forward-Pipe Operator for R", + "Authors@R": "c( person(\"Stefan Milton\", \"Bache\", , \"stefan@stefanbache.dk\", role = c(\"aut\", \"cph\"), comment = \"Original author and creator of magrittr\"), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = \"cre\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", + "Description": "Provides a mechanism for chaining commands with a new forward-pipe operator, %>%. This operator will forward a value, or the result of an expression, into the next function call/expression. There is flexible support for the type of right-hand side expressions. For more information, see package vignette. To quote Rene Magritte, \"Ceci n'est pas un pipe.\"", + "License": "MIT + file LICENSE", + "URL": "https://magrittr.tidyverse.org, https://github.com/tidyverse/magrittr", + "BugReports": "https://github.com/tidyverse/magrittr/issues", + "Depends": [ + "R (>= 3.4.0)" ], - "Hash": "7ce2733a9826b3aeb1775d56fd305472" - }, - "mime": { - "Package": "mime", - "Version": "0.12", - "Source": "Repository", - "Repository": "RSPM", - "Requirements": [ - "tools" + "Suggests": [ + "covr", + "knitr", + "rlang", + "rmarkdown", + "testthat" ], - "Hash": "18e9c28c1d3ca1560ce30658b22ce104" + "VignetteBuilder": "knitr", + "ByteCompile": "Yes", + "Config/Needs/website": "tidyverse/tidytemplate", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.3", + "NeedsCompilation": "yes", + "Author": "Stefan Milton Bache [aut, cph] (Original author and creator of magrittr), Hadley Wickham [aut], Lionel Henry [cre], Posit Software, PBC [cph, fnd] (ROR: )", + "Maintainer": "Lionel Henry ", + "Repository": "CRAN" }, "move2": { "Package": "move2", - "Version": "0.2.7", + "Version": "0.5.0", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "assertthat", - "bit64", - "cli", - "dplyr", + "Title": "Processing and Analysing Animal Trajectories", + "Authors@R": "c( person(\"Bart\", \"Kranstauber\", email = \"b.kranstauber@uva.nl\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0001-8303-780X\")), person('Kamran','Safi', role='aut', comment=c(ORCID=\"0000-0002-8418-6759\")), person('Anne K.','Scharf', role='aut', comment =c(\"ORCID\"= \"0000-0002-3357-8533\")))", + "Description": "Tools to handle, manipulate and explore trajectory data, with an emphasis on data from tracked animals. The package is designed to support large studies with several million location records and keep track of units where possible. Data import directly from 'movebank' and files is facilitated.", + "License": "GPL (>= 3)", + "Encoding": "UTF-8", + "URL": "https://bartk.gitlab.io/move2/", + "BugReports": "https://gitlab.com/bartk/move2/-/issues", + "RoxygenNote": "7.3.3", + "Depends": [ + "R (>= 4.1.0)" + ], + "Imports": [ "methods", + "assertthat", + "sf (>= 1.0.16)", "rlang", - "sf", - "tibble", - "tidyselect", "units", - "vctrs", - "vroom" + "tidyselect", + "dplyr (>= 1.1.0)", + "tibble", + "vroom (>= 1.6.1)", + "cli", + "vctrs (>= 0.5.2)", + "bit64 (>= 4.5.2)" ], - "Hash": "856ea2f33245f8385206aaf5fbfbf692" + "Suggests": [ + "knitr", + "askpass", + "digest", + "keyring", + "xml2", + "curl", + "magrittr", + "purrr", + "ggplot2", + "testthat (>= 3.0.0)", + "rmarkdown", + "lwgeom", + "s2", + "move", + "raster", + "withr", + "lubridate", + "rnaturalearth", + "rnaturalearthdata", + "circular", + "tidyr", + "gganimate", + "prettymapr", + "gifski", + "ggspatial" + ], + "Config/testthat/edition": "3", + "Config/testthat/parallel": "false", + "VignetteBuilder": "knitr", + "NeedsCompilation": "no", + "Author": "Bart Kranstauber [aut, cre] (ORCID: ), Kamran Safi [aut] (ORCID: ), Anne K. Scharf [aut] (ORCID: )", + "Maintainer": "Bart Kranstauber ", + "Repository": "CRAN" }, - "openssl": { - "Package": "openssl", - "Version": "2.1.1", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "askpass" + "moveapps": { + "Package": "moveapps", + "Version": "1.0.3", + "Source": "GitHub", + "Title": "SDK for App Developing on MoveApps", + "Authors@R": "person(\"Clemens\", \"Hahn\", , \"clemens@couchbits.com\", role = c(\"aut\", \"cre\"))", + "Description": "This package provides a comprehensive Software Development Kit (SDK) for R developers to create and deploy applications on the MoveApps platform. It offers essential tools, utilities, and standardized interfaces that streamline the development process of movement ecology applications, enabling developers to build robust data analysis workflows that can seamlessly integrate with the MoveApps ecosystem.", + "License": "GPL (>= 3)", + "Encoding": "UTF-8", + "Imports": [ + "jsonlite", + "fs" + ], + "Suggests": [ + "shiny", + "httr", + "devtools", + "testthat (>= 3.0.0)" ], - "Hash": "2a0dc8c6adfb6f032e4d4af82d258ab5" + "Config/testthat/edition": "3", + "Roxygen": "list(markdown = TRUE)", + "RoxygenNote": "7.3.3", + "Author": "Clemens Hahn [aut, cre]", + "Maintainer": "Clemens Hahn ", + "RemoteType": "github", + "RemoteHost": "api.github.com", + "RemoteUsername": "movestore", + "RemoteRepo": "moveapps-sdk-r-package", + "RemoteRef": "v1.0.3", + "RemoteSha": "ea8259a3bc69cdf9c98cf991e59818f5ad41226f" }, "pillar": { "Package": "pillar", - "Version": "1.9.0", + "Version": "1.11.1", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "cli", - "fansi", + "Title": "Coloured Formatting for Columns", + "Authors@R": "c(person(given = \"Kirill\", family = \"M\\u00fcller\", role = c(\"aut\", \"cre\"), email = \"kirill@cynkra.com\", comment = c(ORCID = \"0000-0002-1416-3412\")), person(given = \"Hadley\", family = \"Wickham\", role = \"aut\"), person(given = \"RStudio\", role = \"cph\"))", + "Description": "Provides 'pillar' and 'colonnade' generics designed for formatting columns of data using the full range of colours provided by modern terminals.", + "License": "MIT + file LICENSE", + "URL": "https://pillar.r-lib.org/, https://github.com/r-lib/pillar", + "BugReports": "https://github.com/r-lib/pillar/issues", + "Imports": [ + "cli (>= 2.3.0)", "glue", "lifecycle", - "rlang", - "utf8", + "rlang (>= 1.0.2)", + "utf8 (>= 1.1.0)", "utils", - "vctrs" + "vctrs (>= 0.5.0)" + ], + "Suggests": [ + "bit64", + "DBI", + "debugme", + "DiagrammeR", + "dplyr", + "formattable", + "ggplot2", + "knitr", + "lubridate", + "nanotime", + "nycflights13", + "palmerpenguins", + "rmarkdown", + "scales", + "stringi", + "survival", + "testthat (>= 3.1.1)", + "tibble", + "units (>= 0.7.2)", + "vdiffr", + "withr" ], - "Hash": "15da5a8412f317beeee6175fbc76f4bb" + "VignetteBuilder": "knitr", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.3.9000", + "Config/testthat/edition": "3", + "Config/testthat/parallel": "true", + "Config/testthat/start-first": "format_multi_fuzz, format_multi_fuzz_2, format_multi, ctl_colonnade, ctl_colonnade_1, ctl_colonnade_2", + "Config/autostyle/scope": "line_breaks", + "Config/autostyle/strict": "true", + "Config/gha/extra-packages": "units=?ignore-before-r=4.3.0", + "Config/Needs/website": "tidyverse/tidytemplate", + "NeedsCompilation": "no", + "Author": "Kirill Müller [aut, cre] (ORCID: ), Hadley Wickham [aut], RStudio [cph]", + "Maintainer": "Kirill Müller ", + "Repository": "CRAN" }, "pkgconfig": { "Package": "pkgconfig", "Version": "2.0.3", "Source": "Repository", - "Repository": "RSPM", - "Requirements": [ + "Title": "Private Configuration for 'R' Packages", + "Author": "Gábor Csárdi", + "Maintainer": "Gábor Csárdi ", + "Description": "Set configuration options on a per-package basis. Options set by a given package only apply to that package, other packages are unaffected.", + "License": "MIT + file LICENSE", + "LazyData": "true", + "Imports": [ "utils" ], - "Hash": "01f28d4278f15c76cddbea05899c5d6f" + "Suggests": [ + "covr", + "testthat", + "disposables (>= 1.0.3)" + ], + "URL": "https://github.com/r-lib/pkgconfig#readme", + "BugReports": "https://github.com/r-lib/pkgconfig/issues", + "Encoding": "UTF-8", + "NeedsCompilation": "no", + "Repository": "CRAN" }, "prettyunits": { "Package": "prettyunits", "Version": "1.2.0", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R" + "Title": "Pretty, Human Readable Formatting of Quantities", + "Authors@R": "c( person(\"Gabor\", \"Csardi\", email=\"csardi.gabor@gmail.com\", role=c(\"aut\", \"cre\")), person(\"Bill\", \"Denney\", email=\"wdenney@humanpredictions.com\", role=c(\"ctb\"), comment=c(ORCID=\"0000-0002-5759-428X\")), person(\"Christophe\", \"Regouby\", email=\"christophe.regouby@free.fr\", role=c(\"ctb\")) )", + "Description": "Pretty, human readable formatting of quantities. Time intervals: '1337000' -> '15d 11h 23m 20s'. Vague time intervals: '2674000' -> 'about a month ago'. Bytes: '1337' -> '1.34 kB'. Rounding: '99' with 3 significant digits -> '99.0' p-values: '0.00001' -> '<0.0001'. Colors: '#FF0000' -> 'red'. Quantities: '1239437' -> '1.24 M'.", + "License": "MIT + file LICENSE", + "URL": "https://github.com/r-lib/prettyunits", + "BugReports": "https://github.com/r-lib/prettyunits/issues", + "Depends": [ + "R(>= 2.10)" + ], + "Suggests": [ + "codetools", + "covr", + "testthat" ], - "Hash": "6b01fc98b1e86c4f705ce9dcfd2f57c7" + "RoxygenNote": "7.2.3", + "Encoding": "UTF-8", + "NeedsCompilation": "no", + "Author": "Gabor Csardi [aut, cre], Bill Denney [ctb] (), Christophe Regouby [ctb]", + "Maintainer": "Gabor Csardi ", + "Repository": "CRAN" }, "progress": { "Package": "progress", "Version": "1.2.3", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "R6", + "Title": "Terminal Progress Bars", + "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Rich\", \"FitzJohn\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Configurable Progress bars, they may include percentage, elapsed time, and/or the estimated completion time. They work in terminals, in 'Emacs' 'ESS', 'RStudio', 'Windows' 'Rgui' and the 'macOS' 'R.app'. The package also provides a 'C++' 'API', that works with or without 'Rcpp'.", + "License": "MIT + file LICENSE", + "URL": "https://github.com/r-lib/progress#readme, http://r-lib.github.io/progress/", + "BugReports": "https://github.com/r-lib/progress/issues", + "Depends": [ + "R (>= 3.6)" + ], + "Imports": [ "crayon", "hms", - "prettyunits" + "prettyunits", + "R6" + ], + "Suggests": [ + "Rcpp", + "testthat (>= 3.0.0)", + "withr" ], - "Hash": "f4625e061cb2865f111b47ff163a5ca6" + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.2.3", + "NeedsCompilation": "no", + "Author": "Gábor Csárdi [aut, cre], Rich FitzJohn [aut], Posit Software, PBC [cph, fnd]", + "Maintainer": "Gábor Csárdi ", + "Repository": "CRAN" }, "proxy": { "Package": "proxy", - "Version": "0.4-27", + "Version": "0.4-29", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", + "Type": "Package", + "Title": "Distance and Similarity Measures", + "Authors@R": "c(person(given = \"David\", family = \"Meyer\", role = c(\"aut\", \"cre\"), email = \"David.Meyer@R-project.org\", comment = c(ORCID = \"0000-0002-5196-3048\")),\t person(given = \"Christian\", family = \"Buchta\", role = \"aut\"))", + "Description": "Provides an extensible framework for the efficient calculation of auto- and cross-proximities, along with implementations of the most popular ones.", + "Depends": [ + "R (>= 3.4.0)" + ], + "Imports": [ "stats", "utils" ], - "Hash": "e0ef355c12942cf7a6b91a6cfaea8b3e" + "Suggests": [ + "cba" + ], + "Collate": "registry.R database.R dist.R similarities.R dissimilarities.R util.R seal.R", + "License": "GPL-2 | GPL-3", + "NeedsCompilation": "yes", + "Author": "David Meyer [aut, cre] (ORCID: ), Christian Buchta [aut]", + "Maintainer": "David Meyer ", + "Repository": "CRAN", + "Encoding": "UTF-8" }, "purrr": { "Package": "purrr", - "Version": "1.0.2", + "Version": "1.2.2", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "cli", - "lifecycle", - "magrittr", - "rlang", - "vctrs" + "Title": "Functional Programming Tools", + "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"https://ror.org/03wc8by49\")) )", + "Description": "A complete and consistent functional programming toolkit for R.", + "License": "MIT + file LICENSE", + "URL": "https://purrr.tidyverse.org/, https://github.com/tidyverse/purrr", + "BugReports": "https://github.com/tidyverse/purrr/issues", + "Depends": [ + "R (>= 4.1)" + ], + "Imports": [ + "cli (>= 3.6.1)", + "lifecycle (>= 1.0.3)", + "magrittr (>= 1.5.0)", + "rlang (>= 1.1.1)", + "vctrs (>= 0.6.3)" + ], + "Suggests": [ + "carrier (>= 0.3.0)", + "covr", + "dplyr (>= 0.7.8)", + "httr", + "knitr", + "lubridate", + "mirai (>= 2.5.1)", + "rmarkdown", + "testthat (>= 3.0.0)", + "tibble", + "tidyselect" + ], + "LinkingTo": [ + "cli" ], - "Hash": "1cba04a4e9414bdefc9dcaa99649a8dc" + "VignetteBuilder": "knitr", + "Biarch": "true", + "Config/build/compilation-database": "true", + "Config/Needs/website": "tidyverse/tidytemplate, tidyr", + "Config/testthat/edition": "3", + "Config/testthat/parallel": "TRUE", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.3", + "NeedsCompilation": "yes", + "Author": "Hadley Wickham [aut, cre] (ORCID: ), Lionel Henry [aut], Posit Software, PBC [cph, fnd] (ROR: )", + "Maintainer": "Hadley Wickham ", + "Repository": "CRAN" }, "renv": { "Package": "renv", - "Version": "1.0.3", + "Version": "1.2.4", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ + "Type": "Package", + "Title": "Project Environments", + "Authors@R": "c( person(\"Kevin\", \"Ushey\", role = c(\"aut\", \"cre\"), email = \"kevin@rstudio.com\", comment = c(ORCID = \"0000-0003-2880-7407\")), person(\"Hadley\", \"Wickham\", role = c(\"aut\"), email = \"hadley@rstudio.com\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "A dependency management toolkit for R. Using 'renv', you can create and manage project-local R libraries, save the state of these libraries to a 'lockfile', and later restore your library as required. Together, these tools can help make your projects more isolated, portable, and reproducible.", + "License": "MIT + file LICENSE", + "URL": "https://rstudio.github.io/renv/, https://github.com/rstudio/renv", + "BugReports": "https://github.com/rstudio/renv/issues", + "Imports": [ "utils" ], - "Hash": "41b847654f567341725473431dd0d5ab" + "Suggests": [ + "BiocManager", + "cli", + "compiler", + "covr", + "cpp11", + "curl", + "devtools", + "generics", + "gitcreds", + "jsonlite", + "jsonvalidate", + "knitr", + "miniUI", + "modules", + "packrat", + "pak", + "R6", + "remotes", + "reticulate", + "rmarkdown", + "rstudioapi", + "shiny", + "testthat", + "uuid", + "waldo", + "yaml", + "webfakes" + ], + "Encoding": "UTF-8", + "VignetteBuilder": "knitr", + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Config/testthat/parallel": "true", + "Config/testthat/start-first": "bioconductor,python,install,restore,snapshot,retrieve,remotes", + "Config/roxygen2/version": "8.0.0", + "NeedsCompilation": "no", + "Author": "Kevin Ushey [aut, cre] (ORCID: ), Hadley Wickham [aut] (ORCID: ), Posit Software, PBC [cph, fnd]", + "Maintainer": "Kevin Ushey ", + "Repository": "RSPM" }, "rlang": { "Package": "rlang", - "Version": "1.1.3", + "Version": "1.3.0", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", + "Title": "Functions for Base Types and Core R and 'Tidyverse' Features", + "Authors@R": "c( person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = c(\"aut\", \"cre\")), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"mikefc\", , , \"mikefc@coolbutuseless.com\", role = \"cph\", comment = \"Hash implementation based on Mike's xxhashlite\"), person(\"Yann\", \"Collet\", role = \"cph\", comment = \"Author of the embedded xxHash library\"), person(\"Posit, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "A toolbox for working with base types, core R features like the condition system, and core 'Tidyverse' features like tidy evaluation.", + "License": "MIT + file LICENSE", + "URL": "https://rlang.r-lib.org, https://github.com/r-lib/rlang", + "BugReports": "https://github.com/r-lib/rlang/issues", + "Depends": [ + "R (>= 4.0.0)" + ], + "Imports": [ "utils" ], - "Hash": "42548638fae05fd9a9b5f3f437fbbbe2" + "Suggests": [ + "cli (>= 3.1.0)", + "covr", + "crayon", + "desc", + "fs", + "glue", + "knitr", + "magrittr", + "methods", + "pillar", + "pkgload", + "rmarkdown", + "stats", + "testthat (>= 3.3.2)", + "tibble", + "usethis", + "vctrs (>= 0.2.3)", + "withr" + ], + "Enhances": [ + "winch" + ], + "Biarch": "true", + "ByteCompile": "true", + "Config/build/compilation-database": "true", + "Config/Needs/website": "dplyr, tidyverse/tidytemplate", + "Config/roxygen2/version": "8.0.0", + "Config/testthat/edition": "3", + "Config/testthat/parallel": "true", + "Encoding": "UTF-8", + "NeedsCompilation": "yes", + "Author": "Lionel Henry [aut, cre], Hadley Wickham [aut], mikefc [cph] (Hash implementation based on Mike's xxhashlite), Yann Collet [cph] (Author of the embedded xxHash library), Posit, PBC [cph, fnd]", + "Maintainer": "Lionel Henry ", + "Repository": "CRAN" }, "s2": { "Package": "s2", - "Version": "1.1.6", + "Version": "1.1.13", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", + "Title": "Spherical Geometry Operators Using the S2 Geometry Library", + "Authors@R": "c( person(given = \"Dewey\", family = \"Dunnington\", role = c(\"aut\"), email = \"dewey@fishandwhistle.net\", comment = c(ORCID = \"0000-0002-9415-4582\")), person(given = \"Edzer\", family = \"Pebesma\", role = c(\"aut\", \"cre\"), email = \"edzer.pebesma@uni-muenster.de\", comment = c(ORCID = \"0000-0001-8049-7069\")), person(\"Ege\", \"Rubak\", email=\"rubak@math.aau.dk\", role = c(\"aut\")), person(\"Jeroen\", \"Ooms\", , \"jeroen.ooms@stat.ucla.edu\", role = \"ctb\", comment = \"configure script\"), person(family = \"Google, Inc.\", role = \"cph\", comment = \"Original s2geometry.io source code\") )", + "Description": "Provides R bindings for Google's s2 library for geometric calculations on the sphere. High-performance constructors and exporters provide high compatibility with existing spatial packages, transformers construct new geometries from existing geometries, predicates provide a means to select geometries based on spatial relationships, and accessors extract information about geometries.", + "License": "Apache License (== 2.0)", + "Encoding": "UTF-8", + "LazyData": "true", + "SystemRequirements": "cmake, OpenSSL >= 1.0.1, Abseil >= 20230802.0", + "LinkingTo": [ "Rcpp", "wk" ], - "Hash": "32f7b1a15bb01ae809022960abad5363" + "Imports": [ + "Rcpp", + "wk (>= 0.6.0)" + ], + "Suggests": [ + "bit64", + "testthat (>= 3.0.0)", + "vctrs" + ], + "URL": "https://r-spatial.github.io/s2/, https://github.com/r-spatial/s2, https://s2geometry.io/", + "BugReports": "https://github.com/r-spatial/s2/issues", + "Depends": [ + "R (>= 3.0.0)" + ], + "Config/testthat/edition": "3", + "Config/roxygen2/version": "8.0.0", + "NeedsCompilation": "yes", + "Author": "Dewey Dunnington [aut] (ORCID: ), Edzer Pebesma [aut, cre] (ORCID: ), Ege Rubak [aut], Jeroen Ooms [ctb] (configure script), Google, Inc. [cph] (Original s2geometry.io source code)", + "Maintainer": "Edzer Pebesma ", + "Repository": "CRAN" }, "sf": { "Package": "sf", - "Version": "1.0-15", + "Version": "1.1-3", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "DBI", - "R", - "Rcpp", - "classInt", - "grDevices", + "Title": "Simple Features for R", + "Authors@R": "c(person(given = \"Edzer\", family = \"Pebesma\", role = c(\"aut\", \"cre\"), email = \"edzer.pebesma@uni-muenster.de\", comment = c(ORCID = \"0000-0001-8049-7069\")), person(given = \"Roger\", family = \"Bivand\", role = \"ctb\", comment = c(ORCID = \"0000-0003-2392-6140\")), person(given = \"Etienne\", family = \"Racine\", role = \"ctb\"), person(given = \"Michael\", family = \"Sumner\", role = \"ctb\"), person(given = \"Ian\", family = \"Cook\", role = \"ctb\"), person(given = \"Tim\", family = \"Keitt\", role = \"ctb\"), person(given = \"Robin\", family = \"Lovelace\", role = \"ctb\"), person(given = \"Hadley\", family = \"Wickham\", role = \"ctb\"), person(given = \"Jeroen\", family = \"Ooms\", role = \"ctb\", comment = c(ORCID = \"0000-0002-4035-0289\")), person(given = \"Kirill\", family = \"M\\u00fcller\", role = \"ctb\"), person(given = \"Thomas Lin\", family = \"Pedersen\", role = \"ctb\"), person(given = \"Dan\", family = \"Baston\", role = \"ctb\"), person(given = \"Dewey\", family = \"Dunnington\", role = \"ctb\", comment = c(ORCID = \"0000-0002-9415-4582\")), person(given = \"Alexandre\", family = \"Courtiol\", role = \"ctb\", comment = c(ORCID = \"0000-0003-0637-2959\")) )", + "Description": "Support for simple feature access, a standardized way to encode and analyze spatial vector data. Binds to 'GDAL' for reading and writing data, to 'GEOS' for geometrical operations, and to 'PROJ' for projection conversions and datum transformations. Uses by default the 's2' package for geometry operations on geodetic (long/lat degree) coordinates.", + "License": "GPL-2 | MIT + file LICENSE", + "URL": "https://r-spatial.github.io/sf/, https://github.com/r-spatial/sf", + "BugReports": "https://github.com/r-spatial/sf/issues", + "Depends": [ + "methods", + "R (>= 4.1.0)" + ], + "Imports": [ + "classInt (>= 0.4-1)", + "DBI (>= 0.8)", "graphics", + "grDevices", "grid", - "magrittr", - "methods", - "s2", + "s2 (>= 1.1.0)", "stats", "tools", - "units", + "units (>= 0.7-0)", "utils" ], - "Hash": "f432b3379fb1a47046e253468b6b6b6d" - }, - "sys": { - "Package": "sys", - "Version": "3.4.2", - "Source": "Repository", - "Repository": "RSPM", - "Hash": "3a1be13d68d47a8cd0bfd74739ca1555" + "Suggests": [ + "blob", + "covr", + "dplyr (>= 1.0.0)", + "ggplot2", + "knitr", + "lwgeom (>= 0.2-14)", + "maps", + "mapview", + "Matrix", + "microbenchmark", + "nanoarrow", + "odbc", + "pbapply", + "pillar", + "pool", + "raster", + "rlang", + "rmarkdown", + "RPostgres (>= 1.1.0)", + "RPostgreSQL", + "RSQLite", + "sp (>= 1.2-4)", + "spatstat (>= 2.0-1)", + "spatstat.geom", + "spatstat.random", + "spatstat.linnet", + "spatstat.utils", + "stars (>= 0.6-0)", + "terra", + "testthat (>= 3.0.0)", + "tibble (>= 1.4.1)", + "tidyr (>= 1.2.0)", + "tidyselect (>= 1.0.0)", + "tmap (>= 2.0)", + "vctrs", + "wk (>= 0.9.0)" + ], + "LinkingTo": [ + "Rcpp" + ], + "VignetteBuilder": "knitr", + "Encoding": "UTF-8", + "Config/testthat/edition": "2", + "Config/needs/coverage": "XML", + "Config/roxygen2/markdown": "TRUE", + "Config/roxygen2/version": "8.1.0", + "SystemRequirements": "GDAL (>= 2.0.1), GEOS (>= 3.4.0), PROJ (>= 4.8.0), sqlite3", + "Collate": "'RcppExports.R' 'init.R' 'import-standalone-s3-register.R' 'crs.R' 'bbox.R' 'read.R' 'db.R' 'sfc.R' 'sfg.R' 'sf.R' 'bind.R' 'wkb.R' 'wkt.R' 'plot.R' 'geom-measures.R' 'geom-predicates.R' 'geom-transformers.R' 'transform.R' 'proj.R' 'sp.R' 'grid.R' 'arith.R' 'tidyverse.R' 'tidyverse-vctrs.R' 'cast_sfg.R' 'cast_sfc.R' 'graticule.R' 'datasets.R' 'aggregate.R' 'agr.R' 'maps.R' 'join.R' 'sample.R' 'valid.R' 'collection_extract.R' 'jitter.R' 'sgbp.R' 'spatstat.R' 'stars.R' 'crop.R' 'gdal_utils.R' 'nearest.R' 'normalize.R' 'sf-package.R' 'defunct.R' 'z_range.R' 'm_range.R' 'shift_longitude.R' 'make_grid.R' 's2.R' 'terra.R' 'geos-overlayng.R' 'break_antimeridian.R'", + "NeedsCompilation": "yes", + "Author": "Edzer Pebesma [aut, cre] (ORCID: ), Roger Bivand [ctb] (ORCID: ), Etienne Racine [ctb], Michael Sumner [ctb], Ian Cook [ctb], Tim Keitt [ctb], Robin Lovelace [ctb], Hadley Wickham [ctb], Jeroen Ooms [ctb] (ORCID: ), Kirill Müller [ctb], Thomas Lin Pedersen [ctb], Dan Baston [ctb], Dewey Dunnington [ctb] (ORCID: ), Alexandre Courtiol [ctb] (ORCID: )", + "Maintainer": "Edzer Pebesma ", + "Repository": "CRAN" }, "tibble": { "Package": "tibble", - "Version": "3.2.1", + "Version": "3.3.1", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "fansi", - "lifecycle", + "Title": "Simple Data Frames", + "Authors@R": "c( person(\"Kirill\", \"Müller\", , \"kirill@cynkra.com\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-1416-3412\")), person(\"Hadley\", \"Wickham\", , \"hadley@rstudio.com\", role = \"aut\"), person(\"Romain\", \"Francois\", , \"romain@r-enthusiasts.com\", role = \"ctb\"), person(\"Jennifer\", \"Bryan\", , \"jenny@rstudio.com\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", + "Description": "Provides a 'tbl_df' class (the 'tibble') with stricter checking and better formatting than the traditional data frame.", + "License": "MIT + file LICENSE", + "URL": "https://tibble.tidyverse.org/, https://github.com/tidyverse/tibble", + "BugReports": "https://github.com/tidyverse/tibble/issues", + "Depends": [ + "R (>= 3.4.0)" + ], + "Imports": [ + "cli", + "lifecycle (>= 1.0.0)", "magrittr", "methods", - "pillar", + "pillar (>= 1.8.1)", "pkgconfig", - "rlang", + "rlang (>= 1.0.2)", "utils", - "vctrs" + "vctrs (>= 0.5.0)" + ], + "Suggests": [ + "bench", + "bit64", + "blob", + "brio", + "callr", + "DiagrammeR", + "dplyr", + "evaluate", + "formattable", + "ggplot2", + "here", + "hms", + "htmltools", + "knitr", + "lubridate", + "nycflights13", + "pkgload", + "purrr", + "rmarkdown", + "stringi", + "testthat (>= 3.0.2)", + "tidyr", + "withr" ], - "Hash": "a84e2cc86d07289b3b6f5069df7a004c" + "VignetteBuilder": "knitr", + "Config/autostyle/rmd": "false", + "Config/autostyle/scope": "line_breaks", + "Config/autostyle/strict": "true", + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Config/testthat/parallel": "true", + "Config/testthat/start-first": "vignette-formats, as_tibble, add, invariants", + "Config/usethis/last-upkeep": "2025-06-07", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.3.9000", + "NeedsCompilation": "yes", + "Author": "Kirill Müller [aut, cre] (ORCID: ), Hadley Wickham [aut], Romain Francois [ctb], Jennifer Bryan [ctb], Posit Software, PBC [cph, fnd] (ROR: )", + "Maintainer": "Kirill Müller ", + "Repository": "CRAN" }, "tidyselect": { "Package": "tidyselect", - "Version": "1.2.0", + "Version": "1.2.1", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "cli", - "glue", - "lifecycle", - "rlang", - "vctrs", + "Title": "Select from a Set of Strings", + "Authors@R": "c( person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = c(\"aut\", \"cre\")), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "A backend for the selecting functions of the 'tidyverse'. It makes it easy to implement select-like functions in your own packages in a way that is consistent with other 'tidyverse' interfaces for selection.", + "License": "MIT + file LICENSE", + "URL": "https://tidyselect.r-lib.org, https://github.com/r-lib/tidyselect", + "BugReports": "https://github.com/r-lib/tidyselect/issues", + "Depends": [ + "R (>= 3.4)" + ], + "Imports": [ + "cli (>= 3.3.0)", + "glue (>= 1.3.0)", + "lifecycle (>= 1.0.3)", + "rlang (>= 1.0.4)", + "vctrs (>= 0.5.2)", "withr" ], - "Hash": "79540e5fcd9e0435af547d885f184fd5" + "Suggests": [ + "covr", + "crayon", + "dplyr", + "knitr", + "magrittr", + "rmarkdown", + "stringr", + "testthat (>= 3.1.1)", + "tibble (>= 2.1.3)" + ], + "VignetteBuilder": "knitr", + "ByteCompile": "true", + "Config/testthat/edition": "3", + "Config/Needs/website": "tidyverse/tidytemplate", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.0.9000", + "NeedsCompilation": "yes", + "Author": "Lionel Henry [aut, cre], Hadley Wickham [aut], Posit Software, PBC [cph, fnd]", + "Maintainer": "Lionel Henry ", + "Repository": "CRAN" }, "tzdb": { "Package": "tzdb", - "Version": "0.4.0", + "Version": "0.5.0", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "cpp11" + "Title": "Time Zone Database Information", + "Authors@R": "c( person(\"Davis\", \"Vaughan\", , \"davis@posit.co\", role = c(\"aut\", \"cre\")), person(\"Howard\", \"Hinnant\", role = \"cph\", comment = \"Author of the included date library\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Provides an up-to-date copy of the Internet Assigned Numbers Authority (IANA) Time Zone Database. It is updated periodically to reflect changes made by political bodies to time zone boundaries, UTC offsets, and daylight saving time rules. Additionally, this package provides a C++ interface for working with the 'date' library. 'date' provides comprehensive support for working with dates and date-times, which this package exposes to make it easier for other R packages to utilize. Headers are provided for calendar specific calculations, along with a limited interface for time zone manipulations.", + "License": "MIT + file LICENSE", + "URL": "https://tzdb.r-lib.org, https://github.com/r-lib/tzdb", + "BugReports": "https://github.com/r-lib/tzdb/issues", + "Depends": [ + "R (>= 4.0.0)" + ], + "Suggests": [ + "covr", + "testthat (>= 3.0.0)" + ], + "LinkingTo": [ + "cpp11 (>= 0.5.2)" ], - "Hash": "f561504ec2897f4d46f0c7657e488ae1" + "Biarch": "yes", + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "NeedsCompilation": "yes", + "Author": "Davis Vaughan [aut, cre], Howard Hinnant [cph] (Author of the included date library), Posit Software, PBC [cph, fnd]", + "Maintainer": "Davis Vaughan ", + "Repository": "CRAN" }, "units": { "Package": "units", - "Version": "0.8-5", + "Version": "1.0-1", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", + "Title": "Measurement Units for R Vectors", + "Authors@R": "c(person(\"Edzer\", \"Pebesma\", role = c(\"aut\", \"cre\"), email = \"edzer.pebesma@uni-muenster.de\", comment = c(ORCID = \"0000-0001-8049-7069\")), person(\"Thomas\", \"Mailund\", role = \"aut\", email = \"mailund@birc.au.dk\"), person(\"Tomasz\", \"Kalinowski\", role = \"aut\"), person(\"James\", \"Hiebert\", role = \"ctb\"), person(\"Iñaki\", \"Ucar\", role = \"aut\", email = \"iucar@fedoraproject.org\", comment = c(ORCID = \"0000-0001-6403-5550\")), person(\"Thomas Lin\", \"Pedersen\", role = \"ctb\") )", + "Depends": [ + "R (>= 3.5.0)" + ], + "Imports": [ "Rcpp" ], - "Hash": "119d19da480e873f72241ff6962ffd83" + "LinkingTo": [ + "Rcpp (>= 1.1.0)" + ], + "Suggests": [ + "NISTunits", + "measurements", + "xml2", + "magrittr", + "pillar (>= 1.3.0)", + "dplyr (>= 1.0.0)", + "vctrs (>= 0.3.1)", + "ggplot2 (> 3.2.1)", + "testthat (>= 3.0.0)", + "vdiffr", + "knitr", + "rvest", + "rmarkdown" + ], + "VignetteBuilder": "knitr", + "Description": "Support for measurement units in R vectors, matrices and arrays: automatic propagation, conversion, derivation and simplification of units; raising errors in case of unit incompatibility. Compatible with the POSIXct, Date and difftime classes. Uses the UNIDATA udunits library and unit database for unit compatibility checking and conversion. Documentation about 'units' is provided in the paper by Pebesma, Mailund & Hiebert (2016, ), included in this package as a vignette; see 'citation(\"units\")' for details.", + "SystemRequirements": "udunits-2", + "License": "GPL-2", + "URL": "https://r-quantities.github.io/units/, https://github.com/r-quantities/units", + "BugReports": "https://github.com/r-quantities/units/issues", + "RoxygenNote": "7.3.3", + "Encoding": "UTF-8", + "Config/testthat/edition": "3", + "NeedsCompilation": "yes", + "Author": "Edzer Pebesma [aut, cre] (ORCID: ), Thomas Mailund [aut], Tomasz Kalinowski [aut], James Hiebert [ctb], Iñaki Ucar [aut] (ORCID: ), Thomas Lin Pedersen [ctb]", + "Maintainer": "Edzer Pebesma ", + "Repository": "CRAN" }, "utf8": { "Package": "utf8", - "Version": "1.2.4", + "Version": "1.2.6", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R" + "Title": "Unicode Text Processing", + "Authors@R": "c(person(given = c(\"Patrick\", \"O.\"), family = \"Perry\", role = c(\"aut\", \"cph\")), person(given = \"Kirill\", family = \"M\\u00fcller\", role = \"cre\", email = \"kirill@cynkra.com\", comment = c(ORCID = \"0000-0002-1416-3412\")), person(given = \"Unicode, Inc.\", role = c(\"cph\", \"dtc\"), comment = \"Unicode Character Database\"))", + "Description": "Process and print 'UTF-8' encoded international text (Unicode). Input, validate, normalize, encode, format, and display.", + "License": "Apache License (== 2.0) | file LICENSE", + "URL": "https://krlmlr.github.io/utf8/, https://github.com/krlmlr/utf8", + "BugReports": "https://github.com/krlmlr/utf8/issues", + "Depends": [ + "R (>= 2.10)" + ], + "Suggests": [ + "cli", + "covr", + "knitr", + "rlang", + "rmarkdown", + "testthat (>= 3.0.0)", + "withr" ], - "Hash": "62b65c52671e6665f803ff02954446e9" + "VignetteBuilder": "knitr, rmarkdown", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2.9000", + "NeedsCompilation": "yes", + "Author": "Patrick O. Perry [aut, cph], Kirill Müller [cre] (ORCID: ), Unicode, Inc. [cph, dtc] (Unicode Character Database)", + "Maintainer": "Kirill Müller ", + "Repository": "CRAN" }, "vctrs": { "Package": "vctrs", - "Version": "0.6.5", + "Version": "0.7.3", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "cli", + "Title": "Vector Helpers", + "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = \"aut\"), person(\"Davis\", \"Vaughan\", , \"davis@posit.co\", role = c(\"aut\", \"cre\")), person(\"data.table team\", role = \"cph\", comment = \"Radix sort based on data.table's forder() and their contribution to R's order()\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "Defines new notions of prototype and size that are used to provide tools for consistent and well-founded type-coercion and size-recycling, and are in turn connected to ideas of type- and size-stability useful for analysing function interfaces.", + "License": "MIT + file LICENSE", + "URL": "https://vctrs.r-lib.org/, https://github.com/r-lib/vctrs", + "BugReports": "https://github.com/r-lib/vctrs/issues", + "Depends": [ + "R (>= 4.0.0)" + ], + "Imports": [ + "cli (>= 3.4.0)", "glue", - "lifecycle", - "rlang" + "lifecycle (>= 1.0.3)", + "rlang (>= 1.1.7)" ], - "Hash": "c03fa420630029418f7e6da3667aac4a" + "Suggests": [ + "bit64", + "covr", + "crayon", + "dplyr (>= 0.8.5)", + "generics", + "knitr", + "pillar (>= 1.4.4)", + "pkgdown (>= 2.0.1)", + "rmarkdown", + "testthat (>= 3.0.0)", + "tibble (>= 3.1.3)", + "waldo (>= 0.2.0)", + "withr", + "xml2", + "zeallot" + ], + "VignetteBuilder": "knitr", + "Config/build/compilation-database": "true", + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Config/testthat/parallel": "true", + "Encoding": "UTF-8", + "KeepSource": "true", + "Language": "en-GB", + "RoxygenNote": "7.3.3", + "NeedsCompilation": "yes", + "Author": "Hadley Wickham [aut], Lionel Henry [aut], Davis Vaughan [aut, cre], data.table team [cph] (Radix sort based on data.table's forder() and their contribution to R's order()), Posit Software, PBC [cph, fnd]", + "Maintainer": "Davis Vaughan ", + "Repository": "CRAN" }, "vroom": { "Package": "vroom", - "Version": "1.6.5", + "Version": "1.7.1", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", + "Title": "Read and Write Rectangular Text Data Quickly", + "Authors@R": "c( person(\"Jim\", \"Hester\", role = \"aut\", comment = c(ORCID = \"0000-0002-2739-7082\")), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Jennifer\", \"Bryan\", , \"jenny@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-6983-2759\")), person(\"Shelby\", \"Bearrows\", role = \"ctb\"), person(\"https://github.com/mandreyel/\", role = \"cph\", comment = \"mio library\"), person(\"Jukka\", \"Jylänki\", role = \"cph\", comment = \"grisu3 implementation\"), person(\"Mikkel\", \"Jørgensen\", role = \"cph\", comment = \"grisu3 implementation\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", + "Description": "The goal of 'vroom' is to read and write data (like 'csv', 'tsv' and 'fwf') quickly. When reading it uses a quick initial indexing step, then reads the values lazily , so only the data you actually use needs to be read. The writer formats the data in parallel and writes to disk asynchronously from formatting.", + "License": "MIT + file LICENSE", + "URL": "https://vroom.tidyverse.org, https://github.com/tidyverse/vroom", + "BugReports": "https://github.com/tidyverse/vroom/issues", + "Depends": [ + "R (>= 4.1)" + ], + "Imports": [ "bit64", - "cli", - "cpp11", + "cli (>= 3.2.0)", "crayon", "glue", "hms", - "lifecycle", + "lifecycle (>= 1.0.3)", "methods", - "progress", - "rlang", + "rlang (>= 1.1.0)", "stats", - "tibble", + "tibble (>= 2.0.0)", "tidyselect", - "tzdb", - "vctrs", + "tzdb (>= 0.1.1)", + "vctrs (>= 0.2.0)", "withr" ], - "Hash": "390f9315bc0025be03012054103d227c" + "Suggests": [ + "archive", + "bench (>= 1.1.0)", + "covr", + "curl", + "dplyr", + "forcats", + "fs", + "ggplot2", + "knitr", + "patchwork", + "prettyunits", + "purrr", + "rmarkdown", + "rstudioapi", + "scales", + "spelling", + "testthat (>= 2.1.0)", + "tidyr", + "utils", + "waldo", + "xml2" + ], + "LinkingTo": [ + "cpp11 (>= 0.2.0)", + "progress (>= 1.2.3)", + "tzdb (>= 0.1.1)" + ], + "VignetteBuilder": "knitr", + "Config/Needs/website": "nycflights13, tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Config/testthat/parallel": "false", + "Config/usethis/last-upkeep": "2025-11-25", + "Copyright": "file COPYRIGHTS", + "Encoding": "UTF-8", + "Language": "en-US", + "RoxygenNote": "7.3.3", + "Config/build/compilation-database": "true", + "NeedsCompilation": "yes", + "Author": "Jim Hester [aut] (ORCID: ), Hadley Wickham [aut] (ORCID: ), Jennifer Bryan [aut, cre] (ORCID: ), Shelby Bearrows [ctb], https://github.com/mandreyel/ [cph] (mio library), Jukka Jylänki [cph] (grisu3 implementation), Mikkel Jørgensen [cph] (grisu3 implementation), Posit Software, PBC [cph, fnd] (ROR: )", + "Maintainer": "Jennifer Bryan ", + "Repository": "CRAN" }, "withr": { "Package": "withr", - "Version": "2.5.2", + "Version": "3.0.3", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "grDevices", + "Title": "Run Code 'With' Temporarily Modified Global State", + "Authors@R": "c( person(\"Jim\", \"Hester\", role = \"aut\"), person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = c(\"aut\", \"cre\")), person(\"Kirill\", \"Müller\", , \"krlmlr+r@mailbox.org\", role = \"aut\"), person(\"Kevin\", \"Ushey\", , \"kevinushey@gmail.com\", role = \"aut\"), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Winston\", \"Chang\", role = \"aut\"), person(\"Jennifer\", \"Bryan\", role = \"ctb\"), person(\"Richard\", \"Cotton\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", + "Description": "A set of functions to run code 'with' safely and temporarily modified global state. Many of these functions were originally a part of the 'devtools' package, this provides a simple package with limited dependencies to provide access to these functions.", + "License": "MIT + file LICENSE", + "URL": "https://withr.r-lib.org, https://github.com/r-lib/withr#readme", + "BugReports": "https://github.com/r-lib/withr/issues", + "Depends": [ + "R (>= 3.6.0)" + ], + "Imports": [ "graphics", - "stats" + "grDevices" + ], + "Suggests": [ + "callr", + "DBI", + "knitr", + "methods", + "rlang", + "rmarkdown (>= 2.12)", + "RSQLite", + "testthat (>= 3.0.0)" ], - "Hash": "4b25e70111b7d644322e9513f403a272" + "VignetteBuilder": "knitr", + "Config/Needs/website": "tidyverse/tidytemplate", + "Config/testthat/edition": "3", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "Collate": "'aaa.R' 'collate.R' 'connection.R' 'db.R' 'defer-exit.R' 'standalone-defer.R' 'defer.R' 'devices.R' 'local_.R' 'with_.R' 'dir.R' 'env.R' 'file.R' 'language.R' 'libpaths.R' 'locale.R' 'makevars.R' 'namespace.R' 'options.R' 'par.R' 'path.R' 'rng.R' 'seed.R' 'wrap.R' 'sink.R' 'tempfile.R' 'timezone.R' 'torture.R' 'utils.R' 'with.R'", + "NeedsCompilation": "no", + "Author": "Jim Hester [aut], Lionel Henry [aut, cre], Kirill Müller [aut], Kevin Ushey [aut], Hadley Wickham [aut], Winston Chang [aut], Jennifer Bryan [ctb], Richard Cotton [ctb], Posit Software, PBC [cph, fnd]", + "Maintainer": "Lionel Henry ", + "Repository": "CRAN" }, "wk": { "Package": "wk", - "Version": "0.9.1", + "Version": "0.9.5", "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R" + "Title": "Lightweight Well-Known Geometry Parsing", + "Authors@R": "c( person(given = \"Dewey\", family = \"Dunnington\", role = c(\"aut\", \"cre\"), email = \"dewey@fishandwhistle.net\", comment = c(ORCID = \"0000-0002-9415-4582\")), person(given = \"Edzer\", family = \"Pebesma\", role = c(\"aut\"), email = \"edzer.pebesma@uni-muenster.de\", comment = c(ORCID = \"0000-0001-8049-7069\")), person(given = \"Anthony\", family = \"North\", email = \"anthony.jl.north@gmail.com\", role = c(\"ctb\")) )", + "Maintainer": "Dewey Dunnington ", + "Description": "Provides a minimal R and C++ API for parsing well-known binary and well-known text representation of geometries to and from R-native formats. Well-known binary is compact and fast to parse; well-known text is human-readable and is useful for writing tests. These formats are useful in R only if the information they contain can be accessed in R, for which high-performance functions are provided here.", + "License": "MIT + file LICENSE", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.2", + "Suggests": [ + "testthat (>= 3.0.0)", + "vctrs (>= 0.3.0)", + "sf", + "tibble", + "readr" + ], + "URL": "https://paleolimbot.github.io/wk/, https://github.com/paleolimbot/wk", + "BugReports": "https://github.com/paleolimbot/wk/issues", + "Config/testthat/edition": "3", + "Depends": [ + "R (>= 2.10)" ], - "Hash": "5d4545e140e36476f35f20d0ca87963e" + "LazyData": "true", + "NeedsCompilation": "yes", + "Author": "Dewey Dunnington [aut, cre] (ORCID: ), Edzer Pebesma [aut] (ORCID: ), Anthony North [ctb]", + "Repository": "CRAN" } } } diff --git a/r/renv/activate.R b/r/renv/activate.R index cb5401f..b9191f3 100644 --- a/r/renv/activate.R +++ b/r/renv/activate.R @@ -2,11 +2,14 @@ local({ # the requested version of renv - version <- "1.0.3" + version <- "1.2.4" + attr(version, "md5") <- "d5a3bd382107712f897630627794f81a" attr(version, "sha") <- NULL # the project directory - project <- getwd() + project <- Sys.getenv("RENV_PROJECT") + if (!nzchar(project)) + project <- getwd() # use start-up diagnostics if enabled diagnostics <- Sys.getenv("RENV_STARTUP_DIAGNOSTICS", unset = "FALSE") @@ -31,8 +34,16 @@ local({ if (!is.null(override)) return(override) + # if we're being run in a context where R_LIBS is already set, + # don't load -- presumably we're being run as a sub-process and + # the parent process has already set up library paths for us + rcmd <- Sys.getenv("R_CMD", unset = NA) + rlibs <- Sys.getenv("R_LIBS", unset = NA) + if (!is.na(rlibs) && !is.na(rcmd)) + return(FALSE) + # next, check environment variables - # TODO: prefer using the configuration one in the future + # prefer using the configuration one in the future envvars <- c( "RENV_CONFIG_AUTOLOADER_ENABLED", "RENV_AUTOLOADER_ENABLED", @@ -50,9 +61,22 @@ local({ }) - if (!enabled) + # bail if we're not enabled + if (!enabled) { + + # if we're not enabled, we might still need to manually load + # the user profile here + profile <- Sys.getenv("R_PROFILE_USER", unset = "~/.Rprofile") + if (file.exists(profile)) { + cfg <- Sys.getenv("RENV_CONFIG_USER_PROFILE", unset = "TRUE") + if (tolower(cfg) %in% c("true", "t", "1")) + sys.source(profile, envir = globalenv()) + } + return(FALSE) + } + # avoid recursion if (identical(getOption("renv.autoloader.running"), TRUE)) { warning("ignoring recursive attempt to run renv autoloader") @@ -75,6 +99,66 @@ local({ unloadNamespace("renv") # load bootstrap tools + ansify <- function(text) { + if (renv_ansify_enabled()) + renv_ansify_enhanced(text) + else + renv_ansify_default(text) + } + + renv_ansify_enabled <- function() { + + override <- Sys.getenv("RENV_ANSIFY_ENABLED", unset = NA) + if (!is.na(override)) + return(as.logical(override)) + + pane <- Sys.getenv("RSTUDIO_CHILD_PROCESS_PANE", unset = NA) + if (identical(pane, "build")) + return(FALSE) + + testthat <- Sys.getenv("TESTTHAT", unset = "false") + if (tolower(testthat) %in% "true") + return(FALSE) + + iderun <- Sys.getenv("R_CLI_HAS_HYPERLINK_IDE_RUN", unset = "false") + if (tolower(iderun) %in% "false") + return(FALSE) + + TRUE + + } + + renv_ansify_default <- function(text) { + text + } + + renv_ansify_enhanced <- function(text) { + + # R help links + pattern <- "`\\?(renv::(?:[^`])+)`" + replacement <- "`\033]8;;x-r-help:\\1\a?\\1\033]8;;\a`" + text <- gsub(pattern, replacement, text, perl = TRUE) + + # runnable code + pattern <- "`(renv::(?:[^`])+)`" + replacement <- "`\033]8;;x-r-run:\\1\a\\1\033]8;;\a`" + text <- gsub(pattern, replacement, text, perl = TRUE) + + # return ansified text + text + + } + + renv_ansify_init <- function() { + + envir <- renv_envir_self() + if (renv_ansify_enabled()) + assign("ansify", renv_ansify_enhanced, envir = envir) + else + assign("ansify", renv_ansify_default, envir = envir) + + } + `%||%` <- function(x, y) { if (is.null(x)) y else x } @@ -85,6 +169,16 @@ local({ if (quiet) return(invisible()) + # also check for config environment variables that should suppress messages + # https://github.com/rstudio/renv/issues/2214 + enabled <- Sys.getenv("RENV_CONFIG_STARTUP_QUIET", unset = NA) + if (!is.na(enabled) && tolower(enabled) %in% c("true", "1")) + return(invisible()) + + enabled <- Sys.getenv("RENV_CONFIG_SYNCHRONIZED_CHECK", unset = NA) + if (!is.na(enabled) && tolower(enabled) %in% c("false", "0")) + return(invisible()) + msg <- sprintf(fmt, ...) cat(msg, file = stdout(), sep = if (appendLF) "\n" else "") @@ -108,8 +202,22 @@ local({ } - startswith <- function(string, prefix) { - substring(string, 1, nchar(prefix)) == prefix + heredoc <- function(text, leave = 0) { + + # remove leading, trailing whitespace + trimmed <- gsub("^\\s*\\n|\\n\\s*$", "", text) + + # split into lines + lines <- strsplit(trimmed, "\n", fixed = TRUE)[[1L]] + + # compute common indent + indent <- regexpr("[^[:space:]]", lines) + common <- min(setdiff(indent, -1L)) - leave + text <- paste(substring(lines, common), collapse = "\n") + + # substitute in ANSI links for executable renv code + ansify(text) + } bootstrap <- function(version, library) { @@ -118,6 +226,20 @@ local({ section <- header(sprintf("Bootstrapping renv %s", friendly)) catf(section) + # ensure the target library path exists; required for file.copy(..., recursive = TRUE) + dir.create(library, showWarnings = FALSE, recursive = TRUE) + + # try to install renv from cache + md5 <- attr(version, "md5", exact = TRUE) + if (length(md5)) { + pkgpath <- renv_bootstrap_find(version) + if (length(pkgpath) && file.exists(pkgpath)) { + ok <- file.copy(pkgpath, library, recursive = TRUE) + if (isTRUE(ok)) + return(invisible()) + } + } + # attempt to download renv catf("- Downloading renv ... ", appendLF = FALSE) withCallingHandlers( @@ -143,7 +265,6 @@ local({ # add empty line to break up bootstrapping from normal output catf("") - return(invisible()) } @@ -160,12 +281,20 @@ local({ repos <- Sys.getenv("RENV_CONFIG_REPOS_OVERRIDE", unset = NA) if (!is.na(repos)) { - # check for RSPM; if set, use a fallback repository for renv - rspm <- Sys.getenv("RSPM", unset = NA) - if (identical(rspm, repos)) - repos <- c(RSPM = rspm, CRAN = cran) + # split on ';' if present + parts <- strsplit(repos, ";", fixed = TRUE)[[1L]] - return(repos) + # split into named repositories if present + idx <- regexpr("=", parts, fixed = TRUE) + keys <- substring(parts, 1L, idx - 1L) + vals <- substring(parts, idx + 1L) + names(vals) <- keys + + # if we have a single unnamed repository, call it CRAN + if (length(vals) == 1L && identical(keys, "")) + names(vals) <- "CRAN" + + return(vals) } @@ -267,8 +396,11 @@ local({ quiet = TRUE ) - if ("headers" %in% names(formals(utils::download.file))) - args$headers <- renv_bootstrap_download_custom_headers(url) + if ("headers" %in% names(formals(utils::download.file))) { + headers <- renv_bootstrap_download_custom_headers(url) + if (length(headers) && is.character(headers)) + args$headers <- headers + } do.call(utils::download.file, args) @@ -347,10 +479,21 @@ local({ for (type in types) { for (repos in renv_bootstrap_repos()) { + # build arguments for utils::available.packages() call + args <- list(type = type, repos = repos) + + # add custom headers if available -- note that + # utils::available.packages() will pass this to download.file() + if ("headers" %in% names(formals(utils::download.file))) { + headers <- renv_bootstrap_download_custom_headers(repos) + if (length(headers) && is.character(headers)) + args$headers <- headers + } + # retrieve package database db <- tryCatch( as.data.frame( - utils::available.packages(type = type, repos = repos), + do.call(utils::available.packages, args), stringsAsFactors = FALSE ), error = identity @@ -400,6 +543,51 @@ local({ } + renv_bootstrap_find <- function(version) { + + path <- renv_bootstrap_find_cache(version) + if (length(path) && file.exists(path)) { + catf("- Using renv %s from global package cache", version) + return(path) + } + + } + + renv_bootstrap_find_cache <- function(version) { + + md5 <- attr(version, "md5", exact = TRUE) + if (is.null(md5)) + return() + + # infer path to renv cache + cache <- Sys.getenv("RENV_PATHS_CACHE", unset = "") + if (!nzchar(cache)) { + root <- Sys.getenv("RENV_PATHS_ROOT", unset = NA) + if (!is.na(root)) + cache <- file.path(root, "cache") + } + + if (!nzchar(cache)) { + tools <- asNamespace("tools") + if (is.function(tools$R_user_dir)) { + root <- tools$R_user_dir("renv", "cache") + cache <- file.path(root, "cache") + } + } + + # start completing path to cache + file.path( + cache, + renv_bootstrap_cache_version(), + renv_bootstrap_platform_prefix(), + "renv", + version, + md5, + "renv" + ) + + } + renv_bootstrap_download_tarball <- function(version) { # if the user has provided the path to a tarball via @@ -432,6 +620,14 @@ local({ } + renv_bootstrap_github_token <- function() { + for (envvar in c("GITHUB_TOKEN", "GITHUB_PAT", "GH_TOKEN")) { + envval <- Sys.getenv(envvar, unset = NA) + if (!is.na(envval)) + return(envval) + } + } + renv_bootstrap_download_github <- function(version) { enabled <- Sys.getenv("RENV_BOOTSTRAP_FROM_GITHUB", unset = "TRUE") @@ -439,16 +635,19 @@ local({ return(FALSE) # prepare download options - pat <- Sys.getenv("GITHUB_PAT") - if (nzchar(Sys.which("curl")) && nzchar(pat)) { + token <- renv_bootstrap_github_token() + if (is.null(token)) + token <- "" + + if (nzchar(Sys.which("curl")) && nzchar(token)) { fmt <- "--location --fail --header \"Authorization: token %s\"" - extra <- sprintf(fmt, pat) + extra <- sprintf(fmt, token) saved <- options("download.file.method", "download.file.extra") options(download.file.method = "curl", download.file.extra = extra) on.exit(do.call(base::options, saved), add = TRUE) - } else if (nzchar(Sys.which("wget")) && nzchar(pat)) { + } else if (nzchar(Sys.which("wget")) && nzchar(token)) { fmt <- "--header=\"Authorization: token %s\"" - extra <- sprintf(fmt, pat) + extra <- sprintf(fmt, token) saved <- options("download.file.method", "download.file.extra") options(download.file.method = "wget", download.file.extra = extra) on.exit(do.call(base::options, saved), add = TRUE) @@ -573,11 +772,19 @@ local({ } - renv_bootstrap_platform_prefix <- function() { + renv_bootstrap_platform_prefix_default <- function() { - # construct version prefix - version <- paste(R.version$major, R.version$minor, sep = ".") - prefix <- paste("R", numeric_version(version)[1, 1:2], sep = "-") + # read version component + version <- Sys.getenv("RENV_PATHS_VERSION", unset = "R-%v") + + # expand placeholders + placeholders <- list( + list("%v", format(getRversion()[1, 1:2])), + list("%V", format(getRversion()[1, 1:3])) + ) + + for (placeholder in placeholders) + version <- gsub(placeholder[[1L]], placeholder[[2L]], version, fixed = TRUE) # include SVN revision for development versions of R # (to avoid sharing platform-specific artefacts with released versions of R) @@ -586,10 +793,19 @@ local({ identical(R.version[["nickname"]], "Unsuffered Consequences") if (devel) - prefix <- paste(prefix, R.version[["svn rev"]], sep = "-r") + version <- paste(version, R.version[["svn rev"]], sep = "-r") + + version + + } + + renv_bootstrap_platform_prefix <- function() { + + # construct version prefix + version <- renv_bootstrap_platform_prefix_default() # build list of path components - components <- c(prefix, R.version$platform) + components <- c(version, R.version$platform) # include prefix if provided by user prefix <- renv_bootstrap_platform_prefix_impl() @@ -610,6 +826,9 @@ local({ # if the user has requested an automatic prefix, generate it auto <- Sys.getenv("RENV_PATHS_PREFIX_AUTO", unset = NA) + if (is.na(auto) && getRversion() >= "4.4.0") + auto <- "TRUE" + if (auto %in% c("TRUE", "True", "true", "1")) return(renv_bootstrap_platform_prefix_auto()) @@ -801,24 +1020,23 @@ local({ # the loaded version of renv doesn't match the requested version; # give the user instructions on how to proceed - remote <- if (!is.null(description[["RemoteSha"]])) { + dev <- identical(description[["RemoteType"]], "github") + remote <- if (dev) paste("rstudio/renv", description[["RemoteSha"]], sep = "@") - } else { + else paste("renv", description[["Version"]], sep = "@") - } # display both loaded version + sha if available friendly <- renv_bootstrap_version_friendly( version = description[["Version"]], - sha = description[["RemoteSha"]] + sha = if (dev) description[["RemoteSha"]] ) - fmt <- paste( - "renv %1$s was loaded from project library, but this project is configured to use renv %2$s.", - "- Use `renv::record(\"%3$s\")` to record renv %1$s in the lockfile.", - "- Use `renv::restore(packages = \"renv\")` to install renv %2$s into the project library.", - sep = "\n" - ) + fmt <- heredoc(" + renv %1$s was loaded from project library, but this project is configured to use renv %2$s. + - Use `renv::record(\"%3$s\")` to record renv %1$s in the lockfile. + - Use `renv::restore(packages = \"renv\")` to install renv %2$s into the project library. + ") catf(fmt, friendly, renv_bootstrap_version_friendly(version), remote) FALSE @@ -826,13 +1044,19 @@ local({ } renv_bootstrap_validate_version_dev <- function(version, description) { + expected <- description[["RemoteSha"]] - is.character(expected) && startswith(expected, version) + if (!is.character(expected)) + return(FALSE) + + pattern <- sprintf("^\\Q%s\\E", version) + grepl(pattern, expected, perl = TRUE) + } renv_bootstrap_validate_version_release <- function(version, description) { expected <- description[["Version"]] - is.character(expected) && identical(expected, version) + is.character(expected) && identical(c(expected), c(version)) } renv_bootstrap_hash_text <- function(text) { @@ -1007,10 +1231,25 @@ local({ renv_bootstrap_exec <- function(project, libpath, version) { if (!renv_bootstrap_load(project, libpath, version)) - renv_bootstrap_run(version, libpath) + renv_bootstrap_run(project, libpath, version) + } + + renv_bootstrap_run <- function(project, libpath, version) { + tryCatch( + renv_bootstrap_run_impl(project, libpath, version), + error = function(e) { + msg <- paste( + "failed to bootstrap renv: the project will not be loaded.", + paste("Reason:", conditionMessage(e)), + "Use `renv::activate()` to re-initialize the project.", + sep = "\n" + ) + warning(msg, call. = FALSE) + } + ) } - renv_bootstrap_run <- function(version, libpath) { + renv_bootstrap_run_impl <- function(project, libpath, version) { # perform bootstrap bootstrap(version, libpath) @@ -1021,7 +1260,7 @@ local({ # try again to load if (requireNamespace("renv", lib.loc = libpath, quietly = TRUE)) { - return(renv::load(project = getwd())) + return(renv::load(project = project)) } # failed to download or load renv; warn the user @@ -1034,6 +1273,18 @@ local({ } + renv_bootstrap_cache_version <- function() { + # NOTE: users should normally not override the cache version; + # this is provided just to make testing easier + Sys.getenv("RENV_CACHE_VERSION", unset = "v5") + } + + renv_bootstrap_cache_version_previous <- function() { + version <- renv_bootstrap_cache_version() + number <- as.integer(substring(version, 2L)) + paste("v", number - 1L, sep = "") + } + renv_json_read <- function(file = NULL, text = NULL) { jlerr <- NULL @@ -1041,7 +1292,7 @@ local({ # if jsonlite is loaded, use that instead if ("jsonlite" %in% loadedNamespaces()) { - json <- catch(renv_json_read_jsonlite(file, text)) + json <- tryCatch(renv_json_read_jsonlite(file, text), error = identity) if (!inherits(json, "error")) return(json) @@ -1050,7 +1301,7 @@ local({ } # otherwise, fall back to the default JSON reader - json <- catch(renv_json_read_default(file, text)) + json <- tryCatch(renv_json_read_default(file, text), error = identity) if (!inherits(json, "error")) return(json) @@ -1063,102 +1314,109 @@ local({ } renv_json_read_jsonlite <- function(file = NULL, text = NULL) { - text <- paste(text %||% read(file), collapse = "\n") + text <- paste(text %||% readLines(file, warn = FALSE), collapse = "\n") jsonlite::fromJSON(txt = text, simplifyVector = FALSE) } - renv_json_read_default <- function(file = NULL, text = NULL) { + renv_json_read_patterns <- function() { - # find strings in the JSON - text <- paste(text %||% read(file), collapse = "\n") - pattern <- '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' - locs <- gregexpr(pattern, text, perl = TRUE)[[1]] + list( - # if any are found, replace them with placeholders - replaced <- text - strings <- character() - replacements <- character() + # objects + list("{", "\t\n\tobject(\t\n\t", TRUE), + list("}", "\t\n\t)\t\n\t", TRUE), - if (!identical(c(locs), -1L)) { + # arrays + list("[", "\t\n\tarray(\t\n\t", TRUE), + list("]", "\n\t\n)\n\t\n", TRUE), - # get the string values - starts <- locs - ends <- locs + attr(locs, "match.length") - 1L - strings <- substring(text, starts, ends) + # maps + list(":", "\t\n\t=\t\n\t", TRUE), - # only keep those requiring escaping - strings <- grep("[[\\]{}:]", strings, perl = TRUE, value = TRUE) + # newlines + list("\\u000a", "\n", FALSE) - # compute replacements - replacements <- sprintf('"\032%i\032"', seq_along(strings)) + ) - # replace the strings - mapply(function(string, replacement) { - replaced <<- sub(string, replacement, replaced, fixed = TRUE) - }, strings, replacements) + } - } + renv_json_read_envir <- function() { - # transform the JSON into something the R parser understands - transformed <- replaced - transformed <- gsub("{}", "`names<-`(list(), character())", transformed, fixed = TRUE) - transformed <- gsub("[[{]", "list(", transformed, perl = TRUE) - transformed <- gsub("[]}]", ")", transformed, perl = TRUE) - transformed <- gsub(":", "=", transformed, fixed = TRUE) - text <- paste(transformed, collapse = "\n") + envir <- new.env(parent = emptyenv()) - # parse it - json <- parse(text = text, keep.source = FALSE, srcfile = NULL)[[1L]] + envir[["+"]] <- `+` + envir[["-"]] <- `-` - # construct map between source strings, replaced strings - map <- as.character(parse(text = strings)) - names(map) <- as.character(parse(text = replacements)) + envir[["object"]] <- function(...) { + result <- list(...) + names(result) <- as.character(names(result)) + result + } - # convert to list - map <- as.list(map) + envir[["array"]] <- list - # remap strings in object - remapped <- renv_json_remap(json, map) + envir[["true"]] <- TRUE + envir[["false"]] <- FALSE + envir[["null"]] <- NULL - # evaluate - eval(remapped, envir = baseenv()) + envir } - renv_json_remap <- function(json, map) { + renv_json_read_remap <- function(object, patterns) { - # fix names - if (!is.null(names(json))) { - lhs <- match(names(json), names(map), nomatch = 0L) - rhs <- match(names(map), names(json), nomatch = 0L) - names(json)[rhs] <- map[lhs] - } + # repair names if necessary + if (!is.null(names(object))) { + + nms <- names(object) + for (pattern in patterns) + nms <- gsub(pattern[[2L]], pattern[[1L]], nms, fixed = TRUE) + names(object) <- nms - # fix values - if (is.character(json)) - return(map[[json]] %||% json) - - # handle true, false, null - if (is.name(json)) { - text <- as.character(json) - if (text == "true") - return(TRUE) - else if (text == "false") - return(FALSE) - else if (text == "null") - return(NULL) } - # recurse - if (is.recursive(json)) { - for (i in seq_along(json)) { - json[i] <- list(renv_json_remap(json[[i]], map)) - } + # repair strings if necessary + if (is.character(object)) { + for (pattern in patterns) + object <- gsub(pattern[[2L]], pattern[[1L]], object, fixed = TRUE) } - json + # recurse for other objects + if (is.recursive(object)) + for (i in seq_along(object)) + object[i] <- list(renv_json_read_remap(object[[i]], patterns)) + + # return remapped object + object + + } + + renv_json_read_default <- function(file = NULL, text = NULL) { + + # read json text + text <- paste(text %||% readLines(file, warn = FALSE), collapse = "\n") + + # convert into something the R parser will understand + patterns <- renv_json_read_patterns() + transformed <- text + for (pattern in patterns) + transformed <- gsub(pattern[[1L]], pattern[[2L]], transformed, fixed = TRUE) + + # parse it + rfile <- tempfile("renv-json-", fileext = ".R") + on.exit(unlink(rfile), add = TRUE) + writeLines(transformed, con = rfile) + json <- parse(rfile, keep.source = FALSE, srcfile = NULL)[[1L]] + + # evaluate in safe environment + result <- eval(json, envir = renv_json_read_envir()) + + # fix up strings if necessary -- do so only with reversible patterns + patterns <- Filter(function(pattern) pattern[[3L]], patterns) + renv_json_read_remap(result, patterns) } + # load the renv profile, if any renv_bootstrap_profile_load(project) diff --git a/r/src/common/logger.R b/r/src/common/logger.R deleted file mode 100644 index e8d2c79..0000000 --- a/r/src/common/logger.R +++ /dev/null @@ -1,57 +0,0 @@ -##### LOGGER - -FATAL <- 1L -names(FATAL) <- "FATAL" -ERROR <- 2L -names(ERROR) <- "ERROR" -WARN <- 4L -names(WARN) <- "WARN" -INFO <- 6L -names(INFO) <- "INFO" -DEBUG <- 8L -names(DEBUG) <- "DEBUG" -TRACE <- 9L -names(TRACE) <- "TRACE" - -logger.layout <- function(level, msg, id='', ...) { - the.time <- format(Sys.time(), "%Y-%m-%d %H:%M:%S") - if (length(list(...)) > 0) { - parsed <- lapply(list(...), function(x) if(is.null(x)) 'NULL' else x ) - msg <- do.call(sprintf, c(msg, parsed)) - } - sprintf("[%s] %s\n", names(level), msg) -} - -logger.log_level <- function(msg, ..., level) -{ - if (level <= logger.threshold) { - message <- logger.layout(level, msg, name, ...) - cat(message) - } -} - -logger.trace <- function(msg, ...) { - logger.log_level(msg, ..., level=TRACE) -} - -logger.debug <- function(msg, ...) { - logger.log_level(msg, ..., level=DEBUG) -} - -logger.info <- function(msg, ...) { - logger.log_level(msg, ..., level=INFO) -} - -logger.warn <- function(msg, ...) { - logger.log_level(msg, ..., level=WARN) -} - -logger.error <- function(msg, ...) { - logger.log_level(msg, ..., level = ERROR) -} - -logger.fatal <- function(msg, ...) { - logger.log_level(msg, ..., level = FATAL) -} - -logger.threshold = Sys.getenv(x = "LOG_LEVEL_SDK", TRACE) diff --git a/r/src/common/runtime_configuration.R b/r/src/common/runtime_configuration.R deleted file mode 100644 index fdbbfea..0000000 --- a/r/src/common/runtime_configuration.R +++ /dev/null @@ -1,15 +0,0 @@ -configuration <- function() { - configurationString <- Sys.getenv(x = "CONFIGURATION", "{}") - - result <- if(configurationString != "") { - jsonlite::fromJSON(txt=configurationString) - } else { - NULL - } - - if (Sys.getenv(x = "PRINT_CONFIGURATION", "no") == "yes") { - logger.debug("parse stored configuration: \'%s\'", configurationString) - logger.info("app will be started with configuration:\n%s", jsonlite::toJSON(result, auto_unbox = TRUE, pretty = TRUE)) - } - result -} diff --git a/r/src/io/app_files.R b/r/src/io/app_files.R deleted file mode 100644 index 0d5a326..0000000 --- a/r/src/io/app_files.R +++ /dev/null @@ -1,36 +0,0 @@ -#' Provides the path to app-files. App-files are files that -#' - are needed by the app to work during run time -#' - gets uploaded by the user of this app during configuration time -#' - there will be no warning if the requested file-set is not available (eg the user did not upload anything and the app developer did not provide any fallback) -#' - can also be provided by the app developer and gets bundled into the app during build time (the fallback) -#' -#' @param appSpecLocalFileSettingId The ID of the requested set of app-files (see counterpart in `appspec.json` (setting[type=LOCAL_FILE].id)) -#' @param fallbackToProvidedFiles Fallback to bundled directory of requested set of app-files (in case the app-developer provided a fallback)? -#' @return Path to the requested set of files (the app-file parent directory). Or `NULL` if user did not upload anything and no fallback was provided -#' -getAppFilePath <- function(appSpecLocalFileSettingId, fallbackToProvidedFiles=TRUE) { - if(!is.null(appSpecLocalFileSettingId) && appSpecLocalFileSettingId != "") { - userUpload <- paste0(Sys.getenv(x = "LOCAL_APP_FILES_DIR", "./uploaded-app-files/"), appSpecLocalFileSettingId, "/") - if (file.exists(userUpload) && list.files(userUpload) > 0) { - # directory exists and is not empty: user provided some files - logger.info(paste0("Detected app-files provided by user for '", appSpecLocalFileSettingId, "'.")) - return(userUpload) - } else if(fallbackToProvidedFiles) { - # fallback to directory provided by app developer - logger.info(paste0("Using fallback files provided by app developer for '", appSpecLocalFileSettingId, "'.")) - return(paste0("./provided-app-files/", appSpecLocalFileSettingId, "/")) - } else { - logger.warn(paste0("No files present for app-files '", appSpecLocalFileSettingId, "': User did not upload anything and the app did not provide fallback files.")) - return(NULL) - } - } -} - -#' Provides the path to an app-artifact. -#' You can write the app generated file to this path and it will become available on MoveApps after each app run. -#' @param artifactName The name of the artifact -#' @return Path for the artifact. Use it to write your data -#' -appArtifactPath <- function(artifactName) { - return(paste0(Sys.getenv(x = "APP_ARTIFACTS_DIR", "/tmp/"), artifactName)) -} \ No newline at end of file diff --git a/r/src/io/io_handler.R b/r/src/io/io_handler.R deleted file mode 100644 index 12f3e8e..0000000 --- a/r/src/io/io_handler.R +++ /dev/null @@ -1,40 +0,0 @@ -readInput <- function(sourceFile) { - inputType <- Sys.getenv(x = "INPUT_TYPE", "move::moveStack") - # for now every input-type can be read via RDS - logger.debug("Read input of type '%s'", inputType) - return(readRdsInput(sourceFile)) -} - -storeResult <- function(result, outputFile) { - outputType <- Sys.getenv(x = "OUTPUT_TYPE", "move::moveStack") - # for now every output-type can be written via RDS - logger.debug("Write output of type '%s'", outputType) - return(storeRdsOutput(result, outputFile)) -} - -storeToFile <- function(result, outputFile) { - if(!is.null(outputFile) && outputFile != "" && !is.null(result)) { - logger.debug("Writing to file %s", outputFile) - write(paste(result), file = outputFile) - } else { - logger.debug("Skip writing to file: no output File or result is missing") - } -} - -sourceFile <- function() { - result <- Sys.getenv(x = "SOURCE_FILE") - logger.debug("sourceFile: %s", result) - result -} - -outputFile <- function() { - result <- Sys.getenv(x = "OUTPUT_FILE", "") - logger.debug("outputFile: %s", result) - result -} - -errorFile <- function() { - result <- Sys.getenv(x = "ERROR_FILE", "") - logger.debug("errorFile: %s", result) - result -} \ No newline at end of file diff --git a/r/src/io/pilot_client.R b/r/src/io/pilot_client.R deleted file mode 100644 index ca03841..0000000 --- a/r/src/io/pilot_client.R +++ /dev/null @@ -1,41 +0,0 @@ -pilotEndpoint <- function() { - Sys.getenv(x = "PILOT_ENDPOINT", "http://localhost:8100") -} - -httpClientLogging <- function() { - httpClientLogging <- Sys.getenv(x = "PILOT_CLIENT_LOG_LEVEL", "NULL") # real NULL not possible (?!) - httpClientLogging != "NULL" -} - -notifyDone <- function(executionType) { - logger.debug("notify done with success") - response <- httr::POST( - paste(pilotEndpoint(), "/pilot/api/v1/copilot/done", sep = ""), - body = jsonlite::toJSON(list("success" = TRUE, "executionType" = executionType), auto_unbox = TRUE), - encode = "json", - httr::content_type_json(), - if (httpClientLogging()) httr::verbose(info = TRUE, data_out = TRUE, data_in = TRUE) - ) -} - -storeConfiguration <- function(configuration) { - logger.debug("Storing configuration in pilot: %s", configuration) - - response <- httr::POST( - paste(pilotEndpoint(), "/pilot/api/v1/copilot/configuration", sep = ""), - body = jsonlite::toJSON(list("configuration" = configuration), auto_unbox = TRUE), - encode = "json", - httr::content_type_json(), - if (httpClientLogging()) httr::verbose(info = TRUE, data_out = TRUE, data_in = TRUE) - ) - - parsedResponse <- httr::content(response, "parsed") - - if (parsedResponse["success"] == TRUE) { - newConfiguration = jsonlite::toJSON(parsedResponse[["configuration"]], auto_unbox = TRUE) - Sys.setenv(CONFIGURATION = newConfiguration) - logger.debug("Set new configuration environment: %s", newConfiguration) - } else { - logger.info("Couldn't store Configuration") - } -} diff --git a/r/src/io/rds.R b/r/src/io/rds.R deleted file mode 100644 index d1fdd6b..0000000 --- a/r/src/io/rds.R +++ /dev/null @@ -1,27 +0,0 @@ -readRdsInput <- function(sourceFile) { - if(!is.null(sourceFile) && sourceFile != "") { - if (file.info(sourceFile)$size == 0) { - # handle the special `null`-input - logger.warn("The App has received invalid input! It cannot process NULL-input. Aborting..") - # codes for exception handling - # 10: abort consuming null-input - stop("The App has received invalid input! It cannot process NULL-input. Check the output of the preceding App or adjust the datasource configuration. [code 10]") - } - logger.debug("Reading RDS from file '%s'", sourceFile) - return(readRDS(file = sourceFile)) - } else { - logger.debug("Skip loading: no source File") - } - - return(NULL) -} - -storeRdsOutput <- function(result, outputFile) { - if(!is.null(outputFile) && outputFile != "" && !is.null(result)) { - logger.debug("Storing RDS to file '%s'", outputFile) - saveRDS(result, file = outputFile) - } else { - logger.debug("Storing the null-result to file %s", outputFile) - file.create(outputFile) # write an empty file (for post-processing etc.) - } -} \ No newline at end of file diff --git a/r/tests/testthat/helper-link.R b/r/tests/testthat/helper-link.R new file mode 100644 index 0000000..2a4e66d --- /dev/null +++ b/r/tests/testthat/helper-link.R @@ -0,0 +1,20 @@ +# the code under test, and the contract cases shared with the python tests +source(file.path("..", "..", "link.R")) + +contractDir <- file.path("..", "..", "..", "test", "contract") +contractCases <- basename(list.dirs(contractDir, recursive = FALSE)) + +readCase <- function(case) readRDS(file.path(contractDir, case, "input.rds")) + +writeTempLines <- function(lines) { + file <- tempfile(fileext = ".csv") + writeLines(lines, file) + file +} + +metaFileFor <- function(tzone = "UTC") { + writeTempLines(c( + "crs,tzone,timeColName,trackIdColName", + sprintf("EPSG:4326,%s,timestamp_utc,track", tzone) + )) +} diff --git a/r/tests/testthat/test-contract.R b/r/tests/testthat/test-contract.R new file mode 100644 index 0000000..9bc546c --- /dev/null +++ b/r/tests/testthat/test-contract.R @@ -0,0 +1,32 @@ +# the CSV pair between R and python, pinned by the goldens the v2.2.0 production images produced +bytesOf <- function(file) readBin(file, what = "raw", n = file.size(file)) + +for (case in contractCases) { + test_that(sprintf("writeLink reproduces the v2.2.0 CSV pair of '%s' byte for byte", case), { + # arrange + buffer <- tempfile(fileext = ".csv") + meta <- tempfile(fileext = ".csv") + + # act + writeLink(data = readCase(case), bufferFile = buffer, metaFile = meta) + + # assert + expect_identical(bytesOf(meta), bytesOf(file.path(contractDir, case, "meta.csv"))) + expect_identical(bytesOf(buffer), bytesOf(file.path(contractDir, case, "link.csv"))) + }) + + test_that(sprintf("readLink turns python's CSV pair of '%s' back into its locations", case), { + # arrange: python keeps only the tracks with at least two fixes + input <- readCase(case) + + # act + actual <- readLink(bufferFile = file.path(contractDir, case, "py", "link.csv"), metaFile = file.path(contractDir, case, "py", "meta.csv")) + + # assert: times to the millisecond, as an absolute difference - expect_equal's tolerance is + # relative, and at 1.4e9 seconds even its default forgives 20 seconds + kept <- input[as.character(mt_track_id(input)) %in% as.character(mt_track_id(actual)), ] + expect_equal(nrow(actual), nrow(kept)) + expect_equal(sf::st_crs(actual), sf::st_crs(input)) + expect_lt(max(abs(sort(as.numeric(mt_time(actual))) - sort(as.numeric(mt_time(kept))))), 5e-4) + }) +} diff --git a/r/tests/testthat/test-link-meta.R b/r/tests/testthat/test-link-meta.R new file mode 100644 index 0000000..86f37eb --- /dev/null +++ b/r/tests/testthat/test-link-meta.R @@ -0,0 +1,32 @@ +test_that("linkMeta names crs, tzone, time and track column", { + # arrange + data <- readCase("latlon-three-tracks") + + # act + actual <- linkMeta(data = data) + + # assert + expect_equal(actual, data.frame(crs = "EPSG:4326", tzone = "UTC", timeColName = "timestamp", trackIdColName = "individual_local_identifier")) +}) + +test_that("linkMeta keeps the crs of a projected input", { + # arrange + data <- readCase("mollweide-midnight") + + # act + actual <- linkMeta(data = data) + + # assert + expect_equal(actual$crs, "ESRI:54009") +}) + +test_that("linkMeta keeps a timezone other than UTC", { + # arrange + data <- readCase("berlin") + + # act + actual <- linkMeta(data = data) + + # assert + expect_equal(actual$tzone, "Europe/Berlin") +}) diff --git a/r/tests/testthat/test-link-table.R b/r/tests/testthat/test-link-table.R new file mode 100644 index 0000000..e650353 --- /dev/null +++ b/r/tests/testthat/test-link-table.R @@ -0,0 +1,92 @@ +test_that("linkTable carries one row per location with its coordinates, without geometry", { + # arrange + data <- readCase("latlon-three-tracks") + + # act + actual <- linkTable(data = data) + + # assert + expect_equal(nrow(actual), 12) + expect_equal(actual$coords_x, unname(sf::st_coordinates(data)[, 1])) + expect_equal(actual$coords_y, unname(sf::st_coordinates(data)[, 2])) + expect_false("geometry" %in% names(actual)) +}) + +test_that("linkTable repeats the track attributes on every row", { + # arrange + data <- readCase("latlon-three-tracks") + + # act + actual <- linkTable(data = data) + + # assert: data.frame() makes the names syntactic, as v2.2.0 did + expect_true(all(make.names(names(mt_track_data(data))) %in% names(actual))) +}) + +test_that("linkTable writes sfc attributes as WKT", { + # arrange + data <- readCase("argos-sfc") + + # act + actual <- linkTable(data = data) + + # assert + expect_type(actual$argos_location_1, "character") + expect_match(actual$argos_location_1, "^POINT \\(") +}) + +test_that("linkTable leaves no list column behind", { + # arrange + data <- readCase("list-columns") + + # act + actual <- linkTable(data = data) + + # assert + expect_false(any(vapply(actual, is.list, logical(1)))) +}) + +test_that("linkTable keeps the milliseconds of a midnight fix", { + # arrange + data <- readCase("mollweide-midnight") + + # act + actual <- linkTable(data = data) + + # assert + expect_true("2021-12-10 00:00:00.000" %in% actual$timestamp) +}) + +test_that("linkTable keeps milliseconds below the second", { + # arrange + data <- readCase("subsecond") + + # act + actual <- linkTable(data = data) + + # assert + expect_equal(actual$timestamp, c("2014-08-06 09:19:35.000", "2014-08-06 09:19:35.999", "2014-08-06 09:19:36.999", "2014-08-06 09:19:38.000")) +}) + +test_that("linkTable writes UTC instants, whatever the tzone", { + # arrange + data <- readCase("berlin") + + # act + actual <- linkTable(data = data) + + # assert: 06:40 UTC is 08:40 in Berlin; link.csv carries the instant, meta.csv the zone + expect_equal(actual$timestamp[1], "2021-07-01 06:40:00.000") +}) + +test_that("linkTable rounds to the millisecond, so a time stored just below its millisecond keeps it", { + # arrange: 35.999 stored as 35.998999..., as 10 of 75 sub-second fixes of input2 are + data <- readCase("subsecond") + data[[mt_time_column(data)]] <- mt_time(data) - 1e-6 + + # act + actual <- linkTable(data = data) + + # assert + expect_equal(actual$timestamp, c("2014-08-06 09:19:35.000", "2014-08-06 09:19:35.999", "2014-08-06 09:19:36.999", "2014-08-06 09:19:38.000")) +}) diff --git a/r/tests/testthat/test-read-link.R b/r/tests/testthat/test-read-link.R new file mode 100644 index 0000000..4cd7c30 --- /dev/null +++ b/r/tests/testthat/test-read-link.R @@ -0,0 +1,91 @@ +test_that("readLink returns NULL for a buffer without rows", { + # arrange + buffer <- writeTempLines("timestamp_utc,track,coords_x,coords_y") + + # act + actual <- readLink(bufferFile = buffer, metaFile = metaFileFor()) + + # assert + expect_null(actual) +}) + +test_that("readLink builds a move2 in the crs of meta.csv, ordered by track and time", { + # arrange + buffer <- writeTempLines(c( + "timestamp_utc,track,coords_x,coords_y", + "2021-07-01 06:46:00,b,1,2", + "2021-07-01 06:40:00,b,3,4", + "2021-07-01 06:40:00,a,5,6" + )) + + # act + actual <- readLink(bufferFile = buffer, metaFile = metaFileFor()) + + # assert + expect_true(mt_is_move2(actual)) + expect_equal(sf::st_crs(actual), sf::st_crs("EPSG:4326")) + expect_equal(as.character(mt_track_id(actual)), c("a", "b", "b")) + expect_equal(format(mt_time(actual), "%H:%M"), c("06:40", "06:40", "06:46")) +}) + +test_that("readLink reads the times in the tzone of meta.csv", { + # arrange + buffer <- writeTempLines(c( + "timestamp_utc,track,coords_x,coords_y", + "2021-07-01 06:40:00,a,1,2", + "2021-07-01 06:46:00,a,1,2" + )) + + # act + actual <- readLink(bufferFile = buffer, metaFile = metaFileFor(tzone = "Asia/Kolkata")) + + # assert: 06:40 in Kolkata (UTC+05:30) is 01:10 UTC + expect_equal(attr(mt_time(actual), "tzone"), "Asia/Kolkata") + expect_equal(format(mt_time(actual)[1], "%Y-%m-%d %H:%M:%S", tz = "UTC"), "2021-07-01 01:10:00") +}) + +test_that("readLink keeps sub-second times", { + # arrange + buffer <- writeTempLines(c( + "timestamp_utc,track,coords_x,coords_y", + "2021-07-01 06:40:00.123,a,1,2", + "2021-07-01 06:40:01.000,a,1,2" + )) + + # act + actual <- readLink(bufferFile = buffer, metaFile = metaFileFor()) + + # assert: an absolute difference - a double at 1.6e9 seconds carries the .123 only to about 1e-7 + expect_lt(abs(as.numeric(mt_time(actual))[1] %% 1 - 0.123), 5e-4) +}) + +test_that("readLink turns python's True, False and missing value back into logical", { + # arrange: pandas writes a missing value as an empty field + buffer <- writeTempLines(c( + "timestamp_utc,track,coords_x,coords_y,visible,comment", + "2021-07-01 06:40:00,a,1,2,True,True story", + "2021-07-01 06:46:00,a,1,2,False,", + "2021-07-01 06:52:00,a,1,2,,x" + )) + + # act + actual <- readLink(bufferFile = buffer, metaFile = metaFileFor()) + + # assert: a column with other text stays text + expect_equal(actual$visible, c(TRUE, FALSE, NA)) + expect_type(actual$comment, "character") +}) + +test_that("readLink returns NULL for an empty buffer file", { + # arrange: python writes one for an empty TrajectoryCollection + buffer <- tempfile(fileext = ".csv") + file.create(buffer) + meta <- tempfile(fileext = ".csv") + file.create(meta) + + # act + actual <- readLink(bufferFile = buffer, metaFile = meta) + + # assert + expect_null(actual) +}) diff --git a/r2python.sh b/r2python.sh index 58a7a60..c8bae07 100755 --- a/r2python.sh +++ b/r2python.sh @@ -1,4 +1,4 @@ -#!/bin/bash --login +#!/bin/bash ######################################################################################################################## # converts from R .rds to Python .pickle ######################################################################################################################## @@ -14,4 +14,4 @@ set -a set +a (cd r && Rscript rds_2_csv.R) -(cd python && conda activate "$HOME"/co-pilot-r/python-env && python csv_2_pickle.py) +(cd python && conda run --no-capture-output --prefix "$ENV_PREFIX" python csv_2_pickle.py) diff --git a/test/Dockerfile b/test/Dockerfile index ad251fd..fd32022 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,7 +1,9 @@ # Runs the Python unit tests in the environment the App uses. -# The base tag is pinned on purpose: in the sibling repo `cargo-agent-python` a floating -# `:latest` silently moved to a base without `adduser` and broke the build. -FROM condaforge/miniforge3:26.3.2-3 +# The base tag is the python co-pilot's own base, whose /opt/conda the App copies; test/coherence.R +# holds CI to it. Pinned on purpose: in the sibling repo `cargo-agent-python` a floating `:latest` +# silently moved to a base without `adduser` and broke the build. +# linux/amd64: python/conda-linux-64.lock holds linux-64 packages only, so an arm64 host emulates. +FROM --platform=linux/amd64 condaforge/miniforge3:26.7.2-0 LABEL org.opencontainers.image.authors="us@couchbits.com" LABEL org.opencontainers.image.vendor="couchbits GmbH" @@ -9,15 +11,26 @@ ENV PROJECT_DIR=/opt/link-r-python ENV ENV_PREFIX=${PROJECT_DIR}/conda WORKDIR $PROJECT_DIR -# the environment first, so its layer caches independently of the sources -COPY python/environment.yml ./python/ -RUN conda env create --prefix ${ENV_PREFIX} --file python/environment.yml && \ +# the environment first, so its layer caches independently of the sources. +# lock (default): what the App installs. fresh: environment.yml resolved today - what the next lock would +# be, for the weekly run. +ARG ENVIRONMENT=lock +COPY python/environment.yml python/conda-linux-64.lock ./python/ +RUN case "$ENVIRONMENT" in \ + lock) conda create --yes --prefix ${ENV_PREFIX} --file python/conda-linux-64.lock ;; \ + fresh) conda env create --prefix ${ENV_PREFIX} --file python/environment.yml ;; \ + *) echo "ENVIRONMENT must be lock or fresh" >&2; exit 1 ;; \ + esac && \ conda clean --all --yes +# what got installed, for the weekly comparison with the lock +RUN conda list --prefix ${ENV_PREFIX} --explicit --md5 | grep '^https' > resolved.txt COPY python/ ./python/ +# the golden CSV pairs the contract test reads, shared with the R tests +COPY test/contract/ ./test/contract/ # `--no-capture-output` or conda swallows the result, including a failure. # The test modules are named explicitly: they use relative imports and there is no # `__init__.py`, so `unittest discover` does not find them reliably. RUN conda run --no-capture-output --prefix ${ENV_PREFIX} python -m unittest \ - python.tests.test_transform_to_pickle python.tests.test_transform_to_csv + python.tests.test_transform_to_pickle python.tests.test_transform_to_csv python.tests.test_contract diff --git a/test/app-test.Dockerfile b/test/app-test.Dockerfile new file mode 100644 index 0000000..4fc0e03 --- /dev/null +++ b/test/app-test.Dockerfile @@ -0,0 +1,20 @@ +# Runs the coherence check, the R unit tests and the round trip inside the real App images. +FROM app-r2python + +# the python -> R entry point; everything else in the two images is identical +COPY --from=app-python2r --chown=$UID:$GID /home/moveapps/co-pilot-r/start-process.sh /home/moveapps/co-pilot-r/start-process-python2r.sh + +# testthat is a test dependency only: installed here, never part of the App's lock +WORKDIR /home/moveapps/co-pilot-r/r +RUN R -q -e 'renv::install("testthat@3.3.2")' + +WORKDIR /home/moveapps/co-pilot-r +COPY --chown=$UID:$GID Dockerfile ./Dockerfile +COPY --chown=$UID:$GID python/environment.yml python/conda-linux-64.lock ./python/ +COPY --chown=$UID:$GID test/ ./test/ +COPY --chown=$UID:$GID r/tests/ ./r/tests/ +COPY --chown=$UID:$GID r/data/raw/ ./r/data/raw/ + +RUN cd r && Rscript ../test/coherence.R .. +RUN cd r && Rscript -e 'testthat::test_dir("tests/testthat")' +RUN test/roundtrip/run.sh diff --git a/test/co-pilot-r.Dockerfile b/test/co-pilot-r.Dockerfile new file mode 100644 index 0000000..a0133a3 --- /dev/null +++ b/test/co-pilot-r.Dockerfile @@ -0,0 +1,42 @@ +######################################################################################################################## +# A public stand-in for the private R co-pilot, so CI can build the real App Dockerfile. +# Mirrors co-pilot-v1/co-pilot-r/r/Dockerfile of movestore-groundcontrol: same rocker digest, user, directories and +# renv setup. Left out: the Java part, and the co-pilot's own renv project - the translator brings `moveapps` through +# r/renv.lock. Keep the digest in step with the co-pilot; test/coherence.R can check the R version, not the digest, +# because the co-pilot's registry is private. +######################################################################################################################## +# rocker/geospatial:4.6.1 +FROM rocker/geospatial@sha256:4cee92a576e26c31546860eb252290f8f259e8236d49600d6bee695dfb96cae5 + +RUN apt-get update && apt-get install -y \ + libcurl4-openssl-dev \ +# packages that build a vendored C++ dependency need it, e.g. s2 (Abseil) + cmake \ + && apt-get clean + +ARG username=moveapps +ARG uid=1001 +ARG gid=staff +ENV USER=$username +ENV UID=$uid +ENV GID=$gid +ENV HOME=/home/$USER + +RUN adduser --disabled-password \ + --gecos "Non-root user" \ + --uid $UID \ + --ingroup $GID \ + --home $HOME \ + $USER +RUN install -d -o moveapps -g staff $HOME/co-pilot-r +RUN install -d -o moveapps -g staff $HOME/.cache/R +USER $USER +WORKDIR $HOME/co-pilot-r + +ENV RENV_PATHS_CACHE=$HOME/.cache/R/renv +# The one deviation from the co-pilot, which compiles from https://cloud.r-project.org: P3M binaries of the very +# versions the lock pins, so a cold CI run takes minutes. A version P3M lacks falls back to source (hence cmake). +ENV RENV_CONFIG_REPOS_OVERRIDE=https://p3m.dev/cran/__linux__/noble/latest +ENV RENV_CONFIG_SANDBOX_ENABLED=FALSE + +RUN R -e "if (!requireNamespace('renv', quietly = TRUE)) install.packages('renv')" diff --git a/test/coherence.R b/test/coherence.R new file mode 100644 index 0000000..1c1197f --- /dev/null +++ b/test/coherence.R @@ -0,0 +1,33 @@ +# The versions pinned in separate files must agree - nothing else compares them. +# Rscript coherence.R (inside the test image: R is the stand-in's) +root <- commandArgs(trailingOnly = TRUE)[1] + +field <- function(text, pattern) { + match <- regmatches(text, regexec(pattern, text))[[1]] + if (length(match) == 2) match[2] else NA_character_ +} + +base <- grep("^ARG BASE=", readLines(file.path(root, "Dockerfile")), value = TRUE) +lock <- jsonlite::fromJSON(file.path(root, "r", "renv.lock")) +bake <- paste(readLines(file.path(root, "test", "docker-bake.hcl")), collapse = "\n") +testDockerfile <- paste(readLines(file.path(root, "test", "Dockerfile")), collapse = "\n") +condaLock <- paste(readLines(file.path(root, "python", "conda-linux-64.lock")), collapse = "\n") + +check <- function(label, left, right) { + ok <- !is.na(left) && !is.na(right) && identical(left, right) + cat(sprintf("%s %s: %s / %s\n", if (ok) "ok " else "FAIL", label, left, right)) + ok +} + +results <- c( + check("R of this image / R in r/renv.lock", as.character(getRversion()), lock$R$Version), + check("R in r/renv.lock / geospatial-X of BASE", lock$R$Version, field(base, "_geospatial-([0-9.]+)_")), + check("moveapps in r/renv.lock / sdk-vX of BASE", lock$Packages$moveapps$Version, field(base, "_sdk-v([0-9.]+)_")), + check("Miniforge CI passes as CONDA / Miniforge of test/Dockerfile", + field(bake, 'default = "condaforge/miniforge3:([0-9A-Za-z._-]+)"'), + field(testDockerfile, "condaforge/miniforge3:([0-9A-Za-z._-]+)")), + check("environment.yml / its md5 in python/conda-linux-64.lock (python/lock.sh)", + unname(tools::md5sum(file.path(root, "python", "environment.yml"))), + field(condaLock, "# environment.yml md5: ([0-9a-f]{32})")) +) +if (!all(results)) quit(status = 1) diff --git a/test/contract/argos-sfc/input.rds b/test/contract/argos-sfc/input.rds new file mode 100644 index 0000000..0e9d2e9 Binary files /dev/null and b/test/contract/argos-sfc/input.rds differ diff --git a/test/contract/argos-sfc/link.csv b/test/contract/argos-sfc/link.csv new file mode 100644 index 0000000..578b53a --- /dev/null +++ b/test/contract/argos-sfc/link.csv @@ -0,0 +1,11 @@ +"sensor_type_id","argos_altitude","argos_best_level","argos_calcul_freq","argos_iq","argos_lc","argos_nb_mes","argos_nb_mes_120","argos_nopc","argos_pass_duration","argos_sensor_1","argos_sensor_2","argos_sensor_3","argos_sensor_4","timestamp","visible","event_id","argos_location_1","argos_location_2","individual_name_deployment_id","tag_id","individual_id","deploy_off_timestamp","deploy_on_timestamp","deployment_local_identifier","sensor_type_ids","capture_location","deploy_on_location","deploy_off_location","individual_comments","death_comments","latest_date_born","individual_local_identifier","ring_id","sex","taxon_canonical_name","individual_number_of_deployments","mortality_location","tag_local_identifier","tag_number_of_deployments","study_id","has_quota","i_am_owner","is_test","license_type","name","study_number_of_deployments","number_of_individuals","number_of_tags","principal_investigator_name","study_objective","study_type","suspend_license_terms","i_can_see_data","there_are_data_which_i_cannot_see","i_have_download_access","i_am_collaborator","study_permission","timestamp_first_deployed_location","timestamp_last_deployed_location","number_of_deployed_locations","taxon_ids","contact_person_name","main_location","deployment_id","coords_x","coords_y" +82798,0,-125,401.6515613,"60","1",4,0,"2",250,146,169,85,170,"1994-08-12 04:14:24.000",TRUE,16578813,"POINT (12.08 52.126)","POINT (30.367 48.455)","Prinzesschen_1930120",1898884,1898896,1995-06-21,1994-07-09,"Prinzesschen-14552","argos-doppler-shift","POINT EMPTY","POINT EMPTY","POINT EMPTY","Tracked for a life time with 5 tags","23-12-2006 by age on wintering grounds",1993-01-01,"Prinzesschen","DEH KA0749","f","Ciconia ciconia",1,"POINT EMPTY","14552",1,1898591,TRUE,TRUE,FALSE,"CC_BY","MPIAB White Stork Prinzesschen",5,1,5,"Peter Berthold","Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",1994-08-12 04:14:24,2006-12-22 22:07:04,3169,"Ciconia ciconia","myotis (Wolfgang Fiedler)","POINT (11.66748 52.14697)",1930120,12.08,52.126 +82798,0,-128,401.651724,"50","1",6,0,"3",557,248,136,78,175,"1994-10-01 13:43:42.000",TRUE,16578887,"POINT (36.413 15.29)","POINT (57.281 20.078)","Prinzesschen_1930120",1898884,1898896,1995-06-21,1994-07-09,"Prinzesschen-14552","argos-doppler-shift","POINT EMPTY","POINT EMPTY","POINT EMPTY","Tracked for a life time with 5 tags","23-12-2006 by age on wintering grounds",1993-01-01,"Prinzesschen","DEH KA0749","f","Ciconia ciconia",1,"POINT EMPTY","14552",1,1898591,TRUE,TRUE,FALSE,"CC_BY","MPIAB White Stork Prinzesschen",5,1,5,"Peter Berthold","Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",1994-08-12 04:14:24,2006-12-22 22:07:04,3169,"Ciconia ciconia","myotis (Wolfgang Fiedler)","POINT (11.66748 52.14697)",1930120,36.413,15.29 +82798,78,-122,401.6524144,"58","1",5,0,"3",435,115,206,243,2,"1997-08-23 02:28:39.000",TRUE,16573774,"POINT (12.094 52.116)","POINT (7.959 52.928)","Prinzesschen_1930121",1898885,1898896,1998-04-28,1997-08-23,"Prinzesschen-14543","argos-doppler-shift","POINT EMPTY","POINT EMPTY","POINT EMPTY","Tracked for a life time with 5 tags","23-12-2006 by age on wintering grounds",1993-01-01,"Prinzesschen","DEH KA0749","f","Ciconia ciconia",1,"POINT EMPTY","14543",1,1898591,TRUE,TRUE,FALSE,"CC_BY","MPIAB White Stork Prinzesschen",5,1,5,"Peter Berthold","Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",1994-08-12 04:14:24,2006-12-22 22:07:04,3169,"Ciconia ciconia","myotis (Wolfgang Fiedler)","POINT (11.66748 52.14697)",1930121,12.094,52.116 +82798,78,-127,401.6527567,"50","1",6,0,"1",360,143,210,171,0,"1997-08-25 00:23:58.000",TRUE,16573800,"POINT (12.094 52.141)","POINT (63.826 41.815)","Prinzesschen_1930121",1898885,1898896,1998-04-28,1997-08-23,"Prinzesschen-14543","argos-doppler-shift","POINT EMPTY","POINT EMPTY","POINT EMPTY","Tracked for a life time with 5 tags","23-12-2006 by age on wintering grounds",1993-01-01,"Prinzesschen","DEH KA0749","f","Ciconia ciconia",1,"POINT EMPTY","14543",1,1898591,TRUE,TRUE,FALSE,"CC_BY","MPIAB White Stork Prinzesschen",5,1,5,"Peter Berthold","Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",1994-08-12 04:14:24,2006-12-22 22:07:04,3169,"Ciconia ciconia","myotis (Wolfgang Fiedler)","POINT (11.66748 52.14697)",1930121,12.094,52.141 +82798,80,-124,401.6535849,"68","1",4,0,"2",318,192,176,247,96,"1998-08-11 09:35:38.000",TRUE,16587151,"POINT (12.174 52.095)","POINT (-12.434 56.662)","Prinzesschen_1930122",1898886,1898896,1999-05-20,1998-08-02,"Prinzesschen-14544","argos-doppler-shift","POINT EMPTY","POINT EMPTY","POINT EMPTY","Tracked for a life time with 5 tags","23-12-2006 by age on wintering grounds",1993-01-01,"Prinzesschen","DEH KA0749","f","Ciconia ciconia",1,"POINT EMPTY","14544",1,1898591,TRUE,TRUE,FALSE,"CC_BY","MPIAB White Stork Prinzesschen",5,1,5,"Peter Berthold","Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",1994-08-12 04:14:24,2006-12-22 22:07:04,3169,"Ciconia ciconia","myotis (Wolfgang Fiedler)","POINT (11.66748 52.14697)",1930122,12.174,52.095 +82798,80,-125,401.6531782,"68","2",4,0,"3",381,0,4,82,1,"1998-08-20 19:20:44.000",TRUE,16587252,"POINT (12.095 52.135)","POINT (-24.834 44.927)","Prinzesschen_1930122",1898886,1898896,1999-05-20,1998-08-02,"Prinzesschen-14544","argos-doppler-shift","POINT EMPTY","POINT EMPTY","POINT EMPTY","Tracked for a life time with 5 tags","23-12-2006 by age on wintering grounds",1993-01-01,"Prinzesschen","DEH KA0749","f","Ciconia ciconia",1,"POINT EMPTY","14544",1,1898591,TRUE,TRUE,FALSE,"CC_BY","MPIAB White Stork Prinzesschen",5,1,5,"Peter Berthold","Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",1994-08-12 04:14:24,2006-12-22 22:07:04,3169,"Ciconia ciconia","myotis (Wolfgang Fiedler)","POINT (11.66748 52.14697)",1930122,12.095,52.135 +82798,0,-123,401.6535394,"68","3",4,0,"3",453,173,204,162,102,"2001-08-06 09:08:16.000",TRUE,16568432,"POINT (12.094 52.108)","POINT (45.047 45.75)","Prinzesschen_1930123",1898887,1898896,2004-07-25,2001-07-31,"Prinzesschen-16906","argos-doppler-shift","POINT EMPTY","POINT EMPTY","POINT EMPTY","Tracked for a life time with 5 tags","23-12-2006 by age on wintering grounds",1993-01-01,"Prinzesschen","DEH KA0749","f","Ciconia ciconia",1,"POINT EMPTY","16906",1,1898591,TRUE,TRUE,FALSE,"CC_BY","MPIAB White Stork Prinzesschen",5,1,5,"Peter Berthold","Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",1994-08-12 04:14:24,2006-12-22 22:07:04,3169,"Ciconia ciconia","myotis (Wolfgang Fiedler)","POINT (11.66748 52.14697)",1930123,12.094,52.108 +82798,0,-125,401.6534961,"58","1",4,0,"3",456,157,152,118,57,"2001-08-06 16:37:49.000",TRUE,16568439,"POINT (12.104 52.116)","POINT (-17.527 46.17)","Prinzesschen_1930123",1898887,1898896,2004-07-25,2001-07-31,"Prinzesschen-16906","argos-doppler-shift","POINT EMPTY","POINT EMPTY","POINT EMPTY","Tracked for a life time with 5 tags","23-12-2006 by age on wintering grounds",1993-01-01,"Prinzesschen","DEH KA0749","f","Ciconia ciconia",1,"POINT EMPTY","16906",1,1898591,TRUE,TRUE,FALSE,"CC_BY","MPIAB White Stork Prinzesschen",5,1,5,"Peter Berthold","Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",1994-08-12 04:14:24,2006-12-22 22:07:04,3169,"Ciconia ciconia","myotis (Wolfgang Fiedler)","POINT (11.66748 52.14697)",1930123,12.104,52.116 +82798,100,-126,401.6518595,"60","2",5,0,"3",660,204,221,2,25,"2004-07-25 11:52:39.000",TRUE,16574225,"POINT (12.137 52.11)","POINT (-30.36 59.728)","Prinzesschen_1930124",1898888,1898896,2006-12-23,2004-07-25,"Prinzesschen-40534","argos-doppler-shift","POINT EMPTY","POINT EMPTY","POINT EMPTY","Tracked for a life time with 5 tags","23-12-2006 by age on wintering grounds",1993-01-01,"Prinzesschen","DEH KA0749","f","Ciconia ciconia",1,"POINT EMPTY","40534",1,1898591,TRUE,TRUE,FALSE,"CC_BY","MPIAB White Stork Prinzesschen",5,1,5,"Peter Berthold","Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",1994-08-12 04:14:24,2006-12-22 22:07:04,3169,"Ciconia ciconia","myotis (Wolfgang Fiedler)","POINT (11.66748 52.14697)",1930124,12.137,52.11 +82798,100,-125,401.651968,"60","3",8,0,"3",660,0,1,1,6,"2004-07-25 17:13:55.000",TRUE,16574231,"POINT (12.129 52.102)","POINT (-44.103 41.37)","Prinzesschen_1930124",1898888,1898896,2006-12-23,2004-07-25,"Prinzesschen-40534","argos-doppler-shift","POINT EMPTY","POINT EMPTY","POINT EMPTY","Tracked for a life time with 5 tags","23-12-2006 by age on wintering grounds",1993-01-01,"Prinzesschen","DEH KA0749","f","Ciconia ciconia",1,"POINT EMPTY","40534",1,1898591,TRUE,TRUE,FALSE,"CC_BY","MPIAB White Stork Prinzesschen",5,1,5,"Peter Berthold","Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",1994-08-12 04:14:24,2006-12-22 22:07:04,3169,"Ciconia ciconia","myotis (Wolfgang Fiedler)","POINT (11.66748 52.14697)",1930124,12.129,52.102 diff --git a/test/contract/argos-sfc/meta.csv b/test/contract/argos-sfc/meta.csv new file mode 100644 index 0000000..ee91cda --- /dev/null +++ b/test/contract/argos-sfc/meta.csv @@ -0,0 +1,2 @@ +"crs","tzone","timeColName","trackIdColName" +"EPSG:4326","UTC","timestamp","individual_name_deployment_id" diff --git a/test/contract/argos-sfc/py/link.csv b/test/contract/argos-sfc/py/link.csv new file mode 100644 index 0000000..dd379b4 --- /dev/null +++ b/test/contract/argos-sfc/py/link.csv @@ -0,0 +1,11 @@ +timestamp_utc,sensor_type_id,argos_altitude,argos_best_level,argos_calcul_freq,argos_iq,argos_lc,argos_nb_mes,argos_nb_mes_120,argos_nopc,argos_pass_duration,argos_sensor_1,argos_sensor_2,argos_sensor_3,argos_sensor_4,timestamp,visible,event_id,argos_location_1,argos_location_2,individual_name_deployment_id,tag_id,individual_id,deploy_off_timestamp,deploy_on_timestamp,deployment_local_identifier,sensor_type_ids,capture_location,deploy_on_location,deploy_off_location,individual_comments,death_comments,latest_date_born,individual_local_identifier,ring_id,sex,taxon_canonical_name,individual_number_of_deployments,mortality_location,tag_local_identifier,tag_number_of_deployments,study_id,has_quota,i_am_owner,is_test,license_type,name,study_number_of_deployments,number_of_individuals,number_of_tags,principal_investigator_name,study_objective,study_type,suspend_license_terms,i_can_see_data,there_are_data_which_i_cannot_see,i_have_download_access,i_am_collaborator,study_permission,timestamp_first_deployed_location,timestamp_last_deployed_location,number_of_deployed_locations,taxon_ids,contact_person_name,main_location,deployment_id,timestamp_tz,geometry,coords_x,coords_y +1994-08-12 04:14:24,82798,0,-125,401.6515613,60,1,4,0,2,250,146,169,85,170,1994-08-12 04:14:24,True,16578813,POINT (12.08 52.126),POINT (30.367 48.455),Prinzesschen_1930120,1898884,1898896,1995-06-21,1994-07-09,Prinzesschen-14552,argos-doppler-shift,POINT EMPTY,POINT EMPTY,POINT EMPTY,Tracked for a life time with 5 tags,23-12-2006 by age on wintering grounds,1993-01-01,Prinzesschen,DEH KA0749,f,Ciconia ciconia,1,POINT EMPTY,14552,1,1898591,True,True,False,CC_BY,MPIAB White Stork Prinzesschen,5,1,5,Peter Berthold,"Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.",research,False,True,False,True,False,data_manager,1994-08-12 04:14:24,2006-12-22 22:07:04,3169,Ciconia ciconia,myotis (Wolfgang Fiedler),POINT (11.66748 52.14697),1930120,1994-08-12 04:14:24+00:00,POINT (12.08 52.126),12.08,52.126 +1994-10-01 13:43:42,82798,0,-128,401.651724,50,1,6,0,3,557,248,136,78,175,1994-10-01 13:43:42,True,16578887,POINT (36.413 15.29),POINT (57.281 20.078),Prinzesschen_1930120,1898884,1898896,1995-06-21,1994-07-09,Prinzesschen-14552,argos-doppler-shift,POINT EMPTY,POINT EMPTY,POINT EMPTY,Tracked for a life time with 5 tags,23-12-2006 by age on wintering grounds,1993-01-01,Prinzesschen,DEH KA0749,f,Ciconia ciconia,1,POINT EMPTY,14552,1,1898591,True,True,False,CC_BY,MPIAB White Stork Prinzesschen,5,1,5,Peter Berthold,"Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.",research,False,True,False,True,False,data_manager,1994-08-12 04:14:24,2006-12-22 22:07:04,3169,Ciconia ciconia,myotis (Wolfgang Fiedler),POINT (11.66748 52.14697),1930120,1994-10-01 13:43:42+00:00,POINT (36.413 15.29),36.413,15.29 +1997-08-23 02:28:39,82798,78,-122,401.6524144,58,1,5,0,3,435,115,206,243,2,1997-08-23 02:28:39,True,16573774,POINT (12.094 52.116),POINT (7.959 52.928),Prinzesschen_1930121,1898885,1898896,1998-04-28,1997-08-23,Prinzesschen-14543,argos-doppler-shift,POINT EMPTY,POINT EMPTY,POINT EMPTY,Tracked for a life time with 5 tags,23-12-2006 by age on wintering grounds,1993-01-01,Prinzesschen,DEH KA0749,f,Ciconia ciconia,1,POINT EMPTY,14543,1,1898591,True,True,False,CC_BY,MPIAB White Stork Prinzesschen,5,1,5,Peter Berthold,"Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.",research,False,True,False,True,False,data_manager,1994-08-12 04:14:24,2006-12-22 22:07:04,3169,Ciconia ciconia,myotis (Wolfgang Fiedler),POINT (11.66748 52.14697),1930121,1997-08-23 02:28:39+00:00,POINT (12.094 52.116),12.094,52.116 +1997-08-25 00:23:58,82798,78,-127,401.6527567,50,1,6,0,1,360,143,210,171,0,1997-08-25 00:23:58,True,16573800,POINT (12.094 52.141),POINT (63.826 41.815),Prinzesschen_1930121,1898885,1898896,1998-04-28,1997-08-23,Prinzesschen-14543,argos-doppler-shift,POINT EMPTY,POINT EMPTY,POINT EMPTY,Tracked for a life time with 5 tags,23-12-2006 by age on wintering grounds,1993-01-01,Prinzesschen,DEH KA0749,f,Ciconia ciconia,1,POINT EMPTY,14543,1,1898591,True,True,False,CC_BY,MPIAB White Stork Prinzesschen,5,1,5,Peter Berthold,"Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.",research,False,True,False,True,False,data_manager,1994-08-12 04:14:24,2006-12-22 22:07:04,3169,Ciconia ciconia,myotis (Wolfgang Fiedler),POINT (11.66748 52.14697),1930121,1997-08-25 00:23:58+00:00,POINT (12.094 52.141),12.094,52.141 +1998-08-11 09:35:38,82798,80,-124,401.6535849,68,1,4,0,2,318,192,176,247,96,1998-08-11 09:35:38,True,16587151,POINT (12.174 52.095),POINT (-12.434 56.662),Prinzesschen_1930122,1898886,1898896,1999-05-20,1998-08-02,Prinzesschen-14544,argos-doppler-shift,POINT EMPTY,POINT EMPTY,POINT EMPTY,Tracked for a life time with 5 tags,23-12-2006 by age on wintering grounds,1993-01-01,Prinzesschen,DEH KA0749,f,Ciconia ciconia,1,POINT EMPTY,14544,1,1898591,True,True,False,CC_BY,MPIAB White Stork Prinzesschen,5,1,5,Peter Berthold,"Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.",research,False,True,False,True,False,data_manager,1994-08-12 04:14:24,2006-12-22 22:07:04,3169,Ciconia ciconia,myotis (Wolfgang Fiedler),POINT (11.66748 52.14697),1930122,1998-08-11 09:35:38+00:00,POINT (12.174 52.095),12.174,52.095 +1998-08-20 19:20:44,82798,80,-125,401.6531782,68,2,4,0,3,381,0,4,82,1,1998-08-20 19:20:44,True,16587252,POINT (12.095 52.135),POINT (-24.834 44.927),Prinzesschen_1930122,1898886,1898896,1999-05-20,1998-08-02,Prinzesschen-14544,argos-doppler-shift,POINT EMPTY,POINT EMPTY,POINT EMPTY,Tracked for a life time with 5 tags,23-12-2006 by age on wintering grounds,1993-01-01,Prinzesschen,DEH KA0749,f,Ciconia ciconia,1,POINT EMPTY,14544,1,1898591,True,True,False,CC_BY,MPIAB White Stork Prinzesschen,5,1,5,Peter Berthold,"Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.",research,False,True,False,True,False,data_manager,1994-08-12 04:14:24,2006-12-22 22:07:04,3169,Ciconia ciconia,myotis (Wolfgang Fiedler),POINT (11.66748 52.14697),1930122,1998-08-20 19:20:44+00:00,POINT (12.095 52.135),12.095,52.135 +2001-08-06 09:08:16,82798,0,-123,401.6535394,68,3,4,0,3,453,173,204,162,102,2001-08-06 09:08:16,True,16568432,POINT (12.094 52.108),POINT (45.047 45.75),Prinzesschen_1930123,1898887,1898896,2004-07-25,2001-07-31,Prinzesschen-16906,argos-doppler-shift,POINT EMPTY,POINT EMPTY,POINT EMPTY,Tracked for a life time with 5 tags,23-12-2006 by age on wintering grounds,1993-01-01,Prinzesschen,DEH KA0749,f,Ciconia ciconia,1,POINT EMPTY,16906,1,1898591,True,True,False,CC_BY,MPIAB White Stork Prinzesschen,5,1,5,Peter Berthold,"Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.",research,False,True,False,True,False,data_manager,1994-08-12 04:14:24,2006-12-22 22:07:04,3169,Ciconia ciconia,myotis (Wolfgang Fiedler),POINT (11.66748 52.14697),1930123,2001-08-06 09:08:16+00:00,POINT (12.094 52.108),12.094,52.108 +2001-08-06 16:37:49,82798,0,-125,401.6534961,58,1,4,0,3,456,157,152,118,57,2001-08-06 16:37:49,True,16568439,POINT (12.104 52.116),POINT (-17.527 46.17),Prinzesschen_1930123,1898887,1898896,2004-07-25,2001-07-31,Prinzesschen-16906,argos-doppler-shift,POINT EMPTY,POINT EMPTY,POINT EMPTY,Tracked for a life time with 5 tags,23-12-2006 by age on wintering grounds,1993-01-01,Prinzesschen,DEH KA0749,f,Ciconia ciconia,1,POINT EMPTY,16906,1,1898591,True,True,False,CC_BY,MPIAB White Stork Prinzesschen,5,1,5,Peter Berthold,"Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.",research,False,True,False,True,False,data_manager,1994-08-12 04:14:24,2006-12-22 22:07:04,3169,Ciconia ciconia,myotis (Wolfgang Fiedler),POINT (11.66748 52.14697),1930123,2001-08-06 16:37:49+00:00,POINT (12.104 52.116),12.104,52.116 +2004-07-25 11:52:39,82798,100,-126,401.6518595,60,2,5,0,3,660,204,221,2,25,2004-07-25 11:52:39,True,16574225,POINT (12.137 52.11),POINT (-30.36 59.728),Prinzesschen_1930124,1898888,1898896,2006-12-23,2004-07-25,Prinzesschen-40534,argos-doppler-shift,POINT EMPTY,POINT EMPTY,POINT EMPTY,Tracked for a life time with 5 tags,23-12-2006 by age on wintering grounds,1993-01-01,Prinzesschen,DEH KA0749,f,Ciconia ciconia,1,POINT EMPTY,40534,1,1898591,True,True,False,CC_BY,MPIAB White Stork Prinzesschen,5,1,5,Peter Berthold,"Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.",research,False,True,False,True,False,data_manager,1994-08-12 04:14:24,2006-12-22 22:07:04,3169,Ciconia ciconia,myotis (Wolfgang Fiedler),POINT (11.66748 52.14697),1930124,2004-07-25 11:52:39+00:00,POINT (12.137 52.11),12.137,52.11 +2004-07-25 17:13:55,82798,100,-125,401.651968,60,3,8,0,3,660,0,1,1,6,2004-07-25 17:13:55,True,16574231,POINT (12.129 52.102),POINT (-44.103 41.37),Prinzesschen_1930124,1898888,1898896,2006-12-23,2004-07-25,Prinzesschen-40534,argos-doppler-shift,POINT EMPTY,POINT EMPTY,POINT EMPTY,Tracked for a life time with 5 tags,23-12-2006 by age on wintering grounds,1993-01-01,Prinzesschen,DEH KA0749,f,Ciconia ciconia,1,POINT EMPTY,40534,1,1898591,True,True,False,CC_BY,MPIAB White Stork Prinzesschen,5,1,5,Peter Berthold,"Tracks of White Stork ""Prinzesschen"" from Loburg stork centre; one f the earliest birds tracked over many years.",research,False,True,False,True,False,data_manager,1994-08-12 04:14:24,2006-12-22 22:07:04,3169,Ciconia ciconia,myotis (Wolfgang Fiedler),POINT (11.66748 52.14697),1930124,2004-07-25 17:13:55+00:00,POINT (12.129 52.102),12.129,52.102 diff --git a/test/contract/argos-sfc/py/meta.csv b/test/contract/argos-sfc/py/meta.csv new file mode 100644 index 0000000..027cb75 --- /dev/null +++ b/test/contract/argos-sfc/py/meta.csv @@ -0,0 +1,2 @@ +crs,tzone,timeColName,trackIdColName +EPSG:4326,UTC,timestamp_utc,individual_name_deployment_id diff --git a/test/contract/berlin/input.rds b/test/contract/berlin/input.rds new file mode 100644 index 0000000..3920459 Binary files /dev/null and b/test/contract/berlin/input.rds differ diff --git a/test/contract/berlin/link.csv b/test/contract/berlin/link.csv new file mode 100644 index 0000000..9906a0b --- /dev/null +++ b/test/contract/berlin/link.csv @@ -0,0 +1,5 @@ +"sensor_type_id","individual_local_identifier","comments","sequence_number","tag_voltage","timestamp","transmission_protocol","transmission_timestamp","event_id","visible","deployment_id","tag_id","individual_id","animal_life_stage","animal_reproductive_condition","attachment_type","deploy_off_timestamp","deploy_on_person","deploy_on_timestamp","sensor_type_ids","capture_location","deploy_on_location","deploy_off_location","nick_name","ring_id","sex","taxon_canonical_name","individual_number_of_deployments","mortality_location","tag_local_identifier","tag_number_of_deployments","study_id","acknowledgements","citation","grants_used","has_quota","i_am_owner","is_test","license_terms","license_type","name","study_number_of_deployments","number_of_individuals","number_of_tags","principal_investigator_name","study_objective","study_type","suspend_license_terms","i_can_see_data","there_are_data_which_i_cannot_see","i_have_download_access","i_am_collaborator","study_permission","timestamp_first_deployed_location","timestamp_last_deployed_location","number_of_deployed_locations","taxon_ids","contact_person_name","main_location","coords_x","coords_y" +653,"Goat.8810","60179d",1945,3.8,"2021-07-01 06:40:00.000",NA,2021-07-01 06:40:59,19196328498,TRUE,1600804509,1600779812,1600804167,"adult","lactating","collar",2022-10-15,"Benito and Giuseppe",2021-07-01,"sigfox-geolocation,gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","Goat-8810","Goat-8810","f","Capra hircus",1,"POINT EMPTY","C3E063",1,1600771155,"MPIAB","MPIAB DAMN","MPIAB",TRUE,TRUE,FALSE,"Data can not be used without prior contact with the PI","CUSTOM","MPIAB Disaster alert Mt Etna 2021",32,22,40,"martin (Martin Wikelski)","Understand possible preparatory movements of animals before natural disasters","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2021-07-01 06:13:00,2024-05-14 20:12:54,318411,"Capra hircus","Georg Heine (Georg Heine)","POINT (15.0472 37.73428)",14.9403333333333,37.8562833333333 +653,"Goat.8810","60179e",1946,3.8,"2021-07-01 06:46:00.000",NA,2021-07-01 06:46:33,19196328499,TRUE,1600804509,1600779812,1600804167,"adult","lactating","collar",2022-10-15,"Benito and Giuseppe",2021-07-01,"sigfox-geolocation,gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","Goat-8810","Goat-8810","f","Capra hircus",1,"POINT EMPTY","C3E063",1,1600771155,"MPIAB","MPIAB DAMN","MPIAB",TRUE,TRUE,FALSE,"Data can not be used without prior contact with the PI","CUSTOM","MPIAB Disaster alert Mt Etna 2021",32,22,40,"martin (Martin Wikelski)","Understand possible preparatory movements of animals before natural disasters","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2021-07-01 06:13:00,2024-05-14 20:12:54,318411,"Capra hircus","Georg Heine (Georg Heine)","POINT (15.0472 37.73428)",14.9404,37.8565 +653,"Goat.8810","60179f",1947,3.8,"2021-07-01 06:52:00.000",NA,2021-07-01 06:52:08,19196328500,TRUE,1600804509,1600779812,1600804167,"adult","lactating","collar",2022-10-15,"Benito and Giuseppe",2021-07-01,"sigfox-geolocation,gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","Goat-8810","Goat-8810","f","Capra hircus",1,"POINT EMPTY","C3E063",1,1600771155,"MPIAB","MPIAB DAMN","MPIAB",TRUE,TRUE,FALSE,"Data can not be used without prior contact with the PI","CUSTOM","MPIAB Disaster alert Mt Etna 2021",32,22,40,"martin (Martin Wikelski)","Understand possible preparatory movements of animals before natural disasters","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2021-07-01 06:13:00,2024-05-14 20:12:54,318411,"Capra hircus","Georg Heine (Georg Heine)","POINT (15.0472 37.73428)",14.9403666666667,37.8564 +653,"Goat.8810","6017a0",1948,3.8,"2021-07-01 06:57:00.000",NA,2021-07-01 06:57:40,19196328501,TRUE,1600804509,1600779812,1600804167,"adult","lactating","collar",2022-10-15,"Benito and Giuseppe",2021-07-01,"sigfox-geolocation,gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","Goat-8810","Goat-8810","f","Capra hircus",1,"POINT EMPTY","C3E063",1,1600771155,"MPIAB","MPIAB DAMN","MPIAB",TRUE,TRUE,FALSE,"Data can not be used without prior contact with the PI","CUSTOM","MPIAB Disaster alert Mt Etna 2021",32,22,40,"martin (Martin Wikelski)","Understand possible preparatory movements of animals before natural disasters","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2021-07-01 06:13:00,2024-05-14 20:12:54,318411,"Capra hircus","Georg Heine (Georg Heine)","POINT (15.0472 37.73428)",14.9402666666667,37.8563333333333 diff --git a/test/contract/berlin/meta.csv b/test/contract/berlin/meta.csv new file mode 100644 index 0000000..c7d24d5 --- /dev/null +++ b/test/contract/berlin/meta.csv @@ -0,0 +1,2 @@ +"crs","tzone","timeColName","trackIdColName" +"EPSG:4326","Europe/Berlin","timestamp","individual_local_identifier" diff --git a/test/contract/berlin/py/link.csv b/test/contract/berlin/py/link.csv new file mode 100644 index 0000000..0e5babe --- /dev/null +++ b/test/contract/berlin/py/link.csv @@ -0,0 +1,5 @@ +timestamp_utc,sensor_type_id,individual_local_identifier,comments,sequence_number,tag_voltage,timestamp,transmission_protocol,transmission_timestamp,event_id,visible,deployment_id,tag_id,individual_id,animal_life_stage,animal_reproductive_condition,attachment_type,deploy_off_timestamp,deploy_on_person,deploy_on_timestamp,sensor_type_ids,capture_location,deploy_on_location,deploy_off_location,nick_name,ring_id,sex,taxon_canonical_name,individual_number_of_deployments,mortality_location,tag_local_identifier,tag_number_of_deployments,study_id,acknowledgements,citation,grants_used,has_quota,i_am_owner,is_test,license_terms,license_type,name,study_number_of_deployments,number_of_individuals,number_of_tags,principal_investigator_name,study_objective,study_type,suspend_license_terms,i_can_see_data,there_are_data_which_i_cannot_see,i_have_download_access,i_am_collaborator,study_permission,timestamp_first_deployed_location,timestamp_last_deployed_location,number_of_deployed_locations,taxon_ids,contact_person_name,main_location,timestamp_tz,geometry,coords_x,coords_y +2021-07-01 06:40:00,653,Goat.8810,60179d,1945,3.8,2021-07-01 06:40:00,,2021-07-01 06:40:59,19196328498,True,1600804509,1600779812,1600804167,adult,lactating,collar,2022-10-15,Benito and Giuseppe,2021-07-01,"sigfox-geolocation,gps",POINT EMPTY,POINT EMPTY,POINT EMPTY,Goat-8810,Goat-8810,f,Capra hircus,1,POINT EMPTY,C3E063,1,1600771155,MPIAB,MPIAB DAMN,MPIAB,True,True,False,Data can not be used without prior contact with the PI,CUSTOM,MPIAB Disaster alert Mt Etna 2021,32,22,40,martin (Martin Wikelski),Understand possible preparatory movements of animals before natural disasters,research,False,True,False,True,False,data_manager,2021-07-01 06:13:00,2024-05-14 20:12:54,318411,Capra hircus,Georg Heine (Georg Heine),POINT (15.0472 37.73428),2021-07-01 08:40:00+02:00,POINT (14.9403333333333 37.8562833333333),14.9403333333333,37.8562833333333 +2021-07-01 06:46:00,653,Goat.8810,60179e,1946,3.8,2021-07-01 06:46:00,,2021-07-01 06:46:33,19196328499,True,1600804509,1600779812,1600804167,adult,lactating,collar,2022-10-15,Benito and Giuseppe,2021-07-01,"sigfox-geolocation,gps",POINT EMPTY,POINT EMPTY,POINT EMPTY,Goat-8810,Goat-8810,f,Capra hircus,1,POINT EMPTY,C3E063,1,1600771155,MPIAB,MPIAB DAMN,MPIAB,True,True,False,Data can not be used without prior contact with the PI,CUSTOM,MPIAB Disaster alert Mt Etna 2021,32,22,40,martin (Martin Wikelski),Understand possible preparatory movements of animals before natural disasters,research,False,True,False,True,False,data_manager,2021-07-01 06:13:00,2024-05-14 20:12:54,318411,Capra hircus,Georg Heine (Georg Heine),POINT (15.0472 37.73428),2021-07-01 08:46:00+02:00,POINT (14.9404 37.8565),14.9404,37.8565 +2021-07-01 06:52:00,653,Goat.8810,60179f,1947,3.8,2021-07-01 06:52:00,,2021-07-01 06:52:08,19196328500,True,1600804509,1600779812,1600804167,adult,lactating,collar,2022-10-15,Benito and Giuseppe,2021-07-01,"sigfox-geolocation,gps",POINT EMPTY,POINT EMPTY,POINT EMPTY,Goat-8810,Goat-8810,f,Capra hircus,1,POINT EMPTY,C3E063,1,1600771155,MPIAB,MPIAB DAMN,MPIAB,True,True,False,Data can not be used without prior contact with the PI,CUSTOM,MPIAB Disaster alert Mt Etna 2021,32,22,40,martin (Martin Wikelski),Understand possible preparatory movements of animals before natural disasters,research,False,True,False,True,False,data_manager,2021-07-01 06:13:00,2024-05-14 20:12:54,318411,Capra hircus,Georg Heine (Georg Heine),POINT (15.0472 37.73428),2021-07-01 08:52:00+02:00,POINT (14.9403666666667 37.8564),14.9403666666667,37.8564 +2021-07-01 06:57:00,653,Goat.8810,6017a0,1948,3.8,2021-07-01 06:57:00,,2021-07-01 06:57:40,19196328501,True,1600804509,1600779812,1600804167,adult,lactating,collar,2022-10-15,Benito and Giuseppe,2021-07-01,"sigfox-geolocation,gps",POINT EMPTY,POINT EMPTY,POINT EMPTY,Goat-8810,Goat-8810,f,Capra hircus,1,POINT EMPTY,C3E063,1,1600771155,MPIAB,MPIAB DAMN,MPIAB,True,True,False,Data can not be used without prior contact with the PI,CUSTOM,MPIAB Disaster alert Mt Etna 2021,32,22,40,martin (Martin Wikelski),Understand possible preparatory movements of animals before natural disasters,research,False,True,False,True,False,data_manager,2021-07-01 06:13:00,2024-05-14 20:12:54,318411,Capra hircus,Georg Heine (Georg Heine),POINT (15.0472 37.73428),2021-07-01 08:57:00+02:00,POINT (14.9402666666667 37.8563333333333),14.9402666666667,37.8563333333333 diff --git a/test/contract/berlin/py/meta.csv b/test/contract/berlin/py/meta.csv new file mode 100644 index 0000000..3587343 --- /dev/null +++ b/test/contract/berlin/py/meta.csv @@ -0,0 +1,2 @@ +crs,tzone,timeColName,trackIdColName +EPSG:4326,UTC,timestamp_utc,individual_local_identifier diff --git a/test/contract/cut.R b/test/contract/cut.R new file mode 100644 index 0000000..482d079 --- /dev/null +++ b/test/contract/cut.R @@ -0,0 +1,23 @@ +# Cuts the contract cases out of the sample inputs: a few rows each, by row number of the full input. +# Rscript cut.R +suppressMessages(library("move2")) + +cases <- list( + "latlon-three-tracks" = list(file = "input4_move2loc_LatLon.rds", rows = c(1:4, 3011:3014, 4506:4509)), + "mollweide-midnight" = list(file = "input1_move2loc_Mollweide.rds", rows = 2728:2731), + "subsecond" = list(file = "input2_move2loc_LatLon.rds", rows = 3443:3446), + "argos-sfc" = list(file = "input3_move2loc_LatLon.rds", rows = c(1, 2, 25, 26, 63, 64, 365, 366, 938, 939)), + "list-columns" = list(file = "input_move2loc_List.rds", rows = 1:5), + "berlin" = list(file = "input1_move2loc_LatLon.rds", rows = 1:4, tzone = "Europe/Berlin") +) + +args <- commandArgs(trailingOnly = TRUE) +for (name in names(cases)) { + spec <- cases[[name]] + data <- readRDS(file.path(args[1], spec$file))[spec$rows, ] + if (!is.null(spec$tzone)) { + attr(data[[mt_time_column(data)]], "tzone") <- spec$tzone + } + dir.create(file.path(args[2], name), recursive = TRUE, showWarnings = FALSE) + saveRDS(data, file.path(args[2], name, "input.rds")) +} diff --git a/test/contract/generate.sh b/test/contract/generate.sh new file mode 100755 index 0000000..4f86da8 --- /dev/null +++ b/test/contract/generate.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Produces the golden CSV pairs of test/contract// from each /input.rds, through two Apps: +# input.rds --R->python--> meta.csv, link.csv, pickle --python->R--> py/meta.csv, py/link.csv +# +# [R2PY_IMAGE=…] [PY2R_IMAGE=…] [CUT=1] [OVERLAY=1] test/contract/generate.sh +# +# Defaults to the v2.2.0 production images (registry access needed). CUT=1 first cuts every +# input.rds out of r/data/raw (see cut.R). OVERLAY: see test/images.sh. +set -o errexit -o nounset -o pipefail + +cd "$(dirname "$0")/../.." +source test/images.sh + +R2PY_IMAGE="${R2PY_IMAGE:-gitlab-registry.mpcdf.mpg.de/moveapps/apps/production-a81e3046-bc48-4fcb-8d28-4291000c77f9-move2loc-to-movingpandas:7}" +PY2R_IMAGE="${PY2R_IMAGE:-gitlab-registry.mpcdf.mpg.de/moveapps/apps/production-ed490356-9d55-4cb3-90d1-b8a2fc18ce10-movingpandas-to-move2loc:6}" +CUT="${CUT:-0}" + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +mkdir -p "$work/in" +cp -R test "$work/in/test" +cp -R r/data/raw "$work/in/raw" +runInImage "$R2PY_IMAGE" "$work/in" \ + "/tmp/in/test/contract/prepare.sh /tmp/in/raw /tmp/out $CUT && /tmp/in/test/roundtrip/r2python.sh ./start-process.sh /tmp/out/inputs /tmp/out/cases" \ + "$work/stage1" + +cp -R test "$work/stage1/test" +runInImage "$PY2R_IMAGE" "$work/stage1" '/tmp/in/test/roundtrip/stage-python2r.sh /tmp/in/cases /tmp/out' "$work/stage2" + +for dir in "$work"/stage1/contract/*/; do + [ -f "$dir/input.rds" ] || continue + case="$(basename "$dir")" + for exit in "$work/stage1/cases/$case/r2python.exit" "$work/stage2/cases/$case/python2r.exit"; do + if [ "$(cat "$exit")" != 0 ]; then + echo "$case: $(basename "$exit" .exit) failed" >&2 + cat "${exit%.exit}.log" >&2 + exit 1 + fi + done + + target="test/contract/$case" + mkdir -p "$target/py" + if [ "$CUT" = 1 ]; then + cp "$dir/input.rds" "$target/input.rds" + fi + for file in meta.csv link.csv; do + cp "$work/stage1/cases/$case/$file" "$target/$file" + cp "$work/stage2/cases/$case/py/$file" "$target/py/$file" + done +done diff --git a/test/contract/latlon-three-tracks/input.rds b/test/contract/latlon-three-tracks/input.rds new file mode 100644 index 0000000..3a2c289 Binary files /dev/null and b/test/contract/latlon-three-tracks/input.rds differ diff --git a/test/contract/latlon-three-tracks/link.csv b/test/contract/latlon-three-tracks/link.csv new file mode 100644 index 0000000..ae153da --- /dev/null +++ b/test/contract/latlon-three-tracks/link.csv @@ -0,0 +1,13 @@ +"sensor_type_id","individual_local_identifier","gps_satellite_count","ground_speed","gt_tx_count","heading","tag_voltage","timestamp","event_id","visible","deployment_id","tag_id","individual_id","deploy_on_person","deploy_on_timestamp","deployment_local_identifier","sensor_type_ids","capture_location","deploy_on_location","deploy_off_location","ring_id","sex","taxon_canonical_name","individual_number_of_deployments","mortality_location","tag_local_identifier","tag_number_of_deployments","study_id","acknowledgements","citation","grants_used","has_quota","i_am_owner","is_test","license_terms","license_type","name","study_number_of_deployments","number_of_individuals","number_of_tags","principal_investigator_name","study_objective","study_type","suspend_license_terms","i_can_see_data","there_are_data_which_i_cannot_see","i_have_download_access","i_am_collaborator","study_permission","timestamp_first_deployed_location","timestamp_last_deployed_location","number_of_deployed_locations","taxon_ids","contact_person_name","main_location","coords_x","coords_y" +653,"X746",4,2.86,5,NA,3.96,"2013-08-07 19:31:26.000",319286626,TRUE,20813885,20813604,20813603,NA,2013-08-07 19:31:26,"0309-746","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","CS007385","m","Anser albifrons",1,"POINT EMPTY","0309",1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121","Max Planck Institute of Animal Behavior and German Aerospace Center (DLR)",TRUE,TRUE,FALSE,"Please do not use any data without first contacting the PI and the Contact Person.","CUSTOM","LifeTrack Geese IG-RAS MPIAB ICARUS",75,75,98,"Pglaz (Peter Glazov)","Determine social movement patterns in white-fronted geese.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,"Anser albifrons","andreakoelzsch (Andrea Kölzsch)","POINT (44.3119 58.8292)",48.774917,69.125833 +653,"X746",6,0.61,6,NA,4,"2013-08-07 20:34:08.000",319286627,TRUE,20813885,20813604,20813603,NA,2013-08-07 19:31:26,"0309-746","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","CS007385","m","Anser albifrons",1,"POINT EMPTY","0309",1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121","Max Planck Institute of Animal Behavior and German Aerospace Center (DLR)",TRUE,TRUE,FALSE,"Please do not use any data without first contacting the PI and the Contact Person.","CUSTOM","LifeTrack Geese IG-RAS MPIAB ICARUS",75,75,98,"Pglaz (Peter Glazov)","Determine social movement patterns in white-fronted geese.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,"Anser albifrons","andreakoelzsch (Andrea Kölzsch)","POINT (44.3119 58.8292)",48.7747,69.125433 +653,"X746",7,0,7,NA,4,"2013-08-07 21:38:08.000",319286628,TRUE,20813885,20813604,20813603,NA,2013-08-07 19:31:26,"0309-746","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","CS007385","m","Anser albifrons",1,"POINT EMPTY","0309",1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121","Max Planck Institute of Animal Behavior and German Aerospace Center (DLR)",TRUE,TRUE,FALSE,"Please do not use any data without first contacting the PI and the Contact Person.","CUSTOM","LifeTrack Geese IG-RAS MPIAB ICARUS",75,75,98,"Pglaz (Peter Glazov)","Determine social movement patterns in white-fronted geese.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,"Anser albifrons","andreakoelzsch (Andrea Kölzsch)","POINT (44.3119 58.8292)",48.77465,69.125483 +653,"X746",7,0,9,NA,3.91,"2013-08-07 23:46:02.000",319286629,TRUE,20813885,20813604,20813603,NA,2013-08-07 19:31:26,"0309-746","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","CS007385","m","Anser albifrons",1,"POINT EMPTY","0309",1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121","Max Planck Institute of Animal Behavior and German Aerospace Center (DLR)",TRUE,TRUE,FALSE,"Please do not use any data without first contacting the PI and the Contact Person.","CUSTOM","LifeTrack Geese IG-RAS MPIAB ICARUS",75,75,98,"Pglaz (Peter Glazov)","Determine social movement patterns in white-fronted geese.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,"Anser albifrons","andreakoelzsch (Andrea Kölzsch)","POINT (44.3119 58.8292)",48.7746,69.126733 +653,"X749",5,0.3,5,NA,3.92,"2013-08-07 18:27:02.000",319300766,TRUE,20813892,20813619,20813618,NA,2013-08-07 18:27:02,"0328-749","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","CS007392","m","Anser albifrons",1,"POINT EMPTY","0328",1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121","Max Planck Institute of Animal Behavior and German Aerospace Center (DLR)",TRUE,TRUE,FALSE,"Please do not use any data without first contacting the PI and the Contact Person.","CUSTOM","LifeTrack Geese IG-RAS MPIAB ICARUS",75,75,98,"Pglaz (Peter Glazov)","Determine social movement patterns in white-fronted geese.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,"Anser albifrons","andreakoelzsch (Andrea Kölzsch)","POINT (44.3119 58.8292)",48.784133,69.12765 +653,"X749",5,0.91,9,NA,3.93,"2013-08-07 21:57:12.000",319300767,TRUE,20813892,20813619,20813618,NA,2013-08-07 18:27:02,"0328-749","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","CS007392","m","Anser albifrons",1,"POINT EMPTY","0328",1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121","Max Planck Institute of Animal Behavior and German Aerospace Center (DLR)",TRUE,TRUE,FALSE,"Please do not use any data without first contacting the PI and the Contact Person.","CUSTOM","LifeTrack Geese IG-RAS MPIAB ICARUS",75,75,98,"Pglaz (Peter Glazov)","Determine social movement patterns in white-fronted geese.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,"Anser albifrons","andreakoelzsch (Andrea Kölzsch)","POINT (44.3119 58.8292)",48.769983,69.123583 +653,"X749",7,0,13,NA,3.98,"2013-08-08 02:03:39.000",319300768,TRUE,20813892,20813619,20813618,NA,2013-08-07 18:27:02,"0328-749","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","CS007392","m","Anser albifrons",1,"POINT EMPTY","0328",1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121","Max Planck Institute of Animal Behavior and German Aerospace Center (DLR)",TRUE,TRUE,FALSE,"Please do not use any data without first contacting the PI and the Contact Person.","CUSTOM","LifeTrack Geese IG-RAS MPIAB ICARUS",75,75,98,"Pglaz (Peter Glazov)","Determine social movement patterns in white-fronted geese.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,"Anser albifrons","andreakoelzsch (Andrea Kölzsch)","POINT (44.3119 58.8292)",48.771483,69.123467 +653,"X749",5,0.91,17,NA,4.16,"2013-08-08 06:10:34.000",319300769,TRUE,20813892,20813619,20813618,NA,2013-08-07 18:27:02,"0328-749","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","CS007392","m","Anser albifrons",1,"POINT EMPTY","0328",1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121","Max Planck Institute of Animal Behavior and German Aerospace Center (DLR)",TRUE,TRUE,FALSE,"Please do not use any data without first contacting the PI and the Contact Person.","CUSTOM","LifeTrack Geese IG-RAS MPIAB ICARUS",75,75,98,"Pglaz (Peter Glazov)","Determine social movement patterns in white-fronted geese.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,"Anser albifrons","andreakoelzsch (Andrea Kölzsch)","POINT (44.3119 58.8292)",48.768583,69.12195 +653,"X742",6,1.8,NA,1.39,4.18,"2013-08-08 06:47:31.000",566120615,TRUE,56853924,56853250,56850841,"Andrea Koelzsch",2013-08-08,"0380-742","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","CS007378","m","Anser albifrons",1,"POINT EMPTY","0380",1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121","Max Planck Institute of Animal Behavior and German Aerospace Center (DLR)",TRUE,TRUE,FALSE,"Please do not use any data without first contacting the PI and the Contact Person.","CUSTOM","LifeTrack Geese IG-RAS MPIAB ICARUS",75,75,98,"Pglaz (Peter Glazov)","Determine social movement patterns in white-fronted geese.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,"Anser albifrons","andreakoelzsch (Andrea Kölzsch)","POINT (44.3119 58.8292)",48.761833,69.128117 +653,"X742",6,4.51,NA,1.35,4.21,"2013-08-08 07:49:34.000",566120616,TRUE,56853924,56853250,56850841,"Andrea Koelzsch",2013-08-08,"0380-742","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","CS007378","m","Anser albifrons",1,"POINT EMPTY","0380",1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121","Max Planck Institute of Animal Behavior and German Aerospace Center (DLR)",TRUE,TRUE,FALSE,"Please do not use any data without first contacting the PI and the Contact Person.","CUSTOM","LifeTrack Geese IG-RAS MPIAB ICARUS",75,75,98,"Pglaz (Peter Glazov)","Determine social movement patterns in white-fronted geese.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,"Anser albifrons","andreakoelzsch (Andrea Kölzsch)","POINT (44.3119 58.8292)",48.7753,69.135267 +653,"X742",5,8.43,NA,1.78,4.22,"2013-08-08 08:20:35.000",566120617,TRUE,56853924,56853250,56850841,"Andrea Koelzsch",2013-08-08,"0380-742","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","CS007378","m","Anser albifrons",1,"POINT EMPTY","0380",1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121","Max Planck Institute of Animal Behavior and German Aerospace Center (DLR)",TRUE,TRUE,FALSE,"Please do not use any data without first contacting the PI and the Contact Person.","CUSTOM","LifeTrack Geese IG-RAS MPIAB ICARUS",75,75,98,"Pglaz (Peter Glazov)","Determine social movement patterns in white-fronted geese.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,"Anser albifrons","andreakoelzsch (Andrea Kölzsch)","POINT (44.3119 58.8292)",48.758833,69.141817 +653,"X742",4,0.3,NA,1.45,4.17,"2013-08-08 16:07:34.000",566120618,TRUE,56853924,56853250,56850841,"Andrea Koelzsch",2013-08-08,"0380-742","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","CS007378","m","Anser albifrons",1,"POINT EMPTY","0380",1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121","Max Planck Institute of Animal Behavior and German Aerospace Center (DLR)",TRUE,TRUE,FALSE,"Please do not use any data without first contacting the PI and the Contact Person.","CUSTOM","LifeTrack Geese IG-RAS MPIAB ICARUS",75,75,98,"Pglaz (Peter Glazov)","Determine social movement patterns in white-fronted geese.","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,"Anser albifrons","andreakoelzsch (Andrea Kölzsch)","POINT (44.3119 58.8292)",48.749483,69.1428 diff --git a/test/contract/latlon-three-tracks/meta.csv b/test/contract/latlon-three-tracks/meta.csv new file mode 100644 index 0000000..5dc61b0 --- /dev/null +++ b/test/contract/latlon-three-tracks/meta.csv @@ -0,0 +1,2 @@ +"crs","tzone","timeColName","trackIdColName" +"EPSG:4326","UTC","timestamp","individual_local_identifier" diff --git a/test/contract/latlon-three-tracks/py/link.csv b/test/contract/latlon-three-tracks/py/link.csv new file mode 100644 index 0000000..686e4b9 --- /dev/null +++ b/test/contract/latlon-three-tracks/py/link.csv @@ -0,0 +1,13 @@ +timestamp_utc,sensor_type_id,individual_local_identifier,gps_satellite_count,ground_speed,gt_tx_count,heading,tag_voltage,timestamp,event_id,visible,deployment_id,tag_id,individual_id,deploy_on_person,deploy_on_timestamp,deployment_local_identifier,sensor_type_ids,capture_location,deploy_on_location,deploy_off_location,ring_id,sex,taxon_canonical_name,individual_number_of_deployments,mortality_location,tag_local_identifier,tag_number_of_deployments,study_id,acknowledgements,citation,grants_used,has_quota,i_am_owner,is_test,license_terms,license_type,name,study_number_of_deployments,number_of_individuals,number_of_tags,principal_investigator_name,study_objective,study_type,suspend_license_terms,i_can_see_data,there_are_data_which_i_cannot_see,i_have_download_access,i_am_collaborator,study_permission,timestamp_first_deployed_location,timestamp_last_deployed_location,number_of_deployed_locations,taxon_ids,contact_person_name,main_location,timestamp_tz,geometry,coords_x,coords_y +2013-08-08 06:47:31,653,X742,6,1.8,,1.39,4.18,2013-08-08 06:47:31,566120615,True,56853924,56853250,56850841,Andrea Koelzsch,2013-08-08,0380-742,gps,POINT EMPTY,POINT EMPTY,POINT EMPTY,CS007378,m,Anser albifrons,1,POINT EMPTY,380,1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121",Max Planck Institute of Animal Behavior and German Aerospace Center (DLR),True,True,False,Please do not use any data without first contacting the PI and the Contact Person.,CUSTOM,LifeTrack Geese IG-RAS MPIAB ICARUS,75,75,98,Pglaz (Peter Glazov),Determine social movement patterns in white-fronted geese.,research,False,True,False,True,False,data_manager,2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,Anser albifrons,andreakoelzsch (Andrea Kölzsch),POINT (44.3119 58.8292),2013-08-08 06:47:31+00:00,POINT (48.761833 69.128117),48.761833,69.128117 +2013-08-08 07:49:34,653,X742,6,4.51,,1.35,4.21,2013-08-08 07:49:34,566120616,True,56853924,56853250,56850841,Andrea Koelzsch,2013-08-08,0380-742,gps,POINT EMPTY,POINT EMPTY,POINT EMPTY,CS007378,m,Anser albifrons,1,POINT EMPTY,380,1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121",Max Planck Institute of Animal Behavior and German Aerospace Center (DLR),True,True,False,Please do not use any data without first contacting the PI and the Contact Person.,CUSTOM,LifeTrack Geese IG-RAS MPIAB ICARUS,75,75,98,Pglaz (Peter Glazov),Determine social movement patterns in white-fronted geese.,research,False,True,False,True,False,data_manager,2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,Anser albifrons,andreakoelzsch (Andrea Kölzsch),POINT (44.3119 58.8292),2013-08-08 07:49:34+00:00,POINT (48.7753 69.135267),48.7753,69.135267 +2013-08-08 08:20:35,653,X742,5,8.43,,1.78,4.22,2013-08-08 08:20:35,566120617,True,56853924,56853250,56850841,Andrea Koelzsch,2013-08-08,0380-742,gps,POINT EMPTY,POINT EMPTY,POINT EMPTY,CS007378,m,Anser albifrons,1,POINT EMPTY,380,1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121",Max Planck Institute of Animal Behavior and German Aerospace Center (DLR),True,True,False,Please do not use any data without first contacting the PI and the Contact Person.,CUSTOM,LifeTrack Geese IG-RAS MPIAB ICARUS,75,75,98,Pglaz (Peter Glazov),Determine social movement patterns in white-fronted geese.,research,False,True,False,True,False,data_manager,2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,Anser albifrons,andreakoelzsch (Andrea Kölzsch),POINT (44.3119 58.8292),2013-08-08 08:20:35+00:00,POINT (48.758833 69.141817),48.758833,69.141817 +2013-08-08 16:07:34,653,X742,4,0.3,,1.45,4.17,2013-08-08 16:07:34,566120618,True,56853924,56853250,56850841,Andrea Koelzsch,2013-08-08,0380-742,gps,POINT EMPTY,POINT EMPTY,POINT EMPTY,CS007378,m,Anser albifrons,1,POINT EMPTY,380,1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121",Max Planck Institute of Animal Behavior and German Aerospace Center (DLR),True,True,False,Please do not use any data without first contacting the PI and the Contact Person.,CUSTOM,LifeTrack Geese IG-RAS MPIAB ICARUS,75,75,98,Pglaz (Peter Glazov),Determine social movement patterns in white-fronted geese.,research,False,True,False,True,False,data_manager,2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,Anser albifrons,andreakoelzsch (Andrea Kölzsch),POINT (44.3119 58.8292),2013-08-08 16:07:34+00:00,POINT (48.749483 69.1428),48.749483,69.1428 +2013-08-07 19:31:26,653,X746,4,2.86,5.0,,3.96,2013-08-07 19:31:26,319286626,True,20813885,20813604,20813603,,2013-08-07 19:31:26,0309-746,gps,POINT EMPTY,POINT EMPTY,POINT EMPTY,CS007385,m,Anser albifrons,1,POINT EMPTY,309,1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121",Max Planck Institute of Animal Behavior and German Aerospace Center (DLR),True,True,False,Please do not use any data without first contacting the PI and the Contact Person.,CUSTOM,LifeTrack Geese IG-RAS MPIAB ICARUS,75,75,98,Pglaz (Peter Glazov),Determine social movement patterns in white-fronted geese.,research,False,True,False,True,False,data_manager,2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,Anser albifrons,andreakoelzsch (Andrea Kölzsch),POINT (44.3119 58.8292),2013-08-07 19:31:26+00:00,POINT (48.774917 69.125833),48.774917,69.125833 +2013-08-07 20:34:08,653,X746,6,0.61,6.0,,4.0,2013-08-07 20:34:08,319286627,True,20813885,20813604,20813603,,2013-08-07 19:31:26,0309-746,gps,POINT EMPTY,POINT EMPTY,POINT EMPTY,CS007385,m,Anser albifrons,1,POINT EMPTY,309,1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121",Max Planck Institute of Animal Behavior and German Aerospace Center (DLR),True,True,False,Please do not use any data without first contacting the PI and the Contact Person.,CUSTOM,LifeTrack Geese IG-RAS MPIAB ICARUS,75,75,98,Pglaz (Peter Glazov),Determine social movement patterns in white-fronted geese.,research,False,True,False,True,False,data_manager,2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,Anser albifrons,andreakoelzsch (Andrea Kölzsch),POINT (44.3119 58.8292),2013-08-07 20:34:08+00:00,POINT (48.7747 69.125433),48.7747,69.125433 +2013-08-07 21:38:08,653,X746,7,0.0,7.0,,4.0,2013-08-07 21:38:08,319286628,True,20813885,20813604,20813603,,2013-08-07 19:31:26,0309-746,gps,POINT EMPTY,POINT EMPTY,POINT EMPTY,CS007385,m,Anser albifrons,1,POINT EMPTY,309,1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121",Max Planck Institute of Animal Behavior and German Aerospace Center (DLR),True,True,False,Please do not use any data without first contacting the PI and the Contact Person.,CUSTOM,LifeTrack Geese IG-RAS MPIAB ICARUS,75,75,98,Pglaz (Peter Glazov),Determine social movement patterns in white-fronted geese.,research,False,True,False,True,False,data_manager,2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,Anser albifrons,andreakoelzsch (Andrea Kölzsch),POINT (44.3119 58.8292),2013-08-07 21:38:08+00:00,POINT (48.77465 69.125483),48.77465,69.125483 +2013-08-07 23:46:02,653,X746,7,0.0,9.0,,3.91,2013-08-07 23:46:02,319286629,True,20813885,20813604,20813603,,2013-08-07 19:31:26,0309-746,gps,POINT EMPTY,POINT EMPTY,POINT EMPTY,CS007385,m,Anser albifrons,1,POINT EMPTY,309,1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121",Max Planck Institute of Animal Behavior and German Aerospace Center (DLR),True,True,False,Please do not use any data without first contacting the PI and the Contact Person.,CUSTOM,LifeTrack Geese IG-RAS MPIAB ICARUS,75,75,98,Pglaz (Peter Glazov),Determine social movement patterns in white-fronted geese.,research,False,True,False,True,False,data_manager,2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,Anser albifrons,andreakoelzsch (Andrea Kölzsch),POINT (44.3119 58.8292),2013-08-07 23:46:02+00:00,POINT (48.7746 69.126733),48.7746,69.126733 +2013-08-07 18:27:02,653,X749,5,0.3,5.0,,3.92,2013-08-07 18:27:02,319300766,True,20813892,20813619,20813618,,2013-08-07 18:27:02,0328-749,gps,POINT EMPTY,POINT EMPTY,POINT EMPTY,CS007392,m,Anser albifrons,1,POINT EMPTY,328,1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121",Max Planck Institute of Animal Behavior and German Aerospace Center (DLR),True,True,False,Please do not use any data without first contacting the PI and the Contact Person.,CUSTOM,LifeTrack Geese IG-RAS MPIAB ICARUS,75,75,98,Pglaz (Peter Glazov),Determine social movement patterns in white-fronted geese.,research,False,True,False,True,False,data_manager,2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,Anser albifrons,andreakoelzsch (Andrea Kölzsch),POINT (44.3119 58.8292),2013-08-07 18:27:02+00:00,POINT (48.784133 69.12765),48.784133,69.12765 +2013-08-07 21:57:12,653,X749,5,0.91,9.0,,3.93,2013-08-07 21:57:12,319300767,True,20813892,20813619,20813618,,2013-08-07 18:27:02,0328-749,gps,POINT EMPTY,POINT EMPTY,POINT EMPTY,CS007392,m,Anser albifrons,1,POINT EMPTY,328,1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121",Max Planck Institute of Animal Behavior and German Aerospace Center (DLR),True,True,False,Please do not use any data without first contacting the PI and the Contact Person.,CUSTOM,LifeTrack Geese IG-RAS MPIAB ICARUS,75,75,98,Pglaz (Peter Glazov),Determine social movement patterns in white-fronted geese.,research,False,True,False,True,False,data_manager,2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,Anser albifrons,andreakoelzsch (Andrea Kölzsch),POINT (44.3119 58.8292),2013-08-07 21:57:12+00:00,POINT (48.769983 69.123583),48.769983,69.123583 +2013-08-08 02:03:39,653,X749,7,0.0,13.0,,3.98,2013-08-08 02:03:39,319300768,True,20813892,20813619,20813618,,2013-08-07 18:27:02,0328-749,gps,POINT EMPTY,POINT EMPTY,POINT EMPTY,CS007392,m,Anser albifrons,1,POINT EMPTY,328,1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121",Max Planck Institute of Animal Behavior and German Aerospace Center (DLR),True,True,False,Please do not use any data without first contacting the PI and the Contact Person.,CUSTOM,LifeTrack Geese IG-RAS MPIAB ICARUS,75,75,98,Pglaz (Peter Glazov),Determine social movement patterns in white-fronted geese.,research,False,True,False,True,False,data_manager,2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,Anser albifrons,andreakoelzsch (Andrea Kölzsch),POINT (44.3119 58.8292),2013-08-08 02:03:39+00:00,POINT (48.771483 69.123467),48.771483,69.123467 +2013-08-08 06:10:34,653,X749,5,0.91,17.0,,4.16,2013-08-08 06:10:34,319300769,True,20813892,20813619,20813618,,2013-08-07 18:27:02,0328-749,gps,POINT EMPTY,POINT EMPTY,POINT EMPTY,CS007392,m,Anser albifrons,1,POINT EMPTY,328,1,9589196,"Collaborative project of Institute of Geography, Russian Academy of Sciences (IG-RAS, Russia) and Max Planck Institute of Animal Behavior (MPIAB, Germany)","Portions of these data are published as
Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2021. Data from: Longer days enable higher diurnal activity for migratory birds [greater white-fronted geese]. Movebank Data Repository. https://doi.org/10.5441/001/1.254rd102

Kölzsch A, Kruckenberg H, Glazov P, Müskens GJDM, Wikelski M. 2016. Data from: Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Movebank Data Repository. https://doi.org/10.5441/001/1.31c2v92f (study ""Migration timing in white-fronted geese (data from Kölzsch et al. 2016)"")

Kölzsch A, Müskens GJDM, Moonen S, Kruckenberg H, Glazov P, Wikelski M. 2019. Data from: Flyway connectivity and exchange primarily driven by moult migration in geese [North Sea population]. Movebank Data Repository. https://doi.org/10.5441/001/1.ct72m82n (study ""North Sea population tracks of greater white-fronted geese 2014-2017 (data from Kölzsch et al. 2019)"")

Papers
Pokrovsky I, Kölzsch A, Sherub S, Fiedler W, Glazov P, Kulikova O, Wikelski M, Flack A. 2021. Longer days enable higher diurnal activity for migratory birds. J Anim Ecol. https://doi.org/10.1111/1365-2656.13484

Kölzsch A, Müskens GJDM, Szinai P, Moonen S, Glazov P, Kruckenberg H, Wikelski M, Nolet BA. 2019. Flyway connectivity and exchange primarily driven by moult migration in geese. Movement Ecology. 7:3. https://doi.org/10.1186/s40462-019-0148-6

Kölzsch A, Müskens GJDM, Kruckenberg H, Glazov P, Weinzierl R, Nolet BA, Wikelski M. 2016. Towards a new understanding of migration timing: slower spring than autumn migration in geese reflects different decision rules for stopover use and departure. Oikos. 125(10): 1496–1507. https://doi.org/10.1111/oik.03121",Max Planck Institute of Animal Behavior and German Aerospace Center (DLR),True,True,False,Please do not use any data without first contacting the PI and the Contact Person.,CUSTOM,LifeTrack Geese IG-RAS MPIAB ICARUS,75,75,98,Pglaz (Peter Glazov),Determine social movement patterns in white-fronted geese.,research,False,True,False,True,False,data_manager,2004-07-05 07:37:43,2016-08-11 17:40:25,4996042,Anser albifrons,andreakoelzsch (Andrea Kölzsch),POINT (44.3119 58.8292),2013-08-08 06:10:34+00:00,POINT (48.768583 69.12195),48.768583,69.12195 diff --git a/test/contract/latlon-three-tracks/py/meta.csv b/test/contract/latlon-three-tracks/py/meta.csv new file mode 100644 index 0000000..3587343 --- /dev/null +++ b/test/contract/latlon-three-tracks/py/meta.csv @@ -0,0 +1,2 @@ +crs,tzone,timeColName,trackIdColName +EPSG:4326,UTC,timestamp_utc,individual_local_identifier diff --git a/test/contract/list-columns/input.rds b/test/contract/list-columns/input.rds new file mode 100644 index 0000000..533271b Binary files /dev/null and b/test/contract/list-columns/input.rds differ diff --git a/test/contract/list-columns/link.csv b/test/contract/list-columns/link.csv new file mode 100644 index 0000000..4820d75 --- /dev/null +++ b/test/contract/list-columns/link.csv @@ -0,0 +1,6 @@ +"deployment_id","sensor_type_id","timestamp","event_id","visible","individual_local_identifier","speed","tag_id","individual_id","deploy_off_timestamp","deploy_on_timestamp","sensor_type_ids","capture_location","deploy_on_location","deploy_off_location","sex","taxon_canonical_name","individual_number_of_deployments","taxon_detail","mortality_location","tag_local_identifier","tag_number_of_deployments","study_id","acknowledgements","citation","grants_used","has_quota","i_am_owner","is_test","license_terms","license_type","name","study_number_of_deployments","number_of_individuals","number_of_tags","principal_investigator_name","study_objective","study_type","suspend_license_terms","i_can_see_data","there_are_data_which_i_cannot_see","i_have_download_access","i_am_collaborator","study_permission","timestamp_first_deployed_location","timestamp_last_deployed_location","number_of_deployed_locations","taxon_ids","contact_person_name","main_location","coords_x","coords_y" +404733942,653,"2009-07-07 22:00:00.000",29681040315,TRUE,"B13",0.0132024737227999,"2099696252,2099696211",404732174,"2010-01-08 02:00:00,2010-08-21 02:00:00","2009-06-26,2010-01-20","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","f","Syncerus caffer",1,"caffer","POINT EMPTY","T5HS-2636,T5HS-1876",1,404717594,"We thank the Botswana Ministry of Environment, Wildlife and Tourism for permission to conduct this study; Dane Hawk, Rob Jackson and Larry Patterson for veterinary support; Laura Atkinson, Janette Baarman, Roz Balen, Shavaughn Davies and Jennifer Gilbert for assistance in the field; and Guy Lobjoit and Duncan Rowles for logistical support.","Bennitt, E. (2012). The ecology of African buffalo (Syncerus caffer) in the Okavango Delta, Botswana, Ph.D. Thesis, University of Bristol. PhD; Bennitt, E., M. C. Bonyongo, et al. (2014). ""Habitat selection by African buffalo (Syncerus caffer) in response to landscape-level fluctuations in water availability on two temporal scales."" PLoS One 9(7): e101346; Bennitt, E., M. C. Bonyongo, et al. (2015). ""Behaviour-related scalar habitat use by Cape buffalo (Syncerus caffer caffer)."" PLoS ONE 10(12): e0145145; Bennitt, E., M. C. Bonyongo, et al. (2016). ""Effects of divergent migratory strategies on access to resources for Cape buffalo (Syncerus caffer caffer)."" Journal of Mammalogy; ; Bennitt, E., M. C. Bonyongo, et al. (2017). ""Cape buffalo (Syncerus caffer caffer) social dynamics in a flood-pulsed environment."" Behavioral Ecology: arx138-arx138","This work was supported by Jenny and Martin Bennitt; the Dulverton Trust; Harry Ferguson; Ian Fuhr; Rodney Fuhr; Dane Hawk; Idea Wild; the North of England Zoological Society; the Roberts Fund; and Wilderness Safaris Wildlife Trust.",TRUE,FALSE,FALSE,"Any person wishing to use these data in a publication must contact me first. All funders and contributors must be acknowledged in any publications","CUSTOM","Cape buffalo movements in the Okavango Delta, Botswana",18,15,23,"EmilyBennitt (Emily Bennitt)","15 Cape buffalo cows were collared in separate herds between 2007-2010 in the Okavango Delta, Botswana","research",FALSE,TRUE,FALSE,TRUE,TRUE,"collaborator",2007-12-09 14:00:00,2010-08-07 20:00:00,11686,"Syncerus caffer","EmilyBennitt (Emily Bennitt)","POINT (23.37891 -19.29559)",23.3392567,-19.486085 +404728739,653,"2010-03-02 06:00:00.000",29681042714,TRUE,"B11",0.00247593692911997,"2099696215",404728730,"2010-07-13 02:00:00","2009-06-25","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","f","Syncerus caffer",1,"caffer","POINT EMPTY","T5HS-1875",1,404717594,"We thank the Botswana Ministry of Environment, Wildlife and Tourism for permission to conduct this study; Dane Hawk, Rob Jackson and Larry Patterson for veterinary support; Laura Atkinson, Janette Baarman, Roz Balen, Shavaughn Davies and Jennifer Gilbert for assistance in the field; and Guy Lobjoit and Duncan Rowles for logistical support.","Bennitt, E. (2012). The ecology of African buffalo (Syncerus caffer) in the Okavango Delta, Botswana, Ph.D. Thesis, University of Bristol. PhD; Bennitt, E., M. C. Bonyongo, et al. (2014). ""Habitat selection by African buffalo (Syncerus caffer) in response to landscape-level fluctuations in water availability on two temporal scales."" PLoS One 9(7): e101346; Bennitt, E., M. C. Bonyongo, et al. (2015). ""Behaviour-related scalar habitat use by Cape buffalo (Syncerus caffer caffer)."" PLoS ONE 10(12): e0145145; Bennitt, E., M. C. Bonyongo, et al. (2016). ""Effects of divergent migratory strategies on access to resources for Cape buffalo (Syncerus caffer caffer)."" Journal of Mammalogy; ; Bennitt, E., M. C. Bonyongo, et al. (2017). ""Cape buffalo (Syncerus caffer caffer) social dynamics in a flood-pulsed environment."" Behavioral Ecology: arx138-arx138","This work was supported by Jenny and Martin Bennitt; the Dulverton Trust; Harry Ferguson; Ian Fuhr; Rodney Fuhr; Dane Hawk; Idea Wild; the North of England Zoological Society; the Roberts Fund; and Wilderness Safaris Wildlife Trust.",TRUE,FALSE,FALSE,"Any person wishing to use these data in a publication must contact me first. All funders and contributors must be acknowledged in any publications","CUSTOM","Cape buffalo movements in the Okavango Delta, Botswana",18,15,23,"EmilyBennitt (Emily Bennitt)","15 Cape buffalo cows were collared in separate herds between 2007-2010 in the Okavango Delta, Botswana","research",FALSE,TRUE,FALSE,TRUE,TRUE,"collaborator",2007-12-09 14:00:00,2010-08-07 20:00:00,11686,"Syncerus caffer","EmilyBennitt (Emily Bennitt)","POINT (23.37891 -19.29559)",23.392955,-19.748145 +404725560,653,"2008-05-20 10:00:00.000",29681035522,TRUE,"B3",0.0390267715007808,"2099696207",404725133,"2009-03-16 02:00:00","2007-12-07","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","f","Syncerus caffer",1,NA,"POINT EMPTY","T5HS-1873",1,404717594,"We thank the Botswana Ministry of Environment, Wildlife and Tourism for permission to conduct this study; Dane Hawk, Rob Jackson and Larry Patterson for veterinary support; Laura Atkinson, Janette Baarman, Roz Balen, Shavaughn Davies and Jennifer Gilbert for assistance in the field; and Guy Lobjoit and Duncan Rowles for logistical support.","Bennitt, E. (2012). The ecology of African buffalo (Syncerus caffer) in the Okavango Delta, Botswana, Ph.D. Thesis, University of Bristol. PhD; Bennitt, E., M. C. Bonyongo, et al. (2014). ""Habitat selection by African buffalo (Syncerus caffer) in response to landscape-level fluctuations in water availability on two temporal scales."" PLoS One 9(7): e101346; Bennitt, E., M. C. Bonyongo, et al. (2015). ""Behaviour-related scalar habitat use by Cape buffalo (Syncerus caffer caffer)."" PLoS ONE 10(12): e0145145; Bennitt, E., M. C. Bonyongo, et al. (2016). ""Effects of divergent migratory strategies on access to resources for Cape buffalo (Syncerus caffer caffer)."" Journal of Mammalogy; ; Bennitt, E., M. C. Bonyongo, et al. (2017). ""Cape buffalo (Syncerus caffer caffer) social dynamics in a flood-pulsed environment."" Behavioral Ecology: arx138-arx138","This work was supported by Jenny and Martin Bennitt; the Dulverton Trust; Harry Ferguson; Ian Fuhr; Rodney Fuhr; Dane Hawk; Idea Wild; the North of England Zoological Society; the Roberts Fund; and Wilderness Safaris Wildlife Trust.",TRUE,FALSE,FALSE,"Any person wishing to use these data in a publication must contact me first. All funders and contributors must be acknowledged in any publications","CUSTOM","Cape buffalo movements in the Okavango Delta, Botswana",18,15,23,"EmilyBennitt (Emily Bennitt)","15 Cape buffalo cows were collared in separate herds between 2007-2010 in the Okavango Delta, Botswana","research",FALSE,TRUE,FALSE,TRUE,TRUE,"collaborator",2007-12-09 14:00:00,2010-08-07 20:00:00,11686,"Syncerus caffer","EmilyBennitt (Emily Bennitt)","POINT (23.37891 -19.29559)",23.309895,-19.380082 +404727443,653,"2008-12-18 06:00:00.000",29681038112,TRUE,"B4",0.230368615982088,"2099696211",404725743,"2009-01-21 02:00:00","2007-12-07","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","f","Syncerus caffer",1,NA,"POINT EMPTY","T5HS-1876",1,404717594,"We thank the Botswana Ministry of Environment, Wildlife and Tourism for permission to conduct this study; Dane Hawk, Rob Jackson and Larry Patterson for veterinary support; Laura Atkinson, Janette Baarman, Roz Balen, Shavaughn Davies and Jennifer Gilbert for assistance in the field; and Guy Lobjoit and Duncan Rowles for logistical support.","Bennitt, E. (2012). The ecology of African buffalo (Syncerus caffer) in the Okavango Delta, Botswana, Ph.D. Thesis, University of Bristol. PhD; Bennitt, E., M. C. Bonyongo, et al. (2014). ""Habitat selection by African buffalo (Syncerus caffer) in response to landscape-level fluctuations in water availability on two temporal scales."" PLoS One 9(7): e101346; Bennitt, E., M. C. Bonyongo, et al. (2015). ""Behaviour-related scalar habitat use by Cape buffalo (Syncerus caffer caffer)."" PLoS ONE 10(12): e0145145; Bennitt, E., M. C. Bonyongo, et al. (2016). ""Effects of divergent migratory strategies on access to resources for Cape buffalo (Syncerus caffer caffer)."" Journal of Mammalogy; ; Bennitt, E., M. C. Bonyongo, et al. (2017). ""Cape buffalo (Syncerus caffer caffer) social dynamics in a flood-pulsed environment."" Behavioral Ecology: arx138-arx138","This work was supported by Jenny and Martin Bennitt; the Dulverton Trust; Harry Ferguson; Ian Fuhr; Rodney Fuhr; Dane Hawk; Idea Wild; the North of England Zoological Society; the Roberts Fund; and Wilderness Safaris Wildlife Trust.",TRUE,FALSE,FALSE,"Any person wishing to use these data in a publication must contact me first. All funders and contributors must be acknowledged in any publications","CUSTOM","Cape buffalo movements in the Okavango Delta, Botswana",18,15,23,"EmilyBennitt (Emily Bennitt)","15 Cape buffalo cows were collared in separate herds between 2007-2010 in the Okavango Delta, Botswana","research",FALSE,TRUE,FALSE,TRUE,TRUE,"collaborator",2007-12-09 14:00:00,2010-08-07 20:00:00,11686,"Syncerus caffer","EmilyBennitt (Emily Bennitt)","POINT (23.37891 -19.29559)",23.986685,-19.485348 +404727443,653,"2008-04-27 18:00:00.000",29681035245,TRUE,"B4",0.0143100749717294,"2099696211",404725743,"2009-01-21 02:00:00","2007-12-07","gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","f","Syncerus caffer",1,NA,"POINT EMPTY","T5HS-1876",1,404717594,"We thank the Botswana Ministry of Environment, Wildlife and Tourism for permission to conduct this study; Dane Hawk, Rob Jackson and Larry Patterson for veterinary support; Laura Atkinson, Janette Baarman, Roz Balen, Shavaughn Davies and Jennifer Gilbert for assistance in the field; and Guy Lobjoit and Duncan Rowles for logistical support.","Bennitt, E. (2012). The ecology of African buffalo (Syncerus caffer) in the Okavango Delta, Botswana, Ph.D. Thesis, University of Bristol. PhD; Bennitt, E., M. C. Bonyongo, et al. (2014). ""Habitat selection by African buffalo (Syncerus caffer) in response to landscape-level fluctuations in water availability on two temporal scales."" PLoS One 9(7): e101346; Bennitt, E., M. C. Bonyongo, et al. (2015). ""Behaviour-related scalar habitat use by Cape buffalo (Syncerus caffer caffer)."" PLoS ONE 10(12): e0145145; Bennitt, E., M. C. Bonyongo, et al. (2016). ""Effects of divergent migratory strategies on access to resources for Cape buffalo (Syncerus caffer caffer)."" Journal of Mammalogy; ; Bennitt, E., M. C. Bonyongo, et al. (2017). ""Cape buffalo (Syncerus caffer caffer) social dynamics in a flood-pulsed environment."" Behavioral Ecology: arx138-arx138","This work was supported by Jenny and Martin Bennitt; the Dulverton Trust; Harry Ferguson; Ian Fuhr; Rodney Fuhr; Dane Hawk; Idea Wild; the North of England Zoological Society; the Roberts Fund; and Wilderness Safaris Wildlife Trust.",TRUE,FALSE,FALSE,"Any person wishing to use these data in a publication must contact me first. All funders and contributors must be acknowledged in any publications","CUSTOM","Cape buffalo movements in the Okavango Delta, Botswana",18,15,23,"EmilyBennitt (Emily Bennitt)","15 Cape buffalo cows were collared in separate herds between 2007-2010 in the Okavango Delta, Botswana","research",FALSE,TRUE,FALSE,TRUE,TRUE,"collaborator",2007-12-09 14:00:00,2010-08-07 20:00:00,11686,"Syncerus caffer","EmilyBennitt (Emily Bennitt)","POINT (23.37891 -19.29559)",23.562212,-19.58811 diff --git a/test/contract/list-columns/meta.csv b/test/contract/list-columns/meta.csv new file mode 100644 index 0000000..5dc61b0 --- /dev/null +++ b/test/contract/list-columns/meta.csv @@ -0,0 +1,2 @@ +"crs","tzone","timeColName","trackIdColName" +"EPSG:4326","UTC","timestamp","individual_local_identifier" diff --git a/test/contract/list-columns/py/link.csv b/test/contract/list-columns/py/link.csv new file mode 100644 index 0000000..9171fea --- /dev/null +++ b/test/contract/list-columns/py/link.csv @@ -0,0 +1,3 @@ +timestamp_utc,deployment_id,sensor_type_id,timestamp,event_id,visible,individual_local_identifier,speed,tag_id,individual_id,deploy_off_timestamp,deploy_on_timestamp,sensor_type_ids,capture_location,deploy_on_location,deploy_off_location,sex,taxon_canonical_name,individual_number_of_deployments,taxon_detail,mortality_location,tag_local_identifier,tag_number_of_deployments,study_id,acknowledgements,citation,grants_used,has_quota,i_am_owner,is_test,license_terms,license_type,name,study_number_of_deployments,number_of_individuals,number_of_tags,principal_investigator_name,study_objective,study_type,suspend_license_terms,i_can_see_data,there_are_data_which_i_cannot_see,i_have_download_access,i_am_collaborator,study_permission,timestamp_first_deployed_location,timestamp_last_deployed_location,number_of_deployed_locations,taxon_ids,contact_person_name,main_location,timestamp_tz,geometry,coords_x,coords_y +2008-04-27 18:00:00,404727443,653,2008-04-27 18:00:00,29681035245,True,B4,0.0143100749717294,2099696211,404725743,2009-01-21 02:00:00,2007-12-07,gps,POINT EMPTY,POINT EMPTY,POINT EMPTY,f,Syncerus caffer,1,,POINT EMPTY,T5HS-1876,1,404717594,"We thank the Botswana Ministry of Environment, Wildlife and Tourism for permission to conduct this study; Dane Hawk, Rob Jackson and Larry Patterson for veterinary support; Laura Atkinson, Janette Baarman, Roz Balen, Shavaughn Davies and Jennifer Gilbert for assistance in the field; and Guy Lobjoit and Duncan Rowles for logistical support.","Bennitt, E. (2012). The ecology of African buffalo (Syncerus caffer) in the Okavango Delta, Botswana, Ph.D. Thesis, University of Bristol. PhD; Bennitt, E., M. C. Bonyongo, et al. (2014). ""Habitat selection by African buffalo (Syncerus caffer) in response to landscape-level fluctuations in water availability on two temporal scales."" PLoS One 9(7): e101346; Bennitt, E., M. C. Bonyongo, et al. (2015). ""Behaviour-related scalar habitat use by Cape buffalo (Syncerus caffer caffer)."" PLoS ONE 10(12): e0145145; Bennitt, E., M. C. Bonyongo, et al. (2016). ""Effects of divergent migratory strategies on access to resources for Cape buffalo (Syncerus caffer caffer)."" Journal of Mammalogy; ; Bennitt, E., M. C. Bonyongo, et al. (2017). ""Cape buffalo (Syncerus caffer caffer) social dynamics in a flood-pulsed environment."" Behavioral Ecology: arx138-arx138",This work was supported by Jenny and Martin Bennitt; the Dulverton Trust; Harry Ferguson; Ian Fuhr; Rodney Fuhr; Dane Hawk; Idea Wild; the North of England Zoological Society; the Roberts Fund; and Wilderness Safaris Wildlife Trust.,True,False,False,Any person wishing to use these data in a publication must contact me first. All funders and contributors must be acknowledged in any publications,CUSTOM,"Cape buffalo movements in the Okavango Delta, Botswana",18,15,23,EmilyBennitt (Emily Bennitt),"15 Cape buffalo cows were collared in separate herds between 2007-2010 in the Okavango Delta, Botswana",research,False,True,False,True,True,collaborator,2007-12-09 14:00:00,2010-08-07 20:00:00,11686,Syncerus caffer,EmilyBennitt (Emily Bennitt),POINT (23.37891 -19.29559),2008-04-27 18:00:00+00:00,POINT (23.562212 -19.58811),23.562212,-19.58811 +2008-12-18 06:00:00,404727443,653,2008-12-18 06:00:00,29681038112,True,B4,0.230368615982088,2099696211,404725743,2009-01-21 02:00:00,2007-12-07,gps,POINT EMPTY,POINT EMPTY,POINT EMPTY,f,Syncerus caffer,1,,POINT EMPTY,T5HS-1876,1,404717594,"We thank the Botswana Ministry of Environment, Wildlife and Tourism for permission to conduct this study; Dane Hawk, Rob Jackson and Larry Patterson for veterinary support; Laura Atkinson, Janette Baarman, Roz Balen, Shavaughn Davies and Jennifer Gilbert for assistance in the field; and Guy Lobjoit and Duncan Rowles for logistical support.","Bennitt, E. (2012). The ecology of African buffalo (Syncerus caffer) in the Okavango Delta, Botswana, Ph.D. Thesis, University of Bristol. PhD; Bennitt, E., M. C. Bonyongo, et al. (2014). ""Habitat selection by African buffalo (Syncerus caffer) in response to landscape-level fluctuations in water availability on two temporal scales."" PLoS One 9(7): e101346; Bennitt, E., M. C. Bonyongo, et al. (2015). ""Behaviour-related scalar habitat use by Cape buffalo (Syncerus caffer caffer)."" PLoS ONE 10(12): e0145145; Bennitt, E., M. C. Bonyongo, et al. (2016). ""Effects of divergent migratory strategies on access to resources for Cape buffalo (Syncerus caffer caffer)."" Journal of Mammalogy; ; Bennitt, E., M. C. Bonyongo, et al. (2017). ""Cape buffalo (Syncerus caffer caffer) social dynamics in a flood-pulsed environment."" Behavioral Ecology: arx138-arx138",This work was supported by Jenny and Martin Bennitt; the Dulverton Trust; Harry Ferguson; Ian Fuhr; Rodney Fuhr; Dane Hawk; Idea Wild; the North of England Zoological Society; the Roberts Fund; and Wilderness Safaris Wildlife Trust.,True,False,False,Any person wishing to use these data in a publication must contact me first. All funders and contributors must be acknowledged in any publications,CUSTOM,"Cape buffalo movements in the Okavango Delta, Botswana",18,15,23,EmilyBennitt (Emily Bennitt),"15 Cape buffalo cows were collared in separate herds between 2007-2010 in the Okavango Delta, Botswana",research,False,True,False,True,True,collaborator,2007-12-09 14:00:00,2010-08-07 20:00:00,11686,Syncerus caffer,EmilyBennitt (Emily Bennitt),POINT (23.37891 -19.29559),2008-12-18 06:00:00+00:00,POINT (23.986685 -19.485348),23.986685,-19.485348 diff --git a/test/contract/list-columns/py/meta.csv b/test/contract/list-columns/py/meta.csv new file mode 100644 index 0000000..3587343 --- /dev/null +++ b/test/contract/list-columns/py/meta.csv @@ -0,0 +1,2 @@ +crs,tzone,timeColName,trackIdColName +EPSG:4326,UTC,timestamp_utc,individual_local_identifier diff --git a/test/contract/mollweide-midnight/input.rds b/test/contract/mollweide-midnight/input.rds new file mode 100644 index 0000000..2943b66 Binary files /dev/null and b/test/contract/mollweide-midnight/input.rds differ diff --git a/test/contract/mollweide-midnight/link.csv b/test/contract/mollweide-midnight/link.csv new file mode 100644 index 0000000..3e08dd4 --- /dev/null +++ b/test/contract/mollweide-midnight/link.csv @@ -0,0 +1,5 @@ +"sensor_type_id","individual_local_identifier","comments","sequence_number","tag_voltage","timestamp","transmission_protocol","transmission_timestamp","event_id","visible","deployment_id","tag_id","individual_id","animal_life_stage","animal_reproductive_condition","attachment_type","deploy_off_timestamp","deploy_on_person","deploy_on_timestamp","sensor_type_ids","capture_location","deploy_on_location","deploy_off_location","nick_name","ring_id","sex","taxon_canonical_name","individual_number_of_deployments","mortality_location","tag_local_identifier","tag_number_of_deployments","study_id","acknowledgements","citation","grants_used","has_quota","i_am_owner","is_test","license_terms","license_type","name","study_number_of_deployments","number_of_individuals","number_of_tags","principal_investigator_name","study_objective","study_type","suspend_license_terms","i_can_see_data","there_are_data_which_i_cannot_see","i_have_download_access","i_am_collaborator","study_permission","timestamp_first_deployed_location","timestamp_last_deployed_location","number_of_deployed_locations","taxon_ids","contact_person_name","main_location","coords_x","coords_y" +653,"Goat.8810","202649",1967,3.4,"2021-11-28 04:09:00.000","callback",2021-11-28 04:09:29,20951080272,TRUE,1600804509,1600779812,1600804167,"adult","lactating","collar",2022-10-15,"Benito and Giuseppe",2021-07-01,"sigfox-geolocation,gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","Goat-8810","Goat-8810","f","Capra hircus",1,"POINT EMPTY","C3E063",1,1600771155,"MPIAB","MPIAB DAMN","MPIAB",TRUE,TRUE,FALSE,"Data can not be used without prior contact with the PI","CUSTOM","MPIAB Disaster alert Mt Etna 2021",32,22,40,"martin (Martin Wikelski)","Understand possible preparatory movements of animals before natural disasters","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2021-07-01 06:13:00,2024-05-14 20:12:54,318411,"Capra hircus","Georg Heine (Georg Heine)","POINT (15.0472 37.73428)",1293132.79799209,4548331.07535749 +653,"Goat.8810","002984",2873,3.2,"2021-12-10 00:00:00.000","callback",2022-04-30 08:09:06,21910236611,TRUE,1600804509,1600779812,1600804167,"adult","lactating","collar",2022-10-15,"Benito and Giuseppe",2021-07-01,"sigfox-geolocation,gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","Goat-8810","Goat-8810","f","Capra hircus",1,"POINT EMPTY","C3E063",1,1600771155,"MPIAB","MPIAB DAMN","MPIAB",TRUE,TRUE,FALSE,"Data can not be used without prior contact with the PI","CUSTOM","MPIAB Disaster alert Mt Etna 2021",32,22,40,"martin (Martin Wikelski)","Understand possible preparatory movements of animals before natural disasters","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2021-07-01 06:13:00,2024-05-14 20:12:54,318411,"Capra hircus","Georg Heine (Georg Heine)","POINT (15.0472 37.73428)",1292494.63012212,4547920.32501199 +653,"Goat.8810","002984",2872,3.2,"2021-12-10 04:00:00.000","callback",2022-04-30 06:38:44,21909981548,TRUE,1600804509,1600779812,1600804167,"adult","lactating","collar",2022-10-15,"Benito and Giuseppe",2021-07-01,"sigfox-geolocation,gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","Goat-8810","Goat-8810","f","Capra hircus",1,"POINT EMPTY","C3E063",1,1600771155,"MPIAB","MPIAB DAMN","MPIAB",TRUE,TRUE,FALSE,"Data can not be used without prior contact with the PI","CUSTOM","MPIAB Disaster alert Mt Etna 2021",32,22,40,"martin (Martin Wikelski)","Understand possible preparatory movements of animals before natural disasters","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2021-07-01 06:13:00,2024-05-14 20:12:54,318411,"Capra hircus","Georg Heine (Georg Heine)","POINT (15.0472 37.73428)",1292492.80480183,4547939.16693721 +653,"Goat.8810","002650",1974,3.2,"2022-01-26 22:07:00.000","callback",2022-01-26 22:08:06,21141380248,TRUE,1600804509,1600779812,1600804167,"adult","lactating","collar",2022-10-15,"Benito and Giuseppe",2021-07-01,"sigfox-geolocation,gps","POINT EMPTY","POINT EMPTY","POINT EMPTY","Goat-8810","Goat-8810","f","Capra hircus",1,"POINT EMPTY","C3E063",1,1600771155,"MPIAB","MPIAB DAMN","MPIAB",TRUE,TRUE,FALSE,"Data can not be used without prior contact with the PI","CUSTOM","MPIAB Disaster alert Mt Etna 2021",32,22,40,"martin (Martin Wikelski)","Understand possible preparatory movements of animals before natural disasters","research",FALSE,TRUE,FALSE,TRUE,FALSE,"data_manager",2021-07-01 06:13:00,2024-05-14 20:12:54,318411,"Capra hircus","Georg Heine (Georg Heine)","POINT (15.0472 37.73428)",1293732.18810285,4548992.40513311 diff --git a/test/contract/mollweide-midnight/meta.csv b/test/contract/mollweide-midnight/meta.csv new file mode 100644 index 0000000..1f59c1b --- /dev/null +++ b/test/contract/mollweide-midnight/meta.csv @@ -0,0 +1,2 @@ +"crs","tzone","timeColName","trackIdColName" +"ESRI:54009","UTC","timestamp","individual_local_identifier" diff --git a/test/contract/mollweide-midnight/py/link.csv b/test/contract/mollweide-midnight/py/link.csv new file mode 100644 index 0000000..3fcc157 --- /dev/null +++ b/test/contract/mollweide-midnight/py/link.csv @@ -0,0 +1,5 @@ +timestamp_utc,sensor_type_id,individual_local_identifier,comments,sequence_number,tag_voltage,timestamp,transmission_protocol,transmission_timestamp,event_id,visible,deployment_id,tag_id,individual_id,animal_life_stage,animal_reproductive_condition,attachment_type,deploy_off_timestamp,deploy_on_person,deploy_on_timestamp,sensor_type_ids,capture_location,deploy_on_location,deploy_off_location,nick_name,ring_id,sex,taxon_canonical_name,individual_number_of_deployments,mortality_location,tag_local_identifier,tag_number_of_deployments,study_id,acknowledgements,citation,grants_used,has_quota,i_am_owner,is_test,license_terms,license_type,name,study_number_of_deployments,number_of_individuals,number_of_tags,principal_investigator_name,study_objective,study_type,suspend_license_terms,i_can_see_data,there_are_data_which_i_cannot_see,i_have_download_access,i_am_collaborator,study_permission,timestamp_first_deployed_location,timestamp_last_deployed_location,number_of_deployed_locations,taxon_ids,contact_person_name,main_location,timestamp_tz,geometry,coords_x,coords_y +2021-11-28 04:09:00,653,Goat.8810,202649,1967,3.4,2021-11-28 04:09:00,callback,2021-11-28 04:09:29,20951080272,True,1600804509,1600779812,1600804167,adult,lactating,collar,2022-10-15,Benito and Giuseppe,2021-07-01,"sigfox-geolocation,gps",POINT EMPTY,POINT EMPTY,POINT EMPTY,Goat-8810,Goat-8810,f,Capra hircus,1,POINT EMPTY,C3E063,1,1600771155,MPIAB,MPIAB DAMN,MPIAB,True,True,False,Data can not be used without prior contact with the PI,CUSTOM,MPIAB Disaster alert Mt Etna 2021,32,22,40,martin (Martin Wikelski),Understand possible preparatory movements of animals before natural disasters,research,False,True,False,True,False,data_manager,2021-07-01 06:13:00,2024-05-14 20:12:54,318411,Capra hircus,Georg Heine (Georg Heine),POINT (15.0472 37.73428),2021-11-28 04:09:00+00:00,POINT (1293132.79799209 4548331.07535749),1293132.79799209,4548331.07535749 +2021-12-10 00:00:00,653,Goat.8810,2984,2873,3.2,2021-12-10 00:00:00,callback,2022-04-30 08:09:06,21910236611,True,1600804509,1600779812,1600804167,adult,lactating,collar,2022-10-15,Benito and Giuseppe,2021-07-01,"sigfox-geolocation,gps",POINT EMPTY,POINT EMPTY,POINT EMPTY,Goat-8810,Goat-8810,f,Capra hircus,1,POINT EMPTY,C3E063,1,1600771155,MPIAB,MPIAB DAMN,MPIAB,True,True,False,Data can not be used without prior contact with the PI,CUSTOM,MPIAB Disaster alert Mt Etna 2021,32,22,40,martin (Martin Wikelski),Understand possible preparatory movements of animals before natural disasters,research,False,True,False,True,False,data_manager,2021-07-01 06:13:00,2024-05-14 20:12:54,318411,Capra hircus,Georg Heine (Georg Heine),POINT (15.0472 37.73428),2021-12-10 00:00:00+00:00,POINT (1292494.63012212 4547920.32501199),1292494.63012212,4547920.32501199 +2021-12-10 04:00:00,653,Goat.8810,2984,2872,3.2,2021-12-10 04:00:00,callback,2022-04-30 06:38:44,21909981548,True,1600804509,1600779812,1600804167,adult,lactating,collar,2022-10-15,Benito and Giuseppe,2021-07-01,"sigfox-geolocation,gps",POINT EMPTY,POINT EMPTY,POINT EMPTY,Goat-8810,Goat-8810,f,Capra hircus,1,POINT EMPTY,C3E063,1,1600771155,MPIAB,MPIAB DAMN,MPIAB,True,True,False,Data can not be used without prior contact with the PI,CUSTOM,MPIAB Disaster alert Mt Etna 2021,32,22,40,martin (Martin Wikelski),Understand possible preparatory movements of animals before natural disasters,research,False,True,False,True,False,data_manager,2021-07-01 06:13:00,2024-05-14 20:12:54,318411,Capra hircus,Georg Heine (Georg Heine),POINT (15.0472 37.73428),2021-12-10 04:00:00+00:00,POINT (1292492.80480183 4547939.16693721),1292492.80480183,4547939.16693721 +2022-01-26 22:07:00,653,Goat.8810,2650,1974,3.2,2022-01-26 22:07:00,callback,2022-01-26 22:08:06,21141380248,True,1600804509,1600779812,1600804167,adult,lactating,collar,2022-10-15,Benito and Giuseppe,2021-07-01,"sigfox-geolocation,gps",POINT EMPTY,POINT EMPTY,POINT EMPTY,Goat-8810,Goat-8810,f,Capra hircus,1,POINT EMPTY,C3E063,1,1600771155,MPIAB,MPIAB DAMN,MPIAB,True,True,False,Data can not be used without prior contact with the PI,CUSTOM,MPIAB Disaster alert Mt Etna 2021,32,22,40,martin (Martin Wikelski),Understand possible preparatory movements of animals before natural disasters,research,False,True,False,True,False,data_manager,2021-07-01 06:13:00,2024-05-14 20:12:54,318411,Capra hircus,Georg Heine (Georg Heine),POINT (15.0472 37.73428),2022-01-26 22:07:00+00:00,POINT (1293732.18810285 4548992.40513311),1293732.18810285,4548992.40513311 diff --git a/test/contract/mollweide-midnight/py/meta.csv b/test/contract/mollweide-midnight/py/meta.csv new file mode 100644 index 0000000..6db6950 --- /dev/null +++ b/test/contract/mollweide-midnight/py/meta.csv @@ -0,0 +1,2 @@ +crs,tzone,timeColName,trackIdColName +ESRI:54009,UTC,timestamp_utc,individual_local_identifier diff --git a/test/contract/prepare.sh b/test/contract/prepare.sh new file mode 100755 index 0000000..a3b4694 --- /dev/null +++ b/test/contract/prepare.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Inside the R -> python image: collects every //input.rds as /inputs/.rds, +# after cutting them from first when is 1. +set -o errexit -o nounset + +here="$(cd "$(dirname "$0")" && pwd)" +raw="$1" out="$2" cut="$3" + +mkdir -p "$out/contract" "$out/inputs" +cp -R "$here/." "$out/contract/" +if [ "$cut" = 1 ]; then + (cd r && Rscript "$here/cut.R" "$raw" "$out/contract") +fi + +for dir in "$out"/contract/*/; do + [ -f "$dir/input.rds" ] || continue + cp "$dir/input.rds" "$out/inputs/$(basename "$dir").rds" +done diff --git a/test/contract/subsecond/input.rds b/test/contract/subsecond/input.rds new file mode 100644 index 0000000..7e93881 Binary files /dev/null and b/test/contract/subsecond/input.rds differ diff --git a/test/contract/subsecond/link.csv b/test/contract/subsecond/link.csv new file mode 100644 index 0000000..4220fb5 --- /dev/null +++ b/test/contract/subsecond/link.csv @@ -0,0 +1,5 @@ +"sensor_type_id","individual_local_identifier","eobs_battery_voltage","eobs_fix_battery_voltage","eobs_horizontal_accuracy_estimate","eobs_key_bin_checksum","eobs_speed_accuracy_estimate","eobs_start_timestamp","eobs_status","eobs_temperature","eobs_type_of_fix","eobs_used_time_to_get_fix","ground_speed","heading","height_above_ellipsoid","timestamp","event_id","visible","deployment_id","tag_id","individual_id","animal_life_stage","animal_mass","attachment_type","deployment_comments","deploy_off_timestamp","deploy_on_person","deploy_on_timestamp","deployment_end_type","duty_cycle","deployment_local_identifier","manipulation_type","outlier_comments","study_site","tag_readout_method","sensor_type_ids","capture_location","deploy_on_location","deploy_off_location","birth_hatch_latitude","birth_hatch_longitude","individual_comments","death_comments","group_id","latest_date_born","mortality_date","mortality_type","parents","ring_id","sex","siblings","taxon_canonical_name","individual_number_of_deployments","mortality_location","beacon_frequency","tag_local_identifier","manufacturer_name","model","serial_no","tag_production_date","weight","tag_number_of_deployments","study_id","acknowledgements","citation","has_quota","i_am_owner","is_test","license_type","name","study_number_of_deployments","number_of_individuals","number_of_tags","principal_investigator_name","study_objective","study_type","suspend_license_terms","i_can_see_data","there_are_data_which_i_cannot_see","i_have_download_access","i_am_collaborator","study_permission","timestamp_first_deployed_location","timestamp_last_deployed_location","number_of_deployed_locations","taxon_ids","contact_person_name","main_location","coords_x","coords_y" +653,"Peaches.....DER.AN917..eobs.3916.",4042,4016,1.54,3862950659,0.14,2014-08-06 09:15:00,"A",5,"3",274,12.59,49.81,802.8,"2014-08-06 09:19:35.000",372705832,TRUE,8854045745,23131236,23461880,"nestling",NA,"harness","867.195; Pinger on 19:00-0:00",2014-09-03,"Wolfgang Fiedler",2014-06-16 09:00:00,"dead","GPS and ACC switched off 20:00 to 02:00 GPS time","3916 on AN917","none","filters run in Movebank in Jan 2024: value range filter (flag records with coordinates 0,0); duplicate filter (key attributes tag-id and timestamp; preference order for eobs:status A, B, C, D, blank); additional visible outliers flagged manually","Radolfzell-Boehringen, SW Germany","multiple","acceleration,gps","POINT EMPTY","POINT EMPTY","POINT EMPTY",47.751444,8.931513,"Birke evang. Kirche, 4 nestlings, this is the second youngest, smallest chick died.","Died on 2014-08-31 near Motril, Spain. Reason unknown. lat: 36.726; long: -3.567. bird was moving around locally for a long time before tag stopped.","Boehringen Birke evang. Kirche",2014-04-15,2014-08-31,"unknown","DER A.....9456,","DER AN917","f","AN916,AN918","Ciconia ciconia",1,"POINT (-3.567 36.726)",867.195,"3916","e-obs GmbH","GPRS Solar Logger (Superburst)","3916","2014",55,2,21231406,"Max Planck Institute of Animal Behavior","Fiedler W, Flack A, Schäfle W, Keeves B, Quetting M, Eid B, Schmid H, Wikelski M. 2024. Data from: Study ""LifeTrack White Stork SW Germany"" (2013-2023). Movebank Data Repository. https://doi.org/10.5441/001/1.ck04mn78_2

These data are described in
Aikens EO, Nourani E, Fiedler W, Wikelski M, Flack A. 2024. Learning shapes the development of migratory behavior. Proc Natl Acad Sci USA. 121(12):e2306389121. https://doi.org/10.1073/pnas.2306389121

Kays R, Davidson SC, Berger M, Bohrer G, Fiedler W, Flack A, Hirt J, Hahn C, Gauggel D, Russell B, et al. 2022. The Movebank system for studying global animal movement and demography. Methods Ecol Evol. 13(2):419-431. https://doi.org/10.1111/2041-210X.13767

Cheng Y, Fiedler W, Wikelski M, Flack A. 2019. ""Closer-to-home"" strategy benefits juvenile survival in a long-distance migratory bird. Ecology and Evolution. 9(16): 8945–8952. https://doi.org/10.1002/ece3.5395

Flack A, Fiedler W, Blas J, Pokrovski I, Kaatz M, Mitropolsky M, Aghababyan K, Fakriadis Y, Makrigianni E, Jerzak L, et al. 2016. Costs of migratory decisions: a comparison across eight white stork populations. Science Advances. 2(1): e1500931. https://doi.org/10.1126/sciadv.1500931

Weinzierl R, Bohrer G, Kranstauber B, Fiedler W, Wikelski M, Flack A. 2016. Wind estimation based on thermal soaring of birds. Ecology and Evolution. 6(24): 8706–8718. https://doi.org/10.1002/ece3.2585

Parts of this dataset are also available in studies ""Fall migration of white storks in 2014"" (published as DOI 10.5441/001/1.bj96m274) and ""MPIO white stork lifetime tracking data (2013-2014)"" (published as DOI 10.5441/001/1.78152p3q).",TRUE,FALSE,FALSE,"CC_BY","LifeTrack White Stork SW Germany",92,94,101,"martin (Martin Wikelski)","Determining migration routes of white storks in Eurasia. Lifetime tracking of storks. Involvement of the public","research",TRUE,TRUE,FALSE,TRUE,FALSE,"na",2013-06-24 06:39:34.998,2026-09-24 10:30:29,13275703,"Ciconia ciconia","myotis (Wolfgang Fiedler)","POINT (9.019775 48.01198)",8.8763209,47.8114328 +653,"Peaches.....DER.AN917..eobs.3916.",4042,4016,1.54,2480015882,0.14,2014-08-06 09:15:00,"A",5,"3",275,11.74,51.12,803.5,"2014-08-06 09:19:35.999",372705833,TRUE,8854045745,23131236,23461880,"nestling",NA,"harness","867.195; Pinger on 19:00-0:00",2014-09-03,"Wolfgang Fiedler",2014-06-16 09:00:00,"dead","GPS and ACC switched off 20:00 to 02:00 GPS time","3916 on AN917","none","filters run in Movebank in Jan 2024: value range filter (flag records with coordinates 0,0); duplicate filter (key attributes tag-id and timestamp; preference order for eobs:status A, B, C, D, blank); additional visible outliers flagged manually","Radolfzell-Boehringen, SW Germany","multiple","acceleration,gps","POINT EMPTY","POINT EMPTY","POINT EMPTY",47.751444,8.931513,"Birke evang. Kirche, 4 nestlings, this is the second youngest, smallest chick died.","Died on 2014-08-31 near Motril, Spain. Reason unknown. lat: 36.726; long: -3.567. bird was moving around locally for a long time before tag stopped.","Boehringen Birke evang. Kirche",2014-04-15,2014-08-31,"unknown","DER A.....9456,","DER AN917","f","AN916,AN918","Ciconia ciconia",1,"POINT (-3.567 36.726)",867.195,"3916","e-obs GmbH","GPRS Solar Logger (Superburst)","3916","2014",55,2,21231406,"Max Planck Institute of Animal Behavior","Fiedler W, Flack A, Schäfle W, Keeves B, Quetting M, Eid B, Schmid H, Wikelski M. 2024. Data from: Study ""LifeTrack White Stork SW Germany"" (2013-2023). Movebank Data Repository. https://doi.org/10.5441/001/1.ck04mn78_2

These data are described in
Aikens EO, Nourani E, Fiedler W, Wikelski M, Flack A. 2024. Learning shapes the development of migratory behavior. Proc Natl Acad Sci USA. 121(12):e2306389121. https://doi.org/10.1073/pnas.2306389121

Kays R, Davidson SC, Berger M, Bohrer G, Fiedler W, Flack A, Hirt J, Hahn C, Gauggel D, Russell B, et al. 2022. The Movebank system for studying global animal movement and demography. Methods Ecol Evol. 13(2):419-431. https://doi.org/10.1111/2041-210X.13767

Cheng Y, Fiedler W, Wikelski M, Flack A. 2019. ""Closer-to-home"" strategy benefits juvenile survival in a long-distance migratory bird. Ecology and Evolution. 9(16): 8945–8952. https://doi.org/10.1002/ece3.5395

Flack A, Fiedler W, Blas J, Pokrovski I, Kaatz M, Mitropolsky M, Aghababyan K, Fakriadis Y, Makrigianni E, Jerzak L, et al. 2016. Costs of migratory decisions: a comparison across eight white stork populations. Science Advances. 2(1): e1500931. https://doi.org/10.1126/sciadv.1500931

Weinzierl R, Bohrer G, Kranstauber B, Fiedler W, Wikelski M, Flack A. 2016. Wind estimation based on thermal soaring of birds. Ecology and Evolution. 6(24): 8706–8718. https://doi.org/10.1002/ece3.2585

Parts of this dataset are also available in studies ""Fall migration of white storks in 2014"" (published as DOI 10.5441/001/1.bj96m274) and ""MPIO white stork lifetime tracking data (2013-2014)"" (published as DOI 10.5441/001/1.78152p3q).",TRUE,FALSE,FALSE,"CC_BY","LifeTrack White Stork SW Germany",92,94,101,"martin (Martin Wikelski)","Determining migration routes of white storks in Eurasia. Lifetime tracking of storks. Involvement of the public","research",TRUE,TRUE,FALSE,TRUE,FALSE,"na",2013-06-24 06:39:34.998,2026-09-24 10:30:29,13275703,"Ciconia ciconia","myotis (Wolfgang Fiedler)","POINT (9.019775 48.01198)",8.876445,47.8115006 +653,"Peaches.....DER.AN917..eobs.3916.",4042,4013,1.54,3479504210,0.15,2014-08-06 09:15:00,"A",5,"3",276,10.97,48.5,803.9,"2014-08-06 09:19:36.999",372705834,TRUE,8854045745,23131236,23461880,"nestling",NA,"harness","867.195; Pinger on 19:00-0:00",2014-09-03,"Wolfgang Fiedler",2014-06-16 09:00:00,"dead","GPS and ACC switched off 20:00 to 02:00 GPS time","3916 on AN917","none","filters run in Movebank in Jan 2024: value range filter (flag records with coordinates 0,0); duplicate filter (key attributes tag-id and timestamp; preference order for eobs:status A, B, C, D, blank); additional visible outliers flagged manually","Radolfzell-Boehringen, SW Germany","multiple","acceleration,gps","POINT EMPTY","POINT EMPTY","POINT EMPTY",47.751444,8.931513,"Birke evang. Kirche, 4 nestlings, this is the second youngest, smallest chick died.","Died on 2014-08-31 near Motril, Spain. Reason unknown. lat: 36.726; long: -3.567. bird was moving around locally for a long time before tag stopped.","Boehringen Birke evang. Kirche",2014-04-15,2014-08-31,"unknown","DER A.....9456,","DER AN917","f","AN916,AN918","Ciconia ciconia",1,"POINT (-3.567 36.726)",867.195,"3916","e-obs GmbH","GPRS Solar Logger (Superburst)","3916","2014",55,2,21231406,"Max Planck Institute of Animal Behavior","Fiedler W, Flack A, Schäfle W, Keeves B, Quetting M, Eid B, Schmid H, Wikelski M. 2024. Data from: Study ""LifeTrack White Stork SW Germany"" (2013-2023). Movebank Data Repository. https://doi.org/10.5441/001/1.ck04mn78_2

These data are described in
Aikens EO, Nourani E, Fiedler W, Wikelski M, Flack A. 2024. Learning shapes the development of migratory behavior. Proc Natl Acad Sci USA. 121(12):e2306389121. https://doi.org/10.1073/pnas.2306389121

Kays R, Davidson SC, Berger M, Bohrer G, Fiedler W, Flack A, Hirt J, Hahn C, Gauggel D, Russell B, et al. 2022. The Movebank system for studying global animal movement and demography. Methods Ecol Evol. 13(2):419-431. https://doi.org/10.1111/2041-210X.13767

Cheng Y, Fiedler W, Wikelski M, Flack A. 2019. ""Closer-to-home"" strategy benefits juvenile survival in a long-distance migratory bird. Ecology and Evolution. 9(16): 8945–8952. https://doi.org/10.1002/ece3.5395

Flack A, Fiedler W, Blas J, Pokrovski I, Kaatz M, Mitropolsky M, Aghababyan K, Fakriadis Y, Makrigianni E, Jerzak L, et al. 2016. Costs of migratory decisions: a comparison across eight white stork populations. Science Advances. 2(1): e1500931. https://doi.org/10.1126/sciadv.1500931

Weinzierl R, Bohrer G, Kranstauber B, Fiedler W, Wikelski M, Flack A. 2016. Wind estimation based on thermal soaring of birds. Ecology and Evolution. 6(24): 8706–8718. https://doi.org/10.1002/ece3.2585

Parts of this dataset are also available in studies ""Fall migration of white storks in 2014"" (published as DOI 10.5441/001/1.bj96m274) and ""MPIO white stork lifetime tracking data (2013-2014)"" (published as DOI 10.5441/001/1.78152p3q).",TRUE,FALSE,FALSE,"CC_BY","LifeTrack White Stork SW Germany",92,94,101,"martin (Martin Wikelski)","Determining migration routes of white storks in Eurasia. Lifetime tracking of storks. Involvement of the public","research",TRUE,TRUE,FALSE,TRUE,FALSE,"na",2013-06-24 06:39:34.998,2026-09-24 10:30:29,13275703,"Ciconia ciconia","myotis (Wolfgang Fiedler)","POINT (9.019775 48.01198)",8.8765587,47.8115663 +653,"Peaches.....DER.AN917..eobs.3916.",4042,4016,1.54,461516296,0.17,2014-08-06 09:15:00,"A",5,"3",277,10.71,39.32,803.5,"2014-08-06 09:19:38.000",372705835,TRUE,8854045745,23131236,23461880,"nestling",NA,"harness","867.195; Pinger on 19:00-0:00",2014-09-03,"Wolfgang Fiedler",2014-06-16 09:00:00,"dead","GPS and ACC switched off 20:00 to 02:00 GPS time","3916 on AN917","none","filters run in Movebank in Jan 2024: value range filter (flag records with coordinates 0,0); duplicate filter (key attributes tag-id and timestamp; preference order for eobs:status A, B, C, D, blank); additional visible outliers flagged manually","Radolfzell-Boehringen, SW Germany","multiple","acceleration,gps","POINT EMPTY","POINT EMPTY","POINT EMPTY",47.751444,8.931513,"Birke evang. Kirche, 4 nestlings, this is the second youngest, smallest chick died.","Died on 2014-08-31 near Motril, Spain. Reason unknown. lat: 36.726; long: -3.567. bird was moving around locally for a long time before tag stopped.","Boehringen Birke evang. Kirche",2014-04-15,2014-08-31,"unknown","DER A.....9456,","DER AN917","f","AN916,AN918","Ciconia ciconia",1,"POINT (-3.567 36.726)",867.195,"3916","e-obs GmbH","GPRS Solar Logger (Superburst)","3916","2014",55,2,21231406,"Max Planck Institute of Animal Behavior","Fiedler W, Flack A, Schäfle W, Keeves B, Quetting M, Eid B, Schmid H, Wikelski M. 2024. Data from: Study ""LifeTrack White Stork SW Germany"" (2013-2023). Movebank Data Repository. https://doi.org/10.5441/001/1.ck04mn78_2

These data are described in
Aikens EO, Nourani E, Fiedler W, Wikelski M, Flack A. 2024. Learning shapes the development of migratory behavior. Proc Natl Acad Sci USA. 121(12):e2306389121. https://doi.org/10.1073/pnas.2306389121

Kays R, Davidson SC, Berger M, Bohrer G, Fiedler W, Flack A, Hirt J, Hahn C, Gauggel D, Russell B, et al. 2022. The Movebank system for studying global animal movement and demography. Methods Ecol Evol. 13(2):419-431. https://doi.org/10.1111/2041-210X.13767

Cheng Y, Fiedler W, Wikelski M, Flack A. 2019. ""Closer-to-home"" strategy benefits juvenile survival in a long-distance migratory bird. Ecology and Evolution. 9(16): 8945–8952. https://doi.org/10.1002/ece3.5395

Flack A, Fiedler W, Blas J, Pokrovski I, Kaatz M, Mitropolsky M, Aghababyan K, Fakriadis Y, Makrigianni E, Jerzak L, et al. 2016. Costs of migratory decisions: a comparison across eight white stork populations. Science Advances. 2(1): e1500931. https://doi.org/10.1126/sciadv.1500931

Weinzierl R, Bohrer G, Kranstauber B, Fiedler W, Wikelski M, Flack A. 2016. Wind estimation based on thermal soaring of birds. Ecology and Evolution. 6(24): 8706–8718. https://doi.org/10.1002/ece3.2585

Parts of this dataset are also available in studies ""Fall migration of white storks in 2014"" (published as DOI 10.5441/001/1.bj96m274) and ""MPIO white stork lifetime tracking data (2013-2014)"" (published as DOI 10.5441/001/1.78152p3q).",TRUE,FALSE,FALSE,"CC_BY","LifeTrack White Stork SW Germany",92,94,101,"martin (Martin Wikelski)","Determining migration routes of white storks in Eurasia. Lifetime tracking of storks. Involvement of the public","research",TRUE,TRUE,FALSE,TRUE,FALSE,"na",2013-06-24 06:39:34.998,2026-09-24 10:30:29,13275703,"Ciconia ciconia","myotis (Wolfgang Fiedler)","POINT (9.019775 48.01198)",8.8766559,47.811638 diff --git a/test/contract/subsecond/meta.csv b/test/contract/subsecond/meta.csv new file mode 100644 index 0000000..5dc61b0 --- /dev/null +++ b/test/contract/subsecond/meta.csv @@ -0,0 +1,2 @@ +"crs","tzone","timeColName","trackIdColName" +"EPSG:4326","UTC","timestamp","individual_local_identifier" diff --git a/test/contract/subsecond/py/link.csv b/test/contract/subsecond/py/link.csv new file mode 100644 index 0000000..8feebef --- /dev/null +++ b/test/contract/subsecond/py/link.csv @@ -0,0 +1,5 @@ +timestamp_utc,sensor_type_id,individual_local_identifier,eobs_battery_voltage,eobs_fix_battery_voltage,eobs_horizontal_accuracy_estimate,eobs_key_bin_checksum,eobs_speed_accuracy_estimate,eobs_start_timestamp,eobs_status,eobs_temperature,eobs_type_of_fix,eobs_used_time_to_get_fix,ground_speed,heading,height_above_ellipsoid,timestamp,event_id,visible,deployment_id,tag_id,individual_id,animal_life_stage,animal_mass,attachment_type,deployment_comments,deploy_off_timestamp,deploy_on_person,deploy_on_timestamp,deployment_end_type,duty_cycle,deployment_local_identifier,manipulation_type,outlier_comments,study_site,tag_readout_method,sensor_type_ids,capture_location,deploy_on_location,deploy_off_location,birth_hatch_latitude,birth_hatch_longitude,individual_comments,death_comments,group_id,latest_date_born,mortality_date,mortality_type,parents,ring_id,sex,siblings,taxon_canonical_name,individual_number_of_deployments,mortality_location,beacon_frequency,tag_local_identifier,manufacturer_name,model,serial_no,tag_production_date,weight,tag_number_of_deployments,study_id,acknowledgements,citation,has_quota,i_am_owner,is_test,license_type,name,study_number_of_deployments,number_of_individuals,number_of_tags,principal_investigator_name,study_objective,study_type,suspend_license_terms,i_can_see_data,there_are_data_which_i_cannot_see,i_have_download_access,i_am_collaborator,study_permission,timestamp_first_deployed_location,timestamp_last_deployed_location,number_of_deployed_locations,taxon_ids,contact_person_name,main_location,timestamp_tz,geometry,coords_x,coords_y +2014-08-06 09:19:35.000,653,Peaches.....DER.AN917..eobs.3916.,4042,4016,1.54,3862950659,0.14,2014-08-06 09:15:00,A,5,3,274,12.59,49.81,802.8,2014-08-06 09:19:35.000,372705832,True,8854045745,23131236,23461880,nestling,,harness,867.195; Pinger on 19:00-0:00,2014-09-03,Wolfgang Fiedler,2014-06-16 09:00:00,dead,GPS and ACC switched off 20:00 to 02:00 GPS time,3916 on AN917,none,"filters run in Movebank in Jan 2024: value range filter (flag records with coordinates 0,0); duplicate filter (key attributes tag-id and timestamp; preference order for eobs:status A, B, C, D, blank); additional visible outliers flagged manually","Radolfzell-Boehringen, SW Germany",multiple,"acceleration,gps",POINT EMPTY,POINT EMPTY,POINT EMPTY,47.751444,8.931513,"Birke evang. Kirche, 4 nestlings, this is the second youngest, smallest chick died.","Died on 2014-08-31 near Motril, Spain. Reason unknown. lat: 36.726; long: -3.567. bird was moving around locally for a long time before tag stopped.",Boehringen Birke evang. Kirche,2014-04-15,2014-08-31,unknown,"DER A.....9456,",DER AN917,f,"AN916,AN918",Ciconia ciconia,1,POINT (-3.567 36.726),867.195,3916,e-obs GmbH,GPRS Solar Logger (Superburst),3916,2014,55,2,21231406,Max Planck Institute of Animal Behavior,"Fiedler W, Flack A, Schäfle W, Keeves B, Quetting M, Eid B, Schmid H, Wikelski M. 2024. Data from: Study ""LifeTrack White Stork SW Germany"" (2013-2023). Movebank Data Repository. https://doi.org/10.5441/001/1.ck04mn78_2

These data are described in
Aikens EO, Nourani E, Fiedler W, Wikelski M, Flack A. 2024. Learning shapes the development of migratory behavior. Proc Natl Acad Sci USA. 121(12):e2306389121. https://doi.org/10.1073/pnas.2306389121

Kays R, Davidson SC, Berger M, Bohrer G, Fiedler W, Flack A, Hirt J, Hahn C, Gauggel D, Russell B, et al. 2022. The Movebank system for studying global animal movement and demography. Methods Ecol Evol. 13(2):419-431. https://doi.org/10.1111/2041-210X.13767

Cheng Y, Fiedler W, Wikelski M, Flack A. 2019. ""Closer-to-home"" strategy benefits juvenile survival in a long-distance migratory bird. Ecology and Evolution. 9(16): 8945–8952. https://doi.org/10.1002/ece3.5395

Flack A, Fiedler W, Blas J, Pokrovski I, Kaatz M, Mitropolsky M, Aghababyan K, Fakriadis Y, Makrigianni E, Jerzak L, et al. 2016. Costs of migratory decisions: a comparison across eight white stork populations. Science Advances. 2(1): e1500931. https://doi.org/10.1126/sciadv.1500931

Weinzierl R, Bohrer G, Kranstauber B, Fiedler W, Wikelski M, Flack A. 2016. Wind estimation based on thermal soaring of birds. Ecology and Evolution. 6(24): 8706–8718. https://doi.org/10.1002/ece3.2585

Parts of this dataset are also available in studies ""Fall migration of white storks in 2014"" (published as DOI 10.5441/001/1.bj96m274) and ""MPIO white stork lifetime tracking data (2013-2014)"" (published as DOI 10.5441/001/1.78152p3q).",True,False,False,CC_BY,LifeTrack White Stork SW Germany,92,94,101,martin (Martin Wikelski),Determining migration routes of white storks in Eurasia. Lifetime tracking of storks. Involvement of the public,research,True,True,False,True,False,na,2013-06-24 06:39:34.998,2026-09-24 10:30:29,13275703,Ciconia ciconia,myotis (Wolfgang Fiedler),POINT (9.019775 48.01198),2014-08-06 09:19:35+00:00,POINT (8.8763209 47.8114328),8.8763209,47.8114328 +2014-08-06 09:19:35.999,653,Peaches.....DER.AN917..eobs.3916.,4042,4016,1.54,2480015882,0.14,2014-08-06 09:15:00,A,5,3,275,11.74,51.12,803.5,2014-08-06 09:19:35.999,372705833,True,8854045745,23131236,23461880,nestling,,harness,867.195; Pinger on 19:00-0:00,2014-09-03,Wolfgang Fiedler,2014-06-16 09:00:00,dead,GPS and ACC switched off 20:00 to 02:00 GPS time,3916 on AN917,none,"filters run in Movebank in Jan 2024: value range filter (flag records with coordinates 0,0); duplicate filter (key attributes tag-id and timestamp; preference order for eobs:status A, B, C, D, blank); additional visible outliers flagged manually","Radolfzell-Boehringen, SW Germany",multiple,"acceleration,gps",POINT EMPTY,POINT EMPTY,POINT EMPTY,47.751444,8.931513,"Birke evang. Kirche, 4 nestlings, this is the second youngest, smallest chick died.","Died on 2014-08-31 near Motril, Spain. Reason unknown. lat: 36.726; long: -3.567. bird was moving around locally for a long time before tag stopped.",Boehringen Birke evang. Kirche,2014-04-15,2014-08-31,unknown,"DER A.....9456,",DER AN917,f,"AN916,AN918",Ciconia ciconia,1,POINT (-3.567 36.726),867.195,3916,e-obs GmbH,GPRS Solar Logger (Superburst),3916,2014,55,2,21231406,Max Planck Institute of Animal Behavior,"Fiedler W, Flack A, Schäfle W, Keeves B, Quetting M, Eid B, Schmid H, Wikelski M. 2024. Data from: Study ""LifeTrack White Stork SW Germany"" (2013-2023). Movebank Data Repository. https://doi.org/10.5441/001/1.ck04mn78_2

These data are described in
Aikens EO, Nourani E, Fiedler W, Wikelski M, Flack A. 2024. Learning shapes the development of migratory behavior. Proc Natl Acad Sci USA. 121(12):e2306389121. https://doi.org/10.1073/pnas.2306389121

Kays R, Davidson SC, Berger M, Bohrer G, Fiedler W, Flack A, Hirt J, Hahn C, Gauggel D, Russell B, et al. 2022. The Movebank system for studying global animal movement and demography. Methods Ecol Evol. 13(2):419-431. https://doi.org/10.1111/2041-210X.13767

Cheng Y, Fiedler W, Wikelski M, Flack A. 2019. ""Closer-to-home"" strategy benefits juvenile survival in a long-distance migratory bird. Ecology and Evolution. 9(16): 8945–8952. https://doi.org/10.1002/ece3.5395

Flack A, Fiedler W, Blas J, Pokrovski I, Kaatz M, Mitropolsky M, Aghababyan K, Fakriadis Y, Makrigianni E, Jerzak L, et al. 2016. Costs of migratory decisions: a comparison across eight white stork populations. Science Advances. 2(1): e1500931. https://doi.org/10.1126/sciadv.1500931

Weinzierl R, Bohrer G, Kranstauber B, Fiedler W, Wikelski M, Flack A. 2016. Wind estimation based on thermal soaring of birds. Ecology and Evolution. 6(24): 8706–8718. https://doi.org/10.1002/ece3.2585

Parts of this dataset are also available in studies ""Fall migration of white storks in 2014"" (published as DOI 10.5441/001/1.bj96m274) and ""MPIO white stork lifetime tracking data (2013-2014)"" (published as DOI 10.5441/001/1.78152p3q).",True,False,False,CC_BY,LifeTrack White Stork SW Germany,92,94,101,martin (Martin Wikelski),Determining migration routes of white storks in Eurasia. Lifetime tracking of storks. Involvement of the public,research,True,True,False,True,False,na,2013-06-24 06:39:34.998,2026-09-24 10:30:29,13275703,Ciconia ciconia,myotis (Wolfgang Fiedler),POINT (9.019775 48.01198),2014-08-06 09:19:35.999000+00:00,POINT (8.876445 47.8115006),8.876445,47.8115006 +2014-08-06 09:19:36.999,653,Peaches.....DER.AN917..eobs.3916.,4042,4013,1.54,3479504210,0.15,2014-08-06 09:15:00,A,5,3,276,10.97,48.5,803.9,2014-08-06 09:19:36.999,372705834,True,8854045745,23131236,23461880,nestling,,harness,867.195; Pinger on 19:00-0:00,2014-09-03,Wolfgang Fiedler,2014-06-16 09:00:00,dead,GPS and ACC switched off 20:00 to 02:00 GPS time,3916 on AN917,none,"filters run in Movebank in Jan 2024: value range filter (flag records with coordinates 0,0); duplicate filter (key attributes tag-id and timestamp; preference order for eobs:status A, B, C, D, blank); additional visible outliers flagged manually","Radolfzell-Boehringen, SW Germany",multiple,"acceleration,gps",POINT EMPTY,POINT EMPTY,POINT EMPTY,47.751444,8.931513,"Birke evang. Kirche, 4 nestlings, this is the second youngest, smallest chick died.","Died on 2014-08-31 near Motril, Spain. Reason unknown. lat: 36.726; long: -3.567. bird was moving around locally for a long time before tag stopped.",Boehringen Birke evang. Kirche,2014-04-15,2014-08-31,unknown,"DER A.....9456,",DER AN917,f,"AN916,AN918",Ciconia ciconia,1,POINT (-3.567 36.726),867.195,3916,e-obs GmbH,GPRS Solar Logger (Superburst),3916,2014,55,2,21231406,Max Planck Institute of Animal Behavior,"Fiedler W, Flack A, Schäfle W, Keeves B, Quetting M, Eid B, Schmid H, Wikelski M. 2024. Data from: Study ""LifeTrack White Stork SW Germany"" (2013-2023). Movebank Data Repository. https://doi.org/10.5441/001/1.ck04mn78_2

These data are described in
Aikens EO, Nourani E, Fiedler W, Wikelski M, Flack A. 2024. Learning shapes the development of migratory behavior. Proc Natl Acad Sci USA. 121(12):e2306389121. https://doi.org/10.1073/pnas.2306389121

Kays R, Davidson SC, Berger M, Bohrer G, Fiedler W, Flack A, Hirt J, Hahn C, Gauggel D, Russell B, et al. 2022. The Movebank system for studying global animal movement and demography. Methods Ecol Evol. 13(2):419-431. https://doi.org/10.1111/2041-210X.13767

Cheng Y, Fiedler W, Wikelski M, Flack A. 2019. ""Closer-to-home"" strategy benefits juvenile survival in a long-distance migratory bird. Ecology and Evolution. 9(16): 8945–8952. https://doi.org/10.1002/ece3.5395

Flack A, Fiedler W, Blas J, Pokrovski I, Kaatz M, Mitropolsky M, Aghababyan K, Fakriadis Y, Makrigianni E, Jerzak L, et al. 2016. Costs of migratory decisions: a comparison across eight white stork populations. Science Advances. 2(1): e1500931. https://doi.org/10.1126/sciadv.1500931

Weinzierl R, Bohrer G, Kranstauber B, Fiedler W, Wikelski M, Flack A. 2016. Wind estimation based on thermal soaring of birds. Ecology and Evolution. 6(24): 8706–8718. https://doi.org/10.1002/ece3.2585

Parts of this dataset are also available in studies ""Fall migration of white storks in 2014"" (published as DOI 10.5441/001/1.bj96m274) and ""MPIO white stork lifetime tracking data (2013-2014)"" (published as DOI 10.5441/001/1.78152p3q).",True,False,False,CC_BY,LifeTrack White Stork SW Germany,92,94,101,martin (Martin Wikelski),Determining migration routes of white storks in Eurasia. Lifetime tracking of storks. Involvement of the public,research,True,True,False,True,False,na,2013-06-24 06:39:34.998,2026-09-24 10:30:29,13275703,Ciconia ciconia,myotis (Wolfgang Fiedler),POINT (9.019775 48.01198),2014-08-06 09:19:36.999000+00:00,POINT (8.8765587 47.8115663),8.8765587,47.8115663 +2014-08-06 09:19:38.000,653,Peaches.....DER.AN917..eobs.3916.,4042,4016,1.54,461516296,0.17,2014-08-06 09:15:00,A,5,3,277,10.71,39.32,803.5,2014-08-06 09:19:38.000,372705835,True,8854045745,23131236,23461880,nestling,,harness,867.195; Pinger on 19:00-0:00,2014-09-03,Wolfgang Fiedler,2014-06-16 09:00:00,dead,GPS and ACC switched off 20:00 to 02:00 GPS time,3916 on AN917,none,"filters run in Movebank in Jan 2024: value range filter (flag records with coordinates 0,0); duplicate filter (key attributes tag-id and timestamp; preference order for eobs:status A, B, C, D, blank); additional visible outliers flagged manually","Radolfzell-Boehringen, SW Germany",multiple,"acceleration,gps",POINT EMPTY,POINT EMPTY,POINT EMPTY,47.751444,8.931513,"Birke evang. Kirche, 4 nestlings, this is the second youngest, smallest chick died.","Died on 2014-08-31 near Motril, Spain. Reason unknown. lat: 36.726; long: -3.567. bird was moving around locally for a long time before tag stopped.",Boehringen Birke evang. Kirche,2014-04-15,2014-08-31,unknown,"DER A.....9456,",DER AN917,f,"AN916,AN918",Ciconia ciconia,1,POINT (-3.567 36.726),867.195,3916,e-obs GmbH,GPRS Solar Logger (Superburst),3916,2014,55,2,21231406,Max Planck Institute of Animal Behavior,"Fiedler W, Flack A, Schäfle W, Keeves B, Quetting M, Eid B, Schmid H, Wikelski M. 2024. Data from: Study ""LifeTrack White Stork SW Germany"" (2013-2023). Movebank Data Repository. https://doi.org/10.5441/001/1.ck04mn78_2

These data are described in
Aikens EO, Nourani E, Fiedler W, Wikelski M, Flack A. 2024. Learning shapes the development of migratory behavior. Proc Natl Acad Sci USA. 121(12):e2306389121. https://doi.org/10.1073/pnas.2306389121

Kays R, Davidson SC, Berger M, Bohrer G, Fiedler W, Flack A, Hirt J, Hahn C, Gauggel D, Russell B, et al. 2022. The Movebank system for studying global animal movement and demography. Methods Ecol Evol. 13(2):419-431. https://doi.org/10.1111/2041-210X.13767

Cheng Y, Fiedler W, Wikelski M, Flack A. 2019. ""Closer-to-home"" strategy benefits juvenile survival in a long-distance migratory bird. Ecology and Evolution. 9(16): 8945–8952. https://doi.org/10.1002/ece3.5395

Flack A, Fiedler W, Blas J, Pokrovski I, Kaatz M, Mitropolsky M, Aghababyan K, Fakriadis Y, Makrigianni E, Jerzak L, et al. 2016. Costs of migratory decisions: a comparison across eight white stork populations. Science Advances. 2(1): e1500931. https://doi.org/10.1126/sciadv.1500931

Weinzierl R, Bohrer G, Kranstauber B, Fiedler W, Wikelski M, Flack A. 2016. Wind estimation based on thermal soaring of birds. Ecology and Evolution. 6(24): 8706–8718. https://doi.org/10.1002/ece3.2585

Parts of this dataset are also available in studies ""Fall migration of white storks in 2014"" (published as DOI 10.5441/001/1.bj96m274) and ""MPIO white stork lifetime tracking data (2013-2014)"" (published as DOI 10.5441/001/1.78152p3q).",True,False,False,CC_BY,LifeTrack White Stork SW Germany,92,94,101,martin (Martin Wikelski),Determining migration routes of white storks in Eurasia. Lifetime tracking of storks. Involvement of the public,research,True,True,False,True,False,na,2013-06-24 06:39:34.998,2026-09-24 10:30:29,13275703,Ciconia ciconia,myotis (Wolfgang Fiedler),POINT (9.019775 48.01198),2014-08-06 09:19:38+00:00,POINT (8.8766559 47.811638),8.8766559,47.811638 diff --git a/test/contract/subsecond/py/meta.csv b/test/contract/subsecond/py/meta.csv new file mode 100644 index 0000000..3587343 --- /dev/null +++ b/test/contract/subsecond/py/meta.csv @@ -0,0 +1,2 @@ +crs,tzone,timeColName,trackIdColName +EPSG:4326,UTC,timestamp_utc,individual_local_identifier diff --git a/test/docker-bake.hcl b/test/docker-bake.hcl new file mode 100644 index 0000000..ea9c77d --- /dev/null +++ b/test/docker-bake.hcl @@ -0,0 +1,99 @@ +# Builds and tests both translator Apps from the real Dockerfile, on public images only: +# +# co-pilot-r (stand-in) --BASE--+--> app-r2python --+ +# miniforge -----------CONDA----+--> app-python2r --+--> test: R unit tests, round trip, coherence +# +# docker buildx bake -f test/docker-bake.hcl test + +# "gha" inside GitHub Actions; empty elsewhere, where no Actions cache exists +variable "CACHE" { + default = "" +} + +# "true" for the runs that must resolve everything from scratch (see .github/workflows/test.yml) +variable "FRESH" { + default = "false" +} + +# the python co-pilot's own base; test/coherence.R holds test/Dockerfile to the same tag. +# Not named CONDA: bake overrides a variable from the environment variable of the same name, and +# GitHub's runners export CONDA=/usr/share/miniconda. +variable "CONDA_IMAGE" { + default = "condaforge/miniforge3:26.7.2-0" +} + +function "cacheFrom" { + params = [scope] + result = CACHE == "gha" && FRESH != "true" ? ["type=gha,scope=${scope}"] : [] +} + +function "cacheTo" { + params = [scope] + result = CACHE == "gha" ? ["type=gha,mode=max,scope=${scope},ignore-error=true"] : [] +} + +target "co-pilot-r" { + context = "." + dockerfile = "test/co-pilot-r.Dockerfile" + platforms = ["linux/amd64"] + no-cache = FRESH == "true" + cache-from = cacheFrom("co-pilot-r") + cache-to = cacheTo("co-pilot-r") +} + +target "app" { + name = "app-${direction}" + matrix = { + direction = ["r2python", "python2r"] + } + context = "." + dockerfile = "Dockerfile" + platforms = ["linux/amd64"] + tags = ["link-r-python:${direction}"] + contexts = { + co-pilot-r = "target:co-pilot-r" + } + args = { + BASE = "co-pilot-r" + CONDA = CONDA_IMAGE + DIRECTION = direction + } + no-cache = FRESH == "true" + cache-from = cacheFrom("app-${direction}") + cache-to = cacheTo("app-${direction}") +} + +target "test" { + context = "." + dockerfile = "test/app-test.Dockerfile" + platforms = ["linux/amd64"] + contexts = { + app-r2python = "target:app-r2python" + app-python2r = "target:app-python2r" + } + no-cache = FRESH == "true" + cache-from = cacheFrom("test") + cache-to = cacheTo("test") +} + +# Both must fail. The platform passes no build arguments, so a Dockerfile copy without its DIRECTION +# edit has to be refused rather than built. +target "app-without-direction" { + context = "." + dockerfile = "Dockerfile" + platforms = ["linux/amd64"] + contexts = { + co-pilot-r = "target:co-pilot-r" + } + args = { + BASE = "co-pilot-r" + CONDA = CONDA_IMAGE + } +} + +target "app-with-invalid-direction" { + inherits = ["app-without-direction"] + args = { + DIRECTION = "sideways" + } +} diff --git a/test/images.sh b/test/images.sh new file mode 100644 index 0000000..6c57c56 --- /dev/null +++ b/test/images.sh @@ -0,0 +1,30 @@ +# Runs a command inside an App image and copies its results out. No bind mounts: Docker Desktop on +# macOS cannot mount the per-user temp directories, and copying leaves the image's own files alone. +# +# runInImage +# -> /tmp/in read only: owned by root inside the container +# runs as the image's user in /home/moveapps/co-pilot-r and writes to /tmp/out +# /tmp/out -> +# +# OVERLAY=1 first replaces the image's r/link.R, r/rds_2_csv.R and r/csv_2_rds.R with this +# checkout's - how a change to the R side is proved against an older image. +runInImage() { + local image="$1" input="$2" command="$3" output="$4" + local container status + + container="$(docker create --platform linux/amd64 --entrypoint bash -w /home/moveapps/co-pilot-r "$image" -c "$command")" + docker cp "$input" "$container:/tmp/in" > /dev/null + + if [ "${OVERLAY:-0}" = 1 ]; then + for file in r/link.R r/rds_2_csv.R r/csv_2_rds.R; do + docker cp "$file" "$container:/home/moveapps/co-pilot-r/$file" > /dev/null + done + fi + + docker start --attach "$container" && status=0 || status=$? + + rm -rf "$output" + docker cp "$container:/tmp/out" "$output" > /dev/null || true + docker rm "$container" > /dev/null + return "$status" +} diff --git a/test/roundtrip/classes.csv b/test/roundtrip/classes.csv new file mode 100644 index 0000000..636c4ce --- /dev/null +++ b/test/roundtrip/classes.csv @@ -0,0 +1,725 @@ +"case","column","class_in","class_out" +"dst_ambiguous","sensor_type_id","integer64","integer" +"dst_ambiguous","individual_local_identifier","character","character" +"dst_ambiguous","comments","character","character" +"dst_ambiguous","sequence_number","units","integer" +"dst_ambiguous","tag_voltage","units","numeric" +"dst_ambiguous","timestamp","POSIXct/POSIXt","character" +"dst_ambiguous","transmission_protocol","factor","logical" +"dst_ambiguous","transmission_timestamp","POSIXct/POSIXt","character" +"dst_ambiguous","event_id","integer64","numeric" +"dst_ambiguous","visible","logical","logical" +"dst_ambiguous","deployment_id","integer64","integer" +"dst_ambiguous","tag_id","integer64","integer" +"dst_ambiguous","individual_id","integer64","integer" +"dst_ambiguous","animal_life_stage","factor","character" +"dst_ambiguous","animal_reproductive_condition","factor","character" +"dst_ambiguous","attachment_type","factor","character" +"dst_ambiguous","deploy_off_timestamp","POSIXct/POSIXt","character" +"dst_ambiguous","deploy_on_person","character","character" +"dst_ambiguous","deploy_on_timestamp","POSIXct/POSIXt","character" +"dst_ambiguous","sensor_type_ids","character","character" +"dst_ambiguous","capture_location","sfc_POINT/sfc","character" +"dst_ambiguous","deploy_on_location","sfc_POINT/sfc","character" +"dst_ambiguous","deploy_off_location","sfc_POINT/sfc","character" +"dst_ambiguous","nick_name","factor","character" +"dst_ambiguous","ring_id","factor","character" +"dst_ambiguous","sex","factor","character" +"dst_ambiguous","taxon_canonical_name","factor","character" +"dst_ambiguous","individual_number_of_deployments","integer","integer" +"dst_ambiguous","mortality_location","sfc_POINT/sfc","character" +"dst_ambiguous","tag_local_identifier","factor","character" +"dst_ambiguous","tag_number_of_deployments","integer","integer" +"dst_ambiguous","study_id","integer64","integer" +"dst_ambiguous","acknowledgements","character","character" +"dst_ambiguous","citation","character","character" +"dst_ambiguous","grants_used","character","character" +"dst_ambiguous","has_quota","logical","logical" +"dst_ambiguous","i_am_owner","logical","logical" +"dst_ambiguous","is_test","logical","logical" +"dst_ambiguous","license_terms","character","character" +"dst_ambiguous","license_type","factor","character" +"dst_ambiguous","name","factor","character" +"dst_ambiguous","study_number_of_deployments","integer","integer" +"dst_ambiguous","number_of_individuals","units","integer" +"dst_ambiguous","number_of_tags","units","integer" +"dst_ambiguous","principal_investigator_name","character","character" +"dst_ambiguous","study_objective","character","character" +"dst_ambiguous","study_type","factor","character" +"dst_ambiguous","suspend_license_terms","logical","logical" +"dst_ambiguous","i_can_see_data","logical","logical" +"dst_ambiguous","there_are_data_which_i_cannot_see","logical","logical" +"dst_ambiguous","i_have_download_access","logical","logical" +"dst_ambiguous","i_am_collaborator","logical","logical" +"dst_ambiguous","study_permission","factor","character" +"dst_ambiguous","timestamp_first_deployed_location","POSIXct/POSIXt","character" +"dst_ambiguous","timestamp_last_deployed_location","POSIXct/POSIXt","character" +"dst_ambiguous","number_of_deployed_locations","units","integer" +"dst_ambiguous","taxon_ids","character","character" +"dst_ambiguous","contact_person_name","factor","character" +"dst_ambiguous","main_location","sfc_POINT/sfc","character" +"dst_ambiguous","timestamp_utc",,"POSIXct/POSIXt" +"dst_ambiguous","timestamp_tz",,"character" +"input1_move2loc_LatLon","sensor_type_id","integer64","integer" +"input1_move2loc_LatLon","individual_local_identifier","character","character" +"input1_move2loc_LatLon","comments","character","character" +"input1_move2loc_LatLon","sequence_number","units","integer" +"input1_move2loc_LatLon","tag_voltage","units","numeric" +"input1_move2loc_LatLon","timestamp","POSIXct/POSIXt","character" +"input1_move2loc_LatLon","transmission_protocol","factor","character" +"input1_move2loc_LatLon","transmission_timestamp","POSIXct/POSIXt","character" +"input1_move2loc_LatLon","event_id","integer64","numeric" +"input1_move2loc_LatLon","visible","logical","logical" +"input1_move2loc_LatLon","deployment_id","integer64","integer" +"input1_move2loc_LatLon","tag_id","integer64","integer" +"input1_move2loc_LatLon","individual_id","integer64","integer" +"input1_move2loc_LatLon","animal_life_stage","factor","character" +"input1_move2loc_LatLon","animal_reproductive_condition","factor","character" +"input1_move2loc_LatLon","attachment_type","factor","character" +"input1_move2loc_LatLon","deploy_off_timestamp","POSIXct/POSIXt","character" +"input1_move2loc_LatLon","deploy_on_person","character","character" +"input1_move2loc_LatLon","deploy_on_timestamp","POSIXct/POSIXt","character" +"input1_move2loc_LatLon","sensor_type_ids","character","character" +"input1_move2loc_LatLon","capture_location","sfc_POINT/sfc","character" +"input1_move2loc_LatLon","deploy_on_location","sfc_POINT/sfc","character" +"input1_move2loc_LatLon","deploy_off_location","sfc_POINT/sfc","character" +"input1_move2loc_LatLon","nick_name","factor","character" +"input1_move2loc_LatLon","ring_id","factor","character" +"input1_move2loc_LatLon","sex","factor","character" +"input1_move2loc_LatLon","taxon_canonical_name","factor","character" +"input1_move2loc_LatLon","individual_number_of_deployments","integer","integer" +"input1_move2loc_LatLon","mortality_location","sfc_POINT/sfc","character" +"input1_move2loc_LatLon","tag_local_identifier","factor","character" +"input1_move2loc_LatLon","tag_number_of_deployments","integer","integer" +"input1_move2loc_LatLon","study_id","integer64","integer" +"input1_move2loc_LatLon","acknowledgements","character","character" +"input1_move2loc_LatLon","citation","character","character" +"input1_move2loc_LatLon","grants_used","character","character" +"input1_move2loc_LatLon","has_quota","logical","logical" +"input1_move2loc_LatLon","i_am_owner","logical","logical" +"input1_move2loc_LatLon","is_test","logical","logical" +"input1_move2loc_LatLon","license_terms","character","character" +"input1_move2loc_LatLon","license_type","factor","character" +"input1_move2loc_LatLon","name","factor","character" +"input1_move2loc_LatLon","study_number_of_deployments","integer","integer" +"input1_move2loc_LatLon","number_of_individuals","units","integer" +"input1_move2loc_LatLon","number_of_tags","units","integer" +"input1_move2loc_LatLon","principal_investigator_name","character","character" +"input1_move2loc_LatLon","study_objective","character","character" +"input1_move2loc_LatLon","study_type","factor","character" +"input1_move2loc_LatLon","suspend_license_terms","logical","logical" +"input1_move2loc_LatLon","i_can_see_data","logical","logical" +"input1_move2loc_LatLon","there_are_data_which_i_cannot_see","logical","logical" +"input1_move2loc_LatLon","i_have_download_access","logical","logical" +"input1_move2loc_LatLon","i_am_collaborator","logical","logical" +"input1_move2loc_LatLon","study_permission","factor","character" +"input1_move2loc_LatLon","timestamp_first_deployed_location","POSIXct/POSIXt","character" +"input1_move2loc_LatLon","timestamp_last_deployed_location","POSIXct/POSIXt","character" +"input1_move2loc_LatLon","number_of_deployed_locations","units","integer" +"input1_move2loc_LatLon","taxon_ids","character","character" +"input1_move2loc_LatLon","contact_person_name","factor","character" +"input1_move2loc_LatLon","main_location","sfc_POINT/sfc","character" +"input1_move2loc_LatLon","timestamp_utc",,"POSIXct/POSIXt" +"input1_move2loc_LatLon","timestamp_tz",,"character" +"input1_move2loc_LatLon_berlin","sensor_type_id","integer64","integer" +"input1_move2loc_LatLon_berlin","individual_local_identifier","character","character" +"input1_move2loc_LatLon_berlin","comments","character","character" +"input1_move2loc_LatLon_berlin","sequence_number","units","integer" +"input1_move2loc_LatLon_berlin","tag_voltage","units","numeric" +"input1_move2loc_LatLon_berlin","timestamp","POSIXct/POSIXt","character" +"input1_move2loc_LatLon_berlin","transmission_protocol","factor","character" +"input1_move2loc_LatLon_berlin","transmission_timestamp","POSIXct/POSIXt","character" +"input1_move2loc_LatLon_berlin","event_id","integer64","numeric" +"input1_move2loc_LatLon_berlin","visible","logical","logical" +"input1_move2loc_LatLon_berlin","deployment_id","integer64","integer" +"input1_move2loc_LatLon_berlin","tag_id","integer64","integer" +"input1_move2loc_LatLon_berlin","individual_id","integer64","integer" +"input1_move2loc_LatLon_berlin","animal_life_stage","factor","character" +"input1_move2loc_LatLon_berlin","animal_reproductive_condition","factor","character" +"input1_move2loc_LatLon_berlin","attachment_type","factor","character" +"input1_move2loc_LatLon_berlin","deploy_off_timestamp","POSIXct/POSIXt","character" +"input1_move2loc_LatLon_berlin","deploy_on_person","character","character" +"input1_move2loc_LatLon_berlin","deploy_on_timestamp","POSIXct/POSIXt","character" +"input1_move2loc_LatLon_berlin","sensor_type_ids","character","character" +"input1_move2loc_LatLon_berlin","capture_location","sfc_POINT/sfc","character" +"input1_move2loc_LatLon_berlin","deploy_on_location","sfc_POINT/sfc","character" +"input1_move2loc_LatLon_berlin","deploy_off_location","sfc_POINT/sfc","character" +"input1_move2loc_LatLon_berlin","nick_name","factor","character" +"input1_move2loc_LatLon_berlin","ring_id","factor","character" +"input1_move2loc_LatLon_berlin","sex","factor","character" +"input1_move2loc_LatLon_berlin","taxon_canonical_name","factor","character" +"input1_move2loc_LatLon_berlin","individual_number_of_deployments","integer","integer" +"input1_move2loc_LatLon_berlin","mortality_location","sfc_POINT/sfc","character" +"input1_move2loc_LatLon_berlin","tag_local_identifier","factor","character" +"input1_move2loc_LatLon_berlin","tag_number_of_deployments","integer","integer" +"input1_move2loc_LatLon_berlin","study_id","integer64","integer" +"input1_move2loc_LatLon_berlin","acknowledgements","character","character" +"input1_move2loc_LatLon_berlin","citation","character","character" +"input1_move2loc_LatLon_berlin","grants_used","character","character" +"input1_move2loc_LatLon_berlin","has_quota","logical","logical" +"input1_move2loc_LatLon_berlin","i_am_owner","logical","logical" +"input1_move2loc_LatLon_berlin","is_test","logical","logical" +"input1_move2loc_LatLon_berlin","license_terms","character","character" +"input1_move2loc_LatLon_berlin","license_type","factor","character" +"input1_move2loc_LatLon_berlin","name","factor","character" +"input1_move2loc_LatLon_berlin","study_number_of_deployments","integer","integer" +"input1_move2loc_LatLon_berlin","number_of_individuals","units","integer" +"input1_move2loc_LatLon_berlin","number_of_tags","units","integer" +"input1_move2loc_LatLon_berlin","principal_investigator_name","character","character" +"input1_move2loc_LatLon_berlin","study_objective","character","character" +"input1_move2loc_LatLon_berlin","study_type","factor","character" +"input1_move2loc_LatLon_berlin","suspend_license_terms","logical","logical" +"input1_move2loc_LatLon_berlin","i_can_see_data","logical","logical" +"input1_move2loc_LatLon_berlin","there_are_data_which_i_cannot_see","logical","logical" +"input1_move2loc_LatLon_berlin","i_have_download_access","logical","logical" +"input1_move2loc_LatLon_berlin","i_am_collaborator","logical","logical" +"input1_move2loc_LatLon_berlin","study_permission","factor","character" +"input1_move2loc_LatLon_berlin","timestamp_first_deployed_location","POSIXct/POSIXt","character" +"input1_move2loc_LatLon_berlin","timestamp_last_deployed_location","POSIXct/POSIXt","character" +"input1_move2loc_LatLon_berlin","number_of_deployed_locations","units","integer" +"input1_move2loc_LatLon_berlin","taxon_ids","character","character" +"input1_move2loc_LatLon_berlin","contact_person_name","factor","character" +"input1_move2loc_LatLon_berlin","main_location","sfc_POINT/sfc","character" +"input1_move2loc_LatLon_berlin","timestamp_utc",,"POSIXct/POSIXt" +"input1_move2loc_LatLon_berlin","timestamp_tz",,"character" +"input1_move2loc_Mollweide","sensor_type_id","integer64","integer" +"input1_move2loc_Mollweide","individual_local_identifier","character","character" +"input1_move2loc_Mollweide","comments","character","character" +"input1_move2loc_Mollweide","sequence_number","units","integer" +"input1_move2loc_Mollweide","tag_voltage","units","numeric" +"input1_move2loc_Mollweide","timestamp","POSIXct/POSIXt","character" +"input1_move2loc_Mollweide","transmission_protocol","factor","character" +"input1_move2loc_Mollweide","transmission_timestamp","POSIXct/POSIXt","character" +"input1_move2loc_Mollweide","event_id","integer64","numeric" +"input1_move2loc_Mollweide","visible","logical","logical" +"input1_move2loc_Mollweide","deployment_id","integer64","integer" +"input1_move2loc_Mollweide","tag_id","integer64","integer" +"input1_move2loc_Mollweide","individual_id","integer64","integer" +"input1_move2loc_Mollweide","animal_life_stage","factor","character" +"input1_move2loc_Mollweide","animal_reproductive_condition","factor","character" +"input1_move2loc_Mollweide","attachment_type","factor","character" +"input1_move2loc_Mollweide","deploy_off_timestamp","POSIXct/POSIXt","character" +"input1_move2loc_Mollweide","deploy_on_person","character","character" +"input1_move2loc_Mollweide","deploy_on_timestamp","POSIXct/POSIXt","character" +"input1_move2loc_Mollweide","sensor_type_ids","character","character" +"input1_move2loc_Mollweide","capture_location","sfc_POINT/sfc","character" +"input1_move2loc_Mollweide","deploy_on_location","sfc_POINT/sfc","character" +"input1_move2loc_Mollweide","deploy_off_location","sfc_POINT/sfc","character" +"input1_move2loc_Mollweide","nick_name","factor","character" +"input1_move2loc_Mollweide","ring_id","factor","character" +"input1_move2loc_Mollweide","sex","factor","character" +"input1_move2loc_Mollweide","taxon_canonical_name","factor","character" +"input1_move2loc_Mollweide","individual_number_of_deployments","integer","integer" +"input1_move2loc_Mollweide","mortality_location","sfc_POINT/sfc","character" +"input1_move2loc_Mollweide","tag_local_identifier","factor","character" +"input1_move2loc_Mollweide","tag_number_of_deployments","integer","integer" +"input1_move2loc_Mollweide","study_id","integer64","integer" +"input1_move2loc_Mollweide","acknowledgements","character","character" +"input1_move2loc_Mollweide","citation","character","character" +"input1_move2loc_Mollweide","grants_used","character","character" +"input1_move2loc_Mollweide","has_quota","logical","logical" +"input1_move2loc_Mollweide","i_am_owner","logical","logical" +"input1_move2loc_Mollweide","is_test","logical","logical" +"input1_move2loc_Mollweide","license_terms","character","character" +"input1_move2loc_Mollweide","license_type","factor","character" +"input1_move2loc_Mollweide","name","factor","character" +"input1_move2loc_Mollweide","study_number_of_deployments","integer","integer" +"input1_move2loc_Mollweide","number_of_individuals","units","integer" +"input1_move2loc_Mollweide","number_of_tags","units","integer" +"input1_move2loc_Mollweide","principal_investigator_name","character","character" +"input1_move2loc_Mollweide","study_objective","character","character" +"input1_move2loc_Mollweide","study_type","factor","character" +"input1_move2loc_Mollweide","suspend_license_terms","logical","logical" +"input1_move2loc_Mollweide","i_can_see_data","logical","logical" +"input1_move2loc_Mollweide","there_are_data_which_i_cannot_see","logical","logical" +"input1_move2loc_Mollweide","i_have_download_access","logical","logical" +"input1_move2loc_Mollweide","i_am_collaborator","logical","logical" +"input1_move2loc_Mollweide","study_permission","factor","character" +"input1_move2loc_Mollweide","timestamp_first_deployed_location","POSIXct/POSIXt","character" +"input1_move2loc_Mollweide","timestamp_last_deployed_location","POSIXct/POSIXt","character" +"input1_move2loc_Mollweide","number_of_deployed_locations","units","integer" +"input1_move2loc_Mollweide","taxon_ids","character","character" +"input1_move2loc_Mollweide","contact_person_name","factor","character" +"input1_move2loc_Mollweide","main_location","sfc_POINT/sfc","character" +"input1_move2loc_Mollweide","timestamp_utc",,"POSIXct/POSIXt" +"input1_move2loc_Mollweide","timestamp_tz",,"character" +"input2_move2loc_LatLon","sensor_type_id","integer64","integer" +"input2_move2loc_LatLon","individual_local_identifier","character","character" +"input2_move2loc_LatLon","eobs_battery_voltage","units","integer" +"input2_move2loc_LatLon","eobs_fix_battery_voltage","units","integer" +"input2_move2loc_LatLon","eobs_horizontal_accuracy_estimate","units","numeric" +"input2_move2loc_LatLon","eobs_key_bin_checksum","integer64","numeric" +"input2_move2loc_LatLon","eobs_speed_accuracy_estimate","units","numeric" +"input2_move2loc_LatLon","eobs_start_timestamp","POSIXct/POSIXt","character" +"input2_move2loc_LatLon","eobs_status","ordered/factor","character" +"input2_move2loc_LatLon","eobs_temperature","units","integer" +"input2_move2loc_LatLon","eobs_type_of_fix","factor","integer" +"input2_move2loc_LatLon","eobs_used_time_to_get_fix","units","integer" +"input2_move2loc_LatLon","ground_speed","units","numeric" +"input2_move2loc_LatLon","heading","units","numeric" +"input2_move2loc_LatLon","height_above_ellipsoid","units","numeric" +"input2_move2loc_LatLon","timestamp","POSIXct/POSIXt","character" +"input2_move2loc_LatLon","event_id","integer64","integer" +"input2_move2loc_LatLon","visible","logical","logical" +"input2_move2loc_LatLon","deployment_id","integer64","numeric" +"input2_move2loc_LatLon","tag_id","integer64","integer" +"input2_move2loc_LatLon","individual_id","integer64","integer" +"input2_move2loc_LatLon","animal_life_stage","factor","character" +"input2_move2loc_LatLon","animal_mass","units","numeric" +"input2_move2loc_LatLon","attachment_type","factor","character" +"input2_move2loc_LatLon","deployment_comments","character","character" +"input2_move2loc_LatLon","deploy_off_timestamp","POSIXct/POSIXt","character" +"input2_move2loc_LatLon","deploy_on_person","character","character" +"input2_move2loc_LatLon","deploy_on_timestamp","POSIXct/POSIXt","character" +"input2_move2loc_LatLon","deployment_end_type","factor","character" +"input2_move2loc_LatLon","duty_cycle","character","character" +"input2_move2loc_LatLon","deployment_local_identifier","factor","character" +"input2_move2loc_LatLon","manipulation_type","factor","character" +"input2_move2loc_LatLon","outlier_comments","character","character" +"input2_move2loc_LatLon","study_site","character","character" +"input2_move2loc_LatLon","tag_readout_method","factor","character" +"input2_move2loc_LatLon","sensor_type_ids","character","character" +"input2_move2loc_LatLon","capture_location","sfc_POINT/sfc","character" +"input2_move2loc_LatLon","deploy_on_location","sfc_POINT/sfc","character" +"input2_move2loc_LatLon","deploy_off_location","sfc_POINT/sfc","character" +"input2_move2loc_LatLon","birth_hatch_latitude","numeric","numeric" +"input2_move2loc_LatLon","birth_hatch_longitude","numeric","numeric" +"input2_move2loc_LatLon","individual_comments","character","character" +"input2_move2loc_LatLon","death_comments","character","character" +"input2_move2loc_LatLon","group_id","factor","character" +"input2_move2loc_LatLon","latest_date_born","POSIXct/POSIXt","character" +"input2_move2loc_LatLon","mortality_date","Date","character" +"input2_move2loc_LatLon","mortality_type","factor","character" +"input2_move2loc_LatLon","parents","factor","character" +"input2_move2loc_LatLon","ring_id","factor","character" +"input2_move2loc_LatLon","sex","factor","character" +"input2_move2loc_LatLon","siblings","factor","character" +"input2_move2loc_LatLon","taxon_canonical_name","factor","character" +"input2_move2loc_LatLon","individual_number_of_deployments","integer","integer" +"input2_move2loc_LatLon","mortality_location","sfc_POINT/sfc","character" +"input2_move2loc_LatLon","beacon_frequency","units","numeric" +"input2_move2loc_LatLon","tag_local_identifier","factor","integer" +"input2_move2loc_LatLon","manufacturer_name","factor","character" +"input2_move2loc_LatLon","model","factor","character" +"input2_move2loc_LatLon","serial_no","factor","integer" +"input2_move2loc_LatLon","tag_production_date","character","integer" +"input2_move2loc_LatLon","weight","units","integer" +"input2_move2loc_LatLon","tag_number_of_deployments","integer","integer" +"input2_move2loc_LatLon","study_id","integer64","integer" +"input2_move2loc_LatLon","acknowledgements","character","character" +"input2_move2loc_LatLon","citation","character","character" +"input2_move2loc_LatLon","has_quota","logical","logical" +"input2_move2loc_LatLon","i_am_owner","logical","logical" +"input2_move2loc_LatLon","is_test","logical","logical" +"input2_move2loc_LatLon","license_type","factor","character" +"input2_move2loc_LatLon","name","factor","character" +"input2_move2loc_LatLon","study_number_of_deployments","integer","integer" +"input2_move2loc_LatLon","number_of_individuals","units","integer" +"input2_move2loc_LatLon","number_of_tags","units","integer" +"input2_move2loc_LatLon","principal_investigator_name","character","character" +"input2_move2loc_LatLon","study_objective","character","character" +"input2_move2loc_LatLon","study_type","factor","character" +"input2_move2loc_LatLon","suspend_license_terms","logical","logical" +"input2_move2loc_LatLon","i_can_see_data","logical","logical" +"input2_move2loc_LatLon","there_are_data_which_i_cannot_see","logical","logical" +"input2_move2loc_LatLon","i_have_download_access","logical","logical" +"input2_move2loc_LatLon","i_am_collaborator","logical","logical" +"input2_move2loc_LatLon","study_permission","factor","character" +"input2_move2loc_LatLon","timestamp_first_deployed_location","POSIXct/POSIXt","character" +"input2_move2loc_LatLon","timestamp_last_deployed_location","POSIXct/POSIXt","character" +"input2_move2loc_LatLon","number_of_deployed_locations","units","integer" +"input2_move2loc_LatLon","taxon_ids","character","character" +"input2_move2loc_LatLon","contact_person_name","factor","character" +"input2_move2loc_LatLon","main_location","sfc_POINT/sfc","character" +"input2_move2loc_LatLon","timestamp_utc",,"POSIXct/POSIXt" +"input2_move2loc_LatLon","timestamp_tz",,"character" +"input2_move2loc_Mollweide","sensor_type_id","integer64","integer" +"input2_move2loc_Mollweide","individual_local_identifier","character","character" +"input2_move2loc_Mollweide","eobs_battery_voltage","units","integer" +"input2_move2loc_Mollweide","eobs_fix_battery_voltage","units","integer" +"input2_move2loc_Mollweide","eobs_horizontal_accuracy_estimate","units","numeric" +"input2_move2loc_Mollweide","eobs_key_bin_checksum","integer64","numeric" +"input2_move2loc_Mollweide","eobs_speed_accuracy_estimate","units","numeric" +"input2_move2loc_Mollweide","eobs_start_timestamp","POSIXct/POSIXt","character" +"input2_move2loc_Mollweide","eobs_status","ordered/factor","character" +"input2_move2loc_Mollweide","eobs_temperature","units","integer" +"input2_move2loc_Mollweide","eobs_type_of_fix","factor","integer" +"input2_move2loc_Mollweide","eobs_used_time_to_get_fix","units","integer" +"input2_move2loc_Mollweide","ground_speed","units","numeric" +"input2_move2loc_Mollweide","heading","units","numeric" +"input2_move2loc_Mollweide","height_above_ellipsoid","units","numeric" +"input2_move2loc_Mollweide","timestamp","POSIXct/POSIXt","character" +"input2_move2loc_Mollweide","event_id","integer64","integer" +"input2_move2loc_Mollweide","visible","logical","logical" +"input2_move2loc_Mollweide","deployment_id","integer64","numeric" +"input2_move2loc_Mollweide","tag_id","integer64","integer" +"input2_move2loc_Mollweide","individual_id","integer64","integer" +"input2_move2loc_Mollweide","animal_life_stage","factor","character" +"input2_move2loc_Mollweide","animal_mass","units","numeric" +"input2_move2loc_Mollweide","attachment_type","factor","character" +"input2_move2loc_Mollweide","deployment_comments","character","character" +"input2_move2loc_Mollweide","deploy_off_timestamp","POSIXct/POSIXt","character" +"input2_move2loc_Mollweide","deploy_on_person","character","character" +"input2_move2loc_Mollweide","deploy_on_timestamp","POSIXct/POSIXt","character" +"input2_move2loc_Mollweide","deployment_end_type","factor","character" +"input2_move2loc_Mollweide","duty_cycle","character","character" +"input2_move2loc_Mollweide","deployment_local_identifier","factor","character" +"input2_move2loc_Mollweide","manipulation_type","factor","character" +"input2_move2loc_Mollweide","outlier_comments","character","character" +"input2_move2loc_Mollweide","study_site","character","character" +"input2_move2loc_Mollweide","tag_readout_method","factor","character" +"input2_move2loc_Mollweide","sensor_type_ids","character","character" +"input2_move2loc_Mollweide","capture_location","sfc_POINT/sfc","character" +"input2_move2loc_Mollweide","deploy_on_location","sfc_POINT/sfc","character" +"input2_move2loc_Mollweide","deploy_off_location","sfc_POINT/sfc","character" +"input2_move2loc_Mollweide","birth_hatch_latitude","numeric","numeric" +"input2_move2loc_Mollweide","birth_hatch_longitude","numeric","numeric" +"input2_move2loc_Mollweide","individual_comments","character","character" +"input2_move2loc_Mollweide","death_comments","character","character" +"input2_move2loc_Mollweide","group_id","factor","character" +"input2_move2loc_Mollweide","latest_date_born","POSIXct/POSIXt","character" +"input2_move2loc_Mollweide","mortality_date","Date","character" +"input2_move2loc_Mollweide","mortality_type","factor","character" +"input2_move2loc_Mollweide","parents","factor","character" +"input2_move2loc_Mollweide","ring_id","factor","character" +"input2_move2loc_Mollweide","sex","factor","character" +"input2_move2loc_Mollweide","siblings","factor","character" +"input2_move2loc_Mollweide","taxon_canonical_name","factor","character" +"input2_move2loc_Mollweide","individual_number_of_deployments","integer","integer" +"input2_move2loc_Mollweide","mortality_location","sfc_POINT/sfc","character" +"input2_move2loc_Mollweide","beacon_frequency","units","numeric" +"input2_move2loc_Mollweide","tag_local_identifier","factor","integer" +"input2_move2loc_Mollweide","manufacturer_name","factor","character" +"input2_move2loc_Mollweide","model","factor","character" +"input2_move2loc_Mollweide","serial_no","factor","integer" +"input2_move2loc_Mollweide","tag_production_date","character","integer" +"input2_move2loc_Mollweide","weight","units","integer" +"input2_move2loc_Mollweide","tag_number_of_deployments","integer","integer" +"input2_move2loc_Mollweide","study_id","integer64","integer" +"input2_move2loc_Mollweide","acknowledgements","character","character" +"input2_move2loc_Mollweide","citation","character","character" +"input2_move2loc_Mollweide","has_quota","logical","logical" +"input2_move2loc_Mollweide","i_am_owner","logical","logical" +"input2_move2loc_Mollweide","is_test","logical","logical" +"input2_move2loc_Mollweide","license_type","factor","character" +"input2_move2loc_Mollweide","name","factor","character" +"input2_move2loc_Mollweide","study_number_of_deployments","integer","integer" +"input2_move2loc_Mollweide","number_of_individuals","units","integer" +"input2_move2loc_Mollweide","number_of_tags","units","integer" +"input2_move2loc_Mollweide","principal_investigator_name","character","character" +"input2_move2loc_Mollweide","study_objective","character","character" +"input2_move2loc_Mollweide","study_type","factor","character" +"input2_move2loc_Mollweide","suspend_license_terms","logical","logical" +"input2_move2loc_Mollweide","i_can_see_data","logical","logical" +"input2_move2loc_Mollweide","there_are_data_which_i_cannot_see","logical","logical" +"input2_move2loc_Mollweide","i_have_download_access","logical","logical" +"input2_move2loc_Mollweide","i_am_collaborator","logical","logical" +"input2_move2loc_Mollweide","study_permission","factor","character" +"input2_move2loc_Mollweide","timestamp_first_deployed_location","POSIXct/POSIXt","character" +"input2_move2loc_Mollweide","timestamp_last_deployed_location","POSIXct/POSIXt","character" +"input2_move2loc_Mollweide","number_of_deployed_locations","units","integer" +"input2_move2loc_Mollweide","taxon_ids","character","character" +"input2_move2loc_Mollweide","contact_person_name","factor","character" +"input2_move2loc_Mollweide","main_location","sfc_POINT/sfc","character" +"input2_move2loc_Mollweide","timestamp_utc",,"POSIXct/POSIXt" +"input2_move2loc_Mollweide","timestamp_tz",,"character" +"input3_move2loc_LatLon","sensor_type_id","integer64","integer" +"input3_move2loc_LatLon","argos_altitude","units","integer" +"input3_move2loc_LatLon","argos_best_level","units","integer" +"input3_move2loc_LatLon","argos_calcul_freq","units","numeric" +"input3_move2loc_LatLon","argos_iq","factor","integer" +"input3_move2loc_LatLon","argos_lc","ordered/factor","integer" +"input3_move2loc_LatLon","argos_nb_mes","integer","integer" +"input3_move2loc_LatLon","argos_nb_mes_120","integer","integer" +"input3_move2loc_LatLon","argos_nopc","ordered/factor","integer" +"input3_move2loc_LatLon","argos_pass_duration","units","integer" +"input3_move2loc_LatLon","argos_sensor_1","integer","integer" +"input3_move2loc_LatLon","argos_sensor_2","integer","integer" +"input3_move2loc_LatLon","argos_sensor_3","integer","integer" +"input3_move2loc_LatLon","argos_sensor_4","integer","integer" +"input3_move2loc_LatLon","timestamp","POSIXct/POSIXt","character" +"input3_move2loc_LatLon","visible","logical","logical" +"input3_move2loc_LatLon","event_id","integer64","integer" +"input3_move2loc_LatLon","argos_location_1","sfc_POINT/sfc","character" +"input3_move2loc_LatLon","argos_location_2","sfc_POINT/sfc","character" +"input3_move2loc_LatLon","individual_name_deployment_id","character","character" +"input3_move2loc_LatLon","tag_id","integer64","integer" +"input3_move2loc_LatLon","individual_id","integer64","integer" +"input3_move2loc_LatLon","deploy_off_timestamp","POSIXct/POSIXt","character" +"input3_move2loc_LatLon","deploy_on_timestamp","POSIXct/POSIXt","character" +"input3_move2loc_LatLon","deployment_local_identifier","factor","character" +"input3_move2loc_LatLon","sensor_type_ids","character","character" +"input3_move2loc_LatLon","capture_location","sfc_POINT/sfc","character" +"input3_move2loc_LatLon","deploy_on_location","sfc_POINT/sfc","character" +"input3_move2loc_LatLon","deploy_off_location","sfc_POINT/sfc","character" +"input3_move2loc_LatLon","individual_comments","character","character" +"input3_move2loc_LatLon","death_comments","character","character" +"input3_move2loc_LatLon","latest_date_born","POSIXct/POSIXt","character" +"input3_move2loc_LatLon","individual_local_identifier","factor","character" +"input3_move2loc_LatLon","ring_id","factor","character" +"input3_move2loc_LatLon","sex","factor","character" +"input3_move2loc_LatLon","taxon_canonical_name","factor","character" +"input3_move2loc_LatLon","individual_number_of_deployments","integer","integer" +"input3_move2loc_LatLon","mortality_location","sfc_POINT/sfc","character" +"input3_move2loc_LatLon","tag_local_identifier","factor","integer" +"input3_move2loc_LatLon","tag_number_of_deployments","integer","integer" +"input3_move2loc_LatLon","study_id","integer64","integer" +"input3_move2loc_LatLon","has_quota","logical","logical" +"input3_move2loc_LatLon","i_am_owner","logical","logical" +"input3_move2loc_LatLon","is_test","logical","logical" +"input3_move2loc_LatLon","license_type","factor","character" +"input3_move2loc_LatLon","name","factor","character" +"input3_move2loc_LatLon","study_number_of_deployments","integer","integer" +"input3_move2loc_LatLon","number_of_individuals","units","integer" +"input3_move2loc_LatLon","number_of_tags","units","integer" +"input3_move2loc_LatLon","principal_investigator_name","character","character" +"input3_move2loc_LatLon","study_objective","character","character" +"input3_move2loc_LatLon","study_type","factor","character" +"input3_move2loc_LatLon","suspend_license_terms","logical","logical" +"input3_move2loc_LatLon","i_can_see_data","logical","logical" +"input3_move2loc_LatLon","there_are_data_which_i_cannot_see","logical","logical" +"input3_move2loc_LatLon","i_have_download_access","logical","logical" +"input3_move2loc_LatLon","i_am_collaborator","logical","logical" +"input3_move2loc_LatLon","study_permission","factor","character" +"input3_move2loc_LatLon","timestamp_first_deployed_location","POSIXct/POSIXt","character" +"input3_move2loc_LatLon","timestamp_last_deployed_location","POSIXct/POSIXt","character" +"input3_move2loc_LatLon","number_of_deployed_locations","units","integer" +"input3_move2loc_LatLon","taxon_ids","character","character" +"input3_move2loc_LatLon","contact_person_name","factor","character" +"input3_move2loc_LatLon","main_location","sfc_POINT/sfc","character" +"input3_move2loc_LatLon","deployment_id","integer64","integer" +"input3_move2loc_LatLon","timestamp_utc",,"POSIXct/POSIXt" +"input3_move2loc_LatLon","timestamp_tz",,"character" +"input3_move2loc_Mollweide","sensor_type_id","integer64","integer" +"input3_move2loc_Mollweide","argos_altitude","units","integer" +"input3_move2loc_Mollweide","argos_best_level","units","integer" +"input3_move2loc_Mollweide","argos_calcul_freq","units","numeric" +"input3_move2loc_Mollweide","argos_iq","factor","integer" +"input3_move2loc_Mollweide","argos_lc","ordered/factor","integer" +"input3_move2loc_Mollweide","argos_nb_mes","integer","integer" +"input3_move2loc_Mollweide","argos_nb_mes_120","integer","integer" +"input3_move2loc_Mollweide","argos_nopc","ordered/factor","integer" +"input3_move2loc_Mollweide","argos_pass_duration","units","integer" +"input3_move2loc_Mollweide","argos_sensor_1","integer","integer" +"input3_move2loc_Mollweide","argos_sensor_2","integer","integer" +"input3_move2loc_Mollweide","argos_sensor_3","integer","integer" +"input3_move2loc_Mollweide","argos_sensor_4","integer","integer" +"input3_move2loc_Mollweide","timestamp","POSIXct/POSIXt","character" +"input3_move2loc_Mollweide","visible","logical","logical" +"input3_move2loc_Mollweide","event_id","integer64","integer" +"input3_move2loc_Mollweide","argos_location_1","sfc_POINT/sfc","character" +"input3_move2loc_Mollweide","argos_location_2","sfc_POINT/sfc","character" +"input3_move2loc_Mollweide","individual_name_deployment_id","character","character" +"input3_move2loc_Mollweide","tag_id","integer64","integer" +"input3_move2loc_Mollweide","individual_id","integer64","integer" +"input3_move2loc_Mollweide","deploy_off_timestamp","POSIXct/POSIXt","character" +"input3_move2loc_Mollweide","deploy_on_timestamp","POSIXct/POSIXt","character" +"input3_move2loc_Mollweide","deployment_local_identifier","factor","character" +"input3_move2loc_Mollweide","sensor_type_ids","character","character" +"input3_move2loc_Mollweide","capture_location","sfc_POINT/sfc","character" +"input3_move2loc_Mollweide","deploy_on_location","sfc_POINT/sfc","character" +"input3_move2loc_Mollweide","deploy_off_location","sfc_POINT/sfc","character" +"input3_move2loc_Mollweide","individual_comments","character","character" +"input3_move2loc_Mollweide","death_comments","character","character" +"input3_move2loc_Mollweide","latest_date_born","POSIXct/POSIXt","character" +"input3_move2loc_Mollweide","individual_local_identifier","factor","character" +"input3_move2loc_Mollweide","ring_id","factor","character" +"input3_move2loc_Mollweide","sex","factor","character" +"input3_move2loc_Mollweide","taxon_canonical_name","factor","character" +"input3_move2loc_Mollweide","individual_number_of_deployments","integer","integer" +"input3_move2loc_Mollweide","mortality_location","sfc_POINT/sfc","character" +"input3_move2loc_Mollweide","tag_local_identifier","factor","integer" +"input3_move2loc_Mollweide","tag_number_of_deployments","integer","integer" +"input3_move2loc_Mollweide","study_id","integer64","integer" +"input3_move2loc_Mollweide","has_quota","logical","logical" +"input3_move2loc_Mollweide","i_am_owner","logical","logical" +"input3_move2loc_Mollweide","is_test","logical","logical" +"input3_move2loc_Mollweide","license_type","factor","character" +"input3_move2loc_Mollweide","name","factor","character" +"input3_move2loc_Mollweide","study_number_of_deployments","integer","integer" +"input3_move2loc_Mollweide","number_of_individuals","units","integer" +"input3_move2loc_Mollweide","number_of_tags","units","integer" +"input3_move2loc_Mollweide","principal_investigator_name","character","character" +"input3_move2loc_Mollweide","study_objective","character","character" +"input3_move2loc_Mollweide","study_type","factor","character" +"input3_move2loc_Mollweide","suspend_license_terms","logical","logical" +"input3_move2loc_Mollweide","i_can_see_data","logical","logical" +"input3_move2loc_Mollweide","there_are_data_which_i_cannot_see","logical","logical" +"input3_move2loc_Mollweide","i_have_download_access","logical","logical" +"input3_move2loc_Mollweide","i_am_collaborator","logical","logical" +"input3_move2loc_Mollweide","study_permission","factor","character" +"input3_move2loc_Mollweide","timestamp_first_deployed_location","POSIXct/POSIXt","character" +"input3_move2loc_Mollweide","timestamp_last_deployed_location","POSIXct/POSIXt","character" +"input3_move2loc_Mollweide","number_of_deployed_locations","units","integer" +"input3_move2loc_Mollweide","taxon_ids","character","character" +"input3_move2loc_Mollweide","contact_person_name","factor","character" +"input3_move2loc_Mollweide","main_location","sfc_POINT/sfc","character" +"input3_move2loc_Mollweide","deployment_id","integer64","integer" +"input3_move2loc_Mollweide","timestamp_utc",,"POSIXct/POSIXt" +"input3_move2loc_Mollweide","timestamp_tz",,"character" +"input4_move2loc_LatLon","sensor_type_id","integer64","integer" +"input4_move2loc_LatLon","individual_local_identifier","character","character" +"input4_move2loc_LatLon","gps_satellite_count","units","integer" +"input4_move2loc_LatLon","ground_speed","units","numeric" +"input4_move2loc_LatLon","gt_tx_count","units","numeric" +"input4_move2loc_LatLon","heading","units","numeric" +"input4_move2loc_LatLon","tag_voltage","units","numeric" +"input4_move2loc_LatLon","timestamp","POSIXct/POSIXt","character" +"input4_move2loc_LatLon","event_id","integer64","integer" +"input4_move2loc_LatLon","visible","logical","logical" +"input4_move2loc_LatLon","deployment_id","integer64","integer" +"input4_move2loc_LatLon","tag_id","integer64","integer" +"input4_move2loc_LatLon","individual_id","integer64","integer" +"input4_move2loc_LatLon","deploy_on_person","character","character" +"input4_move2loc_LatLon","deploy_on_timestamp","POSIXct/POSIXt","character" +"input4_move2loc_LatLon","deployment_local_identifier","factor","character" +"input4_move2loc_LatLon","sensor_type_ids","character","character" +"input4_move2loc_LatLon","capture_location","sfc_POINT/sfc","character" +"input4_move2loc_LatLon","deploy_on_location","sfc_POINT/sfc","character" +"input4_move2loc_LatLon","deploy_off_location","sfc_POINT/sfc","character" +"input4_move2loc_LatLon","ring_id","factor","character" +"input4_move2loc_LatLon","sex","factor","character" +"input4_move2loc_LatLon","taxon_canonical_name","factor","character" +"input4_move2loc_LatLon","individual_number_of_deployments","integer","integer" +"input4_move2loc_LatLon","mortality_location","sfc_POINT/sfc","character" +"input4_move2loc_LatLon","tag_local_identifier","factor","integer" +"input4_move2loc_LatLon","tag_number_of_deployments","integer","integer" +"input4_move2loc_LatLon","study_id","integer64","integer" +"input4_move2loc_LatLon","acknowledgements","character","character" +"input4_move2loc_LatLon","citation","character","character" +"input4_move2loc_LatLon","grants_used","character","character" +"input4_move2loc_LatLon","has_quota","logical","logical" +"input4_move2loc_LatLon","i_am_owner","logical","logical" +"input4_move2loc_LatLon","is_test","logical","logical" +"input4_move2loc_LatLon","license_terms","character","character" +"input4_move2loc_LatLon","license_type","factor","character" +"input4_move2loc_LatLon","name","factor","character" +"input4_move2loc_LatLon","study_number_of_deployments","integer","integer" +"input4_move2loc_LatLon","number_of_individuals","units","integer" +"input4_move2loc_LatLon","number_of_tags","units","integer" +"input4_move2loc_LatLon","principal_investigator_name","character","character" +"input4_move2loc_LatLon","study_objective","character","character" +"input4_move2loc_LatLon","study_type","factor","character" +"input4_move2loc_LatLon","suspend_license_terms","logical","logical" +"input4_move2loc_LatLon","i_can_see_data","logical","logical" +"input4_move2loc_LatLon","there_are_data_which_i_cannot_see","logical","logical" +"input4_move2loc_LatLon","i_have_download_access","logical","logical" +"input4_move2loc_LatLon","i_am_collaborator","logical","logical" +"input4_move2loc_LatLon","study_permission","factor","character" +"input4_move2loc_LatLon","timestamp_first_deployed_location","POSIXct/POSIXt","character" +"input4_move2loc_LatLon","timestamp_last_deployed_location","POSIXct/POSIXt","character" +"input4_move2loc_LatLon","number_of_deployed_locations","units","integer" +"input4_move2loc_LatLon","taxon_ids","character","character" +"input4_move2loc_LatLon","contact_person_name","factor","character" +"input4_move2loc_LatLon","main_location","sfc_POINT/sfc","character" +"input4_move2loc_LatLon","timestamp_utc",,"POSIXct/POSIXt" +"input4_move2loc_LatLon","timestamp_tz",,"character" +"input4_move2loc_Mollweide","sensor_type_id","integer64","integer" +"input4_move2loc_Mollweide","individual_local_identifier","character","character" +"input4_move2loc_Mollweide","gps_satellite_count","units","integer" +"input4_move2loc_Mollweide","ground_speed","units","numeric" +"input4_move2loc_Mollweide","gt_tx_count","units","numeric" +"input4_move2loc_Mollweide","heading","units","numeric" +"input4_move2loc_Mollweide","tag_voltage","units","numeric" +"input4_move2loc_Mollweide","timestamp","POSIXct/POSIXt","character" +"input4_move2loc_Mollweide","event_id","integer64","integer" +"input4_move2loc_Mollweide","visible","logical","logical" +"input4_move2loc_Mollweide","deployment_id","integer64","integer" +"input4_move2loc_Mollweide","tag_id","integer64","integer" +"input4_move2loc_Mollweide","individual_id","integer64","integer" +"input4_move2loc_Mollweide","deploy_on_person","character","character" +"input4_move2loc_Mollweide","deploy_on_timestamp","POSIXct/POSIXt","character" +"input4_move2loc_Mollweide","deployment_local_identifier","factor","character" +"input4_move2loc_Mollweide","sensor_type_ids","character","character" +"input4_move2loc_Mollweide","capture_location","sfc_POINT/sfc","character" +"input4_move2loc_Mollweide","deploy_on_location","sfc_POINT/sfc","character" +"input4_move2loc_Mollweide","deploy_off_location","sfc_POINT/sfc","character" +"input4_move2loc_Mollweide","ring_id","factor","character" +"input4_move2loc_Mollweide","sex","factor","character" +"input4_move2loc_Mollweide","taxon_canonical_name","factor","character" +"input4_move2loc_Mollweide","individual_number_of_deployments","integer","integer" +"input4_move2loc_Mollweide","mortality_location","sfc_POINT/sfc","character" +"input4_move2loc_Mollweide","tag_local_identifier","factor","integer" +"input4_move2loc_Mollweide","tag_number_of_deployments","integer","integer" +"input4_move2loc_Mollweide","study_id","integer64","integer" +"input4_move2loc_Mollweide","acknowledgements","character","character" +"input4_move2loc_Mollweide","citation","character","character" +"input4_move2loc_Mollweide","grants_used","character","character" +"input4_move2loc_Mollweide","has_quota","logical","logical" +"input4_move2loc_Mollweide","i_am_owner","logical","logical" +"input4_move2loc_Mollweide","is_test","logical","logical" +"input4_move2loc_Mollweide","license_terms","character","character" +"input4_move2loc_Mollweide","license_type","factor","character" +"input4_move2loc_Mollweide","name","factor","character" +"input4_move2loc_Mollweide","study_number_of_deployments","integer","integer" +"input4_move2loc_Mollweide","number_of_individuals","units","integer" +"input4_move2loc_Mollweide","number_of_tags","units","integer" +"input4_move2loc_Mollweide","principal_investigator_name","character","character" +"input4_move2loc_Mollweide","study_objective","character","character" +"input4_move2loc_Mollweide","study_type","factor","character" +"input4_move2loc_Mollweide","suspend_license_terms","logical","logical" +"input4_move2loc_Mollweide","i_can_see_data","logical","logical" +"input4_move2loc_Mollweide","there_are_data_which_i_cannot_see","logical","logical" +"input4_move2loc_Mollweide","i_have_download_access","logical","logical" +"input4_move2loc_Mollweide","i_am_collaborator","logical","logical" +"input4_move2loc_Mollweide","study_permission","factor","character" +"input4_move2loc_Mollweide","timestamp_first_deployed_location","POSIXct/POSIXt","character" +"input4_move2loc_Mollweide","timestamp_last_deployed_location","POSIXct/POSIXt","character" +"input4_move2loc_Mollweide","number_of_deployed_locations","units","integer" +"input4_move2loc_Mollweide","taxon_ids","character","character" +"input4_move2loc_Mollweide","contact_person_name","factor","character" +"input4_move2loc_Mollweide","main_location","sfc_POINT/sfc","character" +"input4_move2loc_Mollweide","timestamp_utc",,"POSIXct/POSIXt" +"input4_move2loc_Mollweide","timestamp_tz",,"character" +"input_move2loc_List","deployment_id","integer64","integer" +"input_move2loc_List","sensor_type_id","integer64","integer" +"input_move2loc_List","timestamp","POSIXct/POSIXt","character" +"input_move2loc_List","event_id","integer64","numeric" +"input_move2loc_List","visible","logical","logical" +"input_move2loc_List","individual_local_identifier","character","character" +"input_move2loc_List","speed","units","numeric" +"input_move2loc_List","tag_id","list","integer" +"input_move2loc_List","individual_id","list","integer" +"input_move2loc_List","deploy_off_timestamp","list","character" +"input_move2loc_List","deploy_on_timestamp","list","character" +"input_move2loc_List","sensor_type_ids","list","character" +"input_move2loc_List","capture_location","list","character" +"input_move2loc_List","deploy_on_location","list","character" +"input_move2loc_List","deploy_off_location","list","character" +"input_move2loc_List","sex","list","character" +"input_move2loc_List","taxon_canonical_name","list","character" +"input_move2loc_List","individual_number_of_deployments","list","integer" +"input_move2loc_List","taxon_detail","list","logical" +"input_move2loc_List","mortality_location","list","character" +"input_move2loc_List","tag_local_identifier","list","character" +"input_move2loc_List","tag_number_of_deployments","list","integer" +"input_move2loc_List","study_id","list","integer" +"input_move2loc_List","acknowledgements","list","character" +"input_move2loc_List","citation","list","character" +"input_move2loc_List","grants_used","list","character" +"input_move2loc_List","has_quota","list","logical" +"input_move2loc_List","i_am_owner","list","logical" +"input_move2loc_List","is_test","list","logical" +"input_move2loc_List","license_terms","list","character" +"input_move2loc_List","license_type","list","character" +"input_move2loc_List","name","list","character" +"input_move2loc_List","study_number_of_deployments","list","integer" +"input_move2loc_List","number_of_individuals","list","integer" +"input_move2loc_List","number_of_tags","list","integer" +"input_move2loc_List","principal_investigator_name","list","character" +"input_move2loc_List","study_objective","list","character" +"input_move2loc_List","study_type","list","character" +"input_move2loc_List","suspend_license_terms","list","logical" +"input_move2loc_List","i_can_see_data","list","logical" +"input_move2loc_List","there_are_data_which_i_cannot_see","list","logical" +"input_move2loc_List","i_have_download_access","list","logical" +"input_move2loc_List","i_am_collaborator","list","logical" +"input_move2loc_List","study_permission","list","character" +"input_move2loc_List","timestamp_first_deployed_location","list","character" +"input_move2loc_List","timestamp_last_deployed_location","list","character" +"input_move2loc_List","number_of_deployed_locations","list","integer" +"input_move2loc_List","taxon_ids","list","character" +"input_move2loc_List","contact_person_name","list","character" +"input_move2loc_List","main_location","list","character" +"input_move2loc_List","timestamp_utc",,"POSIXct/POSIXt" +"input_move2loc_List","timestamp_tz",,"character" diff --git a/test/roundtrip/compare.R b/test/roundtrip/compare.R new file mode 100644 index 0000000..b5943d9 --- /dev/null +++ b/test/roundtrip/compare.R @@ -0,0 +1,56 @@ +# Checks a round trip against its pins (see describe.R), then the invariants no pin may relax: the +# track of every row, the coordinates, the crs, and every input column present on the way back. +# Rscript compare.R +here <- dirname(sub("^--file=", "", grep("^--file=", commandArgs(trailingOnly = FALSE), value = TRUE))) +source(file.path(here, "describe.R")) + +args <- commandArgs(trailingOnly = TRUE) +described <- describeAll(cases = args[1]) +failures <- character() + +# the pins, line by line +actualExpected <- tempfile(fileext = ".csv") +actualClasses <- tempfile(fileext = ".csv") +writeDescription(described = described, expectedFile = actualExpected, classesFile = actualClasses) +for (files in list(c(args[2], actualExpected), c(args[3], actualClasses))) { + pinned <- readLines(files[1]) + measured <- readLines(files[2]) + # sprintf, not paste0: paste0 turns an empty difference into one line + failures <- c(failures, + sprintf("%s, pinned but not measured: %s", basename(files[1]), setdiff(pinned, measured)), + sprintf("%s, measured but not pinned: %s", basename(files[1]), setdiff(measured, pinned))) +} + +# the invariants +for (case in described) { + if (is.null(case$aligned)) next + name <- case$summary$case + input <- case$aligned$input + output <- case$aligned$output + + if (nrow(input) != nrow(output)) { + failures <- c(failures, sprintf("%s: %d input rows in the kept tracks, %d rows back", name, nrow(input), nrow(output))) + next + } + if (!identical(as.character(mt_track_id(input)), as.character(mt_track_id(output)))) { + failures <- c(failures, sprintf("%s: the track of a row changed", name)) + } + coordinatesIn <- st_coordinates(input) + coordinatesOut <- st_coordinates(output) + if (any(abs(coordinatesIn - coordinatesOut) > 1e-12 * pmax(1, abs(coordinatesIn)))) { + failures <- c(failures, sprintf("%s: coordinates changed", name)) + } + if (st_crs(case$input) != st_crs(case$output)) { + failures <- c(failures, sprintf("%s: crs changed", name)) + } + lost <- setdiff(case$namesIn, case$namesOut) + if (length(lost) > 0) { + failures <- c(failures, sprintf("%s: columns lost: %s", name, paste(lost, collapse = ", "))) + } +} + +if (length(failures) > 0) { + cat(failures, sep = "\n") + quit(status = 1) +} +cat(sprintf("round trip: %d cases as pinned\n", length(described))) diff --git a/test/roundtrip/describe.R b/test/roundtrip/describe.R new file mode 100644 index 0000000..aff6dd0 --- /dev/null +++ b/test/roundtrip/describe.R @@ -0,0 +1,107 @@ +# Describes what the round trip R -> python -> R did to every case directory in : +# /input.rds what went in (absent for a python-born case) +# /r2python.exit exit code of the R -> python App; /r2python.log its log +# /python2r.exit exit code of the python -> R App +# /output.rds what came back (absent if an App failed, empty for the NULL result) +# +# Sourced by compare.R. Run as a script it pins the description: +# Rscript describe.R + +suppressMessages({ + library("move2") + library("sf") +}) + +readExit <- function(file) { + if (file.exists(file)) as.integer(readLines(file, n = 1)) else NA_integer_ +} + +classOf <- function(x) paste(class(x), collapse = "/") + +# the input rows of the tracks the output kept, and the output, both ordered by track id and time; +# radix ordering is locale-free, so two images with different locales order alike +alignRows <- function(input, output) { + input <- input[as.character(mt_track_id(input)) %in% as.character(mt_track_id(output)), ] + list( + input = input[order(as.character(mt_track_id(input)), as.numeric(mt_time(input)), method = "radix"), ], + output = output[order(as.character(mt_track_id(output)), as.numeric(mt_time(output)), method = "radix"), ] + ) +} + +describeCase <- function(dir) { + case <- basename(dir) + log <- file.path(dir, "r2python.log") + summary <- data.frame( + case = case, + r2python_exit = readExit(file.path(dir, "r2python.exit")), + python2r_exit = readExit(file.path(dir, "python2r.exit")), + rows_out = NA_integer_, + tracks_out = NA_integer_, + max_time_deviation_s = NA_character_, + time_column_out = NA_character_, + tzone_out = NA_character_, + dropped_tracks_warned = if (file.exists(log)) sum(grepl("^\\[WARN\\] track ", readLines(log, warn = FALSE))) else NA_integer_ + ) + described <- list(summary = summary, classes = NULL, aligned = NULL) + + outputFile <- file.path(dir, "output.rds") + if (!file.exists(outputFile)) { + return(described) + } + if (file.size(outputFile) == 0) { + described$summary$rows_out <- 0L + return(described) + } + + output <- readRDS(outputFile) + described$summary$rows_out <- nrow(output) + described$summary$tracks_out <- mt_n_tracks(output) + described$summary$time_column_out <- mt_time_column(output) + described$summary$tzone_out <- attr(mt_time(output), "tzone") + + inputFile <- file.path(dir, "input.rds") + if (!file.exists(inputFile)) { + return(described) + } + input <- readRDS(inputFile) + aligned <- alignRows(input = input, output = output) + if (nrow(aligned$input) == nrow(aligned$output) && nrow(aligned$output) > 0) { + deviation <- abs(as.numeric(mt_time(aligned$input)) - as.numeric(mt_time(aligned$output))) + described$summary$max_time_deviation_s <- sprintf("%.3f", max(deviation)) + } + + # the input as it travels: track attributes become event attributes on the way out + events <- mt_as_event_attribute(input, names(mt_track_data(input))) + namesIn <- setdiff(names(events), attr(events, "sf_column")) + namesOut <- setdiff(names(output), attr(output, "sf_column")) + columns <- union(namesIn, namesOut) + described$classes <- data.frame( + case = case, + column = columns, + class_in = vapply(columns, function(column) if (column %in% namesIn) classOf(events[[column]]) else NA_character_, character(1)), + class_out = vapply(columns, function(column) if (column %in% namesOut) classOf(output[[column]]) else NA_character_, character(1)), + row.names = NULL + ) + + described$aligned <- aligned + described$input <- input + described$output <- output + described$namesIn <- namesIn + described$namesOut <- namesOut + described +} + +describeAll <- function(cases) { + lapply(sort(list.dirs(cases, recursive = FALSE), method = "radix"), describeCase) +} + +# one CSV line per case, and one per case and column +writeDescription <- function(described, expectedFile, classesFile) { + write.csv(do.call(rbind, lapply(described, `[[`, "summary")), expectedFile, row.names = FALSE, na = "") + write.csv(do.call(rbind, lapply(described, `[[`, "classes")), classesFile, row.names = FALSE, na = "") +} + +if (sys.nframe() == 0) { + args <- commandArgs(trailingOnly = TRUE) + writeDescription(described = describeAll(cases = args[1]), expectedFile = args[2], classesFile = args[3]) +} diff --git a/test/roundtrip/empty.sh b/test/roundtrip/empty.sh new file mode 100755 index 0000000..4ad8f68 --- /dev/null +++ b/test/roundtrip/empty.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# The python-born edge case: an empty TrajectoryCollection, as a python App may hand one on. +# /empty_trajectory_collection/output.pickle +set -o errexit -o nounset + +dir="$1/empty_trajectory_collection" +mkdir -p "$dir" + +conda run --no-capture-output --prefix "$ENV_PREFIX" python -c ' +import sys +import movingpandas as mpd +import pandas as pd +pd.to_pickle(mpd.TrajectoryCollection([]), sys.argv[1], compression="gzip") +' "$dir/output.pickle" diff --git a/test/roundtrip/expected.csv b/test/roundtrip/expected.csv new file mode 100644 index 0000000..857a814 --- /dev/null +++ b/test/roundtrip/expected.csv @@ -0,0 +1,13 @@ +"case","r2python_exit","python2r_exit","rows_out","tracks_out","max_time_deviation_s","time_column_out","tzone_out","dropped_tracks_warned" +"dst_ambiguous",0,0,2,1,"0.000","timestamp_utc","UTC",0 +"empty_trajectory_collection",,0,0,,,,, +"input1_move2loc_LatLon",0,0,3243,1,"0.000","timestamp_utc","UTC",0 +"input1_move2loc_LatLon_berlin",0,0,3243,1,"0.000","timestamp_utc","UTC",0 +"input1_move2loc_Mollweide",0,0,3243,1,"0.000","timestamp_utc","UTC",0 +"input2_move2loc_LatLon",0,0,98285,3,"0.000","timestamp_utc","UTC",0 +"input2_move2loc_Mollweide",0,0,98285,3,"0.000","timestamp_utc","UTC",0 +"input3_move2loc_LatLon",0,0,3169,5,"0.000","timestamp_utc","UTC",0 +"input3_move2loc_Mollweide",0,0,3169,5,"0.000","timestamp_utc","UTC",0 +"input4_move2loc_LatLon",0,0,7770,3,"0.000","timestamp_utc","UTC",0 +"input4_move2loc_Mollweide",0,0,7770,3,"0.000","timestamp_utc","UTC",0 +"input_move2loc_List",0,0,2,1,"0.000","timestamp_utc","UTC",3 diff --git a/test/roundtrip/measure.sh b/test/roundtrip/measure.sh new file mode 100755 index 0000000..2366f3e --- /dev/null +++ b/test/roundtrip/measure.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Measures what a round trip R -> python -> R does to every sample input, and pins it: +# test/roundtrip/expected.csv - per case: exit codes, rows and tracks out, time deviation, time column and tzone out +# test/roundtrip/classes.csv - per case and column: class in, class out +# +# [R2PY_IMAGE=…] [PY2R_IMAGE=…] [OVERLAY=1] [KEEP=] test/roundtrip/measure.sh +# +# Defaults to the v2.2.0 production images (registry access needed). KEEP copies the cases out, +# outputs included. OVERLAY: see test/images.sh. +set -o errexit -o nounset -o pipefail + +cd "$(dirname "$0")/../.." +source test/images.sh + +R2PY_IMAGE="${R2PY_IMAGE:-gitlab-registry.mpcdf.mpg.de/moveapps/apps/production-a81e3046-bc48-4fcb-8d28-4291000c77f9-move2loc-to-movingpandas:7}" +PY2R_IMAGE="${PY2R_IMAGE:-gitlab-registry.mpcdf.mpg.de/moveapps/apps/production-ed490356-9d55-4cb3-90d1-b8a2fc18ce10-movingpandas-to-move2loc:6}" + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# R -> python: sample inputs and synthetic cases +mkdir -p "$work/in/raw" +cp r/data/raw/*move2loc*.rds "$work/in/raw/" +cp -R test "$work/in/test" +runInImage "$R2PY_IMAGE" "$work/in" '/tmp/in/test/roundtrip/stage-r2python.sh /tmp/in/raw /tmp/out' "$work/stage1" + +# python -> R, then describe with that image's R +cp -R test "$work/stage1/test" +runInImage "$PY2R_IMAGE" "$work/stage1" \ + '/tmp/in/test/roundtrip/stage-python2r.sh /tmp/in/cases /tmp/out && cd r && Rscript /tmp/in/test/roundtrip/describe.R /tmp/out/cases /tmp/out/expected.csv /tmp/out/classes.csv' \ + "$work/stage2" + +cp "$work/stage2/expected.csv" "$work/stage2/classes.csv" test/roundtrip/ +if [ -n "${KEEP:-}" ]; then + rm -rf "$KEEP" + cp -R "$work/stage2/cases" "$KEEP" +fi diff --git a/test/roundtrip/python2r.sh b/test/roundtrip/python2r.sh new file mode 100755 index 0000000..77c8ba4 --- /dev/null +++ b/test/roundtrip/python2r.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Runs every //output.pickle through the python -> R entry point: +# //{py/meta.csv, py/link.csv, output.rds, python2r.log, python2r.exit} +# A case without a pickle (its R -> python step failed) is skipped. +set -o nounset + +start="$1" cases="$2" + +for dir in "$cases"/*/; do + dir="${dir%/}" + [ -f "$dir/output.pickle" ] || continue + mkdir -p "$dir/py" + + SOURCE_FILE="$dir/output.pickle" OUTPUT_FILE="$dir/output.rds" ERROR_FILE="$dir/py/error.log" \ + LINK_R_PYTHON_BUFFER="$dir/py/link.csv" LINK_R_PYTHON_META="$dir/py/meta.csv" \ + "$start" > "$dir/python2r.log" 2>&1 + echo $? > "$dir/python2r.exit" +done diff --git a/test/roundtrip/r2python.sh b/test/roundtrip/r2python.sh new file mode 100755 index 0000000..274556d --- /dev/null +++ b/test/roundtrip/r2python.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Runs every /.rds through the R -> python entry point, as the platform would: +# //{input.rds, meta.csv, link.csv, output.pickle, r2python.log, r2python.exit} +# Never fails itself: a failing App is a measurement, recorded in r2python.exit. +set -o nounset + +start="$1" inputs="$2" cases="$3" + +for input in "$inputs"/*.rds; do + case="$(basename "$input" .rds)" + dir="$cases/$case" + mkdir -p "$dir" + cp "$input" "$dir/input.rds" + + SOURCE_FILE="$dir/input.rds" OUTPUT_FILE="$dir/output.pickle" ERROR_FILE="$dir/error.log" \ + LINK_R_PYTHON_BUFFER="$dir/link.csv" LINK_R_PYTHON_META="$dir/meta.csv" \ + "$start" > "$dir/r2python.log" 2>&1 + echo $? > "$dir/r2python.exit" +done diff --git a/test/roundtrip/run.sh b/test/roundtrip/run.sh new file mode 100755 index 0000000..c5c3580 --- /dev/null +++ b/test/roundtrip/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Inside the test image, which holds both entry points: the round trip of every case, checked against +# expected.csv and classes.csv. +# r/data/raw/*.rds --start-process.sh--> pickle --start-process-python2r.sh--> rds +set -o errexit -o nounset + +here="$(cd "$(dirname "$0")" && pwd)" +work="$(mktemp -d)" + +"$here/stage-r2python.sh" r/data/raw "$work/stage1" +START=./start-process-python2r.sh "$here/stage-python2r.sh" "$work/stage1/cases" "$work/stage2" + +(cd r && Rscript "$here/compare.R" "$work/stage2/cases" "$here/expected.csv" "$here/classes.csv") diff --git a/test/roundtrip/stage-python2r.sh b/test/roundtrip/stage-python2r.sh new file mode 100755 index 0000000..2b23c9d --- /dev/null +++ b/test/roundtrip/stage-python2r.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# The other half, inside a python -> R App image: every pickle of back through the entry +# point $START (default ./start-process.sh), plus the python-born empty case -> /cases/ +set -o errexit -o nounset + +here="$(cd "$(dirname "$0")" && pwd)" +cases="$1" out="$2" + +mkdir -p "$out" +cp -R "$cases" "$out/cases" +"$here/empty.sh" "$out/cases" + +"$here/python2r.sh" "${START:-./start-process.sh}" "$out/cases" diff --git a/test/roundtrip/stage-r2python.sh b/test/roundtrip/stage-r2python.sh new file mode 100755 index 0000000..bc6dc3c --- /dev/null +++ b/test/roundtrip/stage-r2python.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# One half of the round trip, inside an R -> python App image: the sample inputs of plus the +# synthetic cases through the entry point $START (default ./start-process.sh) -> /cases/ +set -o errexit -o nounset + +here="$(cd "$(dirname "$0")" && pwd)" +raw="$1" out="$2" + +mkdir -p "$out/inputs" +cp "$raw"/*move2loc*.rds "$out/inputs/" +(cd r && Rscript "$here/synthetic.R" "$out/inputs/input1_move2loc_LatLon.rds" "$out/inputs") + +"$here/r2python.sh" "${START:-./start-process.sh}" "$out/inputs" "$out/cases" diff --git a/test/roundtrip/synthetic.R b/test/roundtrip/synthetic.R new file mode 100644 index 0000000..d780b4c --- /dev/null +++ b/test/roundtrip/synthetic.R @@ -0,0 +1,17 @@ +# Derives the non-UTC round-trip cases from input1. Movebank data are UTC; these are the edge cases. +# input1_move2loc_LatLon_berlin.rds - all of input1, displayed in Europe/Berlin +# dst_ambiguous.rds - two fixes at 00:30 and 01:30 UTC on 2021-10-31, both 02:30 local +# Rscript synthetic.R +suppressMessages(library("move2")) + +args <- commandArgs(trailingOnly = TRUE) +input1 <- readRDS(args[1]) + +berlin <- input1 +attr(berlin[[mt_time_column(berlin)]], "tzone") <- "Europe/Berlin" +saveRDS(berlin, file.path(args[2], "input1_move2loc_LatLon_berlin.rds")) + +ambiguous <- input1[1:2, ] +ambiguous[[mt_time_column(ambiguous)]] <- as.POSIXct(c("2021-10-31 00:30:00", "2021-10-31 01:30:00"), tz = "UTC") +attr(ambiguous[[mt_time_column(ambiguous)]], "tzone") <- "Europe/Berlin" +saveRDS(ambiguous, file.path(args[2], "dst_ambiguous.rds"))