From 278682b931951ad6fa3d87b4fdaa1b2d0af80922 Mon Sep 17 00:00:00 2001 From: Seth Parker Date: Thu, 23 Jul 2026 10:51:26 -0400 Subject: [PATCH 1/4] Rename to Preppy; migrate CI to GitHub Actions + GHCR Docker builds Transition from GitLab (origin) to GitHub (github). Rename: - Package/project dri-voyager-preppy -> preppy (setup.cfg name + URL). - Console scripts voyager-* -> preppy / preppy-obj2glb / preppy-merge-items / preppy-check-tools. Updated all live references (README, CLAUDE.md, schema $id/description, docstrings, tests, Node helper package name). - singularity/dri-voyager-preppy.def -> singularity/preppy.def with updated in-file install paths. CI: - Add .github/workflows/ci.yml (pytest across py3.11-3.13 + `preppy -h` smoke test, plus a workflow_dispatch-gated, non-blocking integration job). - Remove .gitlab-ci.yml. Docker: - Add Dockerfile derived from the singularity def (ImageMagick, KTX-Software, gltfpack, Node, editable install with the preview extra). Arch-aware KTX .deb via TARGETARCH for multi-arch; pinned to stable v4.4.2 (no final v5.0.0 tag exists yet). Add .dockerignore. - Add .github/workflows/build_docker.yml: multi-arch (amd64/arm64) buildx build pushed to ghcr.io/educelab/preppy (edge on develop; semver + latest on tags). Co-Authored-By: Claude Opus 4.8 (1M context) --- .dockerignore | 21 +++++ .github/workflows/build_docker.yml | 75 ++++++++++++++++ .github/workflows/ci.yml | 82 +++++++++++++++++ .gitlab-ci.yml | 52 ----------- CLAUDE.md | 18 ++-- Dockerfile | 88 +++++++++++++++++++ README.md | 41 +++++++-- preppy/apps/file_prep.py | 2 +- preppy/apps/obj_to_glb.py | 2 +- preppy/node/package.json | 4 +- preppy/preview.py | 2 +- preppy/tools.py | 4 +- setup.cfg | 12 +-- .../{dri-voyager-preppy.def => preppy.def} | 14 +-- templates/prep-models.schema.json | 4 +- tests/test_integration.py | 4 +- tests/test_preview.py | 2 +- tests/test_tools.py | 2 +- 18 files changed, 333 insertions(+), 96 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/build_docker.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 .gitlab-ci.yml create mode 100644 Dockerfile rename singularity/{dri-voyager-preppy.def => preppy.def} (83%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cca24cc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,21 @@ +# Keep the build context small and reproducible. The image installs deps fresh +# (pip, npm), so none of these belong in it. +.git +.github +.gitlab-ci.yml +.idea +.pytest_cache +.DS_Store +**/__pycache__ +**/*.pyc +**/*.egg-info +**/venv +**/.venv +**/node_modules +singularity +spike +conductor +docs +*.sif +data +logs diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml new file mode 100644 index 0000000..597f7d5 --- /dev/null +++ b/.github/workflows/build_docker.yml @@ -0,0 +1,75 @@ +name: Build Docker image + +on: + push: + branches: ["develop"] + tags: ["v*"] + workflow_dispatch: + +concurrency: docker + +env: + REGISTRY: ghcr.io + REGISTRY_IMAGE: educelab/preppy + +permissions: + contents: read + packages: write + +jobs: + build: + name: Build and push + runs-on: ubuntu-latest + steps: + - name: Clean up disk space + run: | + # The multi-arch build + toolchain is large; reclaim runner space. + df -h / + sudo rm -rf /usr/lib/jvm + sudo rm -rf /usr/local/.ghcup + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/share/swift + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + df -h / + + - name: Checkout + uses: actions/checkout@v4 + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.REGISTRY_IMAGE }} + # `latest` tracks the most recent release tag; `edge` tracks develop. + flavor: latest=false + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=edge + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REGISTRY_IMAGE }}:buildcache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REGISTRY_IMAGE }}:buildcache,mode=max diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5d6b2f0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,82 @@ +name: CI + +on: + push: + branches: ["develop"] + tags: ["v*"] + pull_request: + workflow_dispatch: + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + # =========================================================================== + # Python package (preppy/) + # --------------------------------------------------------------------------- + # The pure-Python logic is covered here across supported interpreters. Tests + # that need the external toolchain (magick/ktx/gltfpack/node), pymeshlab, or an + # offscreen-GL backend self-skip when those are absent, so a bare runner still + # gets full logic coverage plus the historic `preppy -h` smoke test. + # =========================================================================== + test: + name: py ${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install + run: | + python -m pip install --upgrade pip setuptools wheel + pip install '.[test]' + + - name: Run tests + run: python -m pytest tests/ + + - name: Smoke test + run: preppy -h + + # Full end-to-end run over the sample object, exercising the real toolchain. + # Heavy and environment-sensitive, so it is opt-in (manual dispatch) and + # non-blocking (continue-on-error); whatever tools install successfully are + # exercised, the rest self-skip. See README / singularity/preppy.def for the + # authoritative install. + integration: + name: integration (full toolchain) + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install external toolchain + run: | + curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash - + sudo apt-get update + sudo apt-get install -y imagemagick nodejs + npm install -g gltfpack + # KTX-Software (`ktx create`) — install per README for full KTX2 tests; + # absent, the KTX2/embed integration tests skip cleanly. + + - name: Install package + embed helper deps + run: | + pip install '.[test,validate,preview]' + npm install --prefix preppy/node + + - name: Run tests + run: python -m pytest tests/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 3a95efb..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,52 +0,0 @@ -stages: - - test - -# =========================================================================== -# Python package (preppy/) -# --------------------------------------------------------------------------- -# The pure-Python logic is covered here across supported interpreters. Tests -# that need the external toolchain (magick/ktx/gltfpack/node), pymeshlab, or an -# offscreen-GL backend self-skip when those are absent, so a bare runner still -# gets full logic coverage plus the historic `voyager-preppy -h` smoke test. -# =========================================================================== -.py-test: - stage: test - before_script: - - python -m pip install --upgrade pip setuptools wheel - script: - - pip install '.[test]' - - python -m pytest tests/ - - voyager-preppy -h - -py:3.11: - extends: .py-test - image: python:3.11 - -py:3.12: - extends: .py-test - image: python:3.12 - -py:3.13: - extends: .py-test - image: python:3.13 - -# Full end-to-end run over the sample object, exercising the real toolchain. -# Heavy and environment-sensitive, so it is opt-in (manual) and non-blocking; -# whatever tools install successfully are exercised, the rest self-skip. See -# README / singularity/dri-voyager-preppy.def for the authoritative install. -integration: - stage: test - image: python:3.13 - when: manual - allow_failure: true - before_script: - - curl -fsSL https://deb.nodesource.com/setup_24.x | bash - - - apt-get update - - apt-get install -y imagemagick - - npm install -g gltfpack - # KTX-Software (>= v5, `ktx create`) — install per README for full KTX2 tests; - # absent, the KTX2/embed integration tests skip cleanly. - script: - - pip install '.[test,validate,preview]' - - npm install --prefix preppy/node - - python -m pytest tests/ diff --git a/CLAUDE.md b/CLAUDE.md index 0666599..6acb169 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,17 +6,17 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co DRI Voyager Preppy turns source OBJs + textures into web-ready 3D assets for the custom `` web component. Each imaged object has several **variants** (RGB, IR, PGS, …); the pipeline emits **one self-contained `.glb` per variant** (meshopt-compressed geometry with its KTX2 texture(s) embedded) plus a viewer-native per-object **`manifest.json`** and an optional top-level `index.json`. -This replaced the original Smithsonian-Voyager path (`.svx.json` scene descriptors + `items.json`, via `obj2gltf` + `gltf-pipeline`). That legacy path survives only as the deprecated `voyager-obj2glb` tool (`convert.py`). +This replaced the original Smithsonian-Voyager path (`.svx.json` scene descriptors + `items.json`, via `obj2gltf` + `gltf-pipeline`). That legacy path survives only as the deprecated `preppy-obj2glb` tool (`convert.py`). ## External dependencies (not pip-installable) -The pipeline shells out to CLI tools that must be on `PATH` (see README for install; `tools.py` detects them, `voyager-check-tools` reports status). npm-installed CLIs are invoked as `.cmd` on Windows (`platform.system()` check in `tools.py`). +The pipeline shells out to CLI tools that must be on `PATH` (see README for install; `tools.py` detects them, `preppy-check-tools` reports status). npm-installed CLIs are invoked as `.cmd` on Windows (`platform.system()` check in `tools.py`). - **ImageMagick** (`magick`/`mogrify`) — normalize textures to 8-bit sRGB; crop thumbnails. - **`ktx`** (KTX-Software **≥ v5**, `ktx create` — `toktx` was removed in v5) — KTX2/Basis encoding. - **`gltfpack`** (meshoptimizer) — OBJ → decimated, meshopt-compressed geometry glb. - **`node`** (20+) + the bundled `@gltf-transform/core` helper (`preppy/node/embed.mjs`) — embeds KTX2 into the geometry glb. Install its deps once: `npm install --prefix preppy/node`. -- Legacy only: `obj2gltf` + `gltf-pipeline` (for `voyager-obj2glb`). +- Legacy only: `obj2gltf` + `gltf-pipeline` (for `preppy-obj2glb`). - Optional: `pymeshlab` (`.[validate]`) for the Hausdorff decimation gate. - Optional: `trimesh` + `pyrender` (`.[preview]`) for the rendered model-preview thumbnail (`preview.py`). Needs an offscreen GL backend; when absent the thumbnail falls back to a texture center-crop. @@ -28,13 +28,13 @@ Python deps (`natsort`, `Pillow`, `numpy`, `scipy`, `tqdm`) install via `pip ins pip install -e '.[validate,test]' # editable install + optional pymeshlab/pytest npm install --prefix preppy/node # KTX2 embed helper deps (once) -voyager-preppy -i config.json -o out/ # batch: variants → self-contained glbs + manifest.json + index.json -voyager-check-tools # report external toolchain status -voyager-obj2glb -i mesh.obj -o mesh.glb # LEGACY single OBJ → Draco GLB (deprecated) -voyager-merge-items a.json b.json -o merged.json # LEGACY items.json merge (deprecated) +preppy -i config.json -o out/ # batch: variants → self-contained glbs + manifest.json + index.json +preppy-check-tools # report external toolchain status +preppy-obj2glb -i mesh.obj -o mesh.glb # LEGACY single OBJ → Draco GLB (deprecated) +preppy-merge-items a.json b.json -o merged.json # LEGACY items.json merge (deprecated) ``` -There **is** a test suite now (`tests/`, pytest): `python -m pytest tests/`. Tests that need the external tools (or pymeshlab) skip cleanly when they're absent, so a bare run still covers the pure logic. CI (`.gitlab-ci.yml`) runs the suite across Python 3.11–3.13 plus the `voyager-preppy -h` smoke test, with a manual, non-blocking `integration` job that exercises the full external toolchain. The `singularity/dri-voyager-preppy.def` bundles all deps for reproducible/HPC runs. +There **is** a test suite now (`tests/`, pytest): `python -m pytest tests/`. Tests that need the external tools (or pymeshlab) skip cleanly when they're absent, so a bare run still covers the pure logic. CI (`.github/workflows/ci.yml`, GitHub Actions) runs the suite across Python 3.11–3.13 plus the `preppy -h` smoke test, with a manual, non-blocking `integration` job that exercises the full external toolchain. A second workflow (`.github/workflows/build_docker.yml`) builds and publishes the multi-arch Docker image to `ghcr.io/educelab/preppy`. The `Dockerfile` (and `singularity/preppy.def`) bundle all deps for reproducible/container/HPC runs. ## Architecture @@ -52,7 +52,7 @@ Console entrypoints in `preppy/apps/` are thin argparse CLIs over the library mo ### Input config format -The `voyager-preppy` input JSON is a flat array of **objects**, validated by `templates/prep-models.schema.json` (examples: `prep-models-example.json`, `mvs-example.json`): +The `preppy` input JSON is a flat array of **objects**, validated by `templates/prep-models.schema.json` (examples: `prep-models-example.json`, `mvs-example.json`): - An **object** needs `id`, `title`, and a `variants` array. Optional `prefix` (output folder/file prefix; defaults to `id`), `titles`, `inventory`, `description`, `credit`, `date`, `units` (default `cm`), `nodataFill`. - A **variant** needs `suffix` (stable key: names the file + is the manifest variant `id`) and `obj`. Optional `label`, `default`, `nodataFill` (resolved variant ?? object ?? CLI), and per-variant `credit`/`date`/`method`/`description`. Textures are resolved transitively from the OBJ's `map_Kd`. Relative `obj` paths resolve against `--data-root` (default CWD), not the config file's location. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..83d3269 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,88 @@ +# syntax=docker/dockerfile:1 +# +# Preppy delivery-pipeline image. Bundles the full external toolchain +# (ImageMagick, KTX-Software, gltfpack, Node) + the Python package with the +# `preview` extra so the default thumbnail is a rendered model preview. +# +# Derived from singularity/preppy.def. Multi-arch (linux/amd64, linux/arm64): +# the KTX-Software .deb is selected per-arch via the buildx-provided TARGETARCH. +ARG BASE_IMAGE=python:3.11-slim +FROM ${BASE_IMAGE} + +LABEL org.opencontainers.image.title="Preppy" +LABEL org.opencontainers.image.description="Mesh preparation for DRI Voyager: source OBJs + textures -> web-ready self-contained glb per variant + manifest." +LABEL org.opencontainers.image.authors="Seth Parker " +LABEL org.opencontainers.image.source="https://github.com/educelab/preppy" +LABEL org.opencontainers.image.licenses="GPL-3.0-or-later" + +# TARGETARCH is injected by Docker buildx (amd64 | arm64). +ARG TARGETARCH + +ENV DEBIAN_FRONTEND=noninteractive \ + # Headless offscreen GL backend for the pyrender model-preview thumbnail. + # (libosmesa6 is installed below; if GL fails on a host the thumbnail falls + # back to a texture crop.) + PYOPENGL_PLATFORM=osmesa \ + PIP_NO_CACHE_DIR=1 + +# System dependencies: +# imagemagick -> texture normalization (magick/mogrify) +# libosmesa6, libgl1 -> offscreen GL for the rendered model-preview thumbnail +# curl/git/gcc/g++/make -> fetch installers + build any sdist-only wheels +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + curl \ + gcc \ + g++ \ + git \ + make \ + tzdata \ + imagemagick \ + libosmesa6 \ + libgl1 \ + && rm -rf /var/lib/apt/lists/* + +# Node.js (24 LTS) for gltfpack and the KTX2 embed helper. +RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && rm -rf /var/lib/apt/lists/* \ + && node -v + +# KTX-Software (provides `ktx create`; `toktx` is not used). The .def targeted a +# v5 that is not yet released as a final tag, so pin the latest stable that ships +# `ktx create` for both amd64 and arm64. Bump when v5.0.0 ships. +# Assets: https://github.com/KhronosGroup/KTX-Software/releases +ARG KTX_VERSION=4.4.2 +RUN set -eux; \ + case "${TARGETARCH}" in \ + amd64) ktx_arch="x86_64" ;; \ + arm64) ktx_arch="arm64" ;; \ + *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac; \ + curl -fsSL -o /tmp/ktx.deb \ + "https://github.com/KhronosGroup/KTX-Software/releases/download/v${KTX_VERSION}/KTX-Software-${KTX_VERSION}-Linux-${ktx_arch}.deb"; \ + apt-get update; \ + apt-get install -y /tmp/ktx.deb; \ + rm -f /tmp/ktx.deb; \ + rm -rf /var/lib/apt/lists/*; \ + ktx --version + +# Node CLI tools: gltfpack (geometry). obj2gltf/gltf-pipeline are the deprecated +# legacy path, kept until it is removed. +RUN npm install -g gltfpack obj2gltf gltf-pipeline + +WORKDIR /usr/local/educelab/preppy +COPY . . + +# Install the embed helper's npm deps (@gltf-transform/core + meshoptimizer) and +# the Python package with the `preview` extra (trimesh + pyrender). The install +# is editable so the embed helper resolves node_modules relative to the package +# source (assemble.NODE_DIR) — the same reason the .def uses --editable. +RUN npm install --prefix preppy/node \ + && python3 -m pip install --upgrade pip wheel setuptools \ + && python3 -m pip install --editable '.[preview]' + +# No restrictive ENTRYPOINT: any console script (preppy, preppy-check-tools, +# preppy-obj2glb, preppy-merge-items) can be used as the command. Bare +# `docker run ` prints the pipeline's help. +CMD ["preppy", "-h"] diff --git a/README.md b/README.md index 24ba484..eb3cb62 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# DRI Voyager Preppy +# Preppy Mesh preparation for DRI Voyager. @@ -37,8 +37,8 @@ deps installed once as shown above. Verify everything is installed and new enough: ```shell -voyager-check-tools # all CLI tools + optional model-preview backend -voyager-check-tools preview # just probe the model-preview render toolchain +preppy-check-tools # all CLI tools + optional model-preview backend +preppy-check-tools preview # just probe the model-preview render toolchain ``` The `preview` line is informational: it renders a tiny offscreen frame to confirm @@ -47,12 +47,12 @@ pipeline still runs — thumbnails fall back to a texture crop. ## Usage -`voyager-preppy` turns source OBJs + textures into **one self-contained `.glb` +`preppy` turns source OBJs + textures into **one self-contained `.glb` per variant** (meshopt geometry with embedded KTX2) plus a viewer-native per-object manifest. ```shell -voyager-preppy -i config.json -o out/ +preppy -i config.json -o out/ ``` The input config is a flat array of **objects**, each with a flat `variants[]` @@ -66,7 +66,7 @@ Relative `obj` paths resolve against `--data-root` (default: the current working directory), so the config file can live anywhere: ```shell -voyager-preppy -i config.json -o out/ --data-root /path/to/meshes/ +preppy -i config.json -o out/ --data-root /path/to/meshes/ ``` Output layout (per-object directory named by `prefix`, defaults to `id`): @@ -108,7 +108,7 @@ capture of what `` shows. Rendering needs the `preview` extra missing the run falls back to the texture center-crop (`--thumbnail-mode texture`) automatically. -Run `voyager-preppy -h` for the complete list. +Run `preppy -h` for the complete list. ### Hosting / caching @@ -137,7 +137,30 @@ python -m pytest tests/ Tests that need the external toolchain (`magick`/`ktx`/`gltfpack`/`node`), `pymeshlab`, or an offscreen-GL backend **skip cleanly** when those are absent, so a bare run still covers the pure logic. CI runs this across Python 3.11–3.13 (plus a manual `integration` -job that exercises the full toolchain); see `.gitlab-ci.yml`. +job that exercises the full toolchain); see [`.github/workflows/ci.yml`](.github/workflows/ci.yml). + +## Docker + +A prebuilt multi-arch (amd64/arm64) image bundling the full toolchain is published +to the GitHub Container Registry on every push to `develop` (tag `edge`) and on +release tags (`vX.Y.Z`, `latest`): + +```shell +docker pull ghcr.io/educelab/preppy:edge + +# The image's default command is `preppy -h`; run the pipeline over a mounted dir: +docker run --rm -v "$PWD":/data -w /data ghcr.io/educelab/preppy:edge \ + preppy -i config.json -o out/ + +# Any of the console scripts work as the command, e.g.: +docker run --rm ghcr.io/educelab/preppy:edge preppy-check-tools +``` + +To build it locally (see [`Dockerfile`](Dockerfile)): + +```shell +docker build -t preppy . +``` > The `` web component that consumes this pipeline's output lives in a > separate repository (`dri-voyager`), with its own TypeScript/Vitest/Playwright @@ -145,7 +168,7 @@ job that exercises the full toolchain); see `.gitlab-ci.yml`. ### Legacy path (deprecated) -The single-object `voyager-obj2glb` tool (and its `convert.py` core) still uses +The single-object `preppy-obj2glb` tool (and its `convert.py` core) still uses `obj2gltf` + `gltf-pipeline` to emit a Draco-compressed GLB. It is **deprecated** in favor of the delivery pipeline above and will be removed. To keep using it during the transition: diff --git a/preppy/apps/file_prep.py b/preppy/apps/file_prep.py index d05c18e..2ff7f67 100644 --- a/preppy/apps/file_prep.py +++ b/preppy/apps/file_prep.py @@ -1,4 +1,4 @@ -"""``voyager-preppy`` — batch orchestrator for the delivery pipeline. +"""``preppy`` — batch orchestrator for the delivery pipeline. For each object in the input config, and for **each variant independently** (no geometry grouping — ADR-0002 amended), run the validated chain: diff --git a/preppy/apps/obj_to_glb.py b/preppy/apps/obj_to_glb.py index 7546f17..323c3a0 100644 --- a/preppy/apps/obj_to_glb.py +++ b/preppy/apps/obj_to_glb.py @@ -9,7 +9,7 @@ def main(): - print('[deprecated] voyager-obj2glb uses the legacy obj2gltf + gltf-pipeline ' + print('[deprecated] preppy-obj2glb uses the legacy obj2gltf + gltf-pipeline ' 'path, which will be removed once the meshopt/KTX2 delivery pipeline ' 'lands.', file=sys.stderr) diff --git a/preppy/node/package.json b/preppy/node/package.json index cdb7d29..20d9182 100644 --- a/preppy/node/package.json +++ b/preppy/node/package.json @@ -1,9 +1,9 @@ { - "name": "dri-voyager-preppy-node", + "name": "preppy-node", "version": "1.0.0", "private": true, "type": "module", - "description": "Node helper (embed.mjs) that embeds KTX2 textures into the meshopt geometry glb for dri-voyager-preppy. Run `npm install` here before using the delivery pipeline.", + "description": "Node helper (embed.mjs) that embeds KTX2 textures into the meshopt geometry glb for preppy. Run `npm install` here before using the delivery pipeline.", "engines": { "node": ">=20" }, diff --git a/preppy/preview.py b/preppy/preview.py index 893fc99..fca9b3d 100644 --- a/preppy/preview.py +++ b/preppy/preview.py @@ -49,7 +49,7 @@ def probe() -> "tuple[bool, str]": initializes an :class:`~pyrender.OffscreenRenderer` and renders a trivial scene — the GL-context creation is the part that fails on headless nodes, so a bare import check would report a false positive. Used by - ``voyager-check-tools`` to tell whether the rendered thumbnail is available + ``preppy-check-tools`` to tell whether the rendered thumbnail is available (the pipeline falls back to a texture crop when it is not). """ import os diff --git a/preppy/tools.py b/preppy/tools.py index bca3f9d..259528f 100644 --- a/preppy/tools.py +++ b/preppy/tools.py @@ -14,7 +14,7 @@ ``convert.py`` used for ``obj2gltf`` / ``gltf-pipeline``. Native binaries (``ktx``, ``mogrify``) are resolved by :func:`shutil.which` as-is. -Use :func:`check_all` (or the ``voyager-check-tools`` console script) to report +Use :func:`check_all` (or the ``preppy-check-tools`` console script) to report what is available, and :func:`require` in the leaf modules to fail early with an actionable message when a needed tool is missing or too old. """ @@ -34,7 +34,7 @@ #: Default wall-clock limit (seconds) for a single external tool invocation via #: :func:`run`. Generous — encoding/decimating a large mesh legitimately takes a #: while — but bounded so a wedged tool can't hang a whole batch indefinitely. -#: The ``voyager-preppy`` ``--tool-timeout`` flag overrides this at startup. +#: The ``preppy`` ``--tool-timeout`` flag overrides this at startup. DEFAULT_TIMEOUT: Optional[float] = 600.0 #: Short, fixed limit for version probes (:func:`check_tool`); reading a version diff --git a/setup.cfg b/setup.cfg index 1ecc3df..edf3691 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,12 +1,12 @@ [metadata] -name = dri-voyager-preppy +name = preppy version = 1.1.0 author = Seth Parker author_email = c.seth.parker@uky.edu description = Mesh preparation for DRI Voyager long_description = file: README.md long_description_content_type = text/markdown -url = https://gitlab.com/educelab/dri-voyager-preppy +url = https://github.com/educelab/preppy classifiers = Programming Language :: Python :: 3 Programming Language :: Python :: 3.11 @@ -52,7 +52,7 @@ exclude = test [options.entry_points] console_scripts = - voyager-preppy = preppy.apps.file_prep:main - voyager-obj2glb = preppy.apps.obj_to_glb:main - voyager-merge-items = preppy.apps.merge_items:main - voyager-check-tools = preppy.apps.check_tools:main + preppy = preppy.apps.file_prep:main + preppy-obj2glb = preppy.apps.obj_to_glb:main + preppy-merge-items = preppy.apps.merge_items:main + preppy-check-tools = preppy.apps.check_tools:main diff --git a/singularity/dri-voyager-preppy.def b/singularity/preppy.def similarity index 83% rename from singularity/dri-voyager-preppy.def rename to singularity/preppy.def index bddb0ed..83de02c 100644 --- a/singularity/dri-voyager-preppy.def +++ b/singularity/preppy.def @@ -6,7 +6,7 @@ From: python:3.11-slim OS Debian %files - /tmp/dri-voyager-preppy /usr/local/educelab/dri-voyager-preppy + /tmp/preppy /usr/local/educelab/preppy %post -c /bin/bash set -e @@ -48,13 +48,13 @@ From: python:3.11-slim npm install -g gltfpack obj2gltf gltf-pipeline # Install the embed helper's npm deps (@gltf-transform/core + meshoptimizer) - npm install --prefix /usr/local/educelab/dri-voyager-preppy/preppy/node + npm install --prefix /usr/local/educelab/preppy/preppy/node # Create a venv for this project - python3 -m venv /usr/local/educelab/dri-voyager-preppy/.venv/ + python3 -m venv /usr/local/educelab/preppy/.venv/ # Activate virtualenv - source /usr/local/educelab/dri-voyager-preppy/.venv/bin/activate + source /usr/local/educelab/preppy/.venv/bin/activate # Update pip python3 -m pip install --upgrade pip wheel setuptools @@ -63,14 +63,14 @@ From: python:3.11-slim # thumbnail is a rendered model preview. pyrender renders offscreen via OSMesa # (libosmesa6, above) with PYOPENGL_PLATFORM=osmesa set in %environment; if the # GL backend fails on a given node the thumbnail falls back to a texture crop. - python3 -m pip install --editable '/usr/local/educelab/dri-voyager-preppy[preview]' + python3 -m pip install --editable '/usr/local/educelab/preppy[preview]' # Make writable echo "Cleaning up installation directory..." chmod --recursive a+rw /usr/local/educelab/ git config --global credential.helper cache git config --global credential.helper 'cache --timeout=3600' - git config --global --add safe.directory /usr/local/educelab/dri-voyager-preppy/ + git config --global --add safe.directory /usr/local/educelab/preppy/ %environment # Headless offscreen GL backend for the pyrender model-preview thumbnail. @@ -83,7 +83,7 @@ From: python:3.11-slim exit 1 fi - source /usr/local/educelab/dri-voyager-preppy/.venv/bin/activate + source /usr/local/educelab/preppy/.venv/bin/activate exec "$@" %help diff --git a/templates/prep-models.schema.json b/templates/prep-models.schema.json index 6282351..2fa3354 100644 --- a/templates/prep-models.schema.json +++ b/templates/prep-models.schema.json @@ -1,9 +1,9 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://gitlab.com/educelab/dri-voyager-preppy/prep-models.schema.json", + "$id": "https://github.com/educelab/preppy/prep-models.schema.json", "title": "DRI Voyager Data Prep Input File", - "description": "Input to voyager-preppy: a flat array of objects, each declaring a flat variants[] (ADR-0002 amended — no geometry grouping). Each variant becomes one self-contained glb; each object gets one per-object manifest.", + "description": "Input to preppy: a flat array of objects, each declaring a flat variants[] (ADR-0002 amended — no geometry grouping). Each variant becomes one self-contained glb; each object gets one per-object manifest.", "type": "array", "items": { "$ref": "#/$defs/object" }, diff --git a/tests/test_integration.py b/tests/test_integration.py index d147346..8d6bbfb 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,4 +1,4 @@ -"""End-to-end smoke test: run the ``voyager-preppy`` orchestrator over a trimmed, +"""End-to-end smoke test: run the ``preppy`` orchestrator over a trimmed, synthetic multi-material sample (standing in for the large `mvs` data, which can't be committed) and assert the emitted manifest + asset set. @@ -84,7 +84,7 @@ def sample(tmp_path): def _run(config_path, out_dir, *extra): # The sample's obj paths are relative to the config's dir, so point # --data-root there (default is the CWD). - argv = ['voyager-preppy', '-i', str(config_path), '-o', str(out_dir), + argv = ['preppy', '-i', str(config_path), '-o', str(out_dir), '--data-root', str(config_path.parent), *extra] from preppy.apps import file_prep old = sys.argv diff --git a/tests/test_preview.py b/tests/test_preview.py index 19e75dd..1062706 100644 --- a/tests/test_preview.py +++ b/tests/test_preview.py @@ -148,7 +148,7 @@ def _run_check_tools(monkeypatch, argv): import contextlib from preppy.apps import check_tools - monkeypatch.setattr('sys.argv', ['voyager-check-tools', *argv]) + monkeypatch.setattr('sys.argv', ['preppy-check-tools', *argv]) buf = io.StringIO() with contextlib.redirect_stdout(buf): code = check_tools.main() diff --git a/tests/test_tools.py b/tests/test_tools.py index 869d04f..e704055 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1,7 +1,7 @@ """Unit tests for the pure logic in ``preppy.tools`` — version parsing, comparison, executable-name resolution, and report formatting. These are the easy-to-get-wrong bits; the actual subprocess probing is covered by the -``voyager-check-tools`` console script against a real install. +``preppy-check-tools`` console script against a real install. """ import sys From f1f031b4ca474c10d978a0f9db51145e551569e4 Mon Sep 17 00:00:00 2001 From: Seth Parker Date: Thu, 23 Jul 2026 11:03:17 -0400 Subject: [PATCH 2/4] tests: skip magick-dependent preview tests when ImageMagick is absent Two _render_thumbnail tests (texture-crop fallback + texture mode) and the two render_preview tests shell out to `magick` to build/crop fixtures, but lacked the availability guard their sibling had. On a bare runner (GitHub Actions unit matrix installs neither ImageMagick nor the preview extra) they failed with FileNotFoundError instead of skipping, breaking the documented "bare run covers pure logic, tool tests skip cleanly" contract. The full texture path is still exercised by the manual `integration` job, which installs ImageMagick. Add a shared `_magick_available()` helper + `requires_magick` skip marker and apply it to the four magick-dependent tests (folding in the one inline probe). Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_preview.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/test_preview.py b/tests/test_preview.py index 1062706..ae924ce 100644 --- a/tests/test_preview.py +++ b/tests/test_preview.py @@ -35,6 +35,22 @@ def _backend_available() -> bool: return False +def _magick_available() -> bool: + try: + sp.run(['magick', '-version'], check=True, capture_output=True) + return True + except Exception: + return False + + +#: Skip tests that build/crop textures with ImageMagick when `magick` is absent, +#: honoring the "bare runner covers pure logic" contract; the full texture path +#: is exercised by the manual `integration` CI job (which installs ImageMagick). +requires_magick = pytest.mark.skipif( + not _magick_available(), + reason='ImageMagick (magick) not available to build/crop test textures') + + def _write_textured_obj(tmp_path: Path) -> Path: """A 2-material OBJ + normalized PNGs; MTL references source names that do NOT exist on disk, so a successful render proves the resolver swapped in the @@ -59,14 +75,10 @@ def _write_textured_obj(tmp_path: Path) -> Path: return src +@requires_magick @pytest.mark.skipif(not _backend_available(), reason='preview extra (trimesh/pyrender) not installed') def test_render_preview_writes_jpeg_from_normalized_textures(tmp_path): - try: - sp.run(['magick', '-version'], check=True, capture_output=True) - except Exception: - pytest.skip('ImageMagick not available to build the test textures') - src = _write_textured_obj(tmp_path) # source_*.png deliberately absent; only norm_*.png exist. textures = {'source_00.png': src / 'norm_00.png', @@ -174,6 +186,7 @@ def test_check_tools_rejects_unknown_target(monkeypatch): _run_check_tools(monkeypatch, ['bogus']) +@requires_magick @pytest.mark.skipif(not _backend_available(), reason='preview extra (trimesh/pyrender) not installed') def test_render_preview_survives_mixed_face_mesh(tmp_path): @@ -200,6 +213,7 @@ def _min_opts(**over) -> argparse.Namespace: return argparse.Namespace(**base) +@requires_magick def test_render_thumbnail_falls_back_to_texture_crop(tmp_path, monkeypatch): """When the preview backend is unavailable, ``_render_thumbnail`` logs and falls back to the texture center-crop instead of aborting the run.""" @@ -220,6 +234,7 @@ def boom(*a, **k): assert dst.is_file() # produced by the texture-crop fallback +@requires_magick def test_render_thumbnail_texture_mode_skips_render(tmp_path, monkeypatch): """``--thumbnail-mode texture`` must not invoke the renderer at all.""" from preppy.apps import file_prep From bbf2695dee3ba10a6654d1e289fcd829ad7a7ece Mon Sep 17 00:00:00 2001 From: Seth Parker Date: Thu, 23 Jul 2026 14:01:03 -0400 Subject: [PATCH 3/4] ci: install ImageMagick + preview extra in the unit test job The tool-availability skips exist so a bare local-dev machine can still run the pure-logic suite, but CI should actually exercise those paths to catch regressions (as the missing magick guard showed). Install ImageMagick, the `preview` extra (trimesh/pyrender), and OSMesa (PYOPENGL_PLATFORM=osmesa) in the py matrix job so the texture-normalization and rendered-thumbnail tests run rather than skip. The heavier geometry toolchain (ktx/gltfpack/node) and pymeshlab stay in the manual `integration` job. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d6b2f0..4643062 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,14 +15,21 @@ jobs: # =========================================================================== # Python package (preppy/) # --------------------------------------------------------------------------- - # The pure-Python logic is covered here across supported interpreters. Tests - # that need the external toolchain (magick/ktx/gltfpack/node), pymeshlab, or an - # offscreen-GL backend self-skip when those are absent, so a bare runner still - # gets full logic coverage plus the historic `preppy -h` smoke test. + # Runs the suite across supported interpreters. ImageMagick + the `preview` + # extra (trimesh/pyrender, with OSMesa for offscreen GL) are installed so the + # texture-normalization and rendered-thumbnail paths actually execute and + # catch regressions here rather than self-skipping. The heavier geometry + # toolchain (ktx/gltfpack/node) and pymeshlab remain the manual `integration` + # job's job; those tests still self-skip here. The tool-availability skips are + # for bare local-dev machines, not CI. # =========================================================================== test: name: py ${{ matrix.python-version }} runs-on: ubuntu-latest + env: + # Headless offscreen GL backend so the pyrender model-preview tests run + # instead of skipping on a missing GL context. + PYOPENGL_PLATFORM: osmesa strategy: fail-fast: false matrix: @@ -35,10 +42,15 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install system deps (ImageMagick + offscreen GL) + run: | + sudo apt-get update + sudo apt-get install -y imagemagick libosmesa6 libgl1 + - name: Install run: | python -m pip install --upgrade pip setuptools wheel - pip install '.[test]' + pip install '.[test,preview]' - name: Run tests run: python -m pytest tests/ From 369389f2d9b3c3cfd7aecfa35134b1458d82b19b Mon Sep 17 00:00:00 2001 From: Seth Parker Date: Thu, 23 Jul 2026 14:10:35 -0400 Subject: [PATCH 4/4] Bump version to 1.2.0 (preppy + preppy-node in sync) Release covering the GitHub migration: rename to Preppy, GitHub Actions CI, and the multi-arch GHCR Docker build. Keep the Node embed helper (preppy-node) version synced with the Python package. Co-Authored-By: Claude Opus 4.8 (1M context) --- preppy/node/package.json | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/preppy/node/package.json b/preppy/node/package.json index 20d9182..11ebf06 100644 --- a/preppy/node/package.json +++ b/preppy/node/package.json @@ -1,6 +1,6 @@ { "name": "preppy-node", - "version": "1.0.0", + "version": "1.2.0", "private": true, "type": "module", "description": "Node helper (embed.mjs) that embeds KTX2 textures into the meshopt geometry glb for preppy. Run `npm install` here before using the delivery pipeline.", diff --git a/setup.cfg b/setup.cfg index edf3691..adf4e0b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = preppy -version = 1.1.0 +version = 1.2.0 author = Seth Parker author_email = c.seth.parker@uky.edu description = Mesh preparation for DRI Voyager