diff --git a/.agents.txt b/.agents.txt new file mode 100644 index 00000000..a6b5857e --- /dev/null +++ b/.agents.txt @@ -0,0 +1,11 @@ +Repository instructions for all agents + +Never create random or unsolicited Markdown (.md) files in this repository. +Create a new Markdown file only when the user explicitly requests that file. +Update the relevant existing documentation when documenting code changes. +Keep task plans, audit reports, progress notes, summaries, and scratch notes in +the conversation unless the user explicitly requests a saved artifact. +Do not evade this rule by creating unsolicited reports with another extension. +Keep temporary files outside the repository and remove your temporary artifacts. +Preserve existing useful documentation, licenses, and vendored documentation. +Follow these instructions throughout the repository, including subdirectories. diff --git a/.codex/config.toml b/.codex/config.toml new file mode 100644 index 00000000..00809464 --- /dev/null +++ b/.codex/config.toml @@ -0,0 +1 @@ +project_doc_fallback_filenames = [".agents.txt"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 300f6d67..39cc173c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,14 +84,46 @@ jobs: python-version: "3.12" cache: pip - name: Install research codec dependencies - run: python -m pip install -e '.[dev,klt,n4mc,qndf]' - - name: Run real in-process encode and fresh-decode contracts on CPU + run: python -m pip install -e '.[dev,klt,n4mc,qndf,usd,gaussians]' constriction 'DracoPy>=2,<3' + - name: Run default suite with research dependencies + run: python -m pytest + - name: Run native N4MC model and loss contracts run: >- python -m pytest - open4d/codec/tests/test_research_cpu.py - open4d/codec/tests/test_qndf_preprocess.py - open4d/codec/tests/test_qndf_entrypoints.py open4d/codecs/n4mc/tests/test_codec.py + open4d/codecs/n4mc/tests/test_losses.py + + native-numerics: + name: Native tracker and editor / .NET 10 + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 15 + env: + DOTNET_NOLOGO: "1" + DOTNET_CLI_TELEMETRY_OPTOUT: "1" + OPEN4D_TEST_DOTNET: dotnet + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "10.0.x" + - name: Install native regression test dependencies + run: python -m pip install -e '.[dev]' + - name: Build and execute native numerical contracts + run: | + dotnet --info + python -m pytest open4d/codec/tests/test_tracker_numerics.py open4d/codec/tests/test_editor_numerics.py --junitxml="${RUNNER_TEMP}/native-numerics.xml" + python - <<'PY' + import os + from pathlib import Path + from xml.etree import ElementTree + report = ElementTree.parse(Path(os.environ["RUNNER_TEMP"]) / "native-numerics.xml") + cases = report.findall(".//testcase") + assert cases and not any(case.find("skipped") is not None for case in cases), "native numerical contracts must execute" + PY io-tools: name: I/O / real optional formats and benchmark smoke @@ -154,6 +186,37 @@ jobs: if-no-files-found: ignore retention-days: 7 + browser: + name: Browser / streamer and web clients + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - uses: actions/setup-node@v4 + with: + node-version: "24" + cache: npm + cache-dependency-path: | + open4d/webclients/system/Server/package-lock.json + open4d/webclients/system/WebClient/package-lock.json + - name: Install browser test dependencies and standalone streamer + run: python -m pip install -e '.[dev]' -e open4d/streamer 'DracoPy>=2,<3' scikit-image + - name: Check installed streamer and run its headless tests + run: | + python -I -c "import streamer; print(streamer.__file__)" + python -m pytest open4d/streamer/streamer_tests + - name: Install and build browser clients + run: | + npm ci --prefix open4d/webclients/system/Server + npm ci --prefix open4d/webclients/system/WebClient + npm run build --prefix open4d/webclients/system/WebClient + - name: Run web client and server JavaScript tests + run: node --test open4d/webclients/tests/*.js open4d/webclients/system/Server/test_*.js + package: name: Install / ${{ matrix.os }} / Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} diff --git a/README.md b/README.md index fbbe5cba..762e7092 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ texture coordinates and custom attributes. Choose a research codec explicitly. For example, after setting up V-DMC: ```python -encoded = open4d.encode(sequence, "wave.v4d", codec="vdmc") +encoded = open4d.encode(sequence, "wave.vmesh", codec="vdmc") decoded = open4d.decode(encoded) open4d.visualize(decoded) decoded.close() @@ -67,25 +67,49 @@ decoded.close() artifact. Close a decoded mesh sequence when finished, or use `with`: ```python -with open4d.decode("wave.v4d") as decoded: +with open4d.decode("wave.vmesh") as decoded: print(len(decoded), "frames") ``` The [notebook](examples/open4d_sequence_codec.ipynb) walks through these calls, reconstruction and streaming in separate short cells. +Short `.vmesh` examples: + +- [Preserve compressed state through USDC](examples/vmesh/01_container_and_usdc.ipynb) +- [The eight mesh codecs](examples/vmesh/02_mesh_codecs.ipynb) +- [Vega, QUEEN and 3DGStream](examples/vmesh/03_gaussian_codecs.ipynb) +- [ReRF](examples/vmesh/04_rerf.ipynb) | Codec | Input | Output | Backend setup | | --- | --- | --- | --- | -| `vdmc` | Mesh sequence | `.v4d` | Build the V-DMC submodule and configure its encoder and decoder | -| `faster_vdmc` | Mesh sequence | `.v4d` | Build the faster V-DMC submodule and configure its encoder and decoder | -| `tvmc` | Mesh sequence | `.tvmc` directory | [TVMC setup](open4d/codecs/tvmc/README.md) | -| `tsmc` | Mesh sequence | `.tsmc` directory | [TSMC setup](open4d/codecs/tsmc/README.md) | -| `klt` | Mesh sequence converted to TSDF volumes | `.k4d` | Research source and `.[klt]` | -| `n4mc` | Mesh sequence converted to TSDF volumes | `.n4d` | Research source and `.[n4mc]` | -| `qndf`, `qndf-int8` | Mesh frames | `.q4d`, `.qi4d` | Research source and `.[qndf]` | -| `vega` | Gaussian splat frames | `.vega` directory | [Vega CUDA environment](open4d/reconstruction/vega/README.md) | - -QNDF-int8 now writes version 2 artifacts. Version 1 artifacts must be encoded again. +| `vdmc` | Mesh sequence | `.vmesh` | Build the V-DMC submodule and configure its encoder and decoder | +| `faster_vdmc` | Mesh sequence | `.vmesh` | Build the faster V-DMC submodule and configure its encoder and decoder | +| `tvmc` | Mesh sequence | `.vmesh` | [TVMC setup](open4d/codecs/tvmc/README.md) | +| `tsmc` | Mesh sequence | `.vmesh` | [TSMC setup](open4d/codecs/tsmc/README.md) | +| `klt` | Mesh sequence converted to TSDF volumes | `.vmesh` | Research source and `.[klt]` | +| `n4mc` | Mesh sequence converted to TSDF volumes | `.vmesh` | Research source and `.[n4mc]` | +| `qndf`, `qndf-int8` | Mesh frames | `.vmesh` | Research source and `.[qndf]` | +| `vega` | Gaussian splat frames or native run | `.vmesh` | [Vega CUDA environment](open4d/reconstruction/vega/README.md) | +| `queen`, `3dgstream`, `rerf` | Native temporal research output | `.vmesh`, with native USDC interchange | Method-specific CUDA runtime for evaluation | + +All twelve public codecs write standalone `.vmesh` files. The embedded profile +identifies the codec; there is no separate public compressed output type for each +method. Existing native outputs can be packed without recompression. Standard +PLY, OBJ and USD import/export remain available. + +Mesh codecs decode to mesh sequences for ordinary USDC geometry export. +To preserve a compressed artifact exactly, construct `open4d.NativeSequence` +and save it to USDC using the custom `VMESH` prim, as shown in the +[container notebook](examples/vmesh/01_container_and_usdc.ipynb). This works with +every VMESH profile. Gaussian and field methods load as `NativeSequence` and +evaluate their native models explicitly. N4MC shares a model across independent +frame latents; QNDF has independent frame models. +[VMESH](docs/api.md#vmesh-format) is a standalone custom format: its own versioned +header and metadata directly carry native codec payloads. It has no private +Open4D/V3C wrapper and does not claim MPEG V-DMC interoperability. A V-DMC +profile carries the actual encoder bitstream as a native payload. Older artifacts +require explicit migration with `open4d.migrate_legacy`; normal load/save paths +accept VMESH rather than silently interpreting an older compressed type. KLT, N4MC and QNDF run in Python. TVMC, TSMC, V-DMC and Gaussian methods use separate research runtimes. N4MC and QNDF currently process frames independently; @@ -163,7 +187,7 @@ send(mesh_sequence(frames=30)) ``` This sends decoded mesh arrays over TCP, at their recorded frame timing. It is -not a compression method. Both calls default to this computer on port 7000. +not a compression method. Both calls default to this computer on port 47004. Pass `host=` and `port=` for another address. Remote transport needs a trusted network or SSH tunnel; this protocol has no authentication or encryption. Use `realtime=False` to transfer a recorded sequence as fast as possible. @@ -196,8 +220,9 @@ Read splat frames, then encode them with the local Vega adaptation: from open4d import load_gaussians, encode, decode frames = [load_gaussians("frame_0000.ply"), load_gaussians("frame_0001.ply")] -encoded = encode(frames, "capture.vega", codec="vega") -decoded = decode(encoded) +encoded = encode(frames, "capture.vmesh", codec="vega") +native = decode(encoded) # owned native state; inspecting it needs no CUDA +decoded = native.decode() # runs the Vega runtime, returning NeuralGaussianFrame values ``` A `GaussianSplats` frame contains `positions` `(N, 3)`, positive `scales` @@ -230,11 +255,10 @@ requires its native viewer. The Qt viewer and TCP stream currently take meshes. - `open4d demo`, `open4d inspect` and `open4d view` provide command-line access. - `open4d.io.write_sequence` exports mesh folders; `open4d.save` writes OpenUSD - or explicitly selected codec artifacts. There is no default `.o4d` encoder. + or `.vmesh` with an explicitly selected codec. - `open4d.compare_sequences` measures mesh error with the `.[metrics]` extra. - [Viewer examples](examples/visualization/README.md) include GIF export and comparisons. - [Contributor setup and tests](CONTRIBUTING.md) cover optional dependencies and packaging. -The general `.o4d` format is separate work. Existing codec-specific formats -remain in use. Publication is still blocked by the unresolved component rights +Publication is still blocked by the unresolved component rights in [THIRD_PARTY.md](THIRD_PARTY.md); preparing the package does not resolve them. diff --git a/SECURITY.md b/SECURITY.md index 5630a353..5de40b9e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -28,4 +28,39 @@ private data, and identifying capture content. references. - Revoke exposed credentials before removing them from Git history. +## Known limitations + +VMESH framing and hashes validate container integrity; they do not authenticate +the publisher or make native payloads safe to execute. Inspection, packing, +and compressed USD round trips do not invoke native models. Decoding can: + +- [QUEEN's compressed loader](open4d/reconstruction/queen/scene/gaussian_model.py) + uses `pickle.load`, which can execute code from a malicious residual. +- [ReRF's player](open4d/reconstruction/rerf/rerf_stream/bitstream.py) loads + checkpoints with `torch.load` without explicitly selecting `weights_only`; + older Torch runtimes can execute serialized code. Other research entry points + also load executable checkpoints. A worker subprocess is not a security sandbox. + +The [Unity TVMC backend](integrations/unity/TVMCUnity/CPP_Backend/src) has +unchecked OBJ face indices, a decoder access to row 100 without a minimum-row +check, and a `FetchFrame` interface without an output-capacity argument. +Its matrix loader also permits very large allocations before checking that all +declared bytes exist. These are source-level memory safety and resource risks; +runtime exploitation and sanitizer validation have not been performed here. + +The Node control API, decoded-frame TCP transport, and research WebSocket +bridges have no authentication. Origin and Host checks do not authenticate +native clients. Some bridges bind to all interfaces, and static file routes +follow symlinks; their network exposure and asset roots must be controlled. + +Vendored QUEEN and 3DGStream trainer/renderer argument loaders evaluate +`cfg_args`. Some 3DGStream conversion scripts and the ReRF LLFF loader construct +shell commands from paths or options. Use trusted inputs for those entry points; +the VMESH import path uses JSON instead of these evaluated configurations. + +Legacy ZIP/NPZ paths do not consistently bound aggregate expanded bytes. +Repacking a legacy N4MC archive can exhaust disk before VMESH validation, and +neural architecture or volume dimensions can trigger large allocations even +after safe deserialization. + The absence of a warning is not a claim that a component is production-ready. diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index bd55a776..53ac65b5 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -82,6 +82,11 @@ boundaries; they do not resolve any release block above: frame on longer clips, use the last available frame on short clips, and remove unused alignment work that also accessed an unavailable sixth frame. +Additional local patches (2026-09-23): N4MC restores SSIM loss; ReRF validates +inputs before native loading and makes test data configurable; webclients use +a local tile record and default to loopback. Licenses and release boundaries +are unchanged. + ## Required record for new material Any copied source, submodule, binary, model, dataset, paper, image, archive, or diff --git a/docs/api.md b/docs/api.md index e2e7ace2..f8283429 100644 --- a/docs/api.md +++ b/docs/api.md @@ -7,16 +7,17 @@ sequences independently of their storage format: import open4d with open4d.load("capture.usdc") as sequence: - open4d.save(sequence, "capture.o4d") + open4d.save(sequence, "capture-copy.usdc") open4d.visualize(sequence) # A path can go straight to the lazy viewer; it is closed when the window exits. -open4d.visualize("capture.o4d") +open4d.visualize("capture-copy.usdc") ``` `.usd`, `.usda`, `.usdc`, and `.usdz` are OpenUSD interchange containers. -`.o4d` and the registered codec suffixes are codec artifacts. Both carry whole -sequences and use the same `Sequence` interface. `open4d.unload(sequence)` is an +All registered compression methods use `.vmesh`; its descriptor identifies the +codec. Mesh profiles expose `Sequence`, while native Gaussian and field profiles +expose `NativeSequence`. `open4d.unload(sequence)` is an explicit, idempotent alternative to the context manager. Frame folders and individual meshes remain supported as import paths; frames @@ -40,9 +41,9 @@ out of a V-DMC bitstream. `open4d.Representation` is the axis to gate on, and taxonomy but have no concrete type yet; they land with the camera model they need in order to be comparable at a known pose. -The containers and codecs described below are the mesh path, and the one most -completely covered. Check a specific codec before assuming it round-trips -points or Gaussians. +The mesh codecs accept triangle meshes. Native Gaussian and field methods require +their own trained states or calibrated reconstruction inputs; a mesh file alone +does not provide those inputs. ## Writing sequences @@ -55,6 +56,11 @@ Single mesh-file exports require `allow_lossy=True` because that storage cannot preserve sequence timing, metadata, or topology declarations. Trimesh-backed OFF/GLB/glTF color export also requires that opt-in because OFF drops vertex color and GLB/glTF quantize canonical float colors to eight bits. +STL also requires `allow_lossy=True`: it discards unused vertices and vertex +identity, so exported manifests clear correspondence and topology guarantees. +STL/GLB/glTF reject geometry without triangles. Gaussian clouds cannot be +exported through these mesh writers. Ordinary USD geometry writing accepts +triangle meshes; compressed VMESH USD interchange is described below. OpenUSD is the public interchange container. `--pack-usd out.usdc` packs any source into one compressed `.usdc` file carrying the frame rate, the key-frame @@ -77,6 +83,8 @@ For a loaded sequence, pass browser options such as `name="capture"` and decoded-mesh TCP transport and pairs with `open4d.receive`. Existing `open4d.stream(sequence, host, port)` calls, and frame iterables without browser options, retain that TCP behavior without requiring `open4d-streamer`. +The TCP default is `127.0.0.1:47004`; pass port 7000 explicitly when talking to +an older Open4D receiver. `rungs` is the quality ladder. The first is the rendition a client plays by default and the rest are what it can switch to mid-playback, so a one-entry @@ -103,18 +111,52 @@ other. It is not this API and shares no code with it. ## Codecs -Five lossless, in-process reference codecs are included: `raw`, `deflate`, -`bzip2`, `lzma`, and byte-level `rle` (`npz` remains the default DEFLATE alias). -They share a safe NumPy-array container so they compare storage strategies, not -research geometry models. +Every public codec encodes to a single `.vmesh` file: -Source checkouts register in-process adapters for `klt`, `n4mc`, `qndf`, and -`qndf-int8`; the lightweight wheel omits them until their provenance review is -complete. Open4D's separate `temporal-delta` and `temporal-pca` experiments are -not the repository's TVMC or TSMC pipelines. The V-DMC adapters do not execute +```python +open4d.encode("capture.usdc", "capture.vmesh", codec="tvmc") +with open4d.load("capture.vmesh") as sequence: + open4d.save(sequence, "decoded.usdc") +``` + +The embedded codec is detected on load. The standalone [VMESH format](#vmesh-format) +preserves native compressed payloads and timing. See the [native `.vmesh` notebook](../examples/vmesh/01_container_and_usdc.ipynb) +for inspection, packing, extraction, and an exact compressed-state USDC round trip. +The [mesh example](../examples/vmesh/02_mesh_codecs.ipynb) demonstrates encoding +and saving decoded geometry as USDC. + +Vega, QUEEN, 3DGStream and ReRF carry native compressed models. Import a +research run with `open4d.import_native(run, codec="queen", config=...)`, or +use `open4d.encode(run, "capture.vmesh", codec="queen", config=...)`. +Loading returns a `NativeSequence`; `native.decode(runtime=..., python=...)` +evaluates the actual temporal models. Saving it to USDC preserves the exact +compressed representation using the custom `VMESH` USD prim. This route does not +train a neural scene from an arbitrary mesh USD. See the linked guide for +required model/configuration files and representation-specific outputs. + +KLT, N4MC, QNDF and QNDF-int8 accept mesh sequences or mesh USDC, for example +`open4d.encode(..., codec="n4mc")`. Loading reconstructs a mesh `Sequence`. +N4MC's profile identifies independent quantized TSDF latents sharing one model; +it does not claim temporal prediction. KLT carries a shared transform basis and +compressed frame coefficients. QNDF carries coarse meshes and independent +displacement models; QNDF-int8 uses quantized model weights. +The guide also documents exact compressed-state preservation in native USDC. + +`available_codecs()` lists the public research adapters: `klt`, `n4mc`, `qndf`, +`qndf-int8`, `vdmc`, `faster_vdmc`, `tvmc`, `tsmc`, `vega`, `queen`, +`3dgstream`, and `rerf`. Use `.vmesh` for every registered compression method +and USD for interchange. NumPy archives, model checkpoints and Draco files are +native payload encodings within a profile, not alternate public sequence types. + +The lightweight wheel includes the adapters but excludes their research +implementations. KLT, N4MC and QNDF need a source checkout and their optional +dependencies; set `OPEN4D_RESEARCH_ROOT` to that checkout when using an installed +wheel. Internal benchmark experiments are not public codec choices. The V-DMC adapters do not execute shell scripts, but they do invoke configured native encoder and decoder processes once per sequence. Callers can also register another `open4d.codec.Codec`. +Native mesh processes run without a time limit by default; set +`OPEN4D_NATIVE_TIMEOUT` to a positive number of seconds to bound them. For an all-registered-codec attempt using `4d_files/Rafa_Approves_hd_4k`, open [`examples/open4d_sequence_codec.ipynb`](../examples/open4d_sequence_codec.ipynb). @@ -128,10 +170,118 @@ The N4MC and QNDF adapters accept `device="auto"` (CUDA, then Apple Metal/MPS, then CPU), or an explicit `"cuda"`, `"mps"`, or `"cpu"`. QNDF-int8 can train on CUDA or Metal, but its quantized decoder remains CPU-only. Override the notebook selection with `OPEN4D_NOTEBOOK_DEVICE=mps` when needed. +KLT decoding also accepts `device="auto"`. ### Test coverage Normal CI runs dependency-complete CPU encode/fresh-decode contracts for KLT, -N4MC, QNDF, and QNDF-int8. The larger two-format Rafa quality/export matrix is +N4MC, QNDF, and QNDF-int8. The larger Rafa quality/export matrix is an additional CUDA acceptance test gated by `OPEN4D_TEST_RESEARCH_CODECS=1` and `OPEN4D_RAFA_DATASET`; it is not presented as part of ordinary CI coverage. + +## VMESH format + +VMESH is a standalone custom data type for a compressed sequence. Its header, +metadata, codec profiles and records belong to VMESH. It does not manufacture a +V3C stream, insert private SEI messages, or serialize Open4D geometry objects. +It is not an interoperable MPEG V-DMC interchange format. A V-DMC profile can +carry a real encoder-produced V3C bitstream as its native payload. + +The Python API uses `Sequence` or `NativeSequence` to expose a file. Those are +in-memory interfaces, not data types embedded in the file. + +### Binary layout + +All integers in the container are unsigned and big endian. The file starts with +eight bytes: `56 4d 45 53 48 00 01 00` (`VMESH`, zero, version 1, reserved zero). +Each following record is: + +| Field | Bytes | Meaning | +| --- | --- | --- | +| Record length | 4 | Length of the record header plus its data | +| Record version | 1 | Must be 1 | +| Kind | 1 | 0 = manifest, 1 = native payload chunk, 2 = end | +| File ID | 4 | Zero-based native file index; zero for manifest/end | +| Offset | 8 | Offset within the native payload; zero for manifest/end | +| Data | variable | JSON manifest, native bytes, or end digest | + +There is exactly one manifest, followed by all payload chunks in manifest file +order, followed by exactly one end record. The end data is the 32-byte SHA-256 +of the exact manifest bytes. No trailing bytes are permitted. + +The UTF-8 JSON manifest has `schema: "vmesh/1"`, `codec`, `native_version: 1`, +`representation`, `dependency_mode`, `frame_count`, `sequence`, `files`, and +codec-specific `native` settings when required. `sequence` contains version 1, +the codec ID, frame indices, timestamps in seconds, JSON metadata, and optional +normalization/topology fields. It cannot carry an application `schema` field. +Each file record contains `id`, `name`, `role`, byte `size`, and hexadecimal +`sha256`. Filenames and roles must exactly match the selected codec profile; +input cannot supply arbitrary extraction paths. + +The manifest is at most 16 MiB; payload chunks are at most 1 MiB. Frame and +profile file counts are bounded at 65,536 (including the generated adapter +metadata in the profile file limit). Some profiles have lower frame limits +because they require several files per frame. Duplicate JSON keys, non-finite +numbers, reordered chunks, offsets, unknown versions, hash failures and extra +records are rejected. These hashes detect corruption; they do not authenticate +a publisher or make an executable research payload safe to decode. + +### Native profiles + +| Codec | Native payloads | Dependencies | +| --- | --- | --- | +| KLT | `decoder_context.pt`, `_quantized_indices.zst`, `_quantized_metadata.npz` | Shared basis, independent coefficients | +| QNDF / QNDF-int8 | `frame_.pt`: subdivided base mesh, displacement context and model | Independent frames; int8 decode on CPU | +| N4MC | `checkpoint.pt`, `frame_.npz` | Shared model, independent TSDF latents | +| TVMC | `reference.drc`, `displacement_.drc` and `.npy` | Shared mesh reference; displacement and vertex order | +| TSMC | `reference.drc`, `B_matrix.txt`, `T_matrix.txt`, `delta_trajectories_encoded.npy`, `entropy_model.npz` | Whole temporal group | +| V-DMC / faster V-DMC | `sequence.vmesh`, optional `decoder.cfg` | Actual native encoder bitstream | +| Vega | `manifest.json`, `color_model.pt`, `frame_.pt` | Shared appearance and key/residual state | +| QUEEN | `initial.ply`, `frame_.pkl` for residual frames | Previous Gaussian frame | +| 3DGStream | `initial.ply`, `ntc_config.json`, `ntc_.pth`, optional `added_.ply` | Previous frame and transform architecture | +| ReRF | Field/model configuration, `rgb_net.tar`, headers, occupancy/channel entropy files, optional PCA/motion | Previous field and group keys | + +Frame ordinals are zero-padded to six digits except Vega's four-digit native +index. ReRF uses its declared native frame IDs and channel suffixes. The profile +defines exact names and order; this table is not a user-supplied extraction map. +The separate `frames` delivery profile carries standard per-frame PLY, Draco, +splat or image payloads and is not another research compression method. + +Native files are copied verbatim. A directory's `metadata.json` is parsed into +the VMESH manifest; it is **not a carried file**. Unpacking reconstructs a +`metadata.json` only as an input for the existing codec adapters. JSON whitespace +in that generated adapter file is not preserved. Native payload bytes are. + +Use `open4d.codec.inspect_vmesh`, `pack_vmesh`, and `unpack_vmesh` to inspect, +create and extract files without loading native models. Inspection verifies +all hashes. Extraction is staged and published only after complete validation. + +### USD interchange + +Compressed interchange uses a custom prim of type `VMESH` at `/VMESH`, with +`vmesh:schema = "vmesh.usd/1"`, timeline samples in `vmesh:frameIndex`, and +static `vmesh:payload:chunkNNNNNN` byte arrays. `vmesh:payloadManifest` describes +the exact carried VMESH file and its hash. This prim preserves compressed state; +it does not claim to be a renderable `UsdGeomMesh` or Gaussian schema. + +### Compatibility + +Normal load, save, encode and decode accept standalone VMESH for compressed +sequences. Older artifacts require explicit `open4d.migrate_legacy` conversion; +renaming their extension is insufficient. Use `open4d.import_native` for a +supported trained native run and `pack_vmesh` for a validated profile directory. +Neither operation trains a model or turns arbitrary native files into a sequence. +Native research encodings remain inside the VMESH profile. Previously exported +USD compressed state must use the custom `VMESH` prim described above. +Raw native MPEG V-DMC bitstreams remain supported as read/import inputs using +external frame timing; new compressed output is always standalone VMESH. + +```python +open4d.migrate_legacy("older_artifact", "converted.vmesh", fps=30.0) +``` + +Migration covers earlier research-codec exports and packed browser clips. Older +private benchmark artifacts must be re-encoded from their original geometry. +Migration is an explicit compatibility operation. `overwrite=False` protects an +existing destination, and `fps` supplies timing only when the old input did not +carry it. The result is validated VMESH; normal loading does not silently migrate. diff --git a/examples/open4d_sequence_codec.ipynb b/examples/open4d_sequence_codec.ipynb index 82ce2ebe..93a710a4 100644 --- a/examples/open4d_sequence_codec.ipynb +++ b/examples/open4d_sequence_codec.ipynb @@ -54,9 +54,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Encode\n", - "\n", - "After building V-DMC, set `OPEN4D_VDMC_ENCODER` and `OPEN4D_VDMC_DECODER` to its programs. The codec is always chosen explicitly. This writes the existing V-DMC artifact; it does not define the future general `.o4d` format.\n" + "After building V-DMC, set `OPEN4D_VDMC_ENCODER` and `OPEN4D_VDMC_DECODER` to its programs. Choose the codec explicitly. Every public codec writes a standalone `.vmesh` file containing its native payload and timeline." ], "id": "open4d-04" }, @@ -70,7 +68,7 @@ }, "outputs": [], "source": [ - "encoded = open4d.encode(sequence, \"wave.v4d\", codec=\"vdmc\")\n" + "encoded = open4d.encode(sequence, \"wave.vmesh\", codec=\"vdmc\")" ], "id": "open4d-05" }, @@ -209,14 +207,14 @@ "Then run the sender in the other process:\n", "\n", "```python\n", - "from open4d import stream\n", + "from open4d import send\n", "from open4d.demo import mesh_sequence\n", "\n", - "stream(mesh_sequence(frames=30))\n", + "send(mesh_sequence(frames=30))\n", "```\n", "\n", "This sends the mesh frames at their recorded timing over TCP on this computer.\n", - "It does not compress them. Both sides use port 7000; pass `host=` and `port=`\n", + "It does not compress them. Both sides use port 47004; pass `host=` and `port=`\n", "for another address. Use a trusted network or SSH tunnel for remote transport.\n" ], "id": "open4d-15" @@ -231,7 +229,7 @@ "\n", "```python\n", "frames = [open4d.load_gaussians(\"frame_0000.ply\"), open4d.load_gaussians(\"frame_0001.ply\")]\n", - "encoded = open4d.encode(frames, \"capture.vega\", codec=\"vega\")\n", + "encoded = open4d.encode(frames, \"capture.vmesh\", codec=\"vega\")\n", "decoded = open4d.decode(encoded)\n", "```\n", "\n", diff --git a/examples/streaming_demo.py b/examples/streaming_demo.py index bdeeb73a..a8848553 100644 --- a/examples/streaming_demo.py +++ b/examples/streaming_demo.py @@ -7,7 +7,7 @@ python examples/streaming_demo.py Needs three things beyond the base install: the optional `open4d-streamer` -package (``pip install -e open4d/streamer``), `open4d[draco]` for the Draco +package (``pip install -e open4d/streamer``), `DracoPy` for the Draco rungs, and SciPy for the nearest-neighbour search the quality column uses. Output goes to ``examples/out/``, which the repository ignores. diff --git a/examples/visualization/README.md b/examples/visualization/README.md index 390d32aa..2f5f4b8b 100644 --- a/examples/visualization/README.md +++ b/examples/visualization/README.md @@ -110,7 +110,7 @@ one and say so. | Folder of `.obj` or `.ply` frames | nothing | | Folder of `.stl` `.off` `.glb` `.gltf` frames | `.[tools]` | | One USD file (`.usd` `.usda` `.usdc` `.usdz`) | `.[usd]` | -| Raw MPEG V-DMC bitstream (`.vmesh`) | a compatible native V-DMC decoder | +| Standalone `.vmesh` with a mesh profile | the codec's configured decoder/runtime | | One mesh file | as above | Frames are ordered by **the last number in the filename**, so `frame_2.obj` comes @@ -118,9 +118,11 @@ before `frame_10.obj` — but `frame_003_qp9.obj` sorts on 9, not 3. A codec tha puts a parameter last will silently misalign every frame and look far worse than it is; rename before comparing. A frame with no faces is drawn as a point cloud. -### Raw V-DMC bitstreams +### Compressed VMESH sequences -Raw `.vmesh` input needs a compatible native decoder: +The VMESH descriptor selects the codec and preserves frame indices, timestamps +and coordinate metadata. Configure that codec's runtime before loading. For +example, a V-DMC profile needs its native decoder: ```bash export OPEN4D_VDMC_DECODER=/path/to/vmesh-decoder @@ -128,10 +130,10 @@ export OPEN4D_VDMC_DECODER_CONFIG=/path/to/decoder.cfg # optional python examples/visualization/visualize_sequence.py capture.vmesh --info ``` -This path is read-only and geometry-only. It defaults to 30 fps, accepts -`--fps N`, and displays the coordinates emitted by the decoder. Open4D removes -the temporary decoded OBJ files when the sequence closes. Use `.v4d` when the -sequence also needs Open4D timestamps, metadata, and coordinate bounds. +The viewer displays decoded geometry. Open4D removes temporary decoded files +when the sequence closes. `--fps N` overrides playback speed without replacing +the stored VMESH timestamps. Native Gaussian and neural-field profiles use +their representation-specific decode/render paths rather than this mesh viewer. ## Flags @@ -169,7 +171,7 @@ with open4d.load("capture.usdc") as sequence: print(len(sequence), sequence.duration, sequence.fps) open4d.visualize(sequence) -open4d.visualize("capture.o4d") +open4d.visualize("capture.vmesh") ``` Loading and playback are lazy — frames decode on access. Frame directories and diff --git a/examples/visualization/compare_frames.py b/examples/visualization/compare_frames.py index f51192ba..b9e01e90 100644 --- a/examples/visualization/compare_frames.py +++ b/examples/visualization/compare_frames.py @@ -77,6 +77,9 @@ def all_distances(self) -> np.ndarray: def summary(self) -> "DirectionalSummary": """Aggregate the per-frame figures over the whole comparison.""" + aggregate = mesh_metrics.SequenceComparison( + tuple(frame.error for frame in self.frames), (), self.peak, self.metric + ) return DirectionalSummary( symmetric_rms=float( np.sqrt(np.mean([f.error.symmetric_rms ** 2 for f in self.frames])) @@ -85,9 +88,7 @@ def summary(self) -> "DirectionalSummary": worst_frame=int( np.argmax([f.error.symmetric_rms for f in self.frames]) ), - mean_psnr_db=float( - np.mean([f.error.symmetric_psnr_db for f in self.frames]) - ), + mean_psnr_db=aggregate.symmetric_psnr_db, ) @@ -203,7 +204,8 @@ def resolve_clamp( return 0.0 if percentile is None: return float(np.max(distances)) - return float(np.percentile(distances, percentile)) + clamp = float(np.percentile(distances, percentile)) + return clamp if clamp > 0 else float(np.max(distances)) def diffuse_intensity(frame: RenderFrame) -> np.ndarray: diff --git a/examples/visualization/compare_sequences.py b/examples/visualization/compare_sequences.py index 207fd37a..c2714e60 100644 --- a/examples/visualization/compare_sequences.py +++ b/examples/visualization/compare_sequences.py @@ -6,8 +6,8 @@ --metric plane --csv error.csv --save compare.gif This is the comparison viewer; `visualize_sequence.py` is the plain one. Both -read the same sources — `.o4d` codec artifacts, time-sampled USD files, frame -directories, raw V-DMC `.vmesh` bitstreams, or standalone mesh imports — +read the same sources — `.vmesh` mesh profiles, time-sampled USD files, frame +directories, or standalone mesh imports — through the same loader. Two synchronized panes: the reference as geometry, the decoded mesh coloured by @@ -153,7 +153,7 @@ def report_error(comparison) -> None: summary = comparison.summary() print(f"\n sequence symmetric RMS : {summary.symmetric_rms:.6g}") print(f" sequence Hausdorff : {summary.hausdorff:.6g}") - print(f" mean symmetric PSNR : {summary.mean_psnr_db:.2f} dB") + print(f" sequence symmetric PSNR: {summary.mean_psnr_db:.2f} dB") print(f" worst frame : {summary.worst_frame}") diff --git a/examples/visualization/frame_sources.py b/examples/visualization/frame_sources.py index 7332a86b..3a5d07cf 100644 --- a/examples/visualization/frame_sources.py +++ b/examples/visualization/frame_sources.py @@ -6,11 +6,11 @@ from open4d import load as _open_sequence from open4d.codec import available_codecs +from open4d.codec._vmesh_format import probe_codec from open4d.io import available_formats, inspect_sequence DEFAULT_FPS = 30.0 _USD_SUFFIXES = {".usd", ".usda", ".usdc", ".usdz"} -_RAW_CODEC_SUFFIXES = {".vmesh"} def _codec_suffixes(): @@ -30,8 +30,8 @@ def supported_formats() -> str: ) target = sequence_lines if info.id == "usd" else frame_lines target.append(f" {'/'.join(info.suffixes):<24}{extra}".rstrip()) - sequence_lines.append(" research codec files/directories: " + ", ".join(sorted(_codec_suffixes()))) - sequence_lines.append(" .vmesh needs a native V-DMC decoder") + sequence_lines.append(" research codec files: " + ", ".join(sorted(_codec_suffixes())) + + " (needs the selected codec runtime)") return "\n".join((*sequence_lines, *frame_lines)) @@ -41,7 +41,7 @@ def source_kind(path: Path | str) -> str: if not path.exists(): raise SystemExit(f"{path} does not exist") if path.suffix.lower() in ( - _USD_SUFFIXES | _codec_suffixes() | _RAW_CODEC_SUFFIXES + _USD_SUFFIXES | _codec_suffixes() ): return "sequence-file" if path.is_dir(): @@ -56,14 +56,15 @@ def source_kind(path: Path | str) -> str: def open_sequence(path: Path | str, fps: float | None = None): """Load a source, using ``fps`` for manifest-free frame timing.""" path = Path(path) - uses_import_fps = (path.is_dir() and path.suffix.lower() not in _codec_suffixes()) or path.suffix.lower() in _RAW_CODEC_SUFFIXES - return _open_sequence(path, fps=fps if uses_import_fps else None) + uses_import_fps = path.is_dir() and path.suffix.lower() not in _codec_suffixes() + raw_vdmc = path.is_file() and path.suffix.lower() == ".vmesh" and probe_codec(path) is None + return _open_sequence(path, fps=fps if uses_import_fps or raw_vdmc else None) def describe_source(path: Path | str, frame_count: int | None = None) -> str: """Describe a source without eagerly parsing its frame geometry.""" path = Path(path) - if path.suffix.lower() in _codec_suffixes() | _RAW_CODEC_SUFFIXES: + if path.suffix.lower() in _codec_suffixes(): if frame_count is None: with _open_sequence(path) as opened: frame_count = len(opened) diff --git a/examples/visualization/tests/test_compare.py b/examples/visualization/tests/test_compare.py index ed9b0020..9a15155f 100644 --- a/examples/visualization/tests/test_compare.py +++ b/examples/visualization/tests/test_compare.py @@ -521,3 +521,24 @@ def test_comparison_stride_keeps_playback_speed(folders): args = cli.build_parser().parse_args([str(folders[0]), str(folders[1]), "--stride", "2", "--info"]) cli.run(args) assert args.fps == 15 +def test_sparse_errors_do_not_collapse_the_color_scale(): + from types import SimpleNamespace + from compare_frames import resolve_clamp + + distances = np.zeros(1000) + distances[-1] = 1 + frames = [SimpleNamespace(decoded_distances=distances, reference_distances=distances)] + assert resolve_clamp(frames, None, 99) > 0 + + +def test_summary_psnr_uses_aggregate_error(reference_sequence, shifted_sequence): + from compare_frames import compare_sequences, Comparison + from open4d.metrics import SequenceComparison + + exact = compare_sequences(reference_sequence, reference_sequence) + shifted = compare_sequences(reference_sequence, shifted_sequence) + frames = [exact.frames[0], shifted.frames[0]] + combined = Comparison(frames, "point", shifted.peak, shifted.clamp, 99, None) + expected = SequenceComparison(tuple(f.error for f in frames), (0., 1.), shifted.peak, "point") + assert np.isfinite(combined.summary().mean_psnr_db) + assert combined.summary().mean_psnr_db == pytest.approx(expected.symmetric_psnr_db) diff --git a/examples/visualization/tests/test_frame_sources.py b/examples/visualization/tests/test_frame_sources.py index 1457ccd1..f680c0ab 100644 --- a/examples/visualization/tests/test_frame_sources.py +++ b/examples/visualization/tests/test_frame_sources.py @@ -24,13 +24,13 @@ def test_example_helpers_treat_codec_artifacts_as_whole_sequences(tmp_path, monk from open4d.codec import _api from open4d.codec._npz import NumPyZipCodec - monkeypatch.setitem(_api._CODECS, "fixture", NumPyZipCodec()) + monkeypatch.setitem(_api._CODECS, "npz", NumPyZipCodec()) mesh = TriangleMesh( [[0.0, 0, 0], [1, 0, 0], [0, 1, 0]], [[0, 1, 2]] ) artifact = open4d.save( Sequence(MemoryFrameProvider([Frame(0, 0.0, mesh)])), - tmp_path / "capture.o4d", codec="fixture", + tmp_path / "capture.vmesh", codec="npz", ) assert frame_sources.source_kind(artifact) == "sequence-file" @@ -72,9 +72,11 @@ def test_raw_vmesh_fps_is_forwarded_to_the_public_loader(tmp_path, monkeypatch): assert received == {"path": bitstream, "fps": 24} -def test_native_directory_uses_artifact_timing(tmp_path, monkeypatch): - source = tmp_path / "capture.tvmc" - source.mkdir() +def test_standalone_vmesh_uses_artifact_timing(tmp_path, monkeypatch): + from open4d.codec._npz import NumPyZipCodec + mesh = TriangleMesh([[0., 0, 0], [1, 0, 0], [0, 1, 0]], [[0, 1, 2]]) + source = NumPyZipCodec().encode(Sequence(MemoryFrameProvider([Frame(0, 1.25, mesh)])), + tmp_path / "capture.vmesh") received = {} monkeypatch.setattr(frame_sources, "_open_sequence", lambda path, **options: received.update(path=path, **options)) diff --git a/examples/visualization/tests/test_visualize_sequence.py b/examples/visualization/tests/test_visualize_sequence.py index 93ed9517..77a5b9f4 100644 --- a/examples/visualization/tests/test_visualize_sequence.py +++ b/examples/visualization/tests/test_visualize_sequence.py @@ -15,8 +15,8 @@ def test_malformed_codec_source_is_reported_without_a_traceback( tmp_path, monkeypatch ): - path = tmp_path / "broken.o4d" - path.write_bytes(b"not an Open4D artifact") + path = tmp_path / "broken.vmesh" + path.write_bytes(b"not a VMESH artifact") monkeypatch.setattr(sys, "argv", ["visualize_sequence.py", str(path), "--info"]) with pytest.raises(SystemExit) as caught: @@ -62,7 +62,7 @@ def get_frame(self, index): return Frame(index, index / 30, TriangleMesh(positions, [[0, 1, 2]])) sequence = Sequence(Provider()) - path = tmp_path / "capture.o4d" + path = tmp_path / "capture.vmesh" path.touch() captured = {} monkeypatch.setattr(cli, "open_sequence", lambda *args, **kwargs: sequence) diff --git a/examples/visualization/visualize_sequence.py b/examples/visualization/visualize_sequence.py index b9cf29d5..4d6ec685 100644 --- a/examples/visualization/visualize_sequence.py +++ b/examples/visualization/visualize_sequence.py @@ -6,7 +6,7 @@ python examples/visualization/visualize_sequence.py my_capture.usdc Point it at your own data. A source is normally a whole-sequence file such as -`.o4d`, a time-sampled `.usdc`, or a raw V-DMC `.vmesh` bitstream. A folder +`.vmesh` with a mesh codec profile or a time-sampled `.usdc`. A folder holding one mesh per frame and a standalone mesh remain useful import paths. Meshes and point clouds are both handled: a frame with no faces is drawn as a point cloud. @@ -131,7 +131,7 @@ def main() -> None: "path", type=Path, nargs="?", - help="your sequence: an .o4d, USD, or raw .vmesh file; a frame " + help="your sequence: a .vmesh mesh profile or USD file; a frame " "directory; or a standalone mesh import", ) parser.add_argument( @@ -141,8 +141,8 @@ def main() -> None: "--fps", type=float, default=None, - help="override playback; for a manifest-free frame directory or raw " - ".vmesh this also defines imported timestamps (default: 30 fps)", + help="override playback; manifest-free frame directories or raw native " + "inputs also use this for imported timestamps (default: 30 fps)", ) parser.add_argument( "--up", diff --git a/examples/vmesh/01_container_and_usdc.ipynb b/examples/vmesh/01_container_and_usdc.ipynb new file mode 100644 index 00000000..31da1cb0 --- /dev/null +++ b/examples/vmesh/01_container_and_usdc.ipynb @@ -0,0 +1,107 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "01-00", + "metadata": {}, + "source": [ + "# Preserve a .vmesh through USDC\n", + "\n", + "Set the paths below. This works with every supported codec and needs OpenUSD, but no codec runtime.\n", + "The USDC stores compressed native state in a custom VMESH prim; it is not a renderable mesh export." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01-01", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "import hashlib\n", + "from pathlib import Path\n", + "import open4d\n", + "from open4d.codec import inspect_vmesh, pack_vmesh, unpack_vmesh\n", + "\n", + "source = Path(\"motion.vmesh\")\n", + "output = Path(\"output/native-usdc\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01-02", + "metadata": {}, + "outputs": [], + "source": [ + "info = inspect_vmesh(source)\n", + "print(info[\"codec\"], info[\"frame_count\"], \"frames\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01-03", + "metadata": {}, + "outputs": [], + "source": [ + "with open4d.NativeSequence(source) as native:\n", + " open4d.save(native, output / \"native.usdc\", overwrite=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01-04", + "metadata": {}, + "outputs": [], + "source": [ + "with open4d.load(output / \"native.usdc\") as native:\n", + " restored = open4d.save(native, output / \"restored.vmesh\", overwrite=True)\n", + "\n", + "with source.open(\"rb\") as a, restored.open(\"rb\") as b:\n", + " assert hashlib.sha256(a.read()).digest() == hashlib.sha256(b.read()).digest()\n", + "print(\"Identical compressed bytes\")" + ] + }, + { + "cell_type": "markdown", + "id": "01-05", + "metadata": {}, + "source": [ + "To extract and repack the same native payloads without decoding:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01-06", + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "\n", + "with tempfile.TemporaryDirectory() as folder:\n", + " native_files = unpack_vmesh(source, Path(folder) / \"native\")\n", + " repacked = pack_vmesh(native_files, output / \"repacked.vmesh\", overwrite=True)\n", + "print(inspect_vmesh(repacked)[\"codec\"])\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/vmesh/02_mesh_codecs.ipynb b/examples/vmesh/02_mesh_codecs.ipynb new file mode 100644 index 00000000..da274eff --- /dev/null +++ b/examples/vmesh/02_mesh_codecs.ipynb @@ -0,0 +1,104 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "02-00", + "metadata": {}, + "source": [ + "# Mesh USDC → .vmesh → mesh USDC\n", + "\n", + "Set your input and choose `klt`, `n4mc`, `qndf`, `qndf-int8`, `tvmc`, `tsmc`, `vdmc` or `faster_vdmc`.\n", + "Configure its [research backend](../../README.md) first. These examples encode geometry only." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02-01", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import open4d\n", + "from open4d.codec import inspect_vmesh\n", + "\n", + "source = Path(\"input.usdc\")\n", + "output = Path(\"output/mesh\")\n", + "codec = \"tvmc\"\n", + "encode_options = {}\n", + "decode_options = {}" + ] + }, + { + "cell_type": "markdown", + "id": "02-02", + "metadata": {}, + "source": [ + "For N4MC on CPU, set `codec=\"n4mc\"` and both option dictionaries to `{\"device\": \"cpu\"}`.\n", + "Add `resolution` and `epochs` to `encode_options` to control training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02-03", + "metadata": {}, + "outputs": [], + "source": [ + "encoded = open4d.encode(\n", + " source, output / \"motion.vmesh\", codec=codec, overwrite=True, **encode_options)\n", + "print(inspect_vmesh(encoded)[\"frame_count\"], \"frames encoded\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02-04", + "metadata": {}, + "outputs": [], + "source": [ + "with open4d.load(encoded, options=decode_options) as decoded:\n", + " open4d.save(decoded, output / \"reconstructed.usdc\", overwrite=True)\n", + " print(len(decoded), \"frames decoded\")" + ] + }, + { + "cell_type": "markdown", + "id": "02-05", + "metadata": {}, + "source": [ + "Older compressed artifacts require explicit migration. Run the next cell only when you have an older artifact; ordinary encode/decode use `.vmesh`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02-06", + "metadata": { + "tags": [ + "migration" + ] + }, + "outputs": [], + "source": [ + "open4d.migrate_legacy(\"older_artifact\", output / \"migrated.vmesh\", overwrite=True)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/vmesh/03_gaussian_codecs.ipynb b/examples/vmesh/03_gaussian_codecs.ipynb new file mode 100644 index 00000000..1a312dbb --- /dev/null +++ b/examples/vmesh/03_gaussian_codecs.ipynb @@ -0,0 +1,149 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "03-00", + "metadata": {}, + "source": [ + "# Native Gaussian runs → .vmesh\n", + "\n", + "Run the section for your codec after setting its paths. Use the configuration that trained\n", + "the run. Packing needs no CUDA; the final decoding cell requires the codec's configured runtime." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03-01", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import open4d\n", + "\n", + "output = Path(\"output/gaussians\")" + ] + }, + { + "cell_type": "markdown", + "id": "03-02", + "metadata": {}, + "source": [ + "Vega" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03-03", + "metadata": { + "tags": [ + "vega" + ] + }, + "outputs": [], + "source": [ + "with open4d.import_native(\"runs/capture\", codec=\"vega\") as native:\n", + " open4d.save(native, output / \"vega.vmesh\", overwrite=True)\n", + " print(len(native), \"Vega frames packed\")" + ] + }, + { + "cell_type": "markdown", + "id": "03-04", + "metadata": {}, + "source": [ + "QUEEN" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03-05", + "metadata": { + "tags": [ + "queen" + ] + }, + "outputs": [], + "source": [ + "with open4d.import_native(\"runs/queen\", codec=\"queen\", config=\"runs/queen/config.yaml\") as native:\n", + " open4d.save(native, output / \"queen.vmesh\", overwrite=True)\n", + " print(len(native), \"QUEEN frames packed\")" + ] + }, + { + "cell_type": "markdown", + "id": "03-06", + "metadata": {}, + "source": [ + "3DGStream. The run must contain its initial model, NTC checkpoints and architecture configuration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03-07", + "metadata": { + "tags": [ + "3dgstream" + ] + }, + "outputs": [], + "source": [ + "with open4d.import_native(\"runs/3dgstream\", codec=\"3dgstream\",\n", + " ntc_config=\"runs/3dgstream/ntc_config.json\") as native:\n", + " open4d.save(native, output / \"3dgstream.vmesh\", overwrite=True)\n", + " print(len(native), \"3DGStream frames packed\")" + ] + }, + { + "cell_type": "markdown", + "id": "03-08", + "metadata": {}, + "source": [ + "Decode one saved artifact. Set `OPEN4D__ROOT` and `OPEN4D__PYTHON` for its native CUDA runtime." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03-09", + "metadata": { + "tags": [ + "native-cuda" + ] + }, + "outputs": [], + "source": [ + "with open4d.load(output / \"queen.vmesh\") as native:\n", + " frames = native.decode()\n", + " print(len(frames), \"frames;\", len(frames[0].positions), \"Gaussians in the first frame\")" + ] + }, + { + "cell_type": "markdown", + "id": "03-10", + "metadata": {}, + "source": [ + "Use the [USDC example](01_container_and_usdc.ipynb) to preserve any of these compressed artifacts through USDC." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/vmesh/04_rerf.ipynb b/examples/vmesh/04_rerf.ipynb new file mode 100644 index 00000000..091deafc --- /dev/null +++ b/examples/vmesh/04_rerf.ipynb @@ -0,0 +1,120 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "04-00", + "metadata": {}, + "source": [ + "# Native ReRF run → .vmesh\n", + "\n", + "Set the compressed run path. It must contain its entropy files, motion, shared RGB network\n", + "and resolved `decoder_config.json`, including rendering bounds. Packing needs no CUDA." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04-01", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "import json\n", + "from pathlib import Path\n", + "import open4d\n", + "from open4d.codec import inspect_vmesh\n", + "\n", + "source = Path(\"runs/rerf\")\n", + "output = Path(\"output/rerf\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04-02", + "metadata": {}, + "outputs": [], + "source": [ + "with open4d.import_native(source, codec=\"rerf\", config=source / \"decoder_config.json\") as native:\n", + " artifact = open4d.save(native, output / \"rerf.vmesh\", overwrite=True)\n", + "print(inspect_vmesh(artifact)[\"frame_count\"], \"frames packed\")" + ] + }, + { + "cell_type": "markdown", + "id": "04-03", + "metadata": {}, + "source": [ + "Decode using `OPEN4D_RERF_ROOT` and `OPEN4D_RERF_PYTHON` for the native CUDA runtime." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04-04", + "metadata": { + "tags": [ + "native-cuda" + ] + }, + "outputs": [], + "source": [ + "with open4d.load(artifact) as native:\n", + " fields = native.decode()\n", + "print(fields[0].density.shape, fields[0].features.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "04-05", + "metadata": {}, + "source": [ + "Render a calibrated camera. `camera.json` contains `camera_id`, `width`, `height`,\n", + "`fx`, `fy`, `cx`, `cy` and a 4×4 `c2w` matrix for this run." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04-06", + "metadata": { + "tags": [ + "native-cuda" + ] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "camera = json.loads(Path(\"camera.json\").read_text())\n", + "with open4d.load(artifact) as native:\n", + " images = native.render(camera)\n", + "plt.imshow(images[0])\n", + "plt.axis(\"off\");" + ] + }, + { + "cell_type": "markdown", + "id": "04-07", + "metadata": {}, + "source": [ + "Use the [USDC example](01_container_and_usdc.ipynb) for an exact compressed-state round trip." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/integrations/open3d/open4d_open3d.py b/integrations/open3d/open4d_open3d.py index 8c530590..1e96b923 100644 --- a/integrations/open3d/open4d_open3d.py +++ b/integrations/open3d/open4d_open3d.py @@ -10,6 +10,7 @@ from collections.abc import Mapping, Sequence from typing import Any +import warnings import numpy as np from numpy.typing import ArrayLike, NDArray @@ -106,6 +107,15 @@ def _vertex_attribute( def _colors(value: ArrayLike, vertex_count: int) -> NDArray[np.float64]: + raw = np.asarray(value) + if raw.shape == (vertex_count, 4): + alpha = raw[:, 3] + maximum = 255 if np.issubdtype(raw.dtype, np.integer) else 1 + if not np.isfinite(alpha).all() or np.any(alpha < 0) or np.any(alpha > maximum): + raise ValueError("Color alpha must be finite and within the color range") + if np.any(alpha != maximum): + warnings.warn("Open3D legacy geometry does not preserve color alpha", UserWarning, stacklevel=3) + value = raw[:, :3] array = _vertex_attribute(value, vertex_count, "Colors") if np.issubdtype(np.asarray(value).dtype, np.integer): if np.any(array < 0) or np.any(array > 255): diff --git a/integrations/open3d/tests/test_open4d_open3d.py b/integrations/open3d/tests/test_open4d_open3d.py index 402617c6..1909298c 100644 --- a/integrations/open3d/tests/test_open4d_open3d.py +++ b/integrations/open3d/tests/test_open4d_open3d.py @@ -17,6 +17,14 @@ TRIANGLES = np.array([[0, 1, 2]], dtype=np.uint32) +def test_rgba_colors_are_accepted_with_explicit_alpha_warning(): + colors = np.tile([1., 0.5, 0., 0.4], (3, 1)) + frame = Frame(0, 0., TriangleMesh(VERTICES, TRIANGLES, colors=colors)) + with pytest.warns(UserWarning, match="alpha"): + mesh = frame_to_open3d(frame) + np.testing.assert_allclose(np.asarray(mesh.vertex_colors), colors[:, :3]) + + def test_core_frame_conversion() -> None: frame = Frame(7, 0.25, TriangleMesh(VERTICES, TRIANGLES)) diff --git a/open4d/__init__.py b/open4d/__init__.py index 895cf6b7..f3742b26 100644 --- a/open4d/__init__.py +++ b/open4d/__init__.py @@ -25,11 +25,12 @@ ) from ._api import load, save, unload, reconstruct, stream from ._streamer import StreamerDependencyError -from .codec import available_codecs, decode_sequence as decode, encode_sequence as encode +from .codec import available_codecs, decode_sequence as decode, encode_sequence as encode, migrate_legacy from .streaming import receive, send from .gaussians import GaussianSplats, GaussianRun, NeuralGaussianFrame, load_gaussians from .metrics import compare_meshes, compare_sequences from .visualization import visualize +from .native import NativeSequence, NeuralFieldFrame, import_native __all__ = [ "ATTRIBUTE_FLOAT_DTYPE", @@ -41,6 +42,9 @@ "GaussianSplats", "GaussianRun", "NeuralGaussianFrame", + "NativeSequence", + "NeuralFieldFrame", + "import_native", "FrameProvider", "GaussianCloud", "Geometry", @@ -59,6 +63,7 @@ "available_codecs", "decode", "encode", + "migrate_legacy", "compare_meshes", "compare_sequences", "dtypes", diff --git a/open4d/_api.py b/open4d/_api.py index 27d40704..2ca2fd9a 100644 --- a/open4d/_api.py +++ b/open4d/_api.py @@ -10,9 +10,10 @@ from .core import Sequence from .gaussians import NeuralGaussianFrame from .io import open_sequence, write_sequence +from .native import NativeSequence, save_native _USD_SUFFIXES = frozenset((".usd", ".usda", ".usdc", ".usdz")) -_RAW_VMESH_SUFFIX = ".vmesh" +_VMESH_SUFFIX = ".vmesh" def _options(value: Mapping[str, object] | None) -> dict[str, object]: @@ -46,33 +47,39 @@ def load( codec: str | Codec | None = None, fps: float | None = None, options: Mapping[str, object] | None = None, -) -> Sequence | tuple[NeuralGaussianFrame, ...]: - """Open a sequence artifact, raw V-DMC bitstream, or geometry source.""" +) -> Sequence | NativeSequence | tuple[NeuralGaussianFrame, ...]: + """Open VMESH, native V-DMC input, or a geometry source.""" if format is not None and codec is not None: raise TypeError("format and codec are mutually exclusive") values = _options(options) path = Path(source) if codec is not None: - if fps is not None and path.suffix.lower() != _RAW_VMESH_SUFFIX: + if fps is not None and path.suffix.lower() != _VMESH_SUFFIX: raise TypeError("fps applies to I/O sources, not codec artifacts") _set_raw_fps(values, fps) return decode_sequence(path, codec=codec, **values) - if not path.is_dir() and path.suffix.lower() == _RAW_VMESH_SUFFIX: + if not path.is_dir() and path.suffix.lower() == _VMESH_SUFFIX: if format is not None: - raise TypeError("format cannot select a raw V-DMC bitstream") + raise TypeError("format cannot select a .vmesh bitstream") _set_raw_fps(values, fps) - return decode_sequence(path, codec="vdmc", **values) + return decode_sequence(path, **values) if path.suffix.lower() in _codec_suffixes(): if format is not None: raise TypeError("format cannot select a codec artifact") if fps is not None: raise TypeError("fps applies to I/O sources, not codec artifacts") return decode_sequence(path, **values) + if path.suffix.lower() in (".usd", ".usda", ".usdc"): + from .io._native_usd import is_native_usd, read_native_usd + if is_native_usd(path): + if fps is not None or format is not None: + raise TypeError("native USD has its own representation and timestamps") + return read_native_usd(path, **values) return open_sequence(path, format=format, fps=fps, options=values) def save( - sequence: Sequence, + sequence: Sequence | NativeSequence, destination: str | os.PathLike[str], *, codec: str | Codec | None = None, @@ -82,8 +89,14 @@ def save( options: Mapping[str, object] | None = None, ) -> Path: """Write a sequence to an OpenUSD file or a research codec artifact.""" + if isinstance(sequence, NativeSequence): + if fps is not None or up_axis is not None or options: + raise TypeError("native repacking preserves its configuration/timeline and accepts no geometry options") + if codec is not None and (codec if isinstance(codec, str) else codec.id) != sequence.codec: + raise ValueError("codec does not match the native representation") + return save_native(sequence, destination, overwrite=overwrite) if not isinstance(sequence, Sequence): - raise TypeError("sequence must be an open4d.Sequence") + raise TypeError("sequence must be an open4d.Sequence or NativeSequence") if not isinstance(overwrite, bool): raise TypeError("overwrite must be bool") path = Path(destination) @@ -139,8 +152,8 @@ def save( def unload(sequence: Sequence) -> None: """Release resources owned by a loaded sequence.""" - if not isinstance(sequence, Sequence): - raise TypeError("sequence must be an open4d.Sequence") + if not isinstance(sequence, (Sequence, NativeSequence)): + raise TypeError("sequence must be an open4d.Sequence or NativeSequence") sequence.close() diff --git a/open4d/_cli.py b/open4d/_cli.py index d96a8600..8b8a9bb2 100644 --- a/open4d/_cli.py +++ b/open4d/_cli.py @@ -9,7 +9,8 @@ import sys from . import load -from .codec import CodecError, available_codecs +from .codec import CodecError +from .codec._vmesh_format import is_mesh_profile from .core import Sequence from .demo import write_demo from .io import Open4DError @@ -145,8 +146,7 @@ def main(argv: list[str] | None = None) -> int: source = args.source.expanduser() if not source.exists(): raise FileNotFoundError(f"sequence source does not exist: {source}") - if any(source.suffix.lower() in info.suffixes and info.representation != "triangle_mesh" - for info in available_codecs()): + if source.suffix.lower() == ".vmesh" and not is_mesh_profile(source): raise ValueError("this command takes mesh sequences; use open4d.decode for Gaussian artifacts") if args.command == "view": # Check optional dependencies before an expensive codec decode. @@ -154,6 +154,8 @@ def main(argv: list[str] | None = None) -> int: _qt.check_available() with load(source, format=args.format, fps=args.input_fps) as sequence: + if not isinstance(sequence, Sequence): + raise ValueError("this command takes mesh sequences; use the native renderer for Gaussian or field data") if args.command == "inspect": info = _inspect(sequence, source) if args.json: diff --git a/open4d/_files.py b/open4d/_files.py index 89954261..ef7e2002 100644 --- a/open4d/_files.py +++ b/open4d/_files.py @@ -1,6 +1,8 @@ """Publish completed outputs while preserving existing destinations.""" +import errno import os +import shutil from pathlib import Path @@ -9,7 +11,21 @@ def publish_file(temporary: Path, destination: Path, *, overwrite: bool = False) if overwrite: temporary.replace(destination) else: - os.link(temporary, destination) + try: + os.link(temporary, destination) + except OSError as error: + if error.errno not in {errno.EPERM, errno.EOPNOTSUPP, errno.ENOSYS, errno.EXDEV}: + raise + # Filesystems such as exFAT have no hard links. Exclusive creation + # retains no-clobber semantics, though this fallback is not atomic + # for readers observing the destination while it is being copied. + with temporary.open("rb") as source: + with destination.open("xb") as target: + try: + shutil.copyfileobj(source, target) + except BaseException: + destination.unlink(missing_ok=True) + raise temporary.unlink() diff --git a/open4d/_native_worker.py b/open4d/_native_worker.py new file mode 100644 index 00000000..c27907fe --- /dev/null +++ b/open4d/_native_worker.py @@ -0,0 +1,132 @@ +"""Native temporal evaluation in isolated, method-specific Python runtimes. + +Keep this entry point compatible with ReRF's required Python 3.8. It does not +import the public package (whose base interpreter may be newer). +""" +from __future__ import annotations + +from collections import OrderedDict +import json +from pathlib import Path +import sys +from types import SimpleNamespace + +import numpy as np + + +def _save_gaussians(model, path): + def array(value): + return value.detach().float().cpu().numpy() + np.savez(path, positions=array(model.get_xyz), scales=array(model.get_scaling), + rotations=array(model.get_rotation), opacities=array(model.get_opacity).reshape(-1), + spherical_harmonics=array(model.get_features)) + + +def _queen(source, output, metadata): + import torch + from scene.gaussian_model import GaussianModel + from scene.decoders import DecoderIdentity + + profile = metadata["native"] + names = ["xyz", "f_dc", "f_rest", "sc", "rot", "op", "flow"] + # Initial PLY uses identity decoders. Later payloads carry their complete + # quantized decoder architecture and weights; no training setup is needed. + latent = SimpleNamespace(param_names=names, seed=0, gate_params=profile["gate_params"], + quant_type=["none"] * len(names)) + arguments = SimpleNamespace(gate_gamma=-0.5, gate_eta=1.01, gate_lr=0.1, gate_temp=0.3) + model = GaussianModel(profile["sh_degree"], latent, arguments) + model.latent_decoders = OrderedDict((name, DecoderIdentity()) for name in names) + model.gate_params = dict.fromkeys(names, False) + model.load_ply(str(source / "initial.ply")) + with torch.no_grad(): + for index in range(len(metadata["frames"])): + model.frame_idx = index + 1 + if index: + model.load_compressed_pkl(str(source / ("frame_%06d.pkl" % index))) + _save_gaussians(model, output / ("frame_%06d.npz" % index)) + # This is the native compressed renderer's state update, including + # decoded attributes and latent state, after each residual frame. + for name in model.get_atts: + model.prev_atts[name] = model.get_decoded_atts[name].detach().clone() + model.prev_latents[name] = model.get_atts[name].detach().clone() + + +def _gstream(source, output, metadata): + import torch + import tinycudann as tcnn + from scene.gaussian_model import GaussianModel + from ntc import NeuralTransformationCache + + profile = metadata["native"] + architecture = json.loads((source / "ntc_config.json").read_text()) + model = GaussianModel(profile["sh_degree"], profile["rotate_sh"]) + model.load_ply(str(source / "initial.ply"), 1.0) + fields = ("_xyz", "_features_dc", "_features_rest", "_opacity", "_scaling", "_rotation") + with torch.no_grad(): + _save_gaussians(model, output / "frame_000000.npz") + for index in range(1, len(metadata["frames"])): + state = torch.load(source / ("ntc_%06d.pth" % index), weights_only=True, map_location="cuda") + options = dict(n_input_dims=3, n_output_dims=8, network_config=architecture["network"]) + network = (tcnn.Network(**options) if profile["only_mlp"] else + tcnn.NetworkWithInputEncoding(**options, encoding_config=architecture["encoding"])) + model.ntc = NeuralTransformationCache(network.cuda(), state["xyz_bound_min"], state["xyz_bound_max"]) + model.ntc.load_state_dict(state) + model.query_ntc() + model.update_by_ntc() + recurrent = {name: getattr(model, name) for name in fields} + if profile["added"][index]: + added = GaussianModel(profile["sh_degree"]) + added.load_ply(str(source / ("added_%06d.ply" % index)), 1.0) + for name in fields: + setattr(model, name, torch.cat((getattr(model, name), getattr(added, name))).detach()) + _save_gaussians(model, output / ("frame_%06d.npz" % index)) + # Stage-two additions render this frame but are not NTC state. + for name, value in recurrent.items(): + setattr(model, name, value) + + +def _rerf(request, source, output, metadata): + from rerf_stream.bitstream import BitstreamPlayer + from rerf_stream.cameras import Camera + + config = json.loads((source / "decoder_config.json").read_text()) + profile = metadata["native"] + player = BitstreamPlayer(config, source, pca=profile["pca"], + pca_channels=profile["pca_channels"], group_size=profile["group_size"], + render_bounds=config["render"]) + if player.frames != [f["id"] for f in profile["frames"]]: + raise ValueError("ReRF decoded frame index disagrees with profile") + camera = None + if request["operation"] == "render": + camera = Camera(**dict(request["camera"], c2w=np.asarray(request["camera"]["c2w"]))) + for index, frame in enumerate(player.play(loop=False)): + if camera is not None: + np.save(output / ("image_%06d.npy" % index), player.render(camera)) + else: + np.savez(output / ("frame_%06d.npz" % index), + density=frame.model.density.detach().cpu().numpy(), + features=frame.model.k0.k0.detach().cpu().numpy(), + xyz_min=np.asarray(player.model_kwargs["xyz_min"]), + xyz_max=np.asarray(player.model_kwargs["xyz_max"])) + + +def main(): + request = json.loads(Path(sys.argv[1]).read_text()) + root = Path(request["runtime"]) + if not root.is_dir(): + raise FileNotFoundError("research runtime is missing: " + str(root)) + sys.path.insert(0, str(root)) + source, output = Path(request["source"]), Path(request["output"]) + metadata = json.loads((source / "metadata.json").read_text()) + if request["codec"] != metadata["codec"]: + raise ValueError("native codec disagrees with worker request") + if request["operation"] == "decode" and request["codec"] in ("queen", "3dgstream"): + (_queen if request["codec"] == "queen" else _gstream)(source, output, metadata) + elif request["operation"] in ("decode", "render") and request["codec"] == "rerf": + _rerf(request, source, output, metadata) + else: + raise ValueError("unsupported native operation") + + +if __name__ == "__main__": + main() diff --git a/open4d/_streamer.py b/open4d/_streamer.py index 9f4e4b55..2df4595b 100644 --- a/open4d/_streamer.py +++ b/open4d/_streamer.py @@ -81,6 +81,10 @@ def stream( by default and the rest are what it can switch to, so a single-entry list is a fixed-quality stream and says so. + ``fps`` applies only to a path source carrying no timing of its own. A + loaded `Sequence` always has timestamps, so the bundle plays at their rate + and ``fps`` is ignored rather than allowed to contradict them. + With ``out_dir`` omitted the bundle goes to a temporary directory, which is **not** deleted afterwards. An export is minutes of encoding and the directory is the artifact; removing it when the server stops would throw @@ -118,7 +122,6 @@ def stream( out_dir, title=title, source=None if loaded else source, - fps=int(fps) if fps else 30, ) with session: if loaded: diff --git a/open4d/codec/__init__.py b/open4d/codec/__init__.py index 65814ace..2d1ed391 100644 --- a/open4d/codec/__init__.py +++ b/open4d/codec/__init__.py @@ -13,6 +13,8 @@ from ._qndf import QNDFCodec from ._vmesh import VMeshCodec from ._tracked import TrackedMeshCodec +from ._migration import migrate_legacy +from ._vmesh_format import inspect_vmesh, pack_vmesh, unpack_vmesh __all__ = [ "Codec", @@ -27,4 +29,8 @@ "decode_sequence", "encode_sequence", "register_codec", + "inspect_vmesh", + "pack_vmesh", + "migrate_legacy", + "unpack_vmesh", ] diff --git a/open4d/codec/_api.py b/open4d/codec/_api.py index b72dcacc..f5fb13f0 100644 --- a/open4d/codec/_api.py +++ b/open4d/codec/_api.py @@ -5,6 +5,7 @@ from dataclasses import dataclass import os from pathlib import Path +from typing import TYPE_CHECKING from open4d.core import Sequence from open4d.gaussians import GaussianSplats, NeuralGaussianFrame, VEGA_CODEC @@ -13,9 +14,15 @@ from ._klt import KLT_CODEC from ._n4mc import N4MC_CODEC from ._protocol import Codec, CodecError +from ._metadata import require_vmesh_destination from ._qndf import QNDF_CODEC, QNDF_INT8_CODEC from ._tracked import TVMC_CODEC, TSMC_CODEC from ._vmesh import FASTER_VDMC_CODEC, VDMC_CODEC +from ._vmesh_format import probe_codec +from ._native_temporal import QUEEN_CODEC, GSTREAM_CODEC, RERF_CODEC + +if TYPE_CHECKING: + from open4d.native import NativeSequence @dataclass(frozen=True) @@ -32,6 +39,7 @@ class CodecInfo: codec.id: codec for codec in ( KLT_CODEC, N4MC_CODEC, QNDF_CODEC, QNDF_INT8_CODEC, VDMC_CODEC, FASTER_VDMC_CODEC, TVMC_CODEC, TSMC_CODEC, VEGA_CODEC, + QUEEN_CODEC, GSTREAM_CODEC, RERF_CODEC, ) } @@ -79,6 +87,9 @@ def _codec(value: str | Codec | None, path: Path) -> Codec: if callable(getattr(codec, "can_decode", None)) and codec.can_decode(path) ] if not detected: + # can_decode is a boolean probe; surface why a damaged, archived or + # retired-wrapper file was rejected before the generic error. + probe_codec(path) raise CodecError(f"invalid Open4D artifact {path}: no known codec manifest") matches = detected if len(matches) != 1: @@ -95,14 +106,16 @@ def encode_sequence( fps: float | None = None, **options, ) -> Path: - """Encode meshes (Sequence or path), or GaussianSplats frames with Vega.""" + """Encode mesh/Gaussian data, or carry native temporal neural outputs.""" if "overwrite" in options and not isinstance(options["overwrite"], bool): raise TypeError("overwrite must be bool") - path = Path(destination) + path = require_vmesh_destination(destination) implementation = _codec(codec, path) - if getattr(implementation, "representation", "triangle_mesh") == "gaussian_splats": - if input_format is not None or fps is not None: - raise TypeError("input_format and fps apply only to mesh path inputs") + if getattr(implementation, "representation", "triangle_mesh") != "triangle_mesh": + if input_format is not None: + raise TypeError("input_format applies only to mesh path inputs") + if fps is not None: + options["fps"] = fps return implementation.encode(sequence, path, **options) if isinstance(sequence, Sequence): if input_format is not None or fps is not None: @@ -118,7 +131,32 @@ def encode_sequence( def decode_sequence( source: str | Path, *, codec: str | Codec | None = None, **options -) -> Sequence | tuple[NeuralGaussianFrame, ...]: - """Decode a mesh Sequence or Vega frames using a named or inferred codec.""" +) -> Sequence | NativeSequence | tuple[NeuralGaussianFrame, ...]: + """Open mesh reconstructions, native temporal state.""" path = Path(source) + if path.suffix.lower() == ".vmesh" and not path.is_dir(): + detected = probe_codec(path) + if detected is not None: + selected = codec if isinstance(codec, str) else getattr(codec, "id", None) + if selected is not None and selected != detected: + raise CodecError(f".vmesh contains {detected}, but codec={selected!r} was requested") + if "fps" in options: + raise TypeError("VMESH stores frame timestamps; fps cannot override them") + if detected == "frames": + if options: + raise TypeError("frame delivery VMESH accepts no native decoder options") + from open4d._streamer import _require + _require() + from streamer.sequence import open_frames + return open_frames(path) + if detected not in _CODECS and (codec is None or codec == detected): + from ._npz import REFERENCE_CODECS + from ._draco import DRACO_CODEC + from ._temporal import TEMPORAL_DELTA_CODEC, TEMPORAL_PCA_CODEC + private = {item.id: item for item in (*REFERENCE_CODECS, DRACO_CODEC, TEMPORAL_DELTA_CODEC, TEMPORAL_PCA_CODEC)} + codec = private.get(detected, detected) + else: + codec = detected if codec is None else codec + elif codec is None: + codec = "vdmc" return _codec(codec, path).decode(path, **options) diff --git a/open4d/codec/_draco.py b/open4d/codec/_draco.py index 311ca4c5..9d4e80cc 100644 --- a/open4d/codec/_draco.py +++ b/open4d/codec/_draco.py @@ -2,22 +2,20 @@ from __future__ import annotations -from collections.abc import Mapping from importlib import import_module import json from pathlib import Path import tempfile from types import MappingProxyType -from zipfile import BadZipFile, ZIP_STORED, ZipFile import numpy as np from open4d.core import Frame, Sequence, TopologyMode, TriangleMesh -from ._npz import _json_value +from ._metadata import _json_value +from ._vmesh_format import contains_codec, pack_vmesh, probe_codec, unpack_vmesh from ._protocol import CodecError -_SCHEMA = "open4d.draco-sequence/v1" def _backend(): @@ -52,8 +50,8 @@ def _encoder_arrays(mesh: TriangleMesh): class _DracoProvider: - def __init__(self, archive: ZipFile, manifest: dict) -> None: - self.archive = archive + def __init__(self, temporary, native: Path, manifest: dict) -> None: + self.temporary, self.native = temporary, native self.frames = manifest["frames"] self.metadata = MappingProxyType(manifest.get("metadata", {})) self.topology = TopologyMode(manifest.get("topology", "unknown")) @@ -76,7 +74,7 @@ def get_frame(self, index: int) -> Frame: raise IndexError("frame index out of range") record = self.frames[index] try: - decoded = _backend().decode(self.archive.read(record["member"])) + decoded = _backend().decode((self.native / f"frame_{index:06d}.drc").read_bytes()) except Exception as error: raise CodecError(f"could not decode Draco frame {index}: {error}") from error colors = decoded.colors @@ -95,25 +93,20 @@ def get_frame(self, index: int) -> Frame: ) def close(self) -> None: - self.archive.close() + self.temporary.cleanup() class DracoCodec: """Store each mesh frame as a real Google Draco bitstream.""" id = "draco" - suffixes = (".d4d",) + suffixes = (".vmesh",) backend = "python-binding" lossless = False preserves = ("positions", "triangles", "colors", "normals", "texture_coordinates") def can_decode(self, source: Path) -> bool: - try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - return isinstance(manifest, Mapping) and manifest.get("schema") == _SCHEMA - except (OSError, BadZipFile, KeyError, json.JSONDecodeError): - return False + return contains_codec(source, self.id) def encode( self, @@ -124,16 +117,17 @@ def encode( quantization_bits: int = 14, compression_level: int = 7, ) -> Path: - backend = _backend() if not isinstance(sequence, Sequence): raise TypeError("sequence must be an open4d.Sequence") destination = Path(destination).absolute() + if destination.suffix.lower() != ".vmesh": + raise ValueError("Draco destination must have a .vmesh extension") if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") + backend = _backend() destination.parent.mkdir(parents=True, exist_ok=True) manifest = { - "schema": _SCHEMA, - "codec": self.id, + "version": 1, "codec": self.id, "native": {"profile": "draco/1"}, "metadata": _json_value(sequence.metadata, "sequence"), "topology": sequence.topology.value, "has_constant_vertex_count": sequence.has_constant_vertex_count, @@ -143,80 +137,60 @@ def encode( } encoded_vertex_counts = [] split_uv_corners = False - with tempfile.NamedTemporaryFile( - prefix=f".{destination.name}.", suffix=".tmp", - dir=destination.parent, delete=False, - ) as stream: - temporary = Path(stream.name) - try: - with ZipFile(temporary, "w", compression=ZIP_STORED) as archive: - for ordinal, frame in enumerate(sequence): - mesh = frame.geometry - if mesh.attributes: - raise CodecError("Draco custom attributes are not yet supported") - positions, triangles, colors, normals, texture_coordinates = ( - _encoder_arrays(mesh) - ) - encoded_vertex_counts.append(len(positions)) - split_uv_corners |= ( - mesh.texture_coordinates is not None - and mesh.texture_coordinates.ndim == 3 - ) - member = f"frames/{ordinal:06d}.drc" - archive.writestr(member, backend.encode( - positions, - triangles, - colors=colors, - normals=normals, - tex_coord=texture_coordinates, - quantization_bits=quantization_bits, - compression_level=compression_level, - preserve_order=True, - )) - manifest["frames"].append({ - "frame_index": frame.frame_index, - "timestamp": frame.timestamp, - "metadata": _json_value(frame.metadata, f"frame {ordinal}"), - "member": member, - }) - if split_uv_corners: - manifest["topology"] = TopologyMode.UNKNOWN.value - manifest["has_constant_vertex_count"] = ( - len(set(encoded_vertex_counts)) == 1 - ) - manifest["has_vertex_correspondence"] = None - archive.writestr("manifest.json", json.dumps( - manifest, separators=(",", ":"), sort_keys=True, + with tempfile.TemporaryDirectory(prefix="open4d-draco-") as directory: + native = Path(directory) + for ordinal, frame in enumerate(sequence): + mesh = frame.geometry + if mesh.attributes: + raise CodecError("Draco custom attributes are not yet supported") + positions, triangles, colors, normals, texture_coordinates = ( + _encoder_arrays(mesh) + ) + encoded_vertex_counts.append(len(positions)) + split_uv_corners |= ( + mesh.texture_coordinates is not None + and mesh.texture_coordinates.ndim == 3 + ) + (native / f"frame_{ordinal:06d}.drc").write_bytes(backend.encode( + positions, + triangles, + colors=colors, + normals=normals, + tex_coord=texture_coordinates, + quantization_bits=quantization_bits, + compression_level=compression_level, + preserve_order=True, )) - temporary.replace(destination) - except Exception: - temporary.unlink(missing_ok=True) - raise + manifest["frames"].append({ + "frame_index": frame.frame_index, + "timestamp": frame.timestamp, + "metadata": _json_value(frame.metadata, f"frame {ordinal}"), + }) + if split_uv_corners: + manifest["topology"] = TopologyMode.UNKNOWN.value + manifest["has_constant_vertex_count"] = ( + len(set(encoded_vertex_counts)) == 1 + ) + manifest["has_vertex_correspondence"] = None + (native / "metadata.json").write_text(json.dumps(manifest, allow_nan=False), encoding="utf-8") + pack_vmesh(native, destination, overwrite=overwrite) return destination def decode(self, source: Path) -> Sequence: source = Path(source).absolute() - if not source.is_file(): - raise FileNotFoundError(f"codec artifact does not exist: {source}") - archive = None + if source.suffix.lower() != ".vmesh": + raise CodecError("Draco decoding requires .vmesh; re-encode older private artifacts") + if probe_codec(source) != self.id: + raise CodecError("VMESH does not contain Draco") + temporary = tempfile.TemporaryDirectory(prefix="open4d-draco-decode-") try: - archive = ZipFile(source) - manifest = json.loads(archive.read("manifest.json")) - if not isinstance(manifest, Mapping): - raise CodecError("Draco artifact manifest root must be an object") - if manifest.get("schema") != _SCHEMA: - raise CodecError("unsupported Draco artifact schema") - if not isinstance(manifest.get("frames"), list): - raise CodecError("Draco artifact manifest has no frame list") - return Sequence(_DracoProvider(archive, manifest)) - except Exception as error: - if archive is not None: - archive.close() - if isinstance(error, CodecError): - raise - if not isinstance(error, (BadZipFile, KeyError, json.JSONDecodeError)): - raise - raise CodecError(f"invalid Draco artifact {source}: {error}") from error + native = Path(temporary.name) / "native" + unpack_vmesh(source, native) + manifest = json.loads((native / "metadata.json").read_text(encoding="utf-8")) + return Sequence(_DracoProvider(temporary, native, manifest)) + except BaseException: + temporary.cleanup() + raise DRACO_CODEC = DracoCodec() diff --git a/open4d/codec/_klt.py b/open4d/codec/_klt.py index 041b6b12..e8ebdfeb 100644 --- a/open4d/codec/_klt.py +++ b/open4d/codec/_klt.py @@ -1,29 +1,74 @@ -"""In-process adapter for Open4D's KLT TSDF codec.""" +"""In-process KLT TSDF codec using standalone VMESH carriage.""" from __future__ import annotations from contextlib import redirect_stderr, redirect_stdout +from importlib import import_module import json import os from pathlib import Path +import pickle import shutil from types import MappingProxyType, SimpleNamespace import tempfile -from zipfile import BadZipFile, ZIP_DEFLATED, ZipFile +from zipfile import BadZipFile, ZipFile import numpy as np from open4d.core import Frame, Sequence, TopologyMode, TriangleMesh from open4d.io import open_sequence -from ._npz import _json_value, _publish_file, _validate_manifest +from ._metadata import _json_value, _validate_manifest from ._protocol import CodecError from ._research import research_module from ._tsdf import write_tsdf_sequence +from ._torch import torch_device +from ._vmesh_format import contains_codec, pack_vmesh, probe_codec, unpack_vmesh _SCHEMA = "open4d.klt-sequence/v1" +def _extract_legacy_klt(source: Path, destination: Path) -> dict: + """Convert the retired archive explicitly; never used by normal decoding.""" + try: + with ZipFile(source) as archive: + members = archive.infolist() + if len({member.filename for member in members}) != len(members): + raise CodecError("duplicate KLT archive member") + if archive.getinfo("manifest.json").file_size > 16 * 1024 * 1024: + raise CodecError("oversized KLT manifest") + manifest = json.loads(archive.read("manifest.json")) + _validate_manifest(manifest, schema=_SCHEMA, codec="klt") + _normalization(manifest) + metadata = {key: value for key, value in manifest.items() if key != "schema"} + metadata.update(version=1, native={"profile": "klt/1"}) + from ._native_profiles import layout + required = [name for name, _ in layout("klt", len(metadata["frames"]), metadata["native"])[1:]] + if set(archive.namelist()) != {"manifest.json", *required}: + raise CodecError("unexpected KLT archive payloads") + for name in required: + member = archive.getinfo(name) + limit = 16 * 1024 * 1024 if name.endswith((".pt", ".npz")) else 128 * 1024 * 1024 + if not 0 < member.file_size <= limit: + raise CodecError(f"KLT legacy payload outside limits: {name}") + with archive.open(member) as payload, (destination / name).open("xb") as output: + shutil.copyfileobj(payload, output) + torch = import_module("torch") + checkpoint = destination / "decoder_context.pt" + context = torch.load(checkpoint, map_location="cpu", weights_only=True) + if not isinstance(context, dict) or context.pop("schema", None) != "open4d.klt/v1": + raise CodecError("invalid legacy KLT context") + context["version"] = 1 + # Old encoders could advertise more components than their training rank. + context["num_components"] = len(context["basis"]) + _backend().validate_decoder_context(context) + torch.save(context, checkpoint) + (destination / "metadata.json").write_text(json.dumps(metadata, allow_nan=False), encoding="utf-8") + return metadata + except (BadZipFile, KeyError, ValueError, TypeError, RuntimeError, pickle.UnpicklingError) as error: + raise CodecError(f"invalid legacy KLT artifact: {error}") from error + + def _normalization(manifest): try: normalization = manifest["normalization"] @@ -85,18 +130,13 @@ def close(self): class KLTCodec: id = "klt" - suffixes = (".k4d",) + suffixes = (".vmesh",) backend = "python-in-process" lossless = False preserves = ("positions", "triangles") def can_decode(self, source: Path) -> bool: - try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - return isinstance(manifest, dict) and manifest.get("schema") == _SCHEMA - except (OSError, BadZipFile, KeyError, ValueError, TypeError): - return False + return contains_codec(source, self.id) def encode( self, sequence: Sequence, destination: Path, *, overwrite: bool = False, @@ -105,8 +145,24 @@ def encode( verbose: bool = False, ) -> Path: destination = Path(destination).absolute() + if destination.suffix.lower() != ".vmesh": + raise ValueError("KLT destination must have a .vmesh extension") if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") + if type(resolution) is not int or not 7 <= resolution <= 255: + raise ValueError("KLT resolution must be an integer from 7 to 255") + if type(block_size) is not int or not 1 <= block_size <= min(16, resolution + 1): + raise ValueError("KLT block_size must be an integer from 1 to 16 within the grid") + if type(num_components) is not int or not 1 <= num_components <= min(block_size ** 3, 512): + raise ValueError("KLT num_components exceeds the block dimensions") + if type(k_total) is not int or not 1 <= k_total <= 65536: + raise ValueError("KLT k_total must be an integer from 1 to 65536") + training_frames = tuple(training_frames) + if not training_frames or any(type(index) is not int or not 0 <= index < len(sequence) + for index in training_frames): + raise ValueError("KLT training_frames must select existing frames") + if type(frame_rate) not in (int, float) or not np.isfinite(frame_rate) or frame_rate <= 0: + raise ValueError("KLT frame_rate must be finite and positive") for frame in sequence: mesh = frame.geometry if any((mesh.colors is not None, mesh.normals is not None, @@ -114,7 +170,7 @@ def encode( raise CodecError("KLT's TSDF profile cannot preserve mesh attributes") destination.parent.mkdir(parents=True, exist_ok=True) manifest = { - "schema": _SCHEMA, "codec": self.id, + "version": 1, "codec": self.id, "native": {"profile": "klt/1"}, "metadata": _json_value(sequence.metadata, "sequence"), "allow_nonmonotonic_timestamps": sequence.allow_nonmonotonic_timestamps, "frames": [{ @@ -139,37 +195,30 @@ def encode( else: with open(os.devnull, "w") as sink, redirect_stdout(sink), redirect_stderr(sink): backend.run_compression(arguments, verify_decode=False) - with tempfile.NamedTemporaryFile( - prefix=f".{destination.name}.", suffix=".tmp", - dir=destination.parent, delete=False, - ) as stream: - temporary = Path(stream.name) - try: - shutil.copyfile(work / "encoded/compressed_archive.zip", temporary) - with ZipFile(temporary, "a", compression=ZIP_DEFLATED) as archive: - archive.writestr("manifest.json", json.dumps(manifest)) - _publish_file(temporary, destination, overwrite=overwrite) - except Exception: - temporary.unlink(missing_ok=True) - raise + native = work / "encoded/compressed" + (native / "metadata.json").write_text(json.dumps(manifest, allow_nan=False), encoding="utf-8") + pack_vmesh(native, destination, overwrite=overwrite) return destination def decode(self, source: Path, *, device=None) -> Sequence: + if device == "auto": + device = str(torch_device(import_module("torch"), device)) source = Path(source).absolute() + if source.suffix.lower() != ".vmesh": + raise CodecError("KLT decoding requires .vmesh; migrate the legacy artifact explicitly") + if probe_codec(source) != self.id: + raise CodecError("VMESH does not contain KLT") temporary = tempfile.TemporaryDirectory(prefix="open4d-klt-decode-") work = Path(temporary.name) decoded = None try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - _validate_manifest(manifest, schema=_SCHEMA, codec=self.id) - _normalization(manifest) - members = [item for item in archive.infolist() if item.filename != "manifest.json"] - if any(Path(item.filename).is_absolute() or ".." in Path(item.filename).parts - for item in members): - raise CodecError("unsafe path in KLT artifact") - archive.extractall(work / "compressed", members) - _backend().decode_compressed(work / "compressed", work / "decoded", device) + unpack_vmesh(source, work / "compressed") + manifest = json.loads((work / "compressed/metadata.json").read_text(encoding="utf-8")) + _normalization(manifest) + try: + _backend().decode_compressed(work / "compressed", work / "decoded", device) + except (ValueError, KeyError, TypeError, RuntimeError, BadZipFile, pickle.UnpicklingError) as error: + raise CodecError(f"invalid KLT native payload: {error}") from error decoded = open_sequence(work / "decoded") if len(decoded) != len(manifest["frames"]): raise CodecError( diff --git a/open4d/codec/_metadata.py b/open4d/codec/_metadata.py new file mode 100644 index 00000000..d963c52a --- /dev/null +++ b/open4d/codec/_metadata.py @@ -0,0 +1,69 @@ +"""Finite sequence metadata shared by VMESH adapters.""" + +from collections.abc import Mapping +import math +from pathlib import Path +import numpy as np +from ._protocol import CodecError + + +def _json_value(value, name: str): + if value is None or isinstance(value, (str, int, bool)): + return value + if isinstance(value, float): + if not math.isfinite(value): + raise CodecError(f"{name} metadata numbers must be finite") + return value + if isinstance(value, np.generic): + return _json_value(value.item(), name) + if isinstance(value, Path): + return str(value) + if isinstance(value, Mapping): + if any(not isinstance(key, str) for key in value): + raise CodecError(f"{name} metadata keys must be strings") + return {key: _json_value(item, f"{name}.{key}") for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_value(item, name) for item in value] + raise CodecError(f"{name} metadata value {type(value).__name__} is not serializable") + + +def _validate_manifest(manifest, *, schema: str | None, codec: str) -> dict: + if not isinstance(manifest, dict): + raise CodecError("artifact manifest root must be an object") + if manifest.get("schema") != schema or manifest.get("codec") != codec: + raise CodecError(f"unsupported {codec} artifact schema or codec") + frames = manifest.get("frames") + if not isinstance(frames, list): + raise CodecError("artifact manifest must contain a frame list") + nonmonotonic = manifest.get("allow_nonmonotonic_timestamps", False) + if not isinstance(nonmonotonic, bool): + raise CodecError("allow_nonmonotonic_timestamps must be boolean") + for name in ("has_constant_vertex_count", "has_vertex_correspondence"): + if manifest.get(name) is not None and not isinstance(manifest[name], bool): + raise CodecError(f"{name} must be boolean or null") + if not isinstance(manifest.get("metadata", {}), dict): + raise CodecError("sequence metadata must be an object") + _json_value(manifest.get("metadata", {}), "sequence") + previous = None + for ordinal, record in enumerate(frames): + if not isinstance(record, dict): + raise CodecError(f"invalid frame record {ordinal}") + index, timestamp = record.get("frame_index"), record.get("timestamp") + if type(index) is not int or index < 0: + raise CodecError(f"invalid frame index at {ordinal}") + if type(timestamp) not in (int, float) or not math.isfinite(timestamp): + raise CodecError(f"invalid frame timestamp at {ordinal}") + if previous is not None and not nonmonotonic and timestamp < previous: + raise CodecError("frame timestamps must be nondecreasing") + previous = timestamp + if not isinstance(record.get("metadata", {}), dict): + raise CodecError(f"frame {ordinal} metadata must be an object") + _json_value(record.get("metadata", {}), f"frame {ordinal}") + return manifest + + +def require_vmesh_destination(destination: str | Path) -> Path: + path = Path(destination).absolute() + if path.suffix.lower() != ".vmesh": + raise ValueError("compressed output requires a .vmesh extension") + return path diff --git a/open4d/codec/_migration.py b/open4d/codec/_migration.py new file mode 100644 index 00000000..e817a2b1 --- /dev/null +++ b/open4d/codec/_migration.py @@ -0,0 +1,108 @@ +"""Explicit, one-way conversion of retired sequence artifacts to VMESH.""" + +from __future__ import annotations + +import json +import pickle +from pathlib import Path +import shutil +import tempfile +from zipfile import BadZipFile, ZipFile + +from ._metadata import _validate_manifest, require_vmesh_destination +from ._protocol import CodecError +from ._vmesh_format import _json, pack_vmesh + +_MAX_PAYLOAD = 8 * 1024**3 + + +def _extract_vdmc(source: Path, destination: Path) -> None: + from ._vmesh import _position_normalization + + try: + with ZipFile(source) as archive: + names = archive.namelist() + if len(names) != len(set(names)) or archive.getinfo("manifest.json").file_size > 16 * 1024**2: + raise CodecError("duplicate members or oversized migration manifest") + if set(names) - {"manifest.json", "sequence.vmesh", "decoder.cfg"}: + raise CodecError("unexpected V-DMC migration member") + manifest = _json(archive.read("manifest.json")) + codec = manifest.get("codec") + if codec not in ("vdmc", "faster_vdmc"): + raise CodecError("unsupported native V-DMC codec") + _validate_manifest(manifest, schema="open4d.vmesh-sequence/v1", codec=codec) + if not 1 <= len(manifest["frames"]) <= 65536: + raise CodecError("migration frame count outside limits") + _position_normalization(manifest) + required = ["sequence.vmesh"] + if "decoder.cfg" in names: + required.append("decoder.cfg") + for name in required: + size = archive.getinfo(name).file_size + maximum = 16 * 1024**2 if name == "decoder.cfg" else _MAX_PAYLOAD + if not 0 < size <= maximum: + raise CodecError("migration payload size outside limits") + with archive.open(name) as payload, (destination / name).open("xb") as output: + shutil.copyfileobj(payload, output) + manifest.pop("schema") + manifest.update(version=1, native={"profile": f"{codec}/1", "decoder_config": len(required) == 2}) + (destination / "metadata.json").write_text(json.dumps(manifest, allow_nan=False), encoding="utf-8") + except (BadZipFile, KeyError, ValueError, TypeError, AttributeError) as error: + raise CodecError(f"invalid migration input: {error}") from error + + +def migrate_legacy(source: str | Path, destination: str | Path, *, overwrite: bool = False, + fps: float = 30.0) -> Path: + """Convert an older artifact explicitly; normal readers accept VMESH only. + + Native payloads are preserved except redundant checkpoint schema fields. + Tensor checkpoints use restricted weights-only loading. Executable early + QNDF-int8 contexts are deliberately unsupported. + """ + source = Path(source).absolute() + destination = require_vmesh_destination(destination) + if not isinstance(overwrite, bool): + raise TypeError("overwrite must be bool") + if destination.exists() and not overwrite: + raise FileExistsError(destination) + if source.is_symlink(): + raise CodecError("linked migration source") + if source.is_dir(): + if (source / "metadata.json").is_file(): + return pack_vmesh(source, destination, overwrite=overwrite) + from open4d.native import import_native, save_native + with import_native(source, codec="vega", fps=fps) as native: + return save_native(native, destination, overwrite=overwrite) + if source.suffix.lower() == ".seq": + from open4d._streamer import _require + _require() + from streamer.sequence import convert_legacy + return convert_legacy(source, destination, fps=fps, overwrite=overwrite) + with tempfile.TemporaryDirectory(prefix="vmesh-migration-") as folder: + native = Path(folder) + suffix = source.suffix.lower() + if suffix == ".k4d": + from ._klt import _extract_legacy_klt + _extract_legacy_klt(source, native) + elif suffix in (".q4d", ".qi4d"): + from ._qndf import _extract_legacy_qndf + _extract_legacy_qndf(source, native, int8=suffix == ".qi4d") + elif suffix == ".n4d": + from ._n4mc import _extract_n4d, _write_native_metadata + manifest = _extract_n4d(source, native) + from importlib import import_module + torch = import_module("torch") + try: + checkpoint = torch.load(native / "checkpoint.pt", map_location="cpu", weights_only=True) + except (OSError, RuntimeError, ValueError, TypeError, pickle.UnpicklingError) as error: + raise CodecError("invalid N4MC migration checkpoint") from error + if not isinstance(checkpoint, dict): + raise CodecError("invalid N4MC checkpoint") + checkpoint.pop("schema", None) + torch.save(checkpoint, native / "checkpoint.pt") + _write_native_metadata(manifest, native) + elif suffix == ".v4d": + _extract_vdmc(source, native) + else: + raise ValueError("unsupported migration input; import native payload directories explicitly") + return pack_vmesh(native, destination, overwrite=overwrite) diff --git a/open4d/codec/_n4mc.py b/open4d/codec/_n4mc.py index 982f486e..f12bf6ca 100644 --- a/open4d/codec/_n4mc.py +++ b/open4d/codec/_n4mc.py @@ -6,9 +6,10 @@ import json from numbers import Integral from pathlib import Path +import shutil import tempfile import warnings -from zipfile import BadZipFile, ZIP_DEFLATED, ZipFile +from zipfile import BadZipFile, ZipFile import numpy as np @@ -16,15 +17,49 @@ from open4d.io import open_sequence from ._klt import _KLTProvider, _normalization -from ._npz import _json_value, _publish_file, _validate_manifest +from ._metadata import _json_value, _validate_manifest, require_vmesh_destination from ._protocol import CodecError from ._research import research_module from ._torch import torch_device from ._tsdf import write_tsdf_sequence +from ._vmesh_format import _json, contains_codec, pack_vmesh, probe_codec, unpack_vmesh _SCHEMA = "open4d.n4mc-sequence/v1" +def _extract_n4d(source: Path, destination: Path) -> dict: + """Read the legacy adapter's fixed payloads without loading a model.""" + try: + with ZipFile(source) as archive: + names = archive.namelist() + if len(names) != len(set(names)): + raise CodecError("duplicate N4MC archive member") + if archive.getinfo("manifest.json").file_size > 16 * 1024 * 1024: + raise CodecError("oversized N4MC manifest") + manifest = _json(archive.read("manifest.json")) + _validate_manifest(manifest, schema=_SCHEMA, codec="n4mc") + _normalization(manifest) + if archive.getinfo("checkpoint.pt").file_size > 64 * 1024**2: + raise CodecError("N4MC migration checkpoint exceeds 64 MiB") + required = ["checkpoint.pt", *(f"frame_{i:06d}.npz" for i in range(len(manifest["frames"])))] + if len(required) > 65535 or sum(archive.getinfo(name).file_size for name in required) > 8 * 1024**3: + raise CodecError("N4MC migration payload budget exceeded") + for name in required: + if archive.getinfo(name).file_size == 0: + raise CodecError(f"empty N4MC payload: {name}") + with archive.open(name) as payload, (destination / name).open("xb") as output: + shutil.copyfileobj(payload, output) + return manifest + except (BadZipFile, KeyError, ValueError, TypeError) as error: + raise CodecError(f"invalid N4MC archive: {error}") from error + + +def _write_native_metadata(manifest: dict, destination: Path) -> None: + metadata = {key: value for key, value in manifest.items() if key != "schema"} + metadata.update(version=1, native={"profile": "n4mc/1"}) + (destination / "metadata.json").write_text(json.dumps(metadata, allow_nan=False), encoding="utf-8") + + def _device(torch, requested): target = torch_device(torch, requested) if target.type != "mps": @@ -89,20 +124,23 @@ def _filter_components(mesh, min_component_faces: int | None): return import_module("trimesh").util.concatenate(kept) +def _reconstruct_mesh(volume, metrics): + mesh = metrics.reconstruct_mesh_from_tsdf(volume) + if mesh is not None: + # The research helper returns [-1, 1]; our TSDF samples [-1.1, 1.1]. + mesh.vertices *= 1.1 + return mesh + + class N4MCCodec: id = "n4mc" - suffixes = (".n4d",) + suffixes = (".vmesh",) backend = "python-in-process" lossless = False preserves = ("positions", "triangles") def can_decode(self, source: Path) -> bool: - try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - return isinstance(manifest, dict) and manifest.get("schema") == _SCHEMA - except (OSError, BadZipFile, KeyError, ValueError, TypeError): - return False + return contains_codec(source, self.id) def encode( self, sequence: Sequence, destination: Path, *, overwrite: bool = False, @@ -110,7 +148,7 @@ def encode( latent_channels: int = 64, learning_rate: float = 1e-4, device: str | None = None, seed: int = 7, ) -> Path: - destination = Path(destination).absolute() + destination = require_vmesh_destination(destination) if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") for frame in sequence: @@ -137,7 +175,7 @@ def encode( "lambda_band": 2., "lambda_sign": .2, } manifest = { - "schema": _SCHEMA, "codec": self.id, + "version": 1, "codec": self.id, "metadata": _json_value(sequence.metadata, "sequence"), "allow_nonmonotonic_timestamps": sequence.allow_nonmonotonic_timestamps, "frames": [{ @@ -164,10 +202,9 @@ def encode( model.eval() checkpoint = work / "checkpoint.pt" torch.save({ - "schema": "open4d.n4mc/v1", "model": model.state_dict(), + "model": model.state_dict(), "model_config": model_config, }, checkpoint) - packs = [] with torch.inference_mode(): for ordinal, path in enumerate(volumes): encoded = model.encode(_volume(torch, path, target_device).unsqueeze(0)) @@ -178,43 +215,28 @@ def encode( original_shape=encoded["original_shape"].cpu().numpy(), bottleneck_shape=encoded["bottleneck_shape"].cpu().numpy(), ) - packs.append(pack) - with tempfile.NamedTemporaryFile( - prefix=f".{destination.name}.", suffix=".tmp", - dir=destination.parent, delete=False, - ) as stream: - temporary = Path(stream.name) - try: - with ZipFile(temporary, "w", compression=ZIP_DEFLATED) as archive: - archive.write(checkpoint, checkpoint.name) - for pack in packs: - archive.write(pack, pack.name) - archive.writestr("manifest.json", json.dumps(manifest)) - _publish_file(temporary, destination, overwrite=overwrite) - except Exception: - temporary.unlink(missing_ok=True) - raise - return destination + _write_native_metadata(manifest, work) + return pack_vmesh(work, destination, overwrite=overwrite) def decode( self, source: Path, *, device: str | None = None, min_component_faces: int | None = None, ) -> Sequence: - torch, models, _, metrics = _backend() - target_device = _device(torch, device) + source = Path(source).absolute() temporary = tempfile.TemporaryDirectory(prefix="open4d-n4mc-decode-") work = Path(temporary.name) decoded = None try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - _validate_manifest(manifest, schema=_SCHEMA, codec=self.id) - _normalization(manifest) - archive.extract("checkpoint.pt", work) - for ordinal in range(len(manifest["frames"])): - archive.extract(f"frame_{ordinal:06d}.npz", work) + if source.suffix.lower() != ".vmesh" or probe_codec(source) != self.id: + raise CodecError("N4MC decode requires its VMESH profile; use migrate_legacy for older artifacts") + native = unpack_vmesh(source, work / "native") + manifest = json.loads((native / "metadata.json").read_text()) + _validate_manifest(manifest, schema=None, codec=self.id) + _normalization(manifest) + torch, models, _, metrics = _backend() + target_device = _device(torch, device) checkpoint = torch.load( - work / "checkpoint.pt", map_location=target_device, weights_only=True + native / "checkpoint.pt", map_location=target_device, weights_only=True ) model = models.TSDFCompressionAutoencoder( **checkpoint["model_config"] @@ -223,14 +245,17 @@ def decode( model.eval() output = work / "decoded" output.mkdir() - with torch.inference_mode(): + with torch.inference_mode(), torch.backends.cudnn.flags( + enabled=torch.backends.cudnn.enabled, benchmark=False, + deterministic=True, allow_tf32=torch.backends.cudnn.allow_tf32, + ): for ordinal in range(len(manifest["frames"])): - with np.load(work / f"frame_{ordinal:06d}.npz", allow_pickle=False) as pack: + with np.load(native / f"frame_{ordinal:06d}.npz", allow_pickle=False) as pack: latent = torch.from_numpy(pack["quantized_latent"]).unsqueeze(0).to(target_device) volume = model.decode_quantized_latent( latent, pack["bottleneck_shape"], pack["original_shape"] )[0] - mesh = metrics.reconstruct_mesh_from_tsdf(volume) + mesh = _reconstruct_mesh(volume, metrics) if mesh is None: raise CodecError(f"N4MC frame {ordinal} has no decoded surface") mesh = _filter_components(mesh, min_component_faces) diff --git a/open4d/codec/_native.py b/open4d/codec/_native.py new file mode 100644 index 00000000..ddf0f3e2 --- /dev/null +++ b/open4d/codec/_native.py @@ -0,0 +1,52 @@ +"""Bounded native-tool execution with robust diagnostics.""" + +from collections import deque +import math +import os +import signal +import subprocess +import tempfile + +from ._protocol import CodecError + + +def _timeout(): + # Native encodes of long sequences can legitimately run for hours, so the + # bound is opt-in rather than a default that kills working jobs. + value = os.environ.get("OPEN4D_NATIVE_TIMEOUT", "").strip() + if not value: + return None + try: + timeout = float(value) + except ValueError: + timeout = math.nan + if not math.isfinite(timeout) or timeout <= 0: + raise CodecError(f"OPEN4D_NATIVE_TIMEOUT must be a finite positive number of seconds, got {value!r}") + return timeout + + +def run(command, label, *, cwd=None): + timeout = _timeout() + with tempfile.TemporaryFile(mode="w+", encoding="utf-8", errors="replace") as log: + with subprocess.Popen( + command, cwd=cwd, stdout=log, stderr=subprocess.STDOUT, + start_new_session=os.name != "nt", + ) as process: + try: + process.wait(timeout=timeout) + except BaseException as error: + if os.name != "nt": + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + else: + process.kill() + process.wait() + if isinstance(error, subprocess.TimeoutExpired): + raise CodecError(f"{label} timed out after {timeout:g} seconds") from error + raise + if process.returncode: + log.seek(0) + detail = "".join(deque(log, maxlen=16)).strip() + raise CodecError(f"{label} exited {process.returncode}: {detail or 'no diagnostic output'}") diff --git a/open4d/codec/_native_profiles.py b/open4d/codec/_native_profiles.py new file mode 100644 index 00000000..3a2cfbe0 --- /dev/null +++ b/open4d/codec/_native_profiles.py @@ -0,0 +1,175 @@ +"""Closed payload layouts for native research representations. + +This module inspects JSON only. It never imports or deserializes a native model. +All paths are generated here, rather than taken from an archive's filenames. +""" +from __future__ import annotations + +from ._protocol import CodecError + +PROFILES = { + "klt": ("tsdf", "shared-basis-independent-frames"), + "qndf": ("triangle_mesh", "independent"), + "qndf-int8": ("triangle_mesh", "independent"), + "frames": ("frame_payloads", "independent"), + "npz": ("triangle_mesh", "independent"), + "raw": ("triangle_mesh", "independent"), + "deflate": ("triangle_mesh", "independent"), + "bzip2": ("triangle_mesh", "independent"), + "lzma": ("triangle_mesh", "independent"), + "rle": ("triangle_mesh", "independent"), + "draco": ("triangle_mesh", "independent"), + "temporal-delta": ("triangle_mesh", "shared-reference"), + "temporal-pca": ("triangle_mesh", "shared-trajectory-basis"), + "tvmc": ("triangle_mesh", "shared-reference"), + "tsmc": ("triangle_mesh", "whole-group"), + "vega": ("neural_gaussians", "shared-reference"), + "queen": ("gaussian_splats", "previous-frame"), + "3dgstream": ("gaussian_splats", "previous-frame"), + "rerf": ("neural_field", "previous-frame"), + "vdmc": ("triangle_mesh", "native-sequence"), + "faster_vdmc": ("triangle_mesh", "native-sequence"), + # Explicit scope exception: a shared spatial model, with no temporal + # prediction or residual dependency between the quantized frame latents. + "n4mc": ("neural_tsdf", "shared-model-independent-frames"), +} +MAX_FILES = 65536 +MAX_FRAMES = 65536 + +NEURAL_CODECS = frozenset(("vega", "queen", "3dgstream", "rerf")) + + +def _require(condition, message): + if not condition: + raise CodecError(f"invalid native profile: {message}") + + +def layout(codec, count, native=None): + """Validate a profile and return its exact ordered filename/role pairs.""" + _require(codec in PROFILES, f"unsupported codec {codec!r}") + _require(type(count) is int and 0 < count <= MAX_FRAMES, "frame count outside limits") + if codec == "tvmc": + _require(2 + 2 * count <= MAX_FILES, "payload file count outside limits") + elif codec == "klt": + _require(2 + 2 * count <= MAX_FILES, "payload file count outside limits") + elif codec in ("qndf", "qndf-int8", "frames", "npz", "raw", "deflate", "bzip2", "lzma", "rle", "draco"): + _require(1 + count <= MAX_FILES, "payload file count outside limits") + elif codec == "vega": + _require(3 + count <= MAX_FILES, "payload file count outside limits") + elif codec == "n4mc": + _require(2 + count <= MAX_FILES, "payload file count outside limits") + elif codec == "queen": + _require(1 + count <= MAX_FILES, "payload file count outside limits") + files = [("metadata.json", "sequence-metadata")] + if codec in ("tvmc", "tsmc"): + _require(native is None, "unexpected tracked-mesh profile") + files.append(("reference.drc", "reference-mesh")) + if codec == "tvmc": + files += [(f"displacement_{i:06d}.{suffix}", role) for i in range(count) + for suffix, role in (("drc", "frame-displacement"), ("npy", "vertex-order"))] + else: + files += [("B_matrix.txt", "trajectory-basis"), ("T_matrix.txt", "trajectory-offset"), + ("delta_trajectories_encoded.npy", "entropy-coded-trajectories"), + ("entropy_model.npz", "entropy-model")] + return files + _require(isinstance(native, dict) and native.get("profile") == f"{codec}/1", "unsupported profile version") + if codec in NEURAL_CODECS: + _require(count >= 2, "native sequence profiles require at least two frames") + if codec == "n4mc": + _require(set(native) == {"profile"}, "unexpected N4MC configuration") + files.append(("checkpoint.pt", "shared-neural-model")) + files += [(f"frame_{i:06d}.npz", "quantized-tsdf-latent") for i in range(count)] + elif codec == "klt": + _require(set(native) == {"profile"}, "unexpected KLT configuration") + files.append(("decoder_context.pt", "shared-klt-basis")) + files += [(f"{i:06d}_quantized_{suffix}", role) for i in range(count) + for suffix, role in (("indices.zst", "quantized-tsdf-coefficients"), + ("metadata.npz", "tsdf-quantizer"))] + elif codec in ("qndf", "qndf-int8"): + _require(set(native) == {"profile"}, "unexpected QNDF configuration") + files += [(f"frame_{i:06d}.pt", "neural-displacement-context") for i in range(count)] + elif codec in ("npz", "raw", "deflate", "bzip2", "lzma", "rle"): + _require(set(native) == {"profile", "rle"} and type(native["rle"]) is bool, + "unexpected array configuration") + files += [(f"frame_{i:06d}.npz", "mesh-arrays") for i in range(count)] + elif codec == "draco": + _require(set(native) == {"profile"}, "unexpected Draco configuration") + files += [(f"frame_{i:06d}.drc", "frame-mesh") for i in range(count)] + elif codec in ("temporal-delta", "temporal-pca"): + _require(set(native) == {"profile"}, "unexpected temporal configuration") + files.append(("sequence.npz", "quantized-trajectories")) + elif codec == "frames": + _require(set(native) == {"profile", "suffix", "representation"}, "unexpected frame configuration") + suffix = native["suffix"] + _require(suffix in ("ply", "drc", "splat", "jpg", "jpeg", "png"), "unsupported frame payload suffix") + representation = native["representation"] + compatible = {"pixels": ("jpg", "jpeg", "png"), "gaussians": ("ply", "splat"), + "mesh": ("ply", "drc"), "points": ("ply", "drc")} + _require(isinstance(representation, str) and representation in compatible + and suffix in compatible[representation], "incompatible frame representation/suffix") + files += [(f"frame_{i:06d}.{suffix}", "frame-payload") for i in range(count)] + elif codec == "vega": + _require(set(native) == {"profile"}, "unexpected Vega configuration") + files += [("manifest.json", "native-frame-index"), ("color_model.pt", "shared-neural-appearance")] + files += [(f"frame_{i:04d}.pt", "key-state" if i == 0 else "residual-state") for i in range(count)] + elif codec == "queen": + _require(set(native) == {"profile", "sh_degree", "gate_params"}, "unexpected QUEEN configuration") + _require(type(native["sh_degree"]) is int and 0 <= native["sh_degree"] <= 3, "invalid SH degree") + gates = native["gate_params"] + _require(isinstance(gates, list) and len(gates) == 7 and all(g in ("on", "none") for g in gates), "invalid QUEEN gates") + _require(all(g == "none" for g in gates[1:]), "only QUEEN position gating is supported by its native serializer") + files.append(("initial.ply", "initial-gaussians")) + files += [(f"frame_{i:06d}.pkl", "compressed-attribute-residual") for i in range(1, count)] + elif codec == "3dgstream": + _require(set(native) == {"profile", "sh_degree", "rotate_sh", "only_mlp", "added"}, "unexpected 3DGStream configuration") + _require(type(native["sh_degree"]) is int and 0 <= native["sh_degree"] <= 3, "invalid SH degree") + _require(type(native["rotate_sh"]) is bool and type(native["only_mlp"]) is bool, "invalid NTC flags") + _require(not native["rotate_sh"] or native["sh_degree"] == 1, "the native SH rotation requires degree 1") + added = native["added"] + _require(isinstance(added, list) and len(added) == count and all(type(v) is bool for v in added) and not added[0], "invalid added-Gaussian index") + files += [("initial.ply", "initial-gaussians"), ("ntc_config.json", "neural-transform-architecture")] + for i in range(1, count): + _require(len(files) + 1 + int(added[i]) <= MAX_FILES, "payload file count outside limits") + files.append((f"ntc_{i:06d}.pth", "neural-transform")) + if added[i]: + files.append((f"added_{i:06d}.ply", "added-gaussians")) + elif codec == "rerf": + _require(set(native) == {"profile", "group_size", "pca", "pca_channels", "frames"}, "unexpected ReRF configuration") + group, pca, frames = native["group_size"], native["pca"], native["frames"] + _require(type(group) is int and group > 0 and type(pca) is bool, "invalid ReRF group/PCA mode") + channels = native["pca_channels"] + _require(isinstance(channels, list) and len(channels) == 2 and all(type(c) is int and c > 0 for c in channels), "invalid PCA channels") + _require(isinstance(frames, list) and len(frames) == count, "invalid ReRF frame index") + files += [("decoder_config.json", "field-configuration"), ("model_kwargs.json", "field-model"), ("rgb_net.tar", "shared-rgb-network")] + previous = None + for frame in frames: + _require(isinstance(frame, dict) and set(frame) == {"id", "quality", "motion", "channels"}, "invalid ReRF frame record") + index, quality, motion = frame["id"], frame["quality"], frame["motion"] + _require(type(index) is int and index >= 0 and (previous is None or index == previous + 1), "ReRF frames must be contiguous") + _require(previous is not None or index % group == 0, "ReRF sequence must start at a group key") + _require(type(quality) is int and 0 <= quality <= 100 and type(motion) is bool, "invalid ReRF quality/motion") + key = index % group == 0 + _require(motion != key, "ReRF residual frames require both motion payloads") + halves = frame["channels"] + _require(isinstance(halves, list) and len(halves) == (2 if not key and pca else 1) + and all(type(n) is int and 0 < n <= 4096 for n in halves), "invalid ReRF entropy channels") + _require(len(files) + 2 + sum(halves) + int(not key and pca) + 2 * int(motion) + <= MAX_FILES, "payload file count outside limits") + files += [(f"header_{index}.json", "field-frame-header"), (f"mask_{index}.rerf", "occupancy-mask")] + # The native ac_dc_encode2 writes one file per channel; .rerf is + # a prefix, not an aggregate entropy file. + for half, channels_in_half in enumerate(halves): + files += [(f"feature_{index}_{quality-half}.rerf{channel}", "key-features" if key else "residual-features") + for channel in range(channels_in_half)] + if not key and pca: + _require(quality > 0, "PCA residual quality must exceed zero") + files.append((f"pca_m_{index}.npy", "pca-basis")) + if motion: + files += [(f"deform_{index}.npy", "motion-field"), (f"deform_mask_{index}.rerf", "motion-mask")] + previous = index + else: + _require(set(native) == {"profile", "decoder_config"} and type(native["decoder_config"]) is bool, "invalid V-DMC configuration") + files.append(("sequence.vmesh", "native-v3c-sequence")) + if native["decoder_config"]: + files.append(("decoder.cfg", "native-decoder-configuration")) + return files diff --git a/open4d/codec/_native_temporal.py b/open4d/codec/_native_temporal.py new file mode 100644 index 00000000..0d8f8413 --- /dev/null +++ b/open4d/codec/_native_temporal.py @@ -0,0 +1,61 @@ +"""Public codec adapters for native temporal neural research artifacts.""" +from pathlib import Path +import os + +from ._native_profiles import PROFILES +from ._protocol import CodecError +from ._metadata import require_vmesh_destination +from ._vmesh_format import contains_codec + + +def encode_native(source, destination, *, codec, overwrite=False, **options): + destination = require_vmesh_destination(destination) + from open4d.native import NativeSequence, import_native, save_native + if isinstance(source, NativeSequence): + if options: + raise TypeError("native repacking does not accept training/import options") + if source.codec != codec: + raise CodecError(f"native source contains {source.codec}, not {codec}") + return save_native(source, destination, overwrite=overwrite) + if isinstance(source, (str, os.PathLike)) and Path(source).suffix.lower() in (".usd", ".usdc", ".usda", ".vmesh"): + from open4d import load + if options: + raise TypeError("native repacking does not accept training/import options") + with load(source) as opened: + if not isinstance(opened, NativeSequence): + raise TypeError(f"{codec} needs a native temporal USD representation; generic mesh USD is not an encoder input") + return encode_native(opened, destination, codec=codec, overwrite=overwrite) + with import_native(source, codec=codec, **options) as imported: + return save_native(imported, destination, overwrite=overwrite) + + +class NativeTemporalCodec: + suffixes = (".vmesh",) + backend = "research-subprocess" + lossless = False + preserves = ("native_temporal_representation", "timestamps", "metadata") + + def __init__(self, identifier): + self.id = identifier + self.representation = PROFILES[identifier][0] + + def can_decode(self, path): + return contains_codec(path, self.id) + + def encode(self, sequence, destination, **options): + return encode_native(sequence, destination, codec=self.id, **options) + + def decode(self, source, *, runtime=None, python=None): + if Path(source).suffix.lower() != ".vmesh": + raise CodecError(f"{self.id} decode requires a .vmesh extension") + from open4d.native import NativeSequence + result = NativeSequence(source, runtime=runtime, python=python) + if result.codec != self.id: + result.close() + raise CodecError(f"native stream does not contain {self.id}") + return result + + +QUEEN_CODEC = NativeTemporalCodec("queen") +GSTREAM_CODEC = NativeTemporalCodec("3dgstream") +RERF_CODEC = NativeTemporalCodec("rerf") diff --git a/open4d/codec/_npz.py b/open4d/codec/_npz.py index cb046ed0..12fdd9f9 100644 --- a/open4d/codec/_npz.py +++ b/open4d/codec/_npz.py @@ -1,8 +1,7 @@ -"""Lossless reference sequence codec using NumPy arrays inside ZIP.""" +"""Lossless array profiles using standard NPZ payloads inside VMESH.""" from __future__ import annotations -from collections.abc import Mapping from io import BytesIO import json import math @@ -21,89 +20,57 @@ import numpy as np from open4d.core import Frame, Sequence, TopologyMode, TriangleMesh -from open4d._files import publish_file as _publish_file from ._protocol import CodecError +from ._metadata import _json_value +from ._vmesh_format import contains_codec, pack_vmesh, probe_codec, unpack_vmesh -_SCHEMA = "open4d.numpy-zip/v1" +_MAX_ARRAY_BYTES = 256 * 1024 * 1024 +# Worst-case RLE doubles its input, plus its length prefix and the NPY header. +_MAX_RLE_MEMBER_BYTES = 2 * _MAX_ARRAY_BYTES + 8 + 4096 _FIELDS = ("positions", "triangles", "colors", "normals", "texture_coordinates") -def _json_value(value, name: str): - if value is None or isinstance(value, (str, int, bool)): - return value - if isinstance(value, float): - if not math.isfinite(value): - raise CodecError(f"{name} metadata numbers must be finite") - return value - if isinstance(value, np.generic): - return _json_value(value.item(), name) - if isinstance(value, Path): - return str(value) - if isinstance(value, Mapping): - if any(not isinstance(key, str) for key in value): - raise CodecError(f"{name} metadata keys must be strings") - return {key: _json_value(item, f"{name}.{key}") for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [_json_value(item, name) for item in value] - raise CodecError(f"{name} metadata value {type(value).__name__} is not serializable") - - -def _validate_manifest(manifest, *, schema: str | None, codec: str) -> dict: - if not isinstance(manifest, dict): - raise CodecError("artifact manifest root must be an object") - if manifest.get("schema") != schema or manifest.get("codec") != codec: - raise CodecError(f"unsupported {codec} artifact schema or codec") - frames = manifest.get("frames") - if not isinstance(frames, list): - raise CodecError("artifact manifest must contain a frame list") - nonmonotonic = manifest.get("allow_nonmonotonic_timestamps", False) - if not isinstance(nonmonotonic, bool): - raise CodecError("allow_nonmonotonic_timestamps must be boolean") - for name in ("has_constant_vertex_count", "has_vertex_correspondence"): - if manifest.get(name) is not None and not isinstance(manifest[name], bool): - raise CodecError(f"{name} must be boolean or null") - if not isinstance(manifest.get("metadata", {}), dict): - raise CodecError("sequence metadata must be an object") - _json_value(manifest.get("metadata", {}), "sequence") - previous = None - for ordinal, record in enumerate(frames): - if not isinstance(record, dict): - raise CodecError(f"invalid frame record {ordinal}") - index, timestamp = record.get("frame_index"), record.get("timestamp") - if type(index) is not int or index < 0: - raise CodecError(f"invalid frame index at {ordinal}") - if type(timestamp) not in (int, float) or not math.isfinite(timestamp): - raise CodecError(f"invalid frame timestamp at {ordinal}") - if previous is not None and not nonmonotonic and timestamp < previous: - raise CodecError("frame timestamps must be nondecreasing") - previous = timestamp - if not isinstance(record.get("metadata", {}), dict): - raise CodecError(f"frame {ordinal} metadata must be an object") - _json_value(record.get("metadata", {}), f"frame {ordinal}") - return manifest - - def _array_bytes(array: np.ndarray) -> bytes: stream = BytesIO() np.save(stream, array, allow_pickle=False) return stream.getvalue() +def _array_from_bytes(payload: bytes) -> np.ndarray: + stream = BytesIO(payload) + version = np.lib.format.read_magic(stream) + readers = {(1, 0): np.lib.format.read_array_header_1_0, + (2, 0): np.lib.format.read_array_header_2_0} + if version not in readers: + raise CodecError("unsupported array NPY version") + shape, _, dtype = readers[version](stream, max_header_size=4096) + if dtype.hasobject or math.prod(shape) * dtype.itemsize != len(payload) - stream.tell(): + raise CodecError("array NPY size/type disagrees with payload") + return np.load(BytesIO(payload), allow_pickle=False, max_header_size=4096) + + def _read_array(archive: ZipFile, name: str, codec: "NumPyZipCodec") -> np.ndarray: try: - payload = archive.read(name) - except KeyError as error: - raise CodecError(f"artifact is missing {name}") from error - return np.load(BytesIO(codec.unpack(payload)), allow_pickle=False) + member = archive.getinfo(f"{name}.npy") + limit = _MAX_RLE_MEMBER_BYTES if codec.rle else _MAX_ARRAY_BYTES + if not 0 < member.file_size <= limit: + raise CodecError("array payload outside limits") + value = _array_from_bytes(archive.read(member)) + if codec.rle: + if value.dtype != np.uint8 or value.ndim != 1: + raise CodecError("RLE array payload must contain encoded bytes") + value = _array_from_bytes(codec.unpack(value.tobytes())) + return value + except (BadZipFile, KeyError, ValueError, TypeError) as error: + raise CodecError(f"invalid array payload {name}: {error}") from error class _ZipProvider: def __init__( - self, source: Path, archive: ZipFile, manifest: dict, codec: "NumPyZipCodec" + self, temporary, native: Path, manifest: dict, codec: "NumPyZipCodec" ) -> None: - self.source = source - self.archive = archive + self.temporary, self.native = temporary, native self.codec = codec self.frames = manifest["frames"] self.metadata = MappingProxyType(manifest.get("metadata", {})) @@ -133,15 +100,20 @@ def get_frame(self, index: int) -> Frame: except IndexError as error: raise IndexError("frame index out of range") from error arrays = record["arrays"] - values = { - name: _read_array(self.archive, path, self.codec) - for name, path in arrays.items() - if name != "attributes" - } - attributes = { - name: _read_array(self.archive, path, self.codec) - for name, path in arrays.get("attributes", {}).items() - } + try: + archive = ZipFile(self.native / f"frame_{index:06d}.npz") + except BadZipFile as error: + raise CodecError(f"invalid array payload frame {index}: {error}") from error + with archive: + values = { + name: _read_array(archive, path, self.codec) + for name, path in arrays.items() + if name != "attributes" + } + attributes = { + name: _read_array(archive, path, self.codec) + for name, path in arrays.get("attributes", {}).items() + } return Frame( frame_index=record["frame_index"], timestamp=record["timestamp"], @@ -150,13 +122,13 @@ def get_frame(self, index: int) -> Frame: ) def close(self) -> None: - self.archive.close() + self.temporary.cleanup() class NumPyZipCodec: - """Losslessly store canonical Open4D arrays in a portable `.o4d` ZIP.""" + """Losslessly carry standard NumPy frame arrays in standalone VMESH.""" - suffixes = (".o4d",) + suffixes = (".vmesh",) backend = "python" lossless = True preserves = (*_FIELDS, "attributes") @@ -196,18 +168,26 @@ def unpack(self, payload: bytes) -> bytes: raise CodecError("invalid RLE payload") expected = int.from_bytes(payload[:8], "little") pairs = np.frombuffer(payload[8:], dtype=np.uint8).reshape(-1, 2) - decoded = np.repeat(pairs[:, 1], pairs[:, 0]).tobytes() - if len(decoded) != expected: - raise CodecError("RLE payload length does not match its header") - return decoded + if (expected > _MAX_ARRAY_BYTES or np.any(pairs[:, 0] == 0) + or int(pairs[:, 0].sum(dtype=np.int64)) != expected): + raise CodecError("RLE payload length does not match its bounded header") + return np.repeat(pairs[:, 1], pairs[:, 0]).tobytes() + + def _member(self, value: np.ndarray, label: str) -> bytes: + # Enforce the decoder's bound here so encode never publishes an + # artifact that its own decoder would reject. + payload = _array_bytes(value) + if len(payload) > _MAX_ARRAY_BYTES: + raise CodecError( + f"{self.id} array {label} is {len(payload)} bytes; the limit is {_MAX_ARRAY_BYTES}" + ) + payload = self.pack(payload) + if self.rle: + payload = _array_bytes(np.frombuffer(payload, dtype=np.uint8)) + return payload def can_decode(self, source: Path) -> bool: - try: - with ZipFile(source, "r") as archive: - manifest = json.loads(archive.read("manifest.json")) - return isinstance(manifest, Mapping) and manifest.get("codec") == self.id - except (OSError, BadZipFile, KeyError, ValueError, TypeError): - return False + return contains_codec(source, self.id) def encode( self, @@ -223,12 +203,14 @@ def encode( if level is not None and not 0 <= level <= 9: raise ValueError("compression_level must be in [0, 9]") destination = Path(destination).absolute() + if destination.suffix.lower() != ".vmesh": + raise ValueError("array destination must have a .vmesh extension") if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") destination.parent.mkdir(parents=True, exist_ok=True) manifest = { - "schema": _SCHEMA, - "codec": self.id, + "version": 1, "codec": self.id, + "native": {"profile": f"{self.id}/1", "rle": self.rle}, "metadata": _json_value(sequence.metadata, "sequence"), "topology": sequence.topology.value, "has_constant_vertex_count": sequence.has_constant_vertex_count, @@ -236,67 +218,63 @@ def encode( "allow_nonmonotonic_timestamps": sequence.allow_nonmonotonic_timestamps, "frames": [], } - with tempfile.NamedTemporaryFile( - prefix=f".{destination.name}.", suffix=".tmp", - dir=destination.parent, delete=False, - ) as stream: - temporary = Path(stream.name) - try: - with ZipFile( - temporary, "w", compression=self.compression, - compresslevel=level, - ) as archive: - for ordinal, frame in enumerate(sequence): - prefix = f"frames/{ordinal:06d}" - arrays = {} + with tempfile.TemporaryDirectory(prefix="open4d-arrays-") as directory: + native = Path(directory) + for ordinal, frame in enumerate(sequence): + arrays = {} + with ZipFile(native / f"frame_{ordinal:06d}.npz", "w", compression=self.compression, + compresslevel=level) as archive: for name in _FIELDS: value = getattr(frame.geometry, name) if value is not None: - member = f"{prefix}/{name}.npy" - archive.writestr(member, self.pack(_array_bytes(value))) - arrays[name] = member + archive.writestr(f"{name}.npy", self._member(value, f"frame {ordinal} {name}")) + arrays[name] = name attributes = {} for number, (name, value) in enumerate(frame.geometry.attributes.items()): - member = f"{prefix}/attribute_{number:04d}.npy" - archive.writestr(member, self.pack(_array_bytes(value))) - attributes[name] = member + key = f"attribute_{number:04d}" + archive.writestr(f"{key}.npy", self._member(value, f"frame {ordinal} attribute {name!r}")) + attributes[name] = key arrays["attributes"] = attributes - manifest["frames"].append( - { - "frame_index": frame.frame_index, - "timestamp": frame.timestamp, - "metadata": _json_value(frame.metadata, f"frame {ordinal}"), - "arrays": arrays, - } - ) - archive.writestr( - "manifest.json", - json.dumps(manifest, separators=(",", ":"), sort_keys=True), - ) - _publish_file(temporary, destination, overwrite=overwrite) - except Exception: - temporary.unlink(missing_ok=True) - raise + manifest["frames"].append({ + "frame_index": frame.frame_index, "timestamp": frame.timestamp, + "metadata": _json_value(frame.metadata, f"frame {ordinal}"), "arrays": arrays, + }) + (native / "metadata.json").write_text(json.dumps(manifest, allow_nan=False), encoding="utf-8") + pack_vmesh(native, destination, overwrite=overwrite) return destination def decode(self, source: Path) -> Sequence: source = Path(source).absolute() - if not source.is_file(): - raise FileNotFoundError(f"codec artifact does not exist: {source}") - archive = None + if source.suffix.lower() != ".vmesh": + raise CodecError("array decoding requires .vmesh; re-encode older private artifacts") + if probe_codec(source) != self.id: + raise CodecError(f"VMESH does not contain {self.id}") + temporary = tempfile.TemporaryDirectory(prefix="open4d-arrays-decode-") try: - archive = ZipFile(source, "r") - manifest = json.loads(archive.read("manifest.json")) - _validate_manifest(manifest, schema=_SCHEMA, codec=self.id) - return Sequence(_ZipProvider(source, archive, manifest, self)) - except Exception as error: - if archive is not None: - archive.close() - if isinstance(error, CodecError): - raise - if not isinstance(error, (BadZipFile, KeyError, ValueError, TypeError)): - raise - raise CodecError(f"invalid Open4D artifact {source}: {error}") from error + native = Path(temporary.name) / "native" + unpack_vmesh(source, native) + manifest = json.loads((native / "metadata.json").read_text(encoding="utf-8")) + for ordinal, frame in enumerate(manifest["frames"]): + arrays = frame.get("arrays") + if (not isinstance(arrays, dict) or not {"positions", "triangles"} <= set(arrays) + or not set(arrays) <= {*_FIELDS, "attributes"} + or any(value != name for name, value in arrays.items() if name != "attributes")): + raise CodecError("invalid mesh array field index") + attributes = arrays.get("attributes", {}) + if (not isinstance(attributes, dict) or not all(isinstance(name, str) for name in attributes) + or list(attributes.values()) != [f"attribute_{i:04d}" for i in range(len(attributes))]): + raise CodecError("invalid mesh attribute field index") + expected = {f"{key}.npy" for key in arrays if key != "attributes"} + expected.update(f"{key}.npy" for key in attributes.values()) + with ZipFile(native / f"frame_{ordinal:06d}.npz") as archive: + members = archive.namelist() + if len(members) != len(set(members)) or set(members) != expected: + raise CodecError("unexpected NPZ frame arrays") + decoder = NumPyZipCodec(self.id, rle=manifest["native"]["rle"]) + return Sequence(_ZipProvider(temporary, native, manifest, decoder)) + except BaseException: + temporary.cleanup() + raise REFERENCE_CODECS = ( diff --git a/open4d/codec/_protocol.py b/open4d/codec/_protocol.py index d6ca8883..04ace865 100644 --- a/open4d/codec/_protocol.py +++ b/open4d/codec/_protocol.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from open4d.gaussians import GaussianSplats, NeuralGaussianFrame + from open4d.native import NativeSequence class CodecError(RuntimeError): @@ -26,5 +27,7 @@ class Codec(Protocol): def encode(self, sequence: Sequence | Iterable[GaussianSplats], destination: Path, **options) -> Path: """Encode *sequence* into *destination*.""" - def decode(self, source: Path, **options) -> Sequence | tuple[NeuralGaussianFrame, ...]: + def decode( + self, source: Path, **options + ) -> Sequence | NativeSequence | tuple[NeuralGaussianFrame, ...]: """Open a decoded sequence backed by *source*.""" diff --git a/open4d/codec/_qndf.py b/open4d/codec/_qndf.py index bc8e0efb..97ac691a 100644 --- a/open4d/codec/_qndf.py +++ b/open4d/codec/_qndf.py @@ -1,4 +1,4 @@ -"""In-process adapters for Open4D's neural displacement field codecs.""" +"""Neural displacement field codecs with direct standalone VMESH carriage.""" from __future__ import annotations @@ -9,30 +9,41 @@ import math from numbers import Real from pathlib import Path +import pickle import tempfile -from zipfile import BadZipFile, ZIP_DEFLATED, ZipFile +from zipfile import BadZipFile, ZipFile import numpy as np from open4d.core import Frame, MemoryFrameProvider, Sequence, TopologyMode, TriangleMesh -from ._npz import _json_value, _publish_file, _validate_manifest +from ._metadata import _json_value, _validate_manifest from ._protocol import CodecError from ._research import research_module from ._torch import torch_device +from ._vmesh_format import contains_codec, pack_vmesh, probe_codec, unpack_vmesh + +_MAX_CONTEXT_BYTES = 64 * 1024 * 1024 +_MAX_GRAPH_ELEMENTS = 64 * 1024 * 1024 def _backend(): try: return ( - import_module("torch"), - research_module("qndf.compress"), + *_model_backend(), research_module("qndf.build_dataset_open3d"), ) except ImportError as error: raise CodecError("QNDF dependencies are missing; install open4d[qndf]") from error +def _model_backend(): + try: + return import_module("torch"), research_module("qndf.compress") + except ImportError as error: + raise CodecError("QNDF decoding needs PyTorch; install open4d[qndf]") from error + + def _torch_bytes(torch, value) -> bytes: stream = BytesIO() torch.save(value, stream) @@ -57,7 +68,7 @@ def _quantized_engine(torch, required=None): def _manifest(sequence, codec): return { - "schema": f"open4d.{codec}-sequence/v1", "codec": codec, + "version": 1, "codec": codec, "native": {"profile": f"{codec}/1"}, "metadata": _json_value(sequence.metadata, "sequence"), "allow_nonmonotonic_timestamps": sequence.allow_nonmonotonic_timestamps, "frames": [{ @@ -82,6 +93,12 @@ def _validate_model_options(pe_dim, hidden_dim, num_layers, input_scale, output_ raise ValueError(f"{name} must be a positive integer") if pe_dim % 2: raise ValueError("pe_dim must be even") + if pe_dim > 128 or hidden_dim > 4096 or num_layers > 128: + raise ValueError("QNDF model dimensions outside limits") + parameters = (2 * 3 * pe_dim * hidden_dim + (num_layers + 1) * hidden_dim ** 2 + + 3 * (num_layers + 3) * hidden_dim + 3 * hidden_dim + 3) + if parameters > 16 * 1024 * 1024: + raise ValueError("QNDF model parameter count outside limits") for name, value in (("input_scale", input_scale), ("output_scale", output_scale)): if isinstance(value, bool) or not isinstance(value, Real) or not math.isfinite(value): raise ValueError(f"{name} must be a finite number") @@ -89,10 +106,16 @@ def _validate_model_options(pe_dim, hidden_dim, num_layers, input_scale, output_ raise ValueError("output_scale must be nonzero") -def _validate_context(torch, context, schema): +def _validate_context(torch, context, *, int8=False): try: - if not isinstance(context, Mapping) or context.get("schema") != schema: - raise ValueError("unsupported schema or context is not an object") + required = {"version", "coarse_vertices", "coarse_faces", "input_mean", "input_std", + "pe_dim", "hidden_dim", "num_layers", "input_scale", "output_scale", + "normalization", "model_state_dict"} + if int8: + required.add("quantized_engine") + if (not isinstance(context, Mapping) or set(context) != required + or type(context.get("version")) is not int or context["version"] != 1): + raise ValueError("unsupported context/version or fields") _validate_model_options(*(context[name] for name in ( "pe_dim", "hidden_dim", "num_layers", "input_scale", "output_scale", ))) @@ -105,6 +128,8 @@ def _validate_context(torch, context, schema): if ((name == "coarse_vertices" and (len(shape) != 2 or shape[1] != 3 or shape[0] == 0)) or (name != "coarse_vertices" and shape != (1, 3))): raise ValueError(f"invalid {name} dimensions") + if len(context["coarse_vertices"]) * 3 * context["pe_dim"] * 14 > _MAX_GRAPH_ELEMENTS: + raise ValueError("QNDF neighbor graph outside limits") if torch.any(context["input_std"] <= 0): raise ValueError("input_std must be positive") faces = context["coarse_faces"] @@ -122,10 +147,51 @@ def _validate_context(torch, context, schema): raise ValueError("normalization requires finite XYZ bounds and a positive scale") if not isinstance(context["model_state_dict"], Mapping): raise ValueError("model_state_dict must be an object") + if int8 and not isinstance(context["quantized_engine"], str): + raise ValueError("quantized_engine must be a string") except (KeyError, TypeError, ValueError) as error: raise CodecError(f"invalid QNDF frame context: {error}") from error +def _extract_legacy_qndf(source: Path, destination: Path, *, int8: bool) -> dict: + """Convert safe legacy weights explicitly, without carrying their archive.""" + codec = "qndf-int8" if int8 else "qndf" + version = 2 if int8 else 1 + try: + with ZipFile(source) as archive: + members = archive.infolist() + if len({member.filename for member in members}) != len(members): + raise CodecError("duplicate QNDF archive member") + if archive.getinfo("manifest.json").file_size > 16 * 1024 * 1024: + raise CodecError("oversized QNDF manifest") + manifest = json.loads(archive.read("manifest.json")) + if int8 and isinstance(manifest, dict) and manifest.get("schema") == "open4d.qndf-int8-sequence/v1": + raise CodecError("QNDF-int8 v1 contains executable models; re-encode it") + _validate_manifest(manifest, schema=f"open4d.{codec}-sequence/v{version}", codec=codec) + metadata = {key: value for key, value in manifest.items() if key != "schema"} + metadata.update(version=1, native={"profile": f"{codec}/1"}) + from ._native_profiles import layout + layout(codec, len(metadata["frames"]), metadata["native"]) + expected = {"manifest.json", *(f"frames/{i:06d}/context.pt" for i in range(len(metadata["frames"])))} + if set(archive.namelist()) != expected: + raise CodecError("unexpected QNDF archive payloads") + torch = import_module("torch") + for ordinal in range(len(metadata["frames"])): + member = archive.getinfo(f"frames/{ordinal:06d}/context.pt") + if not 0 < member.file_size <= _MAX_CONTEXT_BYTES: + raise CodecError("QNDF legacy context outside limits") + context = torch.load(BytesIO(archive.read(member)), map_location="cpu", weights_only=True) + if not isinstance(context, Mapping) or context.pop("schema", None) != f"open4d.{codec}/v{version}": + raise CodecError("invalid legacy QNDF frame context") + context["version"] = 1 + _validate_context(torch, context, int8=int8) + torch.save(context, destination / f"frame_{ordinal:06d}.pt") + (destination / "metadata.json").write_text(json.dumps(metadata, allow_nan=False), encoding="utf-8") + return metadata + except (BadZipFile, KeyError, ValueError, TypeError, RuntimeError, pickle.UnpicklingError) as error: + raise CodecError(f"invalid legacy {codec} artifact: {error}") from error + + class QNDFCodec: backend = "python-in-process" lossless = False @@ -134,17 +200,10 @@ class QNDFCodec: def __init__(self, *, int8: bool = False) -> None: self.int8 = int8 self.id = "qndf-int8" if int8 else "qndf" - self.suffixes = (".qi4d",) if int8 else (".q4d",) - self.version = 2 if int8 else 1 - self.schema = f"open4d.{self.id}-sequence/v{self.version}" + self.suffixes = (".vmesh",) def can_decode(self, source: Path) -> bool: - try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - return isinstance(manifest, dict) and manifest.get("schema") == self.schema - except (OSError, BadZipFile, KeyError, ValueError, TypeError): - return False + return contains_codec(source, self.id) def encode( self, sequence: Sequence, destination: Path, *, overwrite: bool = False, @@ -163,6 +222,8 @@ def encode( if type(num_subdiv) is not int or num_subdiv < 0: raise ValueError("num_subdiv must be a nonnegative integer") destination = Path(destination).absolute() + if destination.suffix.lower() != ".vmesh": + raise ValueError("QNDF destination must have a .vmesh extension") if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") for frame in sequence: @@ -177,88 +238,84 @@ def encode( torch.cuda.manual_seed_all(seed) destination.parent.mkdir(parents=True, exist_ok=True) manifest = _manifest(sequence, self.id) - manifest["schema"] = self.schema - with tempfile.NamedTemporaryFile( - prefix=f".{destination.name}.", suffix=".tmp", - dir=destination.parent, delete=False, - ) as stream: - temporary = Path(stream.name) - try: - with ZipFile(temporary, "w", compression=ZIP_DEFLATED) as archive: - for ordinal, frame in enumerate(sequence): - low, faces, projected, normalization = preprocessing.build_pair( - frame.geometry.positions, frame.geometry.triangles, - coarse_size, num_subdiv, - ) - coarse = torch.from_numpy(np.asarray(low, dtype=np.float32)).to(target) - triangles = torch.from_numpy(np.asarray(faces, dtype=np.int64)).to(target) - goal = torch.from_numpy(np.asarray(projected, dtype=np.float32)).to(target) - encoded, inputs, mean, std = _inputs( - torch, models, coarse, pe_dim, input_scale - ) - model = models.MLP(3 * pe_dim, hidden_dim, 3, num_layers).to(target) - optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate) - graph = models.MeshDataset( - encoded, inputs, triangles, (goal - coarse) * output_scale, - progress=verbose, - ) - loader = torch.utils.data.DataLoader( - graph, batch_size=batch_size, shuffle=False - ) - model.train() - for _ in range(epochs): - for inputs, neighbors, weights, expected in loader: - optimizer.zero_grad(set_to_none=True) - loss = torch.nn.functional.mse_loss( - model(inputs, neighbors, weights), expected - ) - loss.backward() - optimizer.step() - context = { - "schema": f"open4d.{self.id}/v{self.version}", - "coarse_vertices": coarse.cpu(), "coarse_faces": triangles.cpu(), - "input_mean": mean.cpu(), "input_std": std.cpu(), - "pe_dim": pe_dim, "hidden_dim": hidden_dim, - "num_layers": num_layers, "input_scale": input_scale, - "output_scale": output_scale, "normalization": normalization, - } - if self.int8: - context["quantized_engine"] = _quantized_engine(torch) - cpu_model = torch.ao.quantization.quantize_dynamic( - model.cpu().eval(), {torch.nn.Linear}, dtype=torch.qint8 + with tempfile.TemporaryDirectory(prefix="open4d-qndf-") as directory: + native = Path(directory) + for ordinal, frame in enumerate(sequence): + low, faces, projected, normalization = preprocessing.build_pair( + frame.geometry.positions, frame.geometry.triangles, + coarse_size, num_subdiv, + ) + coarse = torch.from_numpy(np.asarray(low, dtype=np.float32)).to(target) + triangles = torch.from_numpy(np.asarray(faces, dtype=np.int64)).to(target) + goal = torch.from_numpy(np.asarray(projected, dtype=np.float32)).to(target) + encoded, inputs, mean, std = _inputs( + torch, models, coarse, pe_dim, input_scale + ) + model = models.MLP(3 * pe_dim, hidden_dim, 3, num_layers).to(target) + optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate) + graph = models.MeshDataset( + encoded, inputs, triangles, (goal - coarse) * output_scale, + progress=verbose, + ) + loader = torch.utils.data.DataLoader( + graph, batch_size=batch_size, shuffle=False + ) + model.train() + for _ in range(epochs): + for inputs, neighbors, weights, expected in loader: + optimizer.zero_grad(set_to_none=True) + loss = torch.nn.functional.mse_loss( + model(inputs, neighbors, weights), expected ) - context["model_state_dict"] = cpu_model.state_dict() - else: - context["model_state_dict"] = model.state_dict() - _validate_context(torch, context, f"open4d.{self.id}/v{self.version}") - archive.writestr( - f"frames/{ordinal:06d}/context.pt", _torch_bytes(torch, context) + loss.backward() + optimizer.step() + context = { + "version": 1, + "coarse_vertices": coarse.cpu(), "coarse_faces": triangles.cpu(), + "input_mean": mean.cpu(), "input_std": std.cpu(), + "pe_dim": pe_dim, "hidden_dim": hidden_dim, + "num_layers": num_layers, "input_scale": input_scale, + "output_scale": output_scale, "normalization": normalization, + } + if self.int8: + context["quantized_engine"] = _quantized_engine(torch) + cpu_model = torch.ao.quantization.quantize_dynamic( + model.cpu().eval(), {torch.nn.Linear}, dtype=torch.qint8 ) - archive.writestr("manifest.json", json.dumps(manifest)) - _publish_file(temporary, destination, overwrite=overwrite) - except Exception: - temporary.unlink(missing_ok=True) - raise + context["model_state_dict"] = cpu_model.state_dict() + else: + context["model_state_dict"] = model.state_dict() + _validate_context(torch, context, int8=self.int8) + torch.save(context, native / f"frame_{ordinal:06d}.pt") + (native / "metadata.json").write_text(json.dumps(manifest, allow_nan=False), encoding="utf-8") + pack_vmesh(native, destination, overwrite=overwrite) return destination def decode( self, source: Path, *, device: str | None = None, verbose: bool = False ) -> Sequence: - try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - if self.int8 and isinstance(manifest, dict) and manifest.get("schema") == "open4d.qndf-int8-sequence/v1": - raise CodecError("QNDF-int8 v1 contains executable models; re-encode with the current version") - _validate_manifest(manifest, schema=self.schema, codec=self.id) - torch, models, _ = _backend() + source = Path(source).absolute() + if source.suffix.lower() != ".vmesh": + raise CodecError("QNDF decoding requires .vmesh; migrate the legacy artifact explicitly") + if probe_codec(source) != self.id: + raise CodecError(f"VMESH does not contain {self.id}") + with tempfile.TemporaryDirectory(prefix="open4d-qndf-decode-") as directory: + native = Path(directory) / "native" + unpack_vmesh(source, native) + manifest = json.loads((native / "metadata.json").read_text(encoding="utf-8")) + try: + torch, models = _model_backend() target = torch.device("cpu") if self.int8 else torch_device(torch, device) frames = [] for ordinal, record in enumerate(manifest["frames"]): + context_path = native / f"frame_{ordinal:06d}.pt" + if context_path.stat().st_size > _MAX_CONTEXT_BYTES: + raise CodecError("QNDF context outside limits") context = torch.load( - BytesIO(archive.read(f"frames/{ordinal:06d}/context.pt")), + context_path, map_location=target, weights_only=True, ) - _validate_context(torch, context, f"open4d.{self.id}/v{self.version}") + _validate_context(torch, context, int8=self.int8) coarse, faces = context["coarse_vertices"].to(target), context["coarse_faces"].to(target) inputs = (coarse * context["input_scale"] - context["input_mean"].to(target)) inputs /= context["input_std"].to(target) @@ -293,8 +350,8 @@ def decode( positions.cpu().numpy(), faces.cpu().numpy() ), record.get("metadata", {}), )) - except (BadZipFile, KeyError, json.JSONDecodeError) as error: - raise CodecError(f"invalid {self.id} artifact {source}: {error}") from error + except (KeyError, ValueError, TypeError, RuntimeError, pickle.UnpicklingError) as error: + raise CodecError(f"invalid {self.id} native payload: {error}") from error return Sequence(MemoryFrameProvider( frames, metadata=manifest.get("metadata", {}), topology=TopologyMode.CHANGING, has_constant_vertex_count=None, has_vertex_correspondence=False, diff --git a/open4d/codec/_temporal.py b/open4d/codec/_temporal.py index c566e572..e44938b5 100644 --- a/open4d/codec/_temporal.py +++ b/open4d/codec/_temporal.py @@ -6,18 +6,21 @@ import json from pathlib import Path import tempfile +from zipfile import ZipFile import numpy as np from open4d.core import Frame, MemoryFrameProvider, Sequence, TopologyMode, TriangleMesh -from ._npz import _json_value, _publish_file, _validate_manifest +from ._metadata import _json_value +from ._npz import NumPyZipCodec, _read_array +from ._vmesh_format import contains_codec, pack_vmesh, probe_codec, unpack_vmesh from ._protocol import CodecError def _manifest(sequence, codec): return { - "schema": f"open4d.{codec}-sequence/v1", "codec": codec, + "version": 1, "codec": codec, "native": {"profile": f"{codec}/1"}, "metadata": _json_value(sequence.metadata, "sequence"), "allow_nonmonotonic_timestamps": sequence.allow_nonmonotonic_timestamps, "frames": [{ @@ -94,25 +97,21 @@ def __init__(self, identifier: str) -> None: if identifier not in {"temporal-delta", "temporal-pca"}: raise ValueError(f"unknown experimental temporal profile: {identifier}") self.id = identifier - self.suffixes = ((".td4d",) if identifier == "temporal-delta" else (".tp4d",)) - self.schema = f"open4d.{identifier}-sequence/v1" + self.suffixes = (".vmesh",) def can_decode(self, source: Path) -> bool: - try: - with np.load(source, allow_pickle=False) as artifact: - manifest = json.loads(artifact["manifest"].tobytes()) - return isinstance(manifest, dict) and manifest.get("schema") == self.schema - except (OSError, ValueError, TypeError, KeyError): - return False + return contains_codec(source, self.id) def encode( self, sequence: Sequence, destination: Path, *, overwrite: bool = False, face_budget: int = 3000, quantization_bits: int = 16, components: int = 5, ) -> Path: - if face_budget < 1: + if type(face_budget) is not int or face_budget < 1: raise ValueError("face_budget must be positive") destination = Path(destination).absolute() + if destination.suffix.lower() != ".vmesh": + raise ValueError("temporal destination must have a .vmesh extension") if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") reference, faces, fitted = _fit(sequence, face_budget) @@ -140,31 +139,33 @@ def encode( manifest = _manifest(sequence, self.id) manifest["quantization_bits"] = quantization_bits destination.parent.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile( - prefix=f".{destination.name}.", suffix=".tmp", - dir=destination.parent, delete=False, - ) as stream: - temporary = Path(stream.name) - try: - with temporary.open("wb") as stream: - np.savez_compressed( - stream, - manifest=np.frombuffer(json.dumps(manifest).encode(), dtype=np.uint8), - **payload, - ) - _publish_file(temporary, destination, overwrite=overwrite) - except Exception: - temporary.unlink(missing_ok=True) - raise + with tempfile.TemporaryDirectory(prefix="open4d-temporal-") as directory: + native = Path(directory) + np.savez_compressed(native / "sequence.npz", **payload) + (native / "metadata.json").write_text(json.dumps(manifest, allow_nan=False), encoding="utf-8") + pack_vmesh(native, destination, overwrite=overwrite) return destination def decode(self, source: Path, *, device: str | None = None) -> Sequence: if device not in (None, "cpu"): raise ValueError(f"{self.id} decoding is NumPy-based; device must be 'cpu'") + source = Path(source).absolute() + if source.suffix.lower() != ".vmesh": + raise CodecError("temporal decoding requires .vmesh; re-encode older private artifacts") + if probe_codec(source) != self.id: + raise CodecError(f"VMESH does not contain {self.id}") try: - with np.load(source, allow_pickle=False) as artifact: - manifest = json.loads(artifact["manifest"].tobytes()) - _validate_manifest(manifest, schema=self.schema, codec=self.id) + with tempfile.TemporaryDirectory(prefix="open4d-temporal-decode-") as directory: + native = Path(directory) / "native" + unpack_vmesh(source, native) + manifest = json.loads((native / "metadata.json").read_text(encoding="utf-8")) + with ZipFile(native / "sequence.npz") as archive: + members = archive.namelist() + if (len(members) != len(set(members)) + or set(members) != {f"{name}.npy" for name in _temporal_fields(self.id)}): + raise CodecError("unexpected temporal native arrays") + artifact = {name: _read_array(archive, name, _PLAIN_ARRAYS) + for name in _temporal_fields(self.id)} reference, faces = artifact["reference"], artifact["triangles"] reference_mesh = TriangleMesh(reference, faces) count, vertices = len(manifest["frames"]), len(reference_mesh.positions) @@ -214,5 +215,12 @@ def decode(self, source: Path, *, device: str | None = None) -> Sequence: )) +def _temporal_fields(codec): + return ({"reference", "triangles", "displacement", "scale"} if codec == "temporal-delta" + else {"reference", "triangles", "coefficients", "basis", "mean", "scale"}) + + +_PLAIN_ARRAYS = NumPyZipCodec("raw", rle=False) + TEMPORAL_DELTA_CODEC = TemporalMeshCodec("temporal-delta") TEMPORAL_PCA_CODEC = TemporalMeshCodec("temporal-pca") diff --git a/open4d/codec/_tracked.py b/open4d/codec/_tracked.py index c0445ba0..e983d97f 100644 --- a/open4d/codec/_tracked.py +++ b/open4d/codec/_tracked.py @@ -2,23 +2,22 @@ from __future__ import annotations -from collections import deque import json import os from pathlib import Path import shutil -import subprocess import sys import tempfile from types import MappingProxyType -from open4d._files import publish_directory from open4d.core import Frame, Sequence, TopologyMode, TriangleMesh from open4d.io import open_sequence from open4d.io._mesh import write_obj -from ._npz import _json_value, _validate_manifest +from ._metadata import _json_value, _validate_manifest, require_vmesh_destination from ._protocol import CodecError +from ._native import run as _run_native +from ._vmesh_format import contains_codec, pack_vmesh, probe_codec, unpack_vmesh def _executable(value, label: str) -> str: @@ -36,15 +35,7 @@ def _positive_integer(value, name: str) -> int: def _run(python: str, action: str, request: Path) -> None: worker = Path(__file__).with_name("_tracked_worker.py") - with tempfile.TemporaryFile(mode="w+", encoding="utf-8") as log: - result = subprocess.run( - [python, str(worker), action, str(request)], cwd=request.parent, - stdout=log, stderr=subprocess.STDOUT, text=True, check=False, - ) - if result.returncode: - log.seek(0) - detail = "".join(deque(log, maxlen=16)).strip() - raise CodecError(f"{action} failed (exit {result.returncode}):\n{detail}") + _run_native([python, str(worker), action, str(request)], action, cwd=request.parent) def _manifest(source: Path, codec: str) -> dict: @@ -103,14 +94,10 @@ class TrackedMeshCodec: def __init__(self, identifier): self.id = identifier - self.suffixes = (f".{identifier}",) + self.suffixes = (".vmesh",) def can_decode(self, source: Path) -> bool: - try: - _manifest(source, self.id) - except CodecError: - return False - return True + return contains_codec(source, self.id) def _settings(self, backend, python, encoder, decoder, *, encoding): variable = f"OPEN4D_{self.id.upper()}_ROOT" @@ -148,12 +135,12 @@ def encode( grid_resolution=512, key_frame=None, components=None, quantization=None, overwrite=False, ) -> Path: - """Encode geometry into a directory of native payloads and timing metadata.""" - destination = Path(destination).absolute() + """Encode direct native payloads to VMESH.""" + destination = require_vmesh_destination(destination) if destination.exists() and not overwrite: raise FileExistsError(f"destination already exists: {destination}") - if destination.exists() and not destination.is_dir(): - raise NotADirectoryError(destination) + if destination.is_dir(): + raise IsADirectoryError(destination) if len(sequence) < 2: raise CodecError(f"{self.id} requires at least two frames") _positive_integer(num_centers, "num_centers") @@ -228,45 +215,20 @@ def encode( if not (result / name).is_file() or not (result / name).stat().st_size: raise CodecError(f"{self.id} encoder produced no {name}") (result / "metadata.json").write_text(json.dumps(manifest), encoding="utf-8") - previous = None - if destination.exists(): - if not overwrite: - raise FileExistsError(f"destination already exists: {destination}") - previous = Path(tempfile.mkdtemp( - prefix=f".{destination.name}.backup.", dir=destination.parent, - )) - previous.rmdir() - destination.rename(previous) - try: - if overwrite: - result.rename(destination) - else: - publish_directory(result, destination) - except BaseException: - if previous is not None: - try: - previous.rename(destination) - except OSError as error: - raise OSError( - f"Could not restore {destination}; original output remains at {previous}" - ) from error - raise - if previous is not None: - if previous.is_symlink(): - previous.unlink() - else: - shutil.rmtree(previous) - return destination + return pack_vmesh(result, destination, overwrite=overwrite) def decode(self, source: Path, *, backend=None, python=None, decoder=None) -> Sequence: - """Reconstruct frames using only the encoded directory.""" + """Reconstruct frames from VMESH.""" source = Path(source).absolute() - manifest = _manifest(source, self.id) - settings = self._settings(backend, python, None, decoder, encoding=False) temporary = tempfile.TemporaryDirectory(prefix=f"open4d-{self.id}-decode-") decoded = None try: work = Path(temporary.name) + if source.suffix.lower() != ".vmesh" or probe_codec(source) != self.id: + raise CodecError(f"{self.id} decode requires VMESH; import native directories explicitly") + source = unpack_vmesh(source, work / "native") + manifest = _manifest(source, self.id) + settings = self._settings(backend, python, None, decoder, encoding=False) output = work / "decoded" output.mkdir() settings.update(input=str(source), output=str(output), frames=len(manifest["frames"])) diff --git a/open4d/codec/_vmesh.py b/open4d/codec/_vmesh.py index 2de8feba..66e889ee 100644 --- a/open4d/codec/_vmesh.py +++ b/open4d/codec/_vmesh.py @@ -2,16 +2,13 @@ from __future__ import annotations -from collections.abc import Mapping import json import math from numbers import Real import os from pathlib import Path -import subprocess import tempfile from types import MappingProxyType -from zipfile import BadZipFile, ZIP_STORED, ZipFile import numpy as np @@ -19,10 +16,11 @@ from open4d.io import Open4DError, open_sequence from open4d.io._mesh import write_obj -from ._npz import _json_value, _publish_file, _validate_manifest +from ._metadata import _json_value, _validate_manifest, require_vmesh_destination from ._protocol import CodecError +from ._native import run as _run +from ._vmesh_format import contains_codec, pack_vmesh, probe_codec, unpack_vmesh -_SCHEMA = "open4d.vmesh-sequence/v1" _POSITION_BIT_DEPTH = 12 _DEFAULT_RAW_FPS = 30.0 @@ -74,18 +72,6 @@ def _position_normalization(manifest): return bounds, (1 << bits) - 1 -def _run(command: list[str], label: str) -> None: - result = subprocess.run( - command, shell=False, check=False, capture_output=True, text=True - ) - if result.returncode: - detail = (result.stderr or result.stdout).strip().splitlines() - raise CodecError( - f"{label} exited {result.returncode}: " - f"{detail[-1] if detail else 'no diagnostic output'}" - ) - - class _DecodedProvider: def __init__(self, temporary: tempfile.TemporaryDirectory, decoded: Sequence, manifest: dict): self.temporary = temporary @@ -175,7 +161,7 @@ def close(self) -> None: class VMeshCodec: """Invoke one external V-Mesh process per sequence direction.""" - suffixes = (".v4d",) + suffixes = (".vmesh",) backend = "native-sequence" lossless = False preserves = ("positions", "triangles") @@ -185,16 +171,7 @@ def __init__(self, identifier: str) -> None: self._environment = identifier.upper() def can_decode(self, source: Path) -> bool: - try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - return ( - isinstance(manifest, Mapping) - and manifest.get("schema") == _SCHEMA - and manifest.get("codec") == self.id - ) - except (OSError, BadZipFile, KeyError, ValueError, TypeError): - return False + return contains_codec(source, self.id) def encode( self, @@ -206,12 +183,12 @@ def encode( decoder_config: str | os.PathLike[str] | None = None, overwrite: bool = False, ) -> Path: + destination = require_vmesh_destination(destination) executable = _executable(encoder, f"OPEN4D_{self._environment}_ENCODER") configs = [Path(value).absolute() for value in (encoder_config, decoder_config) if value] for config in configs: if not config.is_file(): raise CodecError(f"native codec configuration is missing: {config}") - destination = Path(destination).absolute() if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") if not len(sequence): @@ -224,7 +201,7 @@ def encode( lower = np.minimum(lower, frame.geometry.positions.min(0)) upper = np.maximum(upper, frame.geometry.positions.max(0)) manifest = { - "schema": _SCHEMA, "codec": self.id, + "version": 1, "codec": self.id, "metadata": _json_value(sequence.metadata, "sequence"), "topology": sequence.topology.value, "has_constant_vertex_count": sequence.has_constant_vertex_count, @@ -270,22 +247,12 @@ def encode( _run(command, f"{self.id} encoder") if not stream.is_file() or not stream.stat().st_size: raise CodecError(f"{self.id} encoder produced no bitstream") - with tempfile.NamedTemporaryFile( - prefix=f".{destination.name}.", suffix=".tmp", - dir=destination.parent, delete=False, - ) as temporary_stream: - temporary = Path(temporary_stream.name) - try: - with ZipFile(temporary, "w", compression=ZIP_STORED) as archive: - archive.write(stream, "sequence.vmesh") - if decoder_config: - archive.write(Path(decoder_config).absolute(), "decoder.cfg") - archive.writestr("manifest.json", json.dumps(manifest)) - _publish_file(temporary, destination, overwrite=overwrite) - except Exception: - temporary.unlink(missing_ok=True) - raise - return destination + manifest["native"] = {"profile": f"{self.id}/1", "decoder_config": bool(decoder_config)} + (work / "metadata.json").write_text(json.dumps(manifest), encoding="utf-8") + if decoder_config: + import shutil + shutil.copyfile(Path(decoder_config).absolute(), work / "decoder.cfg") + return pack_vmesh(work, destination, overwrite=overwrite) def decode( self, @@ -296,7 +263,12 @@ def decode( fps: float | None = None, ) -> Sequence: source = Path(source).absolute() - raw = source.suffix.lower() == ".vmesh" + if source.suffix.lower() != ".vmesh": + raise CodecError("V-DMC decode requires VMESH; use migrate_legacy for older artifacts") + carried = probe_codec(source) + if carried is not None and carried != self.id: + raise CodecError(f".vmesh contains {carried}, not {self.id}") + raw = source.suffix.lower() == ".vmesh" and carried is None if not raw and fps is not None: raise TypeError("fps applies only to manifest-free .vmesh bitstreams") raw_rate = _raw_fps(fps) if raw else None @@ -311,23 +283,20 @@ def decode( work = Path(temporary.name) decoded = None try: - if raw: + if carried: + native = unpack_vmesh(source, work / "native") + manifest = json.loads((native / "metadata.json").read_text()) + _validate_manifest(manifest, schema=None, codec=self.id) + _position_normalization(manifest) + stream = native / "sequence.vmesh" + elif raw: stream = source manifest = None - else: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - _validate_manifest(manifest, schema=_SCHEMA, codec=self.id) - _position_normalization(manifest) - archive.extract("sequence.vmesh", work) - if "decoder.cfg" in archive.namelist(): - archive.extract("decoder.cfg", work) - stream = work / "sequence.vmesh" output = work / "decoded" output.mkdir() # The pinned V-DMC decoder parses decTex even with zero attributes. command = [str(executable)] - embedded_config = work / "decoder.cfg" + embedded_config = (work / "native" if carried else work) / "decoder.cfg" config = configured or ( embedded_config if embedded_config.is_file() diff --git a/open4d/codec/_vmesh_format.py b/open4d/codec/_vmesh_format.py new file mode 100644 index 00000000..bc495d11 --- /dev/null +++ b/open4d/codec/_vmesh_format.py @@ -0,0 +1,312 @@ +"""Standalone VMESH container for native codec payloads. + +VMESH owns its framing, sequence metadata and codec profiles. No V3C units, +private SEI messages or serialized Open4D objects are used. Native payloads +are copied verbatim and inspection never deserializes them. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from pathlib import Path +import struct +import tempfile + +from open4d._files import publish_directory, publish_file + +from ._metadata import _validate_manifest +from ._protocol import CodecError +from ._native_profiles import MAX_FRAMES, NEURAL_CODECS, PROFILES, layout as _layout + +_SCHEMA = "vmesh/1" +_CHUNK = 1024 * 1024 +_MAX_JSON = 16 * 1024 * 1024 +_RECORD = struct.Struct(">BBIQ") # version, kind, file ID, byte offset +_U32 = struct.Struct(">I") +_MAGIC = b"VMESH\x00\x01\x00" +_MAX_UNIT = _MAX_JSON + _RECORD.size + + +def _error(message): + return CodecError(f"invalid VMESH: {message}") + + +def _read(stream, size): + data = stream.read(size) + if len(data) != size: + raise _error("truncated stream") + return data + + +def _json(data): + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError(f"duplicate JSON key {key!r}") + result[key] = value + return result + + def constant(value): + raise ValueError(f"non-finite JSON value {value}") + + def real(value): + number = float(value) + if not math.isfinite(number): + raise ValueError("non-finite JSON number") + return number + + try: + return json.loads(data, object_pairs_hook=pairs, parse_constant=constant, parse_float=real) + except (ValueError, UnicodeError, RecursionError) as error: + raise _error(f"invalid JSON: {error}") from error + + +def _sequence_metadata(data, codec): + value = _json(data) + try: + if not isinstance(value, dict) or type(value.get("version")) is not int or value["version"] != 1: + raise ValueError("unsupported native metadata version") + _validate_manifest(value, schema=None, codec=codec) + if "schema" in value: + raise ValueError("VMESH metadata cannot contain an application schema") + if not 0 < len(value["frames"]) <= MAX_FRAMES: + raise ValueError("frame count outside limits") + _layout(codec, len(value["frames"]), value.get("native")) + except (ValueError, TypeError, KeyError) as error: + raise _error(f"invalid sequence metadata: {error}") from error + return value + + +def _descriptor(data): + value = _json(data) + if not isinstance(value, dict) or value.get("schema") != _SCHEMA: + raise _error("unsupported application schema/version") + codec, count, files = value.get("codec"), value.get("frame_count"), value.get("files") + if not isinstance(codec, str) or codec not in PROFILES: + raise _error(f"unsupported native codec {codec!r}") + if (value.get("native_version") != 1 or type(value.get("native_version")) is not int + or value.get("representation") != PROFILES[codec][0] + or value.get("dependency_mode") != PROFILES[codec][1]): + raise _error("unsupported native representation/dependencies") + if type(count) is not int or not 1 <= count <= MAX_FRAMES: + raise _error("invalid frame count") + expected = _layout(codec, count, value.get("native"))[1:] + if not isinstance(files, list) or len(files) != len(expected): + raise _error("invalid native file list") + for index, (record, (name, role)) in enumerate(zip(files, expected)): + if (not isinstance(record, dict) or type(record.get("id")) is not int + or record["id"] != index or record.get("name") != name or record.get("role") != role): + raise _error("unexpected native filename, role or file ID") + size, digest = record.get("size"), record.get("sha256") + if type(size) is not int or not 0 < size < 2**64: + raise _error("invalid native payload size") + if not isinstance(digest, str) or len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest): + raise _error("invalid SHA-256 digest") + sequence = value.get("sequence") + if not isinstance(sequence, dict) or "schema" in sequence: + raise _error("invalid VMESH sequence metadata") + parsed = _sequence_metadata(json.dumps(sequence, allow_nan=False).encode(), codec) + if len(parsed["frames"]) != count or parsed.get("native") != value.get("native"): + raise _error("sequence metadata disagrees with codec profile") + return value + + +def _write_record(stream, kind, file_id, offset, data): + unit = _RECORD.pack(1, kind, file_id, offset) + data + stream.write(_U32.pack(len(unit))) + stream.write(unit) + + +def _read_record(stream): + size = _U32.unpack(_read(stream, 4))[0] + if not _RECORD.size <= size <= _MAX_UNIT: + raise _error("record size outside limits") + version, kind, file_id, offset = _RECORD.unpack(_read(stream, _RECORD.size)) + if version != 1 or kind not in (0, 1, 2): + raise _error("unsupported record version/type") + length = size - _RECORD.size + maximum = (_MAX_JSON, _CHUNK, 32)[kind] + if not 0 < length <= maximum: + raise _error("record payload size outside limits") + return kind, file_id, offset, _read(stream, length) + + +def _start(stream): + if _read(stream, len(_MAGIC)) != _MAGIC: + raise _error("unsupported magic/version") + kind, file_id, offset, data = _read_record(stream) + if (kind, file_id, offset) != (0, 0, 0) or len(data) > _MAX_JSON: + raise _error("missing or oversized application manifest") + return _descriptor(data), hashlib.sha256(data).digest() + + +def probe_codec(source: str | Path) -> str | None: + """Identify standalone VMESH without loading native libraries. + + None means another format, including raw MPEG V-DMC. Recognizable corrupt + VMESH and the retired private O4D/V3C wrapper fail before native decoding. + """ + with Path(source).open("rb") as stream: + prefix = stream.read(128) + if not prefix.startswith(_MAGIC): + if prefix.startswith(b"PK\x03\x04"): + raise _error("archive input requires explicit migration") + if prefix.startswith(b"VMESH") or prefix[1:8] == _MAGIC[1:]: + raise _error("damaged magic/version") + if bytes.fromhex("e23b8c4791354e3490ad421c6c57b63a") in prefix or b"O4D\x01" in prefix: + raise _error("retired O4D/V3C wrapper; repack the original native directory") + return None + stream.seek(0) + descriptor, _ = _start(stream) + return descriptor["codec"] + + +def contains_codec(source: str | Path, codec: str) -> bool: + """Boolean probe for ``Codec.can_decode``; never raises for foreign input. + + Callers that need the reason a file is rejected use probe_codec instead. + """ + if Path(source).suffix.lower() != ".vmesh": + return False + try: + return probe_codec(source) == codec + except (CodecError, OSError): + return False + + +def is_mesh_profile(source: str | Path) -> bool: + """Classify VMESH for mesh-only tools without loading a decoder.""" + codec = probe_codec(source) + if codec in NEURAL_CODECS: + return False + if codec == "frames": + with Path(source).open("rb") as stream: + descriptor, _ = _start(stream) + return descriptor["native"]["representation"] == "mesh" + return True + + +def _consume(source, destination=None): + with Path(source).open("rb") as stream: + descriptor, manifest_hash = _start(stream) + for record in descriptor["files"]: + digest, offset = hashlib.sha256(), 0 + target = (destination / record["name"]).open("xb") if destination is not None else None + try: + while offset < record["size"]: + kind, file_id, position, data = _read_record(stream) + if ((kind, file_id, position) != (1, record["id"], offset) + or not 0 < len(data) <= min(_CHUNK, record["size"] - offset)): + raise _error("missing, reordered or oversized native payload chunk") + if descriptor["codec"] == "frames" and len(data) != min(_CHUNK, record["size"] - offset): + raise _error("frames profile requires fixed-size payload chunks") + digest.update(data) + if target is not None: + target.write(data) + offset += len(data) + finally: + if target is not None: + target.close() + if digest.hexdigest() != record["sha256"]: + raise _error(f"SHA-256 mismatch for {record['name']}") + kind, file_id, offset, data = _read_record(stream) + if (kind, file_id, offset, data) != (2, 0, 0, manifest_hash) or stream.read(1): + raise _error("invalid end record or trailing data") + if destination is not None: + # Adapter input is reconstructed from VMESH metadata. It is not a + # carried Open4D file or object in the container. + (destination / "metadata.json").write_text( + json.dumps(descriptor["sequence"], allow_nan=False), encoding="utf-8") + return descriptor + + +def inspect_vmesh(source: str | Path) -> dict: + """Validate a standalone VMESH and return its manifest and sequence metadata. + + Streams and verifies all payload hashes without extracting/deserializing + native data or requiring the research codecs' optional dependencies. + """ + return _consume(source) + + +def pack_vmesh(source: str | Path, destination: str | Path, *, overwrite: bool = False) -> Path: + """Pack a native codec directory into one .vmesh. + + Required native files are preserved byte for byte. Scratch files and + decoded geometry are excluded. This does not recompress the native data. + """ + if not isinstance(overwrite, bool): + raise TypeError("overwrite must be bool") + source, destination = Path(source).absolute(), Path(destination).absolute() + if destination.suffix.lower() != ".vmesh": + raise ValueError("destination must have a .vmesh extension") + if destination.exists() and not overwrite: + raise FileExistsError(destination) + if destination.is_dir(): + raise IsADirectoryError(destination) + metadata = source / "metadata.json" + if metadata.is_symlink() or not metadata.is_file() or metadata.stat().st_size > _MAX_JSON: + raise _error("missing, linked or oversized metadata.json") + data = metadata.read_bytes() + parsed = _json(data) + codec = parsed.get("codec") if isinstance(parsed, dict) else None + if not isinstance(codec, str) or codec not in PROFILES: + raise _error(f"unsupported native codec {codec!r}") + sequence = _sequence_metadata(data, codec) + descriptor = dict(schema=_SCHEMA, codec=codec, native_version=1, + representation=PROFILES[codec][0], frame_count=len(sequence["frames"]), + dependency_mode=PROFILES[codec][1], sequence=sequence, files=[]) + if "native" in sequence: + descriptor["native"] = sequence["native"] + for index, (name, role) in enumerate(_layout(codec, len(sequence["frames"]), sequence.get("native"))[1:]): + path = source / name + if path.is_symlink() or not path.is_file(): + raise _error(f"missing or linked native payload {name}") + digest, size = hashlib.sha256(), 0 + with path.open("rb") as stream: + while chunk := stream.read(_CHUNK): + digest.update(chunk) + size += len(chunk) + descriptor["files"].append(dict(id=index, name=name, role=role, size=size, sha256=digest.hexdigest())) + manifest = json.dumps(descriptor, separators=(",", ":"), allow_nan=False).encode("utf-8") + if len(manifest) > _MAX_JSON: + raise _error("application manifest exceeds size limit") + _descriptor(manifest) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f".{destination.name}-", dir=destination.parent) as directory: + temporary = Path(directory) / "stream" + with temporary.open("xb") as output: + output.write(_MAGIC) + _write_record(output, 0, 0, 0, manifest) + for record in descriptor["files"]: + digest, offset = hashlib.sha256(), 0 + with (source / record["name"]).open("rb") as stream: + while chunk := stream.read(_CHUNK): + digest.update(chunk) + _write_record(output, 1, record["id"], offset, chunk) + offset += len(chunk) + if offset != record["size"] or digest.hexdigest() != record["sha256"]: + raise _error(f"{record['name']} changed during packing") + _write_record(output, 2, 0, 0, hashlib.sha256(manifest).digest()) + publish_file(temporary, destination, overwrite=overwrite) + return destination + + +def unpack_vmesh(source: str | Path, destination: str | Path) -> Path: + """Validate and recover a native directory without decoding geometry. + + The destination must not exist. Partial or corrupt input is never published. + """ + destination = Path(destination).absolute() + if destination.exists(): + raise FileExistsError(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f".{destination.name}-", dir=destination.parent) as directory: + result = Path(directory) / "native" + result.mkdir() + _consume(source, result) + publish_directory(result, destination) + return destination diff --git a/open4d/codec/tests/editor_numerics.cs b/open4d/codec/tests/editor_numerics.cs new file mode 100644 index 00000000..47a6a85f --- /dev/null +++ b/open4d/codec/tests/editor_numerics.cs @@ -0,0 +1,140 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Numerics; +using TVMEditor.Editing.AffinityCalculation; +using TVMEditor.Editing.CenterDeformation; +using TVMEditor.Editing.SurfaceDeformation; +using TVMEditor.Structures; + +sealed class TestAffinity : IAffinityCalculation +{ + float[,] affinity; + public TestAffinity(float[,] value) { affinity = value; } + public float[,] CalculateCentersAffinity(Vector3[][] centers) { return affinity; } + public float[,] GetCentersAffinity() { return affinity; } + public void SetCentersAffinity(float[,] value) { affinity = value; } +} + +static class EditorNumerics +{ + static readonly Vector3[] Vertices = { + new Vector3(1000f, 1200f, 900f), new Vector3(1010f, 1201f, 901f), + new Vector3(1005f, 1210f, 905f) + }; + static readonly Face[] Faces = {new Face {V1 = 0, V2 = 1, V3 = 2}}; + + static void Require(bool condition, string message) + { + if (!condition) throw new Exception(message); + } + + static Vector3[] ManyCenters() + { + return Enumerable.Range(0, 40).Select(i => new Vector3( + 1000f + 100f * (i % 5), 1200f + 100f * ((i / 5) % 4), 900f + 100f * (i / 20) + )).ToArray(); + } + + static void Deform(Vector3[] centers, bool denseAffinity, Vector3 translation) + { + var affinity = new float[centers.Length, centers.Length]; + if (denseAffinity) + for (int i = 0; i < centers.Length; i++) + for (int j = 0; j < centers.Length; j++) affinity[i, j] = 1f; + var deformation = new CustomSurfaceDeformation(new TestAffinity(affinity)); + var transforms = centers.Select(c => DualQuaternion.Translation(translation)).ToArray(); + var moved = centers.Select(c => c + translation).ToArray(); + var result = deformation.DeformSurface(Vertices, Faces, centers, moved, 0, transforms); + Require(result.Vertices.Length == Vertices.Length && result.Faces.Length == Faces.Length, + "Deformation changed the mesh's element counts"); + for (int i = 0; i < Vertices.Length; i++) + { + var actual = result.Vertices[i]; + Require(float.IsFinite(actual.X) && float.IsFinite(actual.Y) && float.IsFinite(actual.Z), + "Surface deformation produced nonfinite coordinates"); + Require(Vector3.Distance(actual, Vertices[i] + translation) < 0.01f, + "Blended identical transformations failed to preserve the expected translation"); + } + Require(result.Faces[0].V1 == 0 && result.Faces[0].V2 == 1 && result.Faces[0].V3 == 2, + "Surface deformation changed connectivity"); + } + + static void EmptyCenters() + { + try { Deform(Array.Empty(), false, Vector3.Zero); } + catch (ArgumentException) { return; } + catch (AggregateException error) + { + if (error.Flatten().InnerExceptions.All(e => e is ArgumentException)) return; + throw; + } + throw new Exception("Empty deformation centers were accepted"); + } + + static void FewCenters() + { + Deform(new[] {Vertices[0]}, false, new Vector3(1f, 2f, 3f)); + Deform(new[] {Vertices[0], Vertices[1]}, false, new Vector3(1f, 2f, 3f)); + } + + static void Effectors(bool partial) + { + var centers = ManyCenters(); + var affinity = new float[centers.Length, centers.Length]; + for (int i = 0; i < centers.Length; i++) affinity[i, i] = 1f; + var deformation = new AffinityCenterDeformation(new TestAffinity(affinity)); + var indices = partial ? new[] {0, 2} : Array.Empty(); + var transforms = new DualQuaternion[centers.Length]; + var first = new Vector3(1f, 2f, 3f); + var second = new Vector3(-2f, 1f, -4f); + if (partial) + { + transforms[0] = DualQuaternion.Translation(first); + transforms[2] = DualQuaternion.Translation(second); + } + var moved = indices.Select(i => transforms[i].Transform(centers[i])).ToArray(); + var result = deformation.DeformCenters(centers, indices, moved, ref transforms); + for (int i = 0; i < centers.Length; i++) + { + var offset = partial && i == 0 ? first : partial && i == 2 ? second : Vector3.Zero; + Require(Vector3.Distance(result[i], centers[i] + offset) < 0.01f, + "Centers without effector influence must stay unchanged"); + Require(float.IsFinite(transforms[i].Norm()) && Math.Abs(transforms[i].Norm() - 1f) < 0.00001f, + "Center deformation produced a nonfinite or invalid unit transformation"); + } + } + + static void InvalidAffinity() + { + foreach (var invalid in new[] {-1f, float.NaN, float.PositiveInfinity}) + { + var affinity = new float[,] {{invalid}}; + var deformation = new AffinityCenterDeformation(new TestAffinity(affinity)); + var transformations = new[] {DualQuaternion.Identity()}; + try + { + deformation.DeformCenters(new[] {Vertices[0]}, new[] {0}, + new[] {Vertices[0]}, ref transformations); + } + catch (ArgumentException) { continue; } + throw new Exception("Invalid total affinity was accepted"); + } + } + + public static int Main(string[] arguments) + { + var cases = new Dictionary { + {"zero_affinity", () => Deform(ManyCenters(), false, Vector3.Zero)}, + {"few_centers", FewCenters}, + {"duplicate_centers", () => Deform(new[] {Vertices[0], Vertices[0], Vertices[1], Vertices[1]}, false, Vector3.Zero)}, + {"empty_centers", EmptyCenters}, + {"finite_deformation", () => Deform(ManyCenters(), true, new Vector3(3f, -4f, 2f))}, + {"partial_effectors", () => Effectors(true)}, {"empty_effectors", () => Effectors(false)}, + {"invalid_affinity", InvalidAffinity} + }; + cases[arguments[0]](); + Console.WriteLine(arguments[0] + ": passed"); + return 0; + } +} diff --git a/open4d/codec/tests/test_artifact_validation.py b/open4d/codec/tests/test_artifact_validation.py index 6efb0f7f..29847aa9 100644 --- a/open4d/codec/tests/test_artifact_validation.py +++ b/open4d/codec/tests/test_artifact_validation.py @@ -14,6 +14,7 @@ from open4d.codec import CodecError from open4d.codec._npz import NumPyZipCodec from open4d.codec import _klt, _n4mc, _qndf, _temporal, _vmesh +from open4d.codec._vmesh_format import pack_vmesh, unpack_vmesh def sequence(): @@ -23,9 +24,25 @@ def sequence(): ])) +def research_artifact(tmp_path, codec, payload, *, normalization=None): + native = tmp_path / "native" + native.mkdir() + metadata = {"version": 1, "codec": codec, "native": {"profile": f"{codec}/1"}, + "frames": [{"frame_index": 0, "timestamp": 0}]} + if codec == "klt": + metadata["normalization"] = normalization or {"center": [0, 0, 0], "scale": 1} + (native / "decoder_context.pt").write_bytes(payload) + (native / "000000_quantized_indices.zst").write_bytes(b"indices") + (native / "000000_quantized_metadata.npz").write_bytes(b"quantizer") + else: + (native / "frame_000000.pt").write_bytes(payload) + (native / "metadata.json").write_text(json.dumps(metadata)) + return pack_vmesh(native, tmp_path / "take.vmesh") + + def test_reference_encode_does_not_replace_a_file_created_during_encoding(tmp_path, monkeypatch): codec = NumPyZipCodec() - destination = tmp_path / "take.o4d" + destination = tmp_path / "take.vmesh" original = codec.pack def pack(payload): @@ -39,8 +56,8 @@ def pack(payload): def test_vmesh_preserves_neighbor_temporary_file(tmp_path, monkeypatch): - destination = tmp_path / "take.v4d" - neighbor = tmp_path / ".take.v4d.tmp" + destination = tmp_path / "take.vmesh" + neighbor = tmp_path / ".take.vmesh.tmp" neighbor.write_bytes(b"unrelated user file") def run(command, label): @@ -53,13 +70,7 @@ def run(command, label): def test_klt_closes_decoded_provider_if_output_count_is_wrong(tmp_path, monkeypatch): - artifact = tmp_path / "take.k4d" - with ZipFile(artifact, "w") as archive: - archive.writestr("manifest.json", json.dumps({ - "schema": "open4d.klt-sequence/v1", "codec": "klt", - "normalization": {"center": [0, 0, 0], "scale": 1}, - "frames": [{"frame_index": 0, "timestamp": 0}], - })) + artifact = research_artifact(tmp_path, "klt", b"context") decoded = sequence() monkeypatch.setattr(_klt, "_backend", lambda: SimpleNamespace(decode_compressed=lambda *args: None)) monkeypatch.setattr(_klt, "open_sequence", lambda *args: decoded) @@ -71,14 +82,16 @@ def test_klt_closes_decoded_provider_if_output_count_is_wrong(tmp_path, monkeypa def test_temporal_delta_rejects_broadcasted_displacements(tmp_path): from open4d.codec._temporal import TEMPORAL_DELTA_CODEC - artifact = tmp_path / "broken.td4d" - manifest = {"schema": "open4d.temporal-delta-sequence/v1", "codec": "temporal-delta", + native = tmp_path / "native" + native.mkdir() + manifest = {"version": 1, "codec": "temporal-delta", "native": {"profile": "temporal-delta/1"}, "quantization_bits": 16, "frames": [{"frame_index": i, "timestamp": i / 30} for i in range(2)]} - with artifact.open("wb") as stream: - np.savez(stream, manifest=np.frombuffer(json.dumps(manifest).encode(), dtype=np.uint8), - reference=sequence()[0].geometry.positions, triangles=np.array([[0, 1, 2]], dtype=np.uint32), - displacement=np.zeros((2, 1, 3), dtype=np.int16), scale=1.0) + np.savez(native / "sequence.npz", reference=sequence()[0].geometry.positions, + triangles=np.array([[0, 1, 2]], dtype=np.uint32), + displacement=np.zeros((2, 1, 3), dtype=np.int16), scale=1.0) + (native / "metadata.json").write_text(json.dumps(manifest)) + artifact = pack_vmesh(native, tmp_path / "broken.vmesh") with pytest.raises(CodecError, match="shape|displacement"): TEMPORAL_DELTA_CODEC.decode(artifact) @@ -93,27 +106,36 @@ def test_codec_detection_returns_false_for_malformed_manifest(tmp_path, codec, m assert codec.can_decode(artifact) is False +@pytest.mark.parametrize("payload", [b"PK\x03\x04legacy", b"VMESH\x00\x01\x00", b"VMESH\x00\x02\x00"]) +def test_codec_detection_returns_false_for_foreign_vmesh(tmp_path, payload): + from open4d.codec._api import _CODECS, _codec + artifact = tmp_path / "foreign.vmesh" + artifact.write_bytes(payload) + assert all(codec.can_decode(artifact) is False for codec in _CODECS.values()) + with pytest.raises(CodecError, match="invalid VMESH"): + _codec(None, artifact) + + @pytest.mark.parametrize("frame", [{"frame_index": -1, "timestamp": 0}, {"frame_index": 0, "timestamp": float("nan")}, {"frame_index": 0, "timestamp": 0, "metadata": []}]) -def test_reference_decode_rejects_invalid_frame_metadata_without_reading_arrays(tmp_path, frame): - artifact = tmp_path / "invalid.o4d" - with ZipFile(artifact, "w") as archive: - archive.writestr("manifest.json", json.dumps({ - "schema": "open4d.numpy-zip/v1", "codec": "npz", "frames": [frame], - })) +def test_reference_decode_rejects_invalid_frame_metadata_without_reading_arrays(tmp_path, frame, monkeypatch): + from open4d.codec import _npz + from open4d.codec.tests.test_vmesh_format import records, rewrite + artifact = NumPyZipCodec().encode(sequence(), tmp_path / "invalid.vmesh") + items = records(artifact) + manifest = json.loads(items[0][3]) + manifest["sequence"]["frames"][0] = frame + items[0] = 0, 0, 0, json.dumps(manifest).encode() + rewrite(artifact, items) + monkeypatch.setattr(_npz, "_read_array", lambda *args: pytest.fail("invalid metadata read arrays")) with pytest.raises(CodecError): NumPyZipCodec().decode(artifact) def test_klt_rejects_broadcast_normalization_before_decoding(tmp_path, monkeypatch): - artifact = tmp_path / "invalid.k4d" - with ZipFile(artifact, "w") as archive: - archive.writestr("manifest.json", json.dumps({ - "schema": "open4d.klt-sequence/v1", "codec": "klt", - "frames": [{"frame_index": 0, "timestamp": 0}], - "normalization": {"center": 0, "scale": 1}, - })) + artifact = research_artifact(tmp_path, "klt", b"context", + normalization={"center": 0, "scale": 1}) monkeypatch.setattr(_klt, "open_sequence", lambda *args: sequence()[:1]) monkeypatch.setattr(_klt, "_backend", lambda: SimpleNamespace(decode_compressed=lambda *args: None)) with pytest.raises(CodecError, match="normalization"): @@ -135,7 +157,8 @@ def test_vmesh_rejects_invalid_position_normalization(tmp_path, monkeypatch, bou monkeypatch.setattr(_vmesh, "_run", lambda *args: None) monkeypatch.setattr(_vmesh, "open_sequence", lambda *args, **kwargs: sequence()[:1]) with pytest.raises(CodecError, match="position"): - _vmesh.VDMC_CODEC.decode(artifact) + from open4d.codec import migrate_legacy + migrate_legacy(artifact, tmp_path / "converted.vmesh") @pytest.mark.torch @@ -161,13 +184,15 @@ def capture(*args, **kwargs): monkeypatch.setattr(torch.ao.quantization, "quantize_dynamic", capture) codec = _qndf.QNDF_INT8_CODEC - path = codec.encode(sequence(), tmp_path / "take.qi4d", coarse_size=3, + path = codec.encode(sequence(), tmp_path / "take.vmesh", coarse_size=3, num_subdiv=0, pe_dim=2, hidden_dim=4, num_layers=3, epochs=1, batch_size=16, input_scale=input_scale, output_scale=output_scale, device="cpu") - with ZipFile(path) as archive: - assert not any(name.endswith("/model.pt") for name in archive.namelist()) - context = torch.load(BytesIO(archive.read("frames/000000/context.pt")), weights_only=True) + native = tmp_path / "unpacked" + unpack_vmesh(path, native) + assert not any(name.name == "model.pt" for name in native.iterdir()) + context = torch.load(native / "frame_000000.pt", weights_only=True) + assert "schema" not in context coarse = context["coarse_vertices"] encoded, normalized, _, _ = _qndf._inputs(torch, models, coarse, 2, input_scale) graph = models.MeshDataset(encoded, normalized, context["coarse_faces"], torch.zeros_like(coarse), progress=False) @@ -187,7 +212,9 @@ def test_qndf_int8_rejects_executable_artifact_version(tmp_path, monkeypatch): "frames": [{"frame_index": 0, "timestamp": 0}], })) with pytest.raises(CodecError, match="executable|re-encode"): - _qndf.QNDF_INT8_CODEC.decode(path) + destination = tmp_path / "legacy" + destination.mkdir() + _qndf._extract_legacy_qndf(path, destination, int8=True) @pytest.mark.torch @@ -205,7 +232,7 @@ def test_qndf_rejects_malformed_frame_context(tmp_path, monkeypatch, field): monkeypatch.setattr(_qndf, "_backend", lambda: (torch, models, None)) coarse = torch.tensor([[0., 0, 0], [1., 0, 0], [0., 1, 0]]) context = { - "schema": "open4d.qndf/v1", "coarse_vertices": coarse, + "version": 1, "coarse_vertices": coarse, "coarse_faces": torch.tensor([[0, 1, 2]]), "input_mean": torch.zeros((1, 3)), "input_std": torch.ones((1, 3)), "input_scale": 1., "output_scale": 1., "pe_dim": 2, "hidden_dim": 4, "num_layers": 1, @@ -227,13 +254,7 @@ def test_qndf_rejects_malformed_frame_context(tmp_path, monkeypatch, field): context["normalization"][field.split(".")[1]] = bad_values[field] else: context[field.split(".")[0]] = bad_values[field] - path = tmp_path / "invalid.q4d" - with ZipFile(path, "w") as archive: - archive.writestr("manifest.json", json.dumps({ - "schema": "open4d.qndf-sequence/v1", "codec": "qndf", - "frames": [{"frame_index": 0, "timestamp": 0}], - })) - archive.writestr("frames/000000/context.pt", _qndf._torch_bytes(torch, context)) + path = research_artifact(tmp_path, "qndf", _qndf._torch_bytes(torch, context)) with pytest.raises(CodecError, match="context"): _qndf.QNDF_CODEC.decode(path, device="cpu") @@ -248,4 +269,4 @@ def unexpected_backend(): monkeypatch.setattr(_qndf, "_backend", unexpected_backend) with pytest.raises(ValueError): - _qndf.QNDF_CODEC.encode(sequence(), tmp_path / "invalid.q4d", **options) + _qndf.QNDF_CODEC.encode(sequence(), tmp_path / "invalid.vmesh", **options) diff --git a/open4d/codec/tests/test_codec.py b/open4d/codec/tests/test_codec.py index 2a5b20c3..357120af 100644 --- a/open4d/codec/tests/test_codec.py +++ b/open4d/codec/tests/test_codec.py @@ -73,7 +73,7 @@ def sequence() -> Sequence: def test_numpy_zip_round_trip_is_lazy_and_preserves_geometry(tmp_path, monkeypatch): source = sequence() - artifact = encode_sequence(source, tmp_path / "take.o4d", codec=REFERENCE) + artifact = encode_sequence(source, tmp_path / "take.vmesh", codec=REFERENCE) calls = [] import open4d.codec._npz as implementation @@ -107,7 +107,7 @@ def recording_read(*args): def test_numpy_zip_preserves_reversed_view_timing_policy(tmp_path): source = sequence()[::-1] - artifact = encode_sequence(source, tmp_path / "reversed.o4d", codec=REFERENCE) + artifact = encode_sequence(source, tmp_path / "reversed.vmesh", codec=REFERENCE) decoded = decode_sequence(artifact, codec=REFERENCE) @@ -132,25 +132,24 @@ def test_n4mc_component_filter_is_disabled_by_default(): def test_encode_refuses_to_overwrite_and_decode_rejects_corruption(tmp_path): - artifact = encode_sequence(sequence(), tmp_path / "take.o4d", codec=REFERENCE) + artifact = encode_sequence(sequence(), tmp_path / "take.vmesh", codec=REFERENCE) with pytest.raises(FileExistsError): encode_sequence(sequence(), artifact, codec=REFERENCE) - broken = tmp_path / "broken.o4d" + broken = tmp_path / "broken.vmesh" broken.write_bytes(b"not a zip") - with pytest.raises(CodecError, match="invalid Open4D artifact"): + with pytest.raises(CodecError, match="VMESH"): decode_sequence(broken, codec=REFERENCE) -@pytest.mark.parametrize(("suffix", "codec"), ( - (".o4d", REFERENCE), (".d4d", DRACO_CODEC), (".v4d", None), -)) -def test_non_object_codec_manifests_are_codec_errors(tmp_path, suffix, codec): - artifact = tmp_path / f"invalid{suffix}" - with ZipFile(artifact, "w") as archive: - archive.writestr("manifest.json", "[]") - +@pytest.mark.parametrize("codec", (REFERENCE, DRACO_CODEC, None)) +def test_non_object_codec_manifests_are_codec_errors(tmp_path, codec): + from open4d.codec import _vmesh_format + artifact = tmp_path / "invalid.vmesh" + with artifact.open("wb") as stream: + stream.write(_vmesh_format._MAGIC) + _vmesh_format._write_record(stream, 0, 0, 0, b"[]") options = {} if codec is None else {"codec": codec} - with pytest.raises(CodecError, match="manifest|no known codec"): + with pytest.raises(CodecError, match="schema"): decode_sequence(artifact, **options) @@ -158,7 +157,7 @@ def test_encode_failure_removes_partial_artifact(tmp_path): bad = Sequence( MemoryFrameProvider(tuple(sequence()), metadata={"bad": object()}) ) - destination = tmp_path / "bad.o4d" + destination = tmp_path / "bad.vmesh" with pytest.raises(CodecError, match="not serializable"): encode_sequence(bad, destination, codec=REFERENCE) @@ -179,7 +178,7 @@ def test_codec_import_does_not_load_optional_dependencies(): @pytest.mark.parametrize("codec", REFERENCE_CODECS) def test_reference_codecs_remain_usable_privately(tmp_path, codec): source = sequence() - artifact = encode_sequence(source, tmp_path / f"{codec}.o4d", codec=codec) + artifact = encode_sequence(source, tmp_path / f"{codec}.vmesh", codec=codec) decoded = decode_sequence(artifact, codec=codec) assert len(decoded) == len(source) @@ -209,7 +208,7 @@ def test_public_registry_contains_research_codecs_only(): @pytest.mark.parametrize("codec,suffix", ( - (TEMPORAL_DELTA_CODEC, ".td4d"), (TEMPORAL_PCA_CODEC, ".tp4d"), + (TEMPORAL_DELTA_CODEC, ".vmesh"), (TEMPORAL_PCA_CODEC, ".vmesh"), )) def test_temporal_codecs_fresh_decode_without_processes(tmp_path, codec, suffix): frames = [Frame( @@ -246,7 +245,7 @@ def test_encode_accepts_a_supported_path_without_codec_specific_io(tmp_path): "v 0 0 0\nv 1 0 0\nv 0 1 0\nf 1 2 3\n", encoding="ascii" ) - artifact = encode_sequence(source, tmp_path / "frame.o4d", fps=24, codec=REFERENCE) + artifact = encode_sequence(source, tmp_path / "frame.vmesh", fps=24, codec=REFERENCE) decoded = decode_sequence(artifact, codec=REFERENCE) assert len(decoded) == 1 @@ -256,13 +255,14 @@ def test_encode_accepts_a_supported_path_without_codec_specific_io(tmp_path): def test_path_reader_options_are_rejected_for_an_open_sequence(tmp_path): with pytest.raises(TypeError, match="apply only to path inputs"): - encode_sequence(sequence(), tmp_path / "frame.o4d", fps=24, codec=REFERENCE) + encode_sequence(sequence(), tmp_path / "frame.vmesh", fps=24, codec=REFERENCE) -def test_caller_supplied_codec_is_used_without_a_registry(tmp_path): +def test_caller_supplied_codec_is_used_without_a_registry(tmp_path, monkeypatch): + import open4d.codec._api as implementation class Codec: id = "memory-test" - suffixes = (".test",) + suffixes = (".vmesh",) def encode(self, value, destination, **options): self.value = value @@ -275,7 +275,8 @@ def decode(self, source, **options): return sequence() codec = Codec() - destination = tmp_path / "sequence.test" + destination = tmp_path / "sequence.vmesh" + monkeypatch.setattr(implementation, "probe_codec", lambda path: codec.id) assert encode_sequence(sequence(), destination, codec=codec, level=2) == destination assert codec.options == {"level": 2} assert len(decode_sequence(destination, codec=codec, verify=True)) == 2 @@ -289,7 +290,7 @@ def test_registered_codec_can_be_selected_by_name(tmp_path, monkeypatch): class RegisteredCodec: id = "registered-test" - suffixes = (".registered",) + suffixes = (".vmesh",) def encode(self, value, destination, **options): return destination @@ -300,7 +301,8 @@ def decode(self, source, **options): codec = RegisteredCodec() register_codec(codec) - destination = tmp_path / "sequence.registered" + destination = tmp_path / "sequence.vmesh" + monkeypatch.setattr(implementation, "probe_codec", lambda path: codec.id) assert encode_sequence(sequence(), destination, codec=codec.id) == destination assert len(decode_sequence(destination)) == 2 with pytest.raises(ValueError, match="already registered"): @@ -315,7 +317,8 @@ def test_missing_research_source_has_an_actionable_error(tmp_path, monkeypatch): research_module("klt.klt") -def test_vmesh_uses_one_native_call_per_sequence_direction(tmp_path, monkeypatch): +@pytest.mark.parametrize("identifier,suffix", [("vdmc", ".vmesh"), ("faster_vdmc", ".vmesh")]) +def test_vmesh_uses_one_native_call_per_sequence_direction(tmp_path, monkeypatch, identifier, suffix): import open4d.codec._vmesh as implementation mesh = TriangleMesh([[-2.0, 3, 4], [2, 3, 4], [-2, 7, 4]], [[0, 1, 2]]) @@ -346,9 +349,9 @@ def native_call(command, label): ) monkeypatch.setattr(implementation, "_run", native_call) - codec = VMeshCodec("native-test") + codec = VMeshCodec(identifier) artifact = codec.encode( - clean, tmp_path / "sequence.v4d", encoder=executable, + clean, tmp_path / ("sequence" + suffix), encoder=executable, ) decoded = codec.decode(artifact, decoder=executable) @@ -360,7 +363,7 @@ def native_call(command, label): assert decoded.has_vertex_correspondence is None assert decoded.timestamps == clean.timestamps assert [label for _, label in calls] == [ - "native-test encoder", "native-test decoder" + f"{identifier} encoder", f"{identifier} decoder" ] assert all(isinstance(command, list) for command, _ in calls) assert "--encodeDisplacements=1" in calls[0][0] @@ -420,7 +423,8 @@ def native_call(command, label): assert not decoded_directory.exists() -def test_klt_artifact_fresh_decode_uses_saved_payload(tmp_path, monkeypatch): +@pytest.mark.parametrize("requested", ["cpu", "auto"]) +def test_klt_artifact_fresh_decode_uses_saved_payload(tmp_path, monkeypatch, requested): import open4d.codec._klt as implementation source = Sequence(MemoryFrameProvider([ @@ -437,9 +441,11 @@ def prepare(sequence, destination, *, resolution): def compress(args, *, verify_decode): calls.append(("encode", args.num_frames, verify_decode)) - Path(args.output_path).mkdir() - with ZipFile(Path(args.output_path) / "compressed_archive.zip", "w") as archive: - archive.writestr("decoder_context.pt", b"saved-klt-context") + native = Path(args.output_path) / "compressed" + native.mkdir(parents=True) + (native / "decoder_context.pt").write_bytes(b"saved-klt-context") + (native / "000000_quantized_indices.zst").write_bytes(b"indices") + (native / "000000_quantized_metadata.npz").write_bytes(b"quantizer") def decompress(source, destination, device): calls.append(("decode", (source / "decoder_context.pt").read_bytes(), device)) @@ -449,11 +455,15 @@ def decompress(source, destination, device): ) monkeypatch.setattr(implementation, "write_tsdf_sequence", prepare) + if requested == "auto": + torch = pytest.importorskip("torch") + monkeypatch.setattr(torch.cuda, "is_available", lambda: False) + monkeypatch.setattr(torch.backends.mps, "is_available", lambda: False) monkeypatch.setattr(implementation, "_backend", lambda: SimpleNamespace( run_compression=compress, decode_compressed=decompress, )) - artifact = encode_sequence(source, tmp_path / "take.k4d", codec="klt") - decoded = decode_sequence(artifact, device="cpu") + artifact = encode_sequence(source, tmp_path / "take.vmesh", codec="klt") + decoded = decode_sequence(artifact, device=requested) assert calls == [ ("prepare", 1, 63), ("encode", 1, False), @@ -467,7 +477,7 @@ def decompress(source, destination, device): def test_encode_rejects_text_overwrite_flag_without_changing_output(tmp_path): from open4d.codec._npz import NumPyZipCodec - output = tmp_path / "existing.o4d" + output = tmp_path / "existing.vmesh" output.write_bytes(b"original") with pytest.raises(TypeError, match="overwrite"): encode_sequence(sequence(), output, codec=NumPyZipCodec(), overwrite="false") diff --git a/open4d/codec/tests/test_codec_review.py b/open4d/codec/tests/test_codec_review.py new file mode 100644 index 00000000..34559ea2 --- /dev/null +++ b/open4d/codec/tests/test_codec_review.py @@ -0,0 +1,85 @@ +import numpy as np +import pytest + + +def test_native_non_utf8_diagnostic_is_a_codec_error(): + import sys + from open4d.codec import CodecError, _vmesh + + with pytest.raises(CodecError, match="exited 2"): + _vmesh._run([sys.executable, "-c", "import os; os.write(2, b'bad\\xff'); exit(2)"], "native") + + +def test_native_timeout_is_bounded_and_explained(monkeypatch): + import sys + from open4d.codec import CodecError, _vmesh + + monkeypatch.setenv("OPEN4D_NATIVE_TIMEOUT", "0.1") + with pytest.raises(CodecError, match="timed out"): + _vmesh._run([sys.executable, "-c", "import time; time.sleep(1)"], "native") + + +def test_native_timeout_is_opt_in(monkeypatch): + from open4d.codec import _native + + monkeypatch.delenv("OPEN4D_NATIVE_TIMEOUT", raising=False) + assert _native._timeout() is None + + +@pytest.mark.parametrize("value", ["soon", "0", "-5", "inf", "nan"]) +def test_invalid_native_timeout_is_a_codec_error(monkeypatch, value): + import sys + from open4d.codec import CodecError, _vmesh + + monkeypatch.setenv("OPEN4D_NATIVE_TIMEOUT", value) + with pytest.raises(CodecError, match="OPEN4D_NATIVE_TIMEOUT"): + _vmesh._run([sys.executable, "-c", "pass"], "native") + + +@pytest.mark.parametrize("identifier", ["npz", "rle"]) +def test_array_codec_refuses_to_write_arrays_it_cannot_read(tmp_path, monkeypatch, identifier): + from open4d import Frame, MemoryFrameProvider, Sequence, TriangleMesh + from open4d.codec import CodecError, _npz + + codec = {c.id: c for c in _npz.REFERENCE_CODECS}[identifier] + mesh = TriangleMesh(np.random.default_rng(0).random((64, 3)), [[0, 1, 2]]) + source = Sequence(MemoryFrameProvider([Frame(0, 0., mesh)])) + monkeypatch.setattr(_npz, "_MAX_ARRAY_BYTES", 512) + destination = tmp_path / "large.vmesh" + with pytest.raises(CodecError, match="limit"): + codec.encode(source, destination) + assert not destination.exists() + monkeypatch.setattr(_npz, "_MAX_ARRAY_BYTES", 4096) + monkeypatch.setattr(_npz, "_MAX_RLE_MEMBER_BYTES", 2 * 4096 + 8 + 4096) + with codec.decode(codec.encode(source, destination)) as decoded: + np.testing.assert_array_equal(decoded[0].geometry.positions, mesh.positions) + + +def test_n4mc_reconstruction_uses_tsdf_sampling_extent(): + pytest.importorskip("torch") + pytest.importorskip("skimage") + pytest.importorskip("trimesh") + from open4d.codec import _n4mc + + axis = np.linspace(-1.1, 1.1, 48) + grid = np.stack(np.meshgrid(axis, axis, axis, indexing="ij"), axis=-1) + volume = np.linalg.norm(grid, axis=-1) - 0.8 + mesh = _n4mc._reconstruct_mesh(volume, _n4mc._backend()[3]) + radii = np.linalg.norm(mesh.vertices, axis=1) + np.testing.assert_allclose(radii, 0.8, atol=0.002) + + +def test_n4mc_ssim_weight_changes_loss_and_gradients(): + torch = pytest.importorskip("torch") + from open4d.codec._research import research_module + + losses = research_module("n4mc.losses") + prediction = torch.full((1, 1, 8, 8, 8), 0.5, requires_grad=True) + outputs = {"reconstruction": prediction, "rate_bpv": torch.tensor(0.)} + config = dict(lambda_rec=0., lambda_rate=0., lambda_band=0., lambda_sign=0.) + zero, _ = losses.compute_rd_loss(outputs, torch.zeros_like(prediction), {**config, "lambda_ssim": 0.}) + weighted, terms = losses.compute_rd_loss(outputs, torch.zeros_like(prediction), {**config, "lambda_ssim": 0.5}) + assert weighted > zero + assert weighted.item() == pytest.approx(0.5 * terms["ssim_loss"].item()) + weighted.backward() + assert prediction.grad.abs().sum() > 0 diff --git a/open4d/codec/tests/test_draco.py b/open4d/codec/tests/test_draco.py index 69b4fc28..01a55fce 100644 --- a/open4d/codec/tests/test_draco.py +++ b/open4d/codec/tests/test_draco.py @@ -16,7 +16,7 @@ def _round_trip(mesh: TriangleMesh, tmp_path) -> TriangleMesh: source = Sequence(MemoryFrameProvider([Frame(7, 1.25, mesh)])) decoded = DRACO_CODEC.decode( - DRACO_CODEC.encode(source, tmp_path / "attributes.d4d") + DRACO_CODEC.encode(source, tmp_path / "attributes.vmesh") ) assert decoded[0].frame_index == 7 assert decoded[0].timestamp == 1.25 @@ -93,7 +93,7 @@ def test_draco_recomputes_declarations_after_uv_corner_splitting(tmp_path): )) decoded = DRACO_CODEC.decode(DRACO_CODEC.encode( - source, tmp_path / "mixed-layout.d4d" + source, tmp_path / "mixed-layout.vmesh" )) assert [len(frame.geometry.positions) for frame in decoded] == [4, 6] diff --git a/open4d/codec/tests/test_editor_numerics.py b/open4d/codec/tests/test_editor_numerics.py new file mode 100644 index 00000000..d94d5971 --- /dev/null +++ b/open4d/codec/tests/test_editor_numerics.py @@ -0,0 +1,54 @@ +import os +from pathlib import Path +import re +import shutil +import subprocess + +import pytest + +pytestmark = pytest.mark.cpu + + +@pytest.fixture(scope="module", params=("tvmc", "tsmc")) +def editor_program(request, tmp_path_factory): + dotnet = os.environ.get("OPEN4D_TEST_DOTNET") or shutil.which("dotnet") + if not dotnet: + pytest.skip("a .NET SDK is required for native editor regressions") + root = Path(os.environ.get( + "OPEN4D_EDITOR_TEST_SOURCE_ROOT", Path(__file__).resolve().parents[2] / "codecs" + )) + source = root / request.param / "tvm-editing/TVMEditor" + if not (source / "TVMEditor.csproj").is_file(): + pytest.skip(f"{request.param} editor sources are unavailable") + sdk = subprocess.check_output([dotnet, "--list-sdks"], text=True, timeout=15) + installed = [int(value) for value in re.findall(r"^(\d+)\.", sdk, re.MULTILINE)] + if not installed or max(installed) < 3: + pytest.skip("a .NET SDK supporting netstandard2.1 is required") + work = tmp_path_factory.mktemp(f"editor-{request.param}") + shutil.copytree(source, work / "TVMEditor", ignore=shutil.ignore_patterns("bin", "obj")) + shutil.copyfile(Path(__file__).with_name("editor_numerics.cs"), work / "Program.cs") + (work / "EditorRegression.csproj").write_text(f""" +Exenet{max(installed)}.0 +false + + +""") + built = subprocess.run( + [dotnet, "build", str(work / "EditorRegression.csproj"), "--disable-build-servers", + "--nologo", "--verbosity", "quiet", "--output", str(work / "bin")], + text=True, capture_output=True, timeout=180, + ) + assert built.returncode == 0, built.stdout + built.stderr + return dotnet, work / "bin/EditorRegression.dll" + + +@pytest.mark.parametrize("case", ( + "zero_affinity", "few_centers", "duplicate_centers", "empty_centers", "finite_deformation", + "partial_effectors", "empty_effectors", "invalid_affinity", +)) +def test_native_editor_numeric_contracts(editor_program, case): + dotnet, program = editor_program + result = subprocess.run([dotnet, str(program), case], text=True, + capture_output=True, timeout=10) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == f"{case}: passed" diff --git a/open4d/codec/tests/test_klt_quantization.py b/open4d/codec/tests/test_klt_quantization.py new file mode 100644 index 00000000..9596cbee --- /dev/null +++ b/open4d/codec/tests/test_klt_quantization.py @@ -0,0 +1,88 @@ +from types import SimpleNamespace + +import numpy as np +import pytest + +torch = pytest.importorskip("torch") +pytest.importorskip("zstd") +pytest.importorskip("trimesh") + +from open4d.codec._research import research_module + + +def test_compression_quantizes_the_variance_of_each_coefficient_column(tmp_path, monkeypatch): + backend = research_module("klt.klt") + source = tmp_path / "tsdf" + source.mkdir() + grid = np.indices((4, 4, 4), dtype=np.float32) + volume = grid[0] * .2 + grid[1] ** 2 * .03 + grid[2] ** 3 * .01 - .5 + np.savez_compressed(source / "000000.npz", sdf=volume[..., None]) + quantize = backend.quantize_coeffs + calls = [] + + def check_variances(coefficients, variances, **options): + torch.testing.assert_close(variances, coefficients.var(dim=0, unbiased=True)) + calls.append(variances) + return quantize(coefficients, variances, **options) + + monkeypatch.setattr(backend, "quantize_coeffs", check_variances) + backend.run_compression(SimpleNamespace( + input_path=str(source), output_path=str(tmp_path / "encoded"), + num_frames=1, training_frames=[0], block_size=2, num_components=2, + k_total=16, fps=30, + ), verify_decode=False) + assert len(calls) == 1 + + +def test_zero_variance_column_does_not_erase_a_changing_coefficient(): + backend = research_module("klt.klt") + coefficients = torch.tensor([[-2., 0.], [-1., 0.], [1., 0.], [2., 0.]]) + indices, centers, fixed = backend.quantize_coeffs( + coefficients, coefficients.var(dim=0, unbiased=True), K_total=16 + ) + reconstructed = backend.decompress_coeffs(indices, centers, fixed, H=2) + assert fixed == {1: 0.} + torch.testing.assert_close(reconstructed, coefficients) + + +def test_constant_coefficients_round_trip_without_quantization_bins(): + backend = research_module("klt.klt") + coefficients = torch.tensor([[2., -3.], [2., -3.]]) + indices, centers, fixed = backend.quantize_coeffs( + coefficients, coefficients.var(dim=0, unbiased=True), K_total=16 + ) + reconstructed = backend.decompress_coeffs(indices, centers, fixed, H=2) + assert fixed == {0: 2., 1: -3.} + torch.testing.assert_close(reconstructed, coefficients) + + +def test_single_block_compression_round_trip(tmp_path, monkeypatch): + backend = research_module("klt.klt") + source = tmp_path / "tsdf" + source.mkdir() + volume = np.linspace(-1., 1., 64, dtype=np.float32).reshape(4, 4, 4) + np.savez_compressed(source / "000000.npz", sdf=volume[..., None]) + quantize = backend.quantize_coeffs + + def check_single_block(coefficients, variances, **options): + assert torch.isfinite(variances).all() + torch.testing.assert_close(variances, torch.zeros_like(variances)) + return quantize(coefficients, variances, **options) + + monkeypatch.setattr(backend, "quantize_coeffs", check_single_block) + backend.run_compression(SimpleNamespace( + input_path=str(source), output_path=str(tmp_path / "encoded"), + num_frames=1, training_frames=[0], block_size=4, num_components=1, + k_total=16, fps=30, + ), verify_decode=False) + folder = tmp_path / "encoded/compressed" + decoder = torch.load(folder / "decoder_context.pt", weights_only=True) + indices, centers, fixed, shape = backend.load_quantized_coeffs( + str(folder / "000000_quantized"), "cpu" + ) + coefficients = backend.decompress_coeffs(indices, centers, fixed, H=1) + blocks = backend.reconstruct_blocks_torch( + coefficients, decoder["basis"], decoder["mean"], 1 + ) + reconstructed = backend.reconstruct_volume_from_blocks_torch(blocks, shape, 4) + torch.testing.assert_close(reconstructed, torch.from_numpy(volume)) diff --git a/open4d/codec/tests/test_n4mc_determinism.py b/open4d/codec/tests/test_n4mc_determinism.py new file mode 100644 index 00000000..1396ee7a --- /dev/null +++ b/open4d/codec/tests/test_n4mc_determinism.py @@ -0,0 +1,62 @@ +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from open4d.codec import encode_sequence +from open4d.codec.tests.test_research_cpu import moving_cube + + +@pytest.mark.gpu +def test_n4mc_cuda_decode_is_stable_in_fresh_processes(tmp_path): + torch = pytest.importorskip("torch") + pytest.importorskip("point_cloud_utils") + pytest.importorskip("skimage") + if not torch.cuda.is_available(): + pytest.skip("CUDA is unavailable") + artifact = encode_sequence( + moving_cube(), tmp_path / "cube.vmesh", codec="n4mc", device="cuda", + resolution=15, epochs=30, hidden_channels=(4, 8), latent_channels=4, + learning_rate=3e-3, + ) + program = """ +import hashlib, json, sys, torch +from open4d.codec import decode_sequence +from open4d.codec._research import research_module +torch.backends.cudnn.benchmark = True +torch.backends.cudnn.deterministic = False +before = (torch.backends.cudnn.enabled, torch.backends.cudnn.benchmark, + torch.backends.cudnn.deterministic, torch.backends.cudnn.allow_tf32) +model = research_module('n4mc.models').TSDFCompressionAutoencoder +original = model.decode_quantized_latent +def checked(self, *arguments): + assert torch.backends.cudnn.deterministic + assert not torch.backends.cudnn.benchmark + assert torch.backends.cudnn.enabled == before[0] + assert torch.backends.cudnn.allow_tf32 == before[3] + return original(self, *arguments) +model.decode_quantized_latent = checked +with decode_sequence(sys.argv[1], device='cuda') as decoded: + frames = [{'index': f.frame_index, 'timestamp': f.timestamp, + 'positions': hashlib.sha256(f.geometry.positions.tobytes()).hexdigest(), + 'triangles': hashlib.sha256(f.geometry.triangles.tobytes()).hexdigest()} + for f in decoded] +after = (torch.backends.cudnn.enabled, torch.backends.cudnn.benchmark, + torch.backends.cudnn.deterministic, torch.backends.cudnn.allow_tf32) +assert after == before +print(json.dumps(frames)) +""" + root = Path(__file__).resolve().parents[3] + environment = dict(os.environ) + environment["PYTHONPATH"] = os.pathsep.join(filter(None, ( + str(root), environment.get("PYTHONPATH"), + ))) + results = [json.loads(subprocess.check_output( + [sys.executable, "-c", program, str(artifact)], env=environment, text=True, + )) for _ in range(2)] + assert results[0] == results[1] + assert [frame["index"] for frame in results[0]] == [41, 73] + assert [frame["timestamp"] for frame in results[0]] == [1.25, 2.75] diff --git a/open4d/codec/tests/test_n4mc_vmesh.py b/open4d/codec/tests/test_n4mc_vmesh.py new file mode 100644 index 00000000..76518715 --- /dev/null +++ b/open4d/codec/tests/test_n4mc_vmesh.py @@ -0,0 +1,170 @@ +"""N4MC carries a shared model and independent latents, not temporal residuals.""" +import json +from io import BytesIO +from zipfile import ZipFile + +import numpy as np +import pytest + +import open4d +from open4d.codec import CodecError, inspect_vmesh, migrate_legacy, pack_vmesh, unpack_vmesh +from open4d.codec import _n4mc + +pytestmark = pytest.mark.cpu + + +def legacy_archive(path, *, omit=None, duplicate=False, normalization=None, frames=2): + manifest = dict(schema="open4d.n4mc-sequence/v1", codec="n4mc", + normalization=normalization or dict(center=[1, 2, 3], scale=2.), + metadata={"scene": "fixture"}, allow_nonmonotonic_timestamps=False, + frames=[dict(frame_index=7, timestamp=.125, metadata={"key": "first"}), + dict(frame_index=19, timestamp=.875, metadata={"key": "last"})][:frames]) + torch = pytest.importorskip("torch") + checkpoint = BytesIO() + torch.save({"model": {}, "model_config": {}, "schema": "open4d.n4mc/v1"}, checkpoint) + payloads = {"checkpoint.pt": checkpoint.getvalue(), + **{f"frame_{i:06d}.npz": f"opaque latent {i}".encode() for i in range(frames)}} + with ZipFile(path, "w") as archive: + archive.writestr("manifest.json", json.dumps(manifest)) + for name, data in payloads.items(): + if name != omit: + archive.writestr(name, data) + if duplicate: + with pytest.warns(UserWarning, match="Duplicate"): + archive.writestr("checkpoint.pt", b"ambiguous") + archive.writestr("scratch.obj", b"never carry dense meshes") + return manifest, payloads + + +def test_n4d_migration_preserves_native_payloads_without_loading_models(tmp_path, monkeypatch): + source = tmp_path / "legacy.n4d" + manifest, payloads = legacy_archive(source) + monkeypatch.setattr(_n4mc, "_backend", lambda: pytest.fail("repacking must not load a model")) + artifact = migrate_legacy(source, tmp_path / "motion.vmesh") + source.unlink() + assert artifact.read_bytes()[:8] == b"VMESH\x00\x01\x00" + info = inspect_vmesh(artifact) + assert info["codec"] == "n4mc" + assert info["representation"] == "neural_tsdf" + assert info["dependency_mode"] == "shared-model-independent-frames" + assert info["sequence"]["normalization"] == manifest["normalization"] + assert info["sequence"]["frames"] == manifest["frames"] + assert _n4mc.N4MC_CODEC.can_decode(artifact) + recovered = unpack_vmesh(artifact, tmp_path / "recovered") + assert sorted(p.name for p in recovered.iterdir()) == sorted(["metadata.json", *payloads]) + for name, data in payloads.items(): + if name == "checkpoint.pt": + import torch + checkpoint = torch.load(recovered / name, weights_only=True) + assert checkpoint == {"model": {}, "model_config": {}} + else: + assert (recovered / name).read_bytes() == data + repacked = pack_vmesh(recovered, tmp_path / "again.vmesh") + assert artifact.read_bytes() == repacked.read_bytes() + with pytest.raises(CodecError, match="contains n4mc"): + open4d.load(artifact, codec="tsmc") + + +def test_single_frame_n4d_migrates(tmp_path, monkeypatch): + source = tmp_path / "single.n4d" + manifest, _ = legacy_archive(source, frames=1) + monkeypatch.setattr(_n4mc, "_backend", lambda: pytest.fail("repacking must not load a model")) + artifact = migrate_legacy(source, tmp_path / "single.vmesh") + assert inspect_vmesh(artifact)["sequence"]["frames"] == manifest["frames"] + + +@pytest.mark.parametrize("options", [dict(omit="checkpoint.pt"), dict(omit="frame_000001.npz"), + dict(duplicate=True), dict(normalization={"center": [0, 0, 0], "scale": 0})]) +def test_invalid_n4d_never_publishes_vmesh(tmp_path, options): + source = tmp_path / "bad.n4d" + legacy_archive(source, **options) + destination = tmp_path / "result.vmesh" + with pytest.raises(CodecError): + migrate_legacy(source, destination) + assert not destination.exists() + + +def test_n4mc_native_usdc_preserves_compressed_state_without_decoding(tmp_path, monkeypatch): + pytest.importorskip("pxr.Usd") + source = tmp_path / "legacy.n4d" + legacy_archive(source) + artifact = migrate_legacy(source, tmp_path / "motion.vmesh") + monkeypatch.setattr(_n4mc, "_backend", lambda: pytest.fail("native USD must not decode N4MC")) + with open4d.NativeSequence(artifact) as native: + usd = open4d.save(native, tmp_path / "native.usdc") + source.unlink() + expected = artifact.read_bytes() + artifact.unlink() + with open4d.load(usd) as restored: + assert restored.codec == "n4mc" + assert restored.timestamps == (.125, .875) + assert restored.frame_indices == (7, 19) + assert restored.path.read_bytes() == expected + repacked = open4d.save(restored, tmp_path / "restored.vmesh") + assert repacked.read_bytes() == expected + + +def test_corrupt_n4mc_vmesh_fails_before_loading_model(tmp_path, monkeypatch): + source = tmp_path / "original.n4d" + legacy_archive(source) + artifact = migrate_legacy(source, tmp_path / "original.vmesh") + data = artifact.read_bytes() + assert b"opaque latent 0" in data + artifact.write_bytes(data.replace(b"opaque latent 0", b"broken latent 0")) + monkeypatch.setattr(_n4mc, "_backend", lambda: pytest.fail("corrupt carriage must not load a model")) + with pytest.raises(CodecError, match="SHA-256"): + open4d.load(artifact) + + +@pytest.mark.torch +@pytest.mark.slow +def test_real_n4mc_mesh_usdc_vmesh_decode_matches_legacy_latents(tmp_path): + pytest.importorskip("pxr.Usd") + pytest.importorskip("torch") + pytest.importorskip("trimesh") + pytest.importorskip("point_cloud_utils") + pytest.importorskip("skimage") + from open4d.codec.tests.test_research_cpu import moving_cube + + with moving_cube() as source: + usd = open4d.save(source, tmp_path / "source.usdc") + artifact = open4d.encode(usd, tmp_path / "cube.vmesh", codec="n4mc", device="cpu", + resolution=7, epochs=30, hidden_channels=(4, 8), + latent_channels=4, learning_rate=3e-3) + native = unpack_vmesh(artifact, tmp_path / "native") + info = inspect_vmesh(artifact) + import torch + checkpoint = torch.load(native / "checkpoint.pt", weights_only=True) + assert "schema" not in checkpoint + # Reconstruct the legacy packaging from exactly the same native model and + # latents, proving the new carrier does not change neural reconstruction. + legacy = tmp_path / "same-latents.n4d" + metadata = info["sequence"] + manifest = {k: v for k, v in metadata.items() if k not in ("version", "native")} + manifest["schema"] = "open4d.n4mc-sequence/v1" + with ZipFile(legacy, "w") as archive: + archive.writestr("manifest.json", json.dumps(manifest)) + for record in info["files"]: + if record["name"] == "checkpoint.pt": + stream = BytesIO() + torch.save({**checkpoint, "schema": "open4d.n4mc/v1"}, stream) + archive.writestr("checkpoint.pt", stream.getvalue()) + else: + archive.write(native / record["name"], record["name"]) + with open4d.load(artifact, options={"device": "cpu"}) as decoded, open4d.load(migrate_legacy(legacy, tmp_path / "converted.vmesh"), options={"device": "cpu"}) as original: + assert len(decoded) == len(original) == 2 + assert decoded.timestamps == original.timestamps == (1.25, 2.75) + assert decoded.metadata == original.metadata + assert decoded.metadata["capture"] == "synthetic-moving-cube" + for left, right in zip(decoded, original): + assert left.frame_index == right.frame_index + assert left.metadata == right.metadata + assert len(left.geometry.positions) and len(left.geometry.triangles) + np.testing.assert_array_equal(left.geometry.positions, right.geometry.positions) + np.testing.assert_array_equal(left.geometry.triangles, right.geometry.triangles) + recovered = open4d.save(decoded, tmp_path / "decoded.usdc") + with open4d.load(recovered) as reopened: + assert reopened.timestamps == decoded.timestamps + for left, right in zip(reopened, decoded): + np.testing.assert_array_equal(left.geometry.positions, right.geometry.positions) + np.testing.assert_array_equal(left.geometry.triangles, right.geometry.triangles) diff --git a/open4d/codec/tests/test_native_temporal.py b/open4d/codec/tests/test_native_temporal.py new file mode 100644 index 00000000..b8b5982d --- /dev/null +++ b/open4d/codec/tests/test_native_temporal.py @@ -0,0 +1,234 @@ +"""Carriage/interchange tests; opaque fixtures do not claim native codec fidelity.""" +import hashlib +import json +from pathlib import Path +import shutil + +import pytest + +import open4d +from open4d.codec import CodecError, inspect_vmesh, pack_vmesh, unpack_vmesh +from open4d.codec._native_profiles import NEURAL_CODECS, layout + +pytestmark = pytest.mark.cpu + + +def test_gstream_additions_are_visible_only_in_their_own_frame(tmp_path, monkeypatch): + import sys + from types import SimpleNamespace + import open4d._native_worker as worker + + torch = pytest.importorskip("torch") + fields = ("_xyz", "_features_dc", "_features_rest", "_opacity", "_scaling", "_rotation") + queried_counts, rendered = [], [] + + class Model: + def __init__(self, *args): + pass + + def load_ply(self, path, scale): + value = 100 + int(Path(path).stem[6:]) if "added_" in path else 0 + for name in fields: + setattr(self, name, torch.tensor([[float(value)]])) + + def query_ntc(self): + queried_counts.append(len(self._xyz)) + + def update_by_ntc(self): + self._xyz = self._xyz + 1 + + class Network: + def __init__(self, **kwargs): + pass + + def cuda(self): + return self + + class Cache: + def __init__(self, *args): + pass + + def load_state_dict(self, state): + pass + + monkeypatch.setitem(sys.modules, "scene.gaussian_model", SimpleNamespace(GaussianModel=Model)) + monkeypatch.setitem(sys.modules, "tinycudann", SimpleNamespace(NetworkWithInputEncoding=Network)) + monkeypatch.setitem(sys.modules, "ntc", SimpleNamespace(NeuralTransformationCache=Cache)) + monkeypatch.setattr(torch, "load", lambda *a, **k: {"xyz_bound_min": 0, "xyz_bound_max": 1}) + monkeypatch.setattr(worker, "_save_gaussians", lambda model, path: rendered.append(model._xyz.clone())) + (tmp_path / "ntc_config.json").write_text(json.dumps({"network": {}, "encoding": {}})) + metadata = {"frames": [{}, {}, {}], "native": { + "sh_degree": 1, "rotate_sh": False, "only_mlp": False, "added": [False, True, True], + }} + + worker._gstream(tmp_path, tmp_path, metadata) + + assert queried_counts == [1, 1] + assert [frame.flatten().tolist() for frame in rendered] == [[0], [1, 101], [2, 102]] + + +def native_run(tmp_path, codec): + root = tmp_path / codec + root.mkdir() + options = {} + if codec == "vega": + (root / "manifest.json").write_text(json.dumps({"frames": [dict(frame_idx=i, group_id=3, frame_type="key" if i == 0 else "residual", file=f"frame_{i:04d}.pt") for i in range(2)]})) + for name in ("color_model.pt", "frame_0000.pt", "frame_0001.pt"): + (root / name).write_bytes(bytes(range(256)) * 5000) + elif codec == "queen": + for i in (1, 2): + frame = root / f"frames/{i:04d}" + (frame / "compressed").mkdir(parents=True) + (frame / "point_cloud.ply").write_bytes(b"opaque dense fixture") + (frame / "compressed/point_cloud.pkl").write_bytes(b"opaque compressed fixture") + options["config"] = dict(sh_degree=1, gate_params=["on"] + ["none"] * 6) + elif codec == "3dgstream": + (root / "init").mkdir() + (root / "init/point_cloud.ply").write_bytes(b"opaque initial fixture") + frame = root / "frame000001" + frame.mkdir() + (frame / "NTC.pth").write_bytes(b"opaque transform fixture") + (frame / "point_cloud.ply").write_bytes(b"must not include this dense frame") + (root / "cfg_args.json").write_text(json.dumps(dict(sh_degree=1, rotate_sh=True, only_mlp=False, iterations_s2=0))) + (root / "ntc_config.json").write_text(json.dumps(dict(network={}, encoding={}))) + else: + config = dict(native=dict(group_size=2, pca=True), voxel_size=8, + fine_model_and_render=dict(rgbnet_dim=12, stepsize=.5), + data=dict(datadir="", white_bkgd=False, inverse_y=True, flip_x=False, flip_y=False), + render=dict(near=.1, far=10.0)) + (root / "decoder_config.json").write_text(json.dumps(config)) + (root / "model_kwargs.json").write_text('{}') + (root / "rgb_net.tar").write_bytes(b"opaque shared network") + for i in (0, 1): + headers = [dict(quality=8 - half, size=[1, 2, 8, 8, 8]) for half in range(i + 1)] + (root / f"header_{i}.json").write_text(json.dumps({"headers": headers})) + for name in (f"mask_{i}.rerf", f"feature_{i}_8.rerf0", f"feature_{i}_8.rerf1"): + (root / name).write_bytes(b"opaque features") + for name in ("feature_1_7.rerf0", "feature_1_7.rerf1", "pca_m_1.npy", "deform_1.npy", "deform_mask_1.rerf"): + (root / name).write_bytes(b"opaque residual dependencies") + return root, options + + +@pytest.mark.parametrize("codec", sorted(NEURAL_CODECS)) +def test_native_import_excludes_dense_exports_and_survives_source_removal(tmp_path, codec): + source, options = native_run(tmp_path, codec) + with open4d.import_native(source, codec=codec, timestamps=[.125, .875], frame_indices=[12, 40], + metadata={"units": "m"}, frame_metadata=[{"label": "key"}, {"label": "later"}], **options) as native: + expected = native.path.read_bytes() + shutil.rmtree(source) + artifact = open4d.save(native, tmp_path / "isolated.vmesh") + assert artifact.read_bytes() == expected + with open4d.load(artifact) as loaded: + assert isinstance(loaded, open4d.NativeSequence) + assert loaded.codec == codec and loaded.timestamps == (.125, .875) + assert loaded.frame_indices == (12, 40) and loaded.metadata == {"units": "m"} + manifest = loaded.manifest + expected_names = [name for name, _ in layout(codec, 2, manifest["native"])] + extracted = loaded.unpack(tmp_path / "native") + assert sorted(p.name for p in extracted.iterdir()) == sorted(expected_names) + assert not any(p.suffix == ".obj" for p in extracted.iterdir()) + assert not (extracted / "frame000001/point_cloud.ply").exists() + repacked = pack_vmesh(extracted, tmp_path / "repacked.vmesh") + assert repacked.read_bytes() == expected + with pytest.raises(ValueError, match="closed"): + loaded.decode() + + +@pytest.mark.parametrize("codec", sorted(NEURAL_CODECS)) +def test_self_contained_usdc_restores_identical_native_bytes_without_runtime(tmp_path, codec, monkeypatch): + pytest.importorskip("pxr.Usd") + import open4d.native as module + monkeypatch.setattr(module, "run", lambda *a, **k: pytest.fail("USD interchange must not execute native models")) + source, options = native_run(tmp_path, codec) + with open4d.import_native(source, codec=codec, timestamps=[1.125, 3.75], frame_indices=[5, 11], **options) as original: + expected = original.path.read_bytes() + usd = open4d.save(original, tmp_path / "take.usdc") + from pxr import Usd + stage = Usd.Stage.Open(str(usd)) + prim = stage.GetPrimAtPath("/VMESH") + assert prim.GetTypeName() == "VMESH" + assert all(not attr.GetName().startswith("open4d:") for attr in prim.GetAttributes()) + assert not stage.GetPrimAtPath("/Open4DNative") + del stage + shutil.rmtree(source) + with open4d.load(usd) as restored: + assert restored.timestamps == (1.125, 3.75) + assert restored.frame_indices == (5, 11) + assert restored.path.read_bytes() == expected + output = open4d.save(restored, tmp_path / "again.vmesh") + assert output.read_bytes() == expected + output2 = open4d.encode(usd, tmp_path / "encode.vmesh", codec=codec) + assert output2.read_bytes() == expected + + +@pytest.mark.parametrize("codec", sorted(NEURAL_CODECS)) +def test_public_encode_accepts_native_research_outputs(tmp_path, codec): + source, options = native_run(tmp_path, codec) + result = open4d.encode(source, tmp_path / "encoded.vmesh", codec=codec, fps=12, **options) + assert inspect_vmesh(result)["sequence"]["frames"][1]["timestamp"] == 1 / 12 + with pytest.raises(CodecError, match="contains"): + open4d.load(result, codec="tsmc") + with pytest.raises(TypeError, match="timestamps"): + open4d.load(result, fps=24) + + +@pytest.mark.parametrize("codec,missing", [("queen", "frames/0002/compressed/point_cloud.pkl"), + ("3dgstream", "frame000001/NTC.pth"), + ("rerf", "deform_mask_1.rerf"), + ("rerf", "feature_1_7.rerf1"), + ("rerf", "pca_m_1.npy")]) +def test_dense_frames_cannot_substitute_for_native_dependencies(tmp_path, codec, missing): + source, options = native_run(tmp_path, codec) + (source / missing).unlink() + target = tmp_path / "result.vmesh" + with pytest.raises((CodecError, FileNotFoundError)): + open4d.encode(source, target, codec=codec, **options) + assert not target.exists() + + +def test_usd_rejects_tampered_payload_and_timeline(tmp_path): + Usd = pytest.importorskip("pxr.Usd") + source, options = native_run(tmp_path, "queen") + with open4d.import_native(source, codec="queen", **options) as native: + usd = open4d.save(native, tmp_path / "take.usdc") + stage = Usd.Stage.Open(str(usd)) + prim = stage.GetPrimAtPath("/VMESH") + prim.GetAttribute("vmesh:frameIndex").Set(999, 0) + stage.GetRootLayer().Save() + with pytest.raises(CodecError, match="timeline"): + open4d.load(usd) + payload = prim.GetAttribute("vmesh:payload:chunk000000") + values = payload.Get() + values[100] ^= 1 + payload.Set(values) + stage.GetRootLayer().Save() + with pytest.raises(CodecError, match="SHA-256"): + open4d.load(usd) + + +def test_unrecognized_independent_research_codecs_are_not_native_profiles(tmp_path): + for codec in ("klt", "qndf", "qndf_int8", "obj"): + with pytest.raises(ValueError, match="codec"): + open4d.import_native(tmp_path, codec=codec) + + +def test_native_overwrite_is_atomic_and_explicit(tmp_path): + source, options = native_run(tmp_path, "queen") + with open4d.import_native(source, codec="queen", **options) as native: + target = tmp_path / "keep.vmesh" + target.write_bytes(b"keep") + with pytest.raises(FileExistsError): + open4d.save(native, target) + assert target.read_bytes() == b"keep" + open4d.save(native, target, overwrite=True) + assert target.read_bytes() == native.path.read_bytes() + + +def test_native_usd_rejects_duplicate_times_before_publishing(tmp_path): + pytest.importorskip("pxr.Usd") + source, options = native_run(tmp_path, "queen") + destination = tmp_path / "duplicate.usdc" + with open4d.import_native(source, codec="queen", timestamps=[0, 0], **options) as native: + with pytest.raises(CodecError, match="strictly increasing"): + open4d.save(native, destination) + assert not destination.exists() diff --git a/open4d/codec/tests/test_qndf_preprocess.py b/open4d/codec/tests/test_qndf_preprocess.py index 2b46858c..d6d326bb 100644 --- a/open4d/codec/tests/test_qndf_preprocess.py +++ b/open4d/codec/tests/test_qndf_preprocess.py @@ -72,4 +72,39 @@ def test_simplification_retains_every_disconnected_component(): assert len(component_counts) == 2 assert sum(item["allocated_coarse_faces"] for item in metadata["components"]) == 8 assert all(item["actual_coarse_faces"] > 0 for item in metadata["components"]) + assert all(item["simplification_target_faces"] == item["allocated_coarse_faces"] + for item in metadata["components"]) + assert metadata["actual_coarse_faces"] == 8 + assert metadata["coarse_face_budget_overshoot"] == 0 assert target.shape == low.shape + + +@pytest.mark.open3d +def test_simplification_retries_collapsing_real_component_and_reports_overshoot(): + o3d = pytest.importorskip("open3d") + from open4d.codecs.qndf.build_dataset_open3d import build_pair + + # C5/mitch frame 100, component 62 after the builder's normalization. + vertices = np.array([ + [0.3591478930135873, 0.08521303404473901, 0.04937343443180466], + [0.3676691964180612, 0.060651630114196595, 0.04761904843676592], + [0.36140354932215, 0.0571428581241191, 0.04987468757324431], + [0.35639101790775357, 0.059899750402037134, 0.04135338416877041], + [0, 0, 0], [1, 0, 0], [0, 1, 0], + ], dtype=np.float64) + faces = np.array([[0, 1, 2], [0, 2, 3], [0, 3, 1], [4, 5, 6]], dtype=np.int32) + + low, low_faces, target, metadata = build_pair(vertices, faces, coarse_size=2, num_subdiv=0) + mesh = o3d.geometry.TriangleMesh( + o3d.utility.Vector3dVector(low), o3d.utility.Vector3iVector(low_faces) + ) + assert len(mesh.cluster_connected_triangles()[1]) == 2 + component = metadata["components"][0] + assert component["original_faces"] == 3 + assert component["allocated_coarse_faces"] == 1 + assert component["simplification_target_faces"] == 2 + assert component["actual_coarse_faces"] == 2 + assert metadata["actual_coarse_faces"] == 3 + assert metadata["coarse_face_budget_overshoot"] == 1 + assert np.isfinite(low).all() and np.isfinite(target).all() + assert np.min(target[:component["training_vertices"], 2]) > 0.04 diff --git a/open4d/codec/tests/test_queen_training_exports.py b/open4d/codec/tests/test_queen_training_exports.py new file mode 100644 index 00000000..e91a8adb --- /dev/null +++ b/open4d/codec/tests/test_queen_training_exports.py @@ -0,0 +1,68 @@ +import ast +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +def export_function(): + tree = ast.parse(Path(__file__).parents[2].joinpath("reconstruction/queen/train.py").read_text()) + function = next(node for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "_save_frame_outputs") + namespace = {} + exec(compile(ast.Module(body=[function], type_ignores=[]), "train.py", "exec"), namespace) + return namespace[function.name] + + +class Gate: + training = True + + def eval(self): + self.training = False + + def train(self, value): + self.training = value + + +@pytest.mark.parametrize("frame_idx", [1, 2]) +def test_dense_and_compressed_exports_use_final_state(frame_idx): + gate = Gate() + state = SimpleNamespace(points=4681, gate_atts=gate) + saved = [] + + def save(iteration, **kwargs): + assert not gate.training + assert kwargs == {"save_point_cloud": True} + saved.append(("ply", state.points)) + + def save_compressed(iteration, parameters): + assert not gate.training + saved.append(("compressed", state.points)) + + scene = SimpleNamespace(save=save, save_compressed=save_compressed) + export_function()(scene, state, SimpleNamespace(log_ply=True, log_compressed=True), + frame_idx, 70, None) + expected = [("ply", 4681)] + ([("compressed", 4681)] if frame_idx > 1 else []) + assert saved == expected + assert gate.training + + +def test_failed_export_restores_gate_mode(): + gate = Gate() + + def fail(*args, **kwargs): + raise OSError("disk full") + + with pytest.raises(OSError, match="disk full"): + export_function()(SimpleNamespace(save=fail), SimpleNamespace(gate_atts=gate), + SimpleNamespace(log_ply=True, log_compressed=True), 1, 70, None) + assert gate.training + + +def test_camera_iteration_does_not_publish_canonical_state(): + tree = ast.parse(Path(__file__).parents[2].joinpath("reconstruction/queen/train.py").read_text()) + training = next(node for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "training") + canonical = [node for node in ast.walk(training) if isinstance(node, ast.Call) + and any(key.arg == "save_point_cloud" for key in node.keywords)] + assert canonical == [] diff --git a/open4d/codec/tests/test_real_dataset.py b/open4d/codec/tests/test_real_dataset.py index 260c24b4..3babe8bc 100644 --- a/open4d/codec/tests/test_real_dataset.py +++ b/open4d/codec/tests/test_real_dataset.py @@ -29,7 +29,7 @@ def test_rafa_obj_frames_encode_and_decode(tmp_path): source = open_sequence(dataset, fps=30) sample = source[:2] - artifact = NumPyZipCodec().encode(sample, tmp_path / "rafa.o4d") + artifact = NumPyZipCodec().encode(sample, tmp_path / "rafa.vmesh") decoded = NumPyZipCodec().decode(artifact) assert len(source) >= 2 @@ -52,7 +52,7 @@ def test_real_draco_round_trip_on_rafa(tmp_path): pytest.skip("Rafa_Approves_hd_4k is not available") source = open_sequence(dataset, fps=30)[:2] - artifact = encode_sequence(source, tmp_path / "rafa.d4d", codec=DRACO_CODEC) + artifact = encode_sequence(source, tmp_path / "rafa.vmesh", codec=DRACO_CODEC) decoded = DRACO_CODEC.decode(artifact) assert len(decoded) == len(source) diff --git a/open4d/codec/tests/test_reference_vmesh.py b/open4d/codec/tests/test_reference_vmesh.py new file mode 100644 index 00000000..2301a43a --- /dev/null +++ b/open4d/codec/tests/test_reference_vmesh.py @@ -0,0 +1,60 @@ +from io import BytesIO +import json + +import numpy as np +import pytest + +from open4d.codec import CodecError, decode_sequence, inspect_vmesh, unpack_vmesh +from open4d.codec._npz import NumPyZipCodec, REFERENCE_CODECS, _array_from_bytes +from open4d.codec._temporal import TEMPORAL_DELTA_CODEC, TEMPORAL_PCA_CODEC +from open4d.codec.tests.test_codec import sequence + + +@pytest.mark.parametrize("codec", REFERENCE_CODECS) +def test_reference_vmesh_carries_standard_npz_arrays_without_application_manifest(tmp_path, codec): + source = sequence() + artifact = codec.encode(source, tmp_path / "take.vmesh") + manifest = inspect_vmesh(artifact) + assert artifact.read_bytes().startswith(b"VMESH\x00\x01\x00") + assert manifest["codec"] == codec.id and "open4d." not in json.dumps(manifest) + assert [record["name"] for record in manifest["files"]] == ["frame_000000.npz", "frame_000001.npz"] + native = tmp_path / "native" + unpack_vmesh(artifact, native) + with np.load(native / "frame_000000.npz", allow_pickle=False) as arrays: + assert "manifest" not in arrays.files + if codec.rle: + assert arrays["positions"].dtype == np.uint8 + else: + np.testing.assert_array_equal(arrays["positions"], source[0].geometry.positions) + with decode_sequence(artifact) as decoded: + np.testing.assert_array_equal(decoded[0].geometry.positions, source[0].geometry.positions) + np.testing.assert_array_equal(decoded[0].geometry.attributes["labels"], source[0].geometry.attributes["labels"]) + + +@pytest.mark.parametrize("payload", ( + (2 ** 40).to_bytes(8, "little") + bytes([255, 1]), + (1).to_bytes(8, "little") + bytes([255, 1]), + (1).to_bytes(8, "little") + bytes([0, 1]), +)) +def test_rle_rejects_invalid_length_before_repeat_allocates(payload, monkeypatch): + monkeypatch.setattr(np, "repeat", lambda *args: pytest.fail("invalid RLE allocated decoded values")) + with pytest.raises(CodecError, match="RLE"): + NumPyZipCodec("rle", rle=True).unpack(payload) + + +def test_npy_shape_bomb_fails_before_numpy_load_allocates(monkeypatch): + stream = BytesIO() + np.lib.format.write_array_header_1_0(stream, { + "descr": " +Exe{target} +false + + +""") + built = subprocess.run( + [dotnet, "build", str(work / "TrackerRegression.csproj"), "--disable-build-servers", + "--nologo", "--verbosity", "quiet", "--output", str(work / "bin")], + text=True, capture_output=True, timeout=180, + ) + assert built.returncode == 0, built.stdout + built.stderr + return dotnet, work / "bin/TrackerRegression.dll" + + +@pytest.mark.parametrize("case", ( + "self_distance", "empty_weighted", "zero_weights", "empty_unweighted", + "nonfinite_kd", "finite_kd", +)) +def test_native_tracker_numeric_contracts(tracker_program, case): + dotnet, program = tracker_program + result = subprocess.run([dotnet, str(program), case], text=True, + capture_output=True, timeout=10) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == f"{case}: passed" diff --git a/open4d/codec/tests/test_vmesh_format.py b/open4d/codec/tests/test_vmesh_format.py new file mode 100644 index 00000000..24832326 --- /dev/null +++ b/open4d/codec/tests/test_vmesh_format.py @@ -0,0 +1,399 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import shutil +import sys + +import numpy as np +import pytest + +import open4d +from open4d.codec import CodecError, decode_sequence, inspect_vmesh, pack_vmesh, unpack_vmesh +from open4d.codec import _api, _tracked, _vmesh_format +from open4d.io._mesh import write_obj + +pytestmark = pytest.mark.cpu + + +def native_directory(root, codec, *, large=False): + source = root / f"native.{codec}" + source.mkdir() + metadata = { + "codec": codec, "version": 1, "metadata": {"subject": "motion", "units": "m"}, + "frames": [{"frame_index": 10, "timestamp": 0.125, "metadata": {"key": True}}, + {"frame_index": 42, "timestamp": 0.375, "metadata": {"key": False}}], + } + (source / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") + # Opaque bytes test carriage; native Draco/ANS validity is tested separately. + (source / "reference.drc").write_bytes(bytes(range(256)) * (5000 if large else 1)) + if codec == "tvmc": + for index in range(2): + (source / f"displacement_{index:06d}.drc").write_bytes(b"displacement" + bytes([index])) + np.save(source / f"displacement_{index:06d}.npy", np.arange(3, dtype=np.uint32)) + else: + np.savetxt(source / "B_matrix.txt", np.arange(12).reshape(2, 6)) + np.savetxt(source / "T_matrix.txt", np.zeros((1, 6))) + np.save(source / "delta_trajectories_encoded.npy", np.array([1234, 5678], dtype=np.uint32)) + np.savez(source / "entropy_model.npz", shape=[3, 2], means=[0., 0.], stds=[1., 1.]) + return source, metadata + + +def records(path): + with path.open("rb") as stream: + stream.seek(len(_vmesh_format._MAGIC)) + result = [] + while stream.peek(1): + result.append(_vmesh_format._read_record(stream)) + return result + + +def rewrite(path, items): + with path.open("wb") as stream: + stream.write(_vmesh_format._MAGIC) + for item in items: + _vmesh_format._write_record(stream, *item) + + +@pytest.mark.parametrize("codec", ["tvmc", "tsmc"]) +def test_native_bytes_and_irregular_timing_survive_single_file_carriage(tmp_path, codec): + source, metadata = native_directory(tmp_path, codec, large=True) + expected = {p.name: p.read_bytes() for p in source.iterdir()} + (source / "frame_000.obj").write_text("scratch OBJ must not be packaged") + path = pack_vmesh(source, tmp_path / "take.vmesh") + shutil.rmtree(source) + assert path.is_file() + assert _vmesh_format.probe_codec(path) == codec + manifest = inspect_vmesh(path) + assert manifest["sequence"] == metadata + assert manifest["frame_count"] == 2 + assert manifest["dependency_mode"] == ("shared-reference" if codec == "tvmc" else "whole-group") + for record in manifest["files"]: + assert hashlib.sha256(expected[record["name"]]).hexdigest() == record["sha256"] + recovered = unpack_vmesh(path, tmp_path / "recovered") + assert json.loads((recovered / "metadata.json").read_text()) == metadata + assert {p.name: p.read_bytes() for p in recovered.iterdir() if p.name != "metadata.json"} == { + k: v for k, v in expected.items() if k != "metadata.json"} + with pytest.raises(FileExistsError): + unpack_vmesh(path, recovered) + + +def test_wire_is_standalone_vmesh_with_direct_native_payloads(tmp_path): + source, metadata = native_directory(tmp_path, "tvmc") + data = pack_vmesh(source, tmp_path / "take.vmesh").read_bytes() + assert data[:8] == b"VMESH\x00\x01\x00" + pos, messages = 8, [] + while pos < len(data): + size = int.from_bytes(data[pos:pos + 4], "big") + unit = data[pos + 4:pos + 4 + size] + assert len(unit) == size + assert unit[0] == 1 # record version + messages.append(unit[1]) + pos += 4 + size + assert pos == len(data) and messages == [0] + [1] * 5 + [2] + manifest = json.loads(records(tmp_path / "take.vmesh")[0][3]) + assert manifest["schema"] == "vmesh/1" + assert manifest["sequence"]["frames"] == metadata["frames"] + assert "schema" not in manifest["sequence"] + assert "metadata.json" not in [f["name"] for f in manifest["files"]] + assert b"open4d." not in data + assert b"O4D\x01" not in data + assert bytes.fromhex("e23b8c4791354e3490ad421c6c57b63a") not in data + + +@pytest.mark.parametrize("length", [224, 225, 226, 480, 481, 1024 * 1024, 1024 * 1024 + 1]) +def test_record_lengths_and_chunk_boundaries(tmp_path, length): + source, _ = native_directory(tmp_path, "tvmc") + payload = bytes(range(256)) * (length // 256) + bytes(range(length % 256)) + (source / "reference.drc").write_bytes(payload) + path = pack_vmesh(source, tmp_path / "take.vmesh") + chunks = [item for item in records(path) if item[0] == 1 and item[1] == 0] + assert b"".join(item[3] for item in chunks) == payload + assert all(len(item[3]) <= 1024 * 1024 for item in chunks) + recovered = unpack_vmesh(path, tmp_path / "native") + assert (recovered / "reference.drc").read_bytes() == payload + + +@pytest.mark.parametrize("damage", ["truncated", "no-end", "trailing", "hash", "order", "offset", + "bootstrap", "huge-unit", "wrong-record", "wrong-end"]) +def test_malformed_stream_never_publishes_or_launches_native_decode(tmp_path, monkeypatch, damage): + source, _ = native_directory(tmp_path, "tvmc") + path = pack_vmesh(source, tmp_path / "take.vmesh") + items = records(path) + if damage == "no-end": + rewrite(path, items[:-1]) + elif damage in ("hash", "order", "offset", "wrong-end"): + if damage == "hash": + kind, file_id, offset, blob = items[2] + items[2] = kind, file_id, offset, bytes([blob[0] ^ 1]) + blob[1:] + elif damage == "order": + items[1], items[2] = items[2], items[1] + elif damage == "offset": + kind, file_id, offset, blob = items[2] + items[2] = kind, file_id, offset + 1, blob + else: + items[-1] = (2, 0, 0, bytes(32)) + rewrite(path, items) + else: + data = bytearray(path.read_bytes()) + if damage == "truncated": + del data[-1] + elif damage == "trailing": + data.extend(b"junk") + elif damage == "bootstrap": + data[0] ^= 1 + elif damage == "huge-unit": + data[len(_vmesh_format._MAGIC):len(_vmesh_format._MAGIC) + 4] = b"\xff" * 4 + else: + data[len(_vmesh_format._MAGIC) + 4] ^= 2 + path.write_bytes(data) + monkeypatch.setattr(_tracked, "_run", lambda *args: pytest.fail("native worker called on corrupt input")) + with pytest.raises(CodecError): + inspect_vmesh(path) + with pytest.raises(CodecError): + unpack_vmesh(path, tmp_path / "recovered") + with pytest.raises(CodecError): + decode_sequence(path, codec="tvmc") + assert not (tmp_path / "recovered").exists() + assert not list(tmp_path.glob(".recovered-*")) + + +@pytest.mark.parametrize("change", ["path", "duplicate", "size", "version", "codec", "count", "mode", "metadata"]) +def test_manifest_rejects_unsafe_or_inconsistent_native_descriptions(tmp_path, change): + source, _ = native_directory(tmp_path, "tvmc") + path = pack_vmesh(source, tmp_path / "take.vmesh") + items = records(path) + manifest = json.loads(items[0][3]) + if change == "path": + manifest["files"][1]["name"] = "../escaped.drc" + elif change == "duplicate": + manifest["files"][1] = manifest["files"][0] + elif change == "size": + manifest["files"][1]["size"] = -1 + elif change == "version": + manifest["schema"] = "open4d.vmesh.native/2" + elif change == "codec": + manifest["codec"] = "obj-sequence" + elif change == "count": + manifest["frame_count"] = True + elif change == "mode": + manifest["dependency_mode"] = "independent-frames" + else: + manifest["sequence"]["frames"][1]["timestamp"] = float("nan") + blob = json.dumps(manifest).encode() + items[0], items[-1] = (0, 0, 0, blob), (2, 0, 0, hashlib.sha256(blob).digest()) + rewrite(path, items) + with pytest.raises(CodecError): + unpack_vmesh(path, tmp_path / "recovered") + assert not (tmp_path / "escaped.drc").exists() + + +def test_pack_preserves_existing_file_on_failure_and_requires_explicit_overwrite(tmp_path, monkeypatch): + source, _ = native_directory(tmp_path, "tsmc") + path = tmp_path / "take.vmesh" + path.write_bytes(b"original") + with pytest.raises(FileExistsError): + pack_vmesh(source, path) + original_writer = _vmesh_format._write_record + + def failing_writer(stream, kind, file_id, offset, data): + if kind == 1: + raise OSError("disk full") + original_writer(stream, kind, file_id, offset, data) + + monkeypatch.setattr(_vmesh_format, "_write_record", failing_writer) + with pytest.raises(OSError, match="disk full"): + pack_vmesh(source, path, overwrite=True) + assert path.read_bytes() == b"original" + assert not list(tmp_path.glob(".take.vmesh-*")) + monkeypatch.setattr(_vmesh_format, "_write_record", original_writer) + pack_vmesh(source, path, overwrite=True) + assert inspect_vmesh(path)["codec"] == "tsmc" + + +@pytest.mark.parametrize("case", ["missing", "symlink", "empty"]) +def test_pack_requires_self_contained_native_payload(tmp_path, case): + source, _ = native_directory(tmp_path, "tsmc") + path = source / "entropy_model.npz" + path.unlink() + if case == "symlink": + path.symlink_to(source / "reference.drc") + elif case == "empty": + path.touch() + with pytest.raises(CodecError): + pack_vmesh(source, tmp_path / "take.vmesh") + assert not (tmp_path / "take.vmesh").exists() + + +@pytest.mark.parametrize("codec", ["tvmc", "tsmc"]) +def test_dispatch_selects_embedded_codec_and_rejects_conflicts(tmp_path, monkeypatch, codec): + source, _ = native_directory(tmp_path, codec) + path = pack_vmesh(source, tmp_path / "take.vmesh") + calls = [] + implementation = _api._CODECS[codec] + monkeypatch.setattr(implementation, "decode", lambda source, **kw: calls.append((source, kw))) + open4d.load(path) + decode_sequence(path, codec=codec) + assert calls == [(path, {}), (path, {})] + with pytest.raises(CodecError, match="contains"): + decode_sequence(path, codec="vdmc") + with pytest.raises(TypeError, match="timestamps"): + open4d.load(path, fps=24) + + +def test_raw_vdmc_vmesh_keeps_existing_decoder_and_fps(tmp_path, monkeypatch): + path = tmp_path / "raw.vmesh" + path.write_bytes(b"ordinary VDMC stream (decoder stub)") + calls = [] + monkeypatch.setattr(_api.VDMC_CODEC, "decode", lambda source, **kw: calls.append((source, kw))) + open4d.load(path, fps=24) + assert calls == [(path, {"fps": 24})] + + +def test_tsmc_real_entropy_stream_decodes_after_packing_and_removing_source(tmp_path): + constriction = pytest.importorskip("constriction") + from open4d.codec import _tracked_worker + + source, _ = native_directory(tmp_path, "tsmc") + delta = np.array([[0.125, 4.0], [-0.75, 4.0], [0.33333, 4.0]]) + original = tmp_path / "original.npy" + np.save(original, delta) + model_path = source / "entropy_model.npz" + _tracked_worker._save_entropy_model(original, model_path) + expected = np.round(delta * 10000).astype(np.int32) + with np.load(model_path, allow_pickle=False) as model: + distribution = constriction.stream.model.QuantizedGaussian(int(model["minimum"]), int(model["maximum"])) + encoder = constriction.stream.stack.AnsCoder() + encoder.encode_reverse(expected.ravel(), distribution, + np.tile(model["means"], 3), np.tile(model["stds"], 3)) + np.save(source / "delta_trajectories_encoded.npy", encoder.get_compressed()) + path = pack_vmesh(source, tmp_path / "tsmc.vmesh") + original.unlink() + shutil.rmtree(source) + recovered = unpack_vmesh(path, tmp_path / "recovered") + actual = _tracked_worker._decode_entropy(recovered / "delta_trajectories_encoded.npy", + recovered / "entropy_model.npz") + np.testing.assert_array_equal(actual, expected / 10000) + + +@pytest.mark.parametrize("codec", ["tvmc", "tsmc"]) +def test_usdc_vmesh_usdc_api_round_trip_with_native_worker_stub(tmp_path, monkeypatch, codec): + pytest.importorskip("pxr.Usd") + source, metadata = native_directory(tmp_path, codec) + expected = {p.name: p.read_bytes() for p in source.iterdir() if p.name != "metadata.json"} + sequence = open4d.Sequence(open4d.MemoryFrameProvider([ + open4d.Frame(record["frame_index"], record["timestamp"], open4d.TriangleMesh( + [[float(i), 0, 0], [i + 1.0, 0, 0], [float(i), 1, 0]], [[0, 1, 2]]), record["metadata"]) + for i, record in enumerate(metadata["frames"]) + ], metadata=metadata["metadata"])) + usd = open4d.save(sequence, tmp_path / "input.usdc") + calls = [] + + def run(python, action, request): + settings = json.loads(request.read_text()) + calls.append(settings) + output = Path(settings["output"]) + if action == "encode": + for name, payload in expected.items(): + (output / name).write_bytes(payload) + else: + assert {p.name: p.read_bytes() for p in Path(settings["input"]).iterdir() + if p.name != "metadata.json"} == expected + for i in range(settings["frames"]): + write_obj(output / f"frame_{i:06d}.obj", sequence[i].geometry.positions, + sequence[i].geometry.triangles) + + monkeypatch.setattr(_tracked, "_run", run) + monkeypatch.setattr(_api._CODECS[codec], "_settings", lambda *a, **kw: {"codec": codec, "python": sys.executable}) + with open4d.load(usd) as opened: + path = open4d.save(opened, tmp_path / "take.vmesh", codec=codec, options={"dotnet": sys.executable}) + timestamps, indices = opened.timestamps, [f.frame_index for f in opened] + input_metadata = dict(opened.metadata) + shutil.rmtree(source) + usd.unlink() + with open4d.load(path) as decoded: + assert decoded.timestamps == timestamps + assert [f.frame_index for f in decoded] == indices + assert decoded.metadata == input_metadata + assert decoded[1].metadata == sequence[1].metadata + reconstructed = open4d.save(decoded, tmp_path / "output.usdc") + private = Path(calls[-1]["input"]).parent + assert private.exists() + assert not private.exists() + with open4d.load(reconstructed) as result: + assert result.timestamps == timestamps + np.testing.assert_allclose(result[1].geometry.positions, sequence[1].geometry.positions) + + +def test_retired_o4d_wrapper_is_rejected_before_native_decode(tmp_path): + path = tmp_path / 'old.vmesh' + path.write_bytes(b'\x60' + bytes(20) + bytes.fromhex('e23b8c4791354e3490ad421c6c57b63a') + b'O4D\x01') + with pytest.raises(CodecError, match='retired O4D/V3C'): + decode_sequence(path) + + +def test_nested_open4d_schema_is_not_a_vmesh_metadata_type(tmp_path): + source, _ = native_directory(tmp_path, 'tvmc') + metadata = json.loads((source / 'metadata.json').read_text()) + metadata['schema'] = 'open4d.some-custom-type/1' + (source / 'metadata.json').write_text(json.dumps(metadata)) + with pytest.raises(CodecError): + pack_vmesh(source, tmp_path / 'bad.vmesh') + + +def test_profile_expansion_is_bounded_before_allocating_payload_names(): + from open4d.codec._native_profiles import layout + with pytest.raises(CodecError, match='outside limits'): + layout('tvmc', 65536) + # Sixteen small JSON frame records could otherwise expand to 131k names. + profile = dict(profile='rerf/1', group_size=1, pca=False, + pca_channels=[7, 13], frames=[dict(id=i, quality=90, motion=False, + channels=[4096]) for i in range(16)]) + with pytest.raises(CodecError, match='file count outside limits'): + layout('rerf', 16, profile) + + +@pytest.mark.parametrize('data', [b'{"x":1e999}', b'{"x":NaN}', b'{"x":1,"x":2}']) +def test_json_rejects_overflow_and_ambiguous_values(data): + with pytest.raises(CodecError): + _vmesh_format._json(data) + + +@pytest.mark.parametrize('codec', ['vdmc', 'faster_vdmc']) +def test_vdmc_adapter_carries_actual_backend_bytes_and_restores_timing(tmp_path, monkeypatch, codec): + from open4d.codec import _vmesh + frames = [open4d.Frame(i + 7, t, open4d.TriangleMesh( + [[float(i), 0, 0], [i + 1., 0, 0], [float(i), 1, 0]], [[0, 1, 2]])) + for i, t in enumerate((.125, .875))] + sequence = open4d.Sequence(open4d.MemoryFrameProvider(frames)) + native_bytes = b'\x60native encoder-produced bitstream fixture' + config = tmp_path / 'decoder.cfg' + config.write_text('native decoder configuration') + normalized = [] + + def run(command, label): + options = dict(item[2:].split('=', 1) for item in command[1:] if item.startswith('--')) + stream = Path(options['compressed']) + if label.endswith('encoder'): + with open4d.load(Path(options['srcMesh']).parent) as input_sequence: + normalized.extend(f.geometry for f in input_sequence) + stream.write_bytes(native_bytes) + else: + assert stream.read_bytes() == native_bytes + assert Path(options['config']).read_text() == config.read_text() + for i, mesh in enumerate(normalized): + write_obj(Path(options['decMesh'] % i), mesh.positions, mesh.triangles) + + monkeypatch.setattr(_vmesh, '_run', run) + artifact = open4d.encode(sequence, tmp_path / 'take.vmesh', codec=codec, + encoder=sys.executable, decoder_config=config) + info = inspect_vmesh(artifact) + assert info['codec'] == codec + assert [record['name'] for record in info['files']] == ['sequence.vmesh', 'decoder.cfg'] + extracted = unpack_vmesh(artifact, tmp_path / 'native') + assert (extracted / 'sequence.vmesh').read_bytes() == native_bytes + with open4d.decode(artifact, decoder=sys.executable) as decoded: + assert decoded.timestamps == sequence.timestamps + for expected, actual in zip(sequence, decoded): + np.testing.assert_allclose(actual.geometry.positions, expected.geometry.positions, atol=1 / 4095) diff --git a/open4d/codec/tests/test_vmesh_only.py b/open4d/codec/tests/test_vmesh_only.py new file mode 100644 index 00000000..9ef42d94 --- /dev/null +++ b/open4d/codec/tests/test_vmesh_only.py @@ -0,0 +1,129 @@ +import pytest + +from open4d.codec import available_codecs, encode_sequence +from open4d.codec._api import _CODECS + + +def test_public_codecs_share_one_custom_output_extension(): + assert len(available_codecs()) == 12 + assert all(info.suffixes == (".vmesh",) for info in available_codecs()) + + +@pytest.mark.parametrize("identifier", sorted(_CODECS)) +def test_invalid_destination_fails_before_codec_backend(identifier, tmp_path): + from open4d.codec.tests.test_research_cpu import moving_cube + + destination = tmp_path / "retired.custom" + with moving_cube() as sequence: + with pytest.raises(ValueError, match=".vmesh"): + _CODECS[identifier].encode(sequence, destination) + with pytest.raises(ValueError, match=".vmesh"): + encode_sequence(sequence, destination, codec=identifier) + assert not destination.exists() + + +def test_pack_is_not_an_implicit_archive_converter(tmp_path): + from open4d.codec import CodecError, pack_vmesh + + source = tmp_path / "archive.n4d" + source.write_bytes(b"not a native directory") + destination = tmp_path / "output.vmesh" + with pytest.raises(CodecError): + pack_vmesh(source, destination) + assert not destination.exists() + + +@pytest.mark.parametrize("identifier", ("vega", "queen", "3dgstream", "rerf")) +def test_native_codec_rejects_a_retired_extension_before_loading(identifier, tmp_path): + from open4d.codec import CodecError + + source = tmp_path / "renamed.retired" + source.write_bytes(b"VMESH\x00\x01\x00") + assert not _CODECS[identifier].can_decode(source) + with pytest.raises(CodecError, match=".vmesh"): + _CODECS[identifier].decode(source) + + +@pytest.mark.parametrize("identifier", ("npz", "raw", "deflate", "bzip2", "lzma", "rle")) +def test_private_profile_can_be_selected_explicitly_without_registration(identifier, tmp_path): + from open4d.codec import decode_sequence + from open4d.codec._npz import REFERENCE_CODECS + from open4d.codec.tests.test_codec import sequence + + codec = next(item for item in REFERENCE_CODECS if item.id == identifier) + with sequence() as source: + artifact = codec.encode(source, tmp_path / "arrays.vmesh") + with decode_sequence(artifact, codec=identifier) as decoded: + assert len(decoded) == len(source) + assert decoded.timestamps == source.timestamps + + +@pytest.fixture +def installed_streamer_namespace(monkeypatch): + import sys + from pathlib import Path + + monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / "streamer")) + for name in tuple(sys.modules): + if name.startswith("open4d.streamer."): + monkeypatch.delitem(sys.modules, name) + monkeypatch.setitem(sys.modules, "open4d.streamer", None) + + +def browser_ply(): + return (b"ply\nformat ascii 1.0\nelement vertex 3\nproperty float x\n" + b"property float y\nproperty float z\nelement face 1\n" + b"property list uchar int vertex_indices\nend_header\n" + b"0 0 0\n1 0 0\n0 1 0\n3 0 1 2\n") + + +def legacy_browser_clip(path): + import struct + + payload = browser_ply() + path.write_bytes(struct.pack("<8sIII", b"O4DSEQ\x00\x00", 1, 1, 3) + + b"ply" + struct.pack("; + +static class TrackerNumerics +{ + static void Require(bool condition, string message) + { + if (!condition) throw new Exception(message); + } + + static void Reject(Action action) + { + try { action(); } + catch (ArgumentException) { return; } + throw new Exception("Invalid input was accepted"); + } + + static void SelfDistance() + { + var points = new List(); + for (int i = 0; i < 22; i++) + points.Add(new Vector4(1000f + i * 0.125f, 1200f + i * 0.25f, 900f + i * 0.375f, 0f)); + var distance = new TransformDistance(points); + var rotation = Matrix.Build.DenseOfArray(new float[,] { + {0.9553365f, -0.2955202f, 0f}, {0.2955202f, 0.9553365f, 0f}, {0f, 0f, 1f} + }); + var translation = FloatVector.Build.DenseOfArray(new float[] {33f, -52f, 7f}); + Require(distance.GetTransformDistance(rotation, rotation, translation, translation) == 0f, + "A transform's distance to itself must be exactly zero at large coordinates"); + Require(distance.GetTransformDistance(rotation, rotation.Clone(), translation, translation.Clone()) == 0f, + "Value-equal cloned transforms must also have zero distance"); + var origin = new TransformDistance(new[] {Vector4.Zero}); + var identity = Matrix.Build.DenseIdentity(3); + var zero = FloatVector.Build.Dense(3); + var offset = FloatVector.Build.DenseOfArray(new float[] {3f, -4f, 0f}); + Require(origin.GetTransformDistance(identity, identity.Clone(), zero, offset) == 25f, + "Different translations must retain their squared transform distance"); + } + + static void EmptyWeighted() + { + Reject(() => ARAP.GetTransform(Array.Empty(), Array.Empty(), Array.Empty())); + } + + static void ZeroWeights() + { + var source = new[] {Vector4.Zero}; + var target = new[] {new Vector4(1f, 2f, 3f, 0f)}; + foreach (var value in new[] {0f, -1f, float.NaN, float.PositiveInfinity}) + Reject(() => ARAP.GetTransform(source, target, new[] {value})); + var valid = ARAP.GetTransform(source, target, new[] {1f}); + Require(Vector4.Distance(valid.Apply(source[0]), target[0]) == 0f, + "A valid isolated point must retain its translation"); + } + + static void EmptyUnweighted() + { + Reject(() => ARAP.GetTransformUnweighted(new[] {Vector4.Zero}, new[] {Vector4.Zero}, new List())); + } + + static void NonfiniteKD() + { + foreach (var point in new[] { + new Vector4(float.NaN, 0f, 0f, 0f), + new Vector4(0f, float.PositiveInfinity, 0f, 0f), + new Vector4(0f, 0f, float.NegativeInfinity, 0f) + }) Reject(() => new KDTree(new[] {point})); + } + + static void FiniteKD() + { + var points = new[] { + new Vector4(-7f, 1f, 0f, 0f), new Vector4(2f, 8f, 4f, 0f), + new Vector4(3f, 2f, -1f, 0f), new Vector4(1f, -9f, 1f, 0f), + new Vector4(5f, 0f, 7f, 0f) + }; + var tree = new KDTree(points); + Require(tree.FindNearest(points[2], out float exact) == 2 && exact == 0f, + "Finite exact nearest-neighbor query failed"); + Require(tree.FindNearest(points[2] + new Vector4(0.01f, 0f, 0f, 0f), out float near) == 2 + && Math.Abs(near - 0.01f) < 0.00001f, "Finite nearby query failed"); + var duplicates = new KDTree(new[] {Vector4.One, Vector4.One, Vector4.One}); + Require(duplicates.FindNearest(Vector4.One, out float same) >= 0 && same == 0f, + "Finite duplicate coordinates must remain supported"); + } + + public static int Main(string[] arguments) + { + var cases = new Dictionary { + {"self_distance", SelfDistance}, {"empty_weighted", EmptyWeighted}, + {"zero_weights", ZeroWeights}, {"empty_unweighted", EmptyUnweighted}, + {"nonfinite_kd", NonfiniteKD}, {"finite_kd", FiniteKD} + }; + cases[arguments[0]](); + Console.WriteLine(arguments[0] + ": passed"); + return 0; + } +} diff --git a/open4d/codecs/klt/klt.py b/open4d/codecs/klt/klt.py index 2477504d..c515f166 100644 --- a/open4d/codecs/klt/klt.py +++ b/open4d/codecs/klt/klt.py @@ -19,6 +19,9 @@ """ import argparse +from collections.abc import Mapping +from io import BytesIO +import math import os import time import zipfile @@ -28,6 +31,7 @@ import torch import trimesh import zstd +import zstandard from tqdm import tqdm try: @@ -37,6 +41,56 @@ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +_MAX_VOXELS = 256 ** 3 +_MAX_INDEX_BYTES = 128 * 1024 * 1024 +_MAX_CONTEXT_BYTES = 16 * 1024 * 1024 +_MAX_METADATA_BYTES = 16 * 1024 * 1024 + + +def validate_decoder_context(context): + if (not isinstance(context, Mapping) or set(context) != { + "version", "basis", "mean", "block_size", "num_components"} + or type(context["version"]) is not int or context["version"] != 1): + raise ValueError("invalid KLT decoder context/version") + block, count = context["block_size"], context["num_components"] + if (type(block) is not int or not 1 <= block <= 16 + or type(count) is not int or not 1 <= count <= min(block ** 3, 512)): + raise ValueError("KLT decoder dimensions outside limits") + for name, shape in (("basis", (count, block ** 3)), ("mean", (1, block ** 3))): + value = context[name] + if (not isinstance(value, torch.Tensor) or value.layout != torch.strided + or value.dtype != torch.float32 or tuple(value.shape) != shape + or not torch.isfinite(value).all()): + raise ValueError(f"invalid KLT decoder {name}") + + +def _quantizer_metadata(path): + """Read bounded, non-object NPY members before NumPy allocates their arrays.""" + if os.path.getsize(path) > _MAX_METADATA_BYTES: + raise ValueError("KLT quantizer file outside limits") + result = {} + with zipfile.ZipFile(path) as archive: + members = archive.infolist() + if (len(members) > 517 or sum(member.file_size for member in members) > _MAX_METADATA_BYTES + or len({member.filename for member in members}) != len(members)): + raise ValueError("KLT quantizer metadata outside limits") + for member in members: + name = member.filename + if not name.endswith(".npy") or "/" in name or "\\" in name: + raise ValueError("invalid KLT quantizer member") + data = archive.read(member) + stream = BytesIO(data) + version = np.lib.format.read_magic(stream) + readers = {(1, 0): np.lib.format.read_array_header_1_0, + (2, 0): np.lib.format.read_array_header_2_0} + if version not in readers: + raise ValueError("unsupported KLT NPY version") + shape, _, dtype = readers[version](stream, max_header_size=4096) + if (dtype.hasobject or dtype.kind not in "iufUS" + or math.prod(shape) * dtype.itemsize != len(data) - stream.tell()): + raise ValueError("invalid KLT NPY payload size/type") + result[name[:-4]] = np.load(BytesIO(data), allow_pickle=False, max_header_size=4096) + return result def extract_training_blocks_torch(tsdf_volumes, block_size=4): @@ -138,11 +192,11 @@ def quantize_coeffs(coeffs, eigenvalues, K_total=256, max_iterations=100, tol=1e eigenvalues = eigenvalues.clamp(min=0) V_total = eigenvalues.sum() if V_total == 0: - raise ValueError("All eigenvalues are zero; cannot assign bins.") - stds = torch.sqrt(eigenvalues) - std_total = torch.sqrt(V_total) - K_h = torch.floor(K_total * stds / std_total).long() - K_h = torch.clamp(K_h, min=1) + K_h = torch.ones_like(eigenvalues, dtype=torch.long) + else: + stds = torch.sqrt(eigenvalues) + std_total = torch.sqrt(V_total) + K_h = torch.floor(K_total * stds / std_total).long().clamp(min=1) # Identify fixed dimensions (zero variance) fixed_dims = {} @@ -244,20 +298,66 @@ def save_quantized_coeffs( np.savez_compressed(f"{output_path}_metadata.npz", **metadata) -def load_quantized_coeffs(output_path, target_device=device): - metadata = np.load(f"{output_path}_metadata.npz", allow_pickle=False) +def load_quantized_coeffs(output_path, target_device=device, *, decoder=None): + metadata = _quantizer_metadata(f"{output_path}_metadata.npz") + for name, dimensions in (("indices_shape", 2), ("volume_shape", 3)): + value = metadata.get(name) + if (not isinstance(value, np.ndarray) or value.dtype != np.int64 + or value.shape != (dimensions,) or np.any(value <= 0)): + raise ValueError(f"invalid KLT {name}") shape = tuple(int(value) for value in metadata["indices_shape"]) - dtype = np.dtype(str(metadata["indices_dtype"])) - with open(f"{output_path}_indices.zst", "rb") as stream: - values = np.frombuffer(zstd.decompress(stream.read()), dtype=dtype).reshape(shape) - centers = [ - torch.from_numpy(metadata[f"bin_centers_{h}"].copy()).to(target_device) - for h in range(shape[1]) - ] - fixed_dims = dict(zip( - metadata["fixed_indices"].tolist(), metadata["fixed_values"].tolist() - )) volume_shape = tuple(int(value) for value in metadata["volume_shape"]) + if (not 1 <= shape[1] <= 512 or len(set(volume_shape)) != 1 + or not 2 <= volume_shape[0] <= 256 or math.prod(volume_shape) > _MAX_VOXELS): + raise ValueError("KLT frame dimensions outside limits") + value = metadata.get("indices_dtype") + if not isinstance(value, np.ndarray) or value.shape != () or value.dtype.kind not in "US": + raise ValueError("invalid KLT index dtype") + dtype = np.dtype(str(value)) + if dtype not in (np.dtype("uint8"), np.dtype("uint16"), np.dtype("uint32")): + raise ValueError("unsupported KLT index dtype") + expected = math.prod(shape) * dtype.itemsize + if expected > _MAX_INDEX_BYTES: + raise ValueError("KLT coefficient storage outside limits") + if decoder is not None: + block = decoder["block_size"] + blocks = math.prod((axis + block - 1) // block for axis in volume_shape) + if shape != (blocks, decoder["num_components"]): + raise ValueError("KLT coefficient dimensions disagree with decoder/grid") + required = {"indices_shape", "indices_dtype", "volume_shape", "fixed_indices", "fixed_values", + *(f"bin_centers_{h}" for h in range(shape[1]))} + if set(metadata) != required: + raise ValueError("unexpected KLT quantizer metadata") + arrays = [metadata[f"bin_centers_{h}"] for h in range(shape[1])] + if any(value.dtype != np.float32 or value.ndim != 1 or not 0 < len(value) <= 65536 + or not np.isfinite(value).all() for value in arrays): + raise ValueError("invalid KLT quantizer centers") + fixed, fixed_values = metadata["fixed_indices"], metadata["fixed_values"] + if (fixed.dtype != np.int64 or fixed.ndim != 1 or len(fixed) > shape[1] + or np.any(fixed < 0) or np.any(fixed >= shape[1]) or len(set(fixed.tolist())) != len(fixed) + or fixed_values.dtype != np.float32 or fixed_values.shape != fixed.shape + or not np.isfinite(fixed_values).all()): + raise ValueError("invalid KLT fixed coefficients") + with open(f"{output_path}_indices.zst", "rb") as stream: + data = stream.read(_MAX_INDEX_BYTES + 1) + if len(data) > _MAX_INDEX_BYTES: + raise ValueError("KLT compressed coefficients outside limits") + try: + declared = zstandard.frame_content_size(data) + if declared not in (expected, zstandard.CONTENTSIZE_UNKNOWN): + raise ValueError("KLT coefficient frame size disagrees with quantizer") + decoded = zstandard.ZstdDecompressor(max_window_size=_MAX_INDEX_BYTES // 1024).decompress( + data, max_output_size=expected, allow_extra_data=False + ) + except zstandard.ZstdError as error: + raise ValueError(f"invalid KLT coefficient stream: {error}") from error + if len(decoded) != expected: + raise ValueError("KLT coefficient output size disagrees with quantizer") + values = np.frombuffer(decoded, dtype=dtype).reshape(shape) + if any(np.any(values[:, h] >= len(centers)) for h, centers in enumerate(arrays)): + raise ValueError("KLT coefficient index outside quantizer") + centers = [torch.from_numpy(value.copy()).to(target_device) for value in arrays] + fixed_dims = dict(zip(fixed.tolist(), fixed_values.tolist())) return torch.from_numpy(values.astype(np.int64)).to(target_device), centers, fixed_dims, volume_shape @@ -327,13 +427,15 @@ def decode_compressed(input_path, output_path, target_device=None): """Decode saved KLT coefficients without access to encoder-side tensors.""" target_device = torch.device(target_device or device) input_path, output_path = os.fspath(input_path), os.fspath(output_path) + context_path = os.path.join(input_path, "decoder_context.pt") + if os.path.getsize(context_path) > _MAX_CONTEXT_BYTES: + raise ValueError("KLT decoder context outside limits") decoder = torch.load( - os.path.join(input_path, "decoder_context.pt"), + context_path, map_location=target_device, weights_only=True, ) - if decoder.get("schema") != "open4d.klt/v1": - raise ValueError(f"Unsupported KLT decoder context: {decoder.get('schema')!r}") + validate_decoder_context(decoder) os.makedirs(output_path, exist_ok=True) metadata_paths = sorted(glob(os.path.join(input_path, "*_quantized_metadata.npz"))) if not metadata_paths: @@ -342,7 +444,7 @@ def decode_compressed(input_path, output_path, target_device=None): for metadata_path in metadata_paths: prefix = metadata_path.removesuffix("_metadata.npz") indices, centers, fixed, volume_shape = load_quantized_coeffs( - prefix, target_device + prefix, target_device, decoder=decoder ) coefficients = decompress_coeffs( indices, centers, fixed, H=indices.shape[1] @@ -385,13 +487,14 @@ def run_compression(args, *, verify_decode=True): training_tsdfs = [load_tsdf_tensor(tsdf_paths[i]) for i in args.training_frames] training_blocks = extract_training_blocks_torch(training_tsdfs, block_size=args.block_size) klt_basis, mean_vec = compute_klt_basis_torch(training_blocks) + components = min(args.num_components, len(klt_basis)) context_path = os.path.join(compressed_folder, "decoder_context.pt") torch.save({ - "schema": "open4d.klt/v1", - "basis": klt_basis[:args.num_components].cpu(), + "version": 1, + "basis": klt_basis[:components].cpu(), "mean": mean_vec.cpu(), "block_size": args.block_size, - "num_components": args.num_components, + "num_components": components, }, context_path) decoder = ( torch.load(context_path, map_location=device, weights_only=True) @@ -413,7 +516,7 @@ def run_compression(args, *, verify_decode=True): num_components=args.num_components) # 2) Quantize + save - eigenvalues = torch.linalg.eigvalsh(coeffs.T @ coeffs / (coeffs.shape[0] - 1)) + eigenvalues = coeffs.var(dim=0, unbiased=coeffs.shape[0] > 1) quantized_indices, bin_centers, fixed_dims = quantize_coeffs( coeffs, eigenvalues, K_total=args.k_total) diff --git a/open4d/codecs/n4mc/README.md b/open4d/codecs/n4mc/README.md index 9e764ce6..d05c01ae 100644 --- a/open4d/codecs/n4mc/README.md +++ b/open4d/codecs/n4mc/README.md @@ -128,3 +128,22 @@ python -m evaluation.restore_sequence \ --normalization datasets/basketball_normalized/normalization.npz \ --output-dir outputs/basketball_sequence_n4mc/original_scale ``` + +## O4D `.vmesh` carriage + +The public N4MC adapter can write `.vmesh` as well as `.n4d`: + +```python +import open4d +from open4d.codec import pack_vmesh + +open4d.encode("input.usdc", "output.vmesh", codec="n4mc", device="cpu") +with open4d.load("output.vmesh", options={"device": "cpu"}) as decoded: + open4d.save(decoded, "reconstructed.usdc") +pack_vmesh("existing.n4d", "migrated.vmesh") +``` + +This carries the shared checkpoint and independent quantized TSDF latents, +including normalization and timing. It does not add temporal prediction. +See the [mesh codec notebook](../../../examples/vmesh/02_mesh_codecs.ipynb) for the explicit native USDC +route that preserves compressed bytes without neural evaluation. diff --git a/open4d/codecs/n4mc/losses/tsdf_losses.py b/open4d/codecs/n4mc/losses/tsdf_losses.py index a950b4be..c6d797cd 100644 --- a/open4d/codecs/n4mc/losses/tsdf_losses.py +++ b/open4d/codecs/n4mc/losses/tsdf_losses.py @@ -119,14 +119,13 @@ def compute_rd_loss(outputs: dict, target: torch.Tensor, config: dict) -> tuple[ target=target, temperature=float(config.get("sign_temperature", 0.1)), ) - ''' + lambda_ssim = float(config.get("lambda_ssim", 0.1)) ssim = ssim_loss( prediction=prediction, target=target, window_size=int(config.get("ssim_window_size", 5)), sigma=float(config.get("ssim_sigma", 1.5)), - ) - ''' + ) if lambda_ssim else prediction.new_zeros(()) rate = outputs["rate_bpv"].float() total = ( @@ -134,7 +133,7 @@ def compute_rd_loss(outputs: dict, target: torch.Tensor, config: dict) -> tuple[ + float(config.get("lambda_rec", 1.0)) * rec + float(config.get("lambda_band", 1.0)) * band + float(config.get("lambda_sign", 0.1)) * sign - #+ float(config.get("lambda_ssim", 0.1)) * ssim + + lambda_ssim * ssim ) return total, { "total_loss": total.detach(), @@ -142,5 +141,5 @@ def compute_rd_loss(outputs: dict, target: torch.Tensor, config: dict) -> tuple[ "rec_loss": rec.detach(), "band_loss": band.detach(), "sign_loss": sign.detach(), - #"ssim_loss": ssim.detach(), + "ssim_loss": ssim.detach(), } diff --git a/open4d/codecs/n4mc/tests/test_losses.py b/open4d/codecs/n4mc/tests/test_losses.py index 145adad9..977ece7b 100644 --- a/open4d/codecs/n4mc/tests/test_losses.py +++ b/open4d/codecs/n4mc/tests/test_losses.py @@ -1,6 +1,8 @@ import torch -from losses.tsdf_losses import compute_rd_loss, narrow_band_loss, sign_consistency_loss, ssim_loss +from open4d.codecs.n4mc.losses.tsdf_losses import ( + compute_rd_loss, narrow_band_loss, sign_consistency_loss, ssim_loss, +) def test_narrow_band_loss_focuses_on_surface(): diff --git a/open4d/codecs/qndf/README.md b/open4d/codecs/qndf/README.md index 7933320a..49214e74 100644 --- a/open4d/codecs/qndf/README.md +++ b/open4d/codecs/qndf/README.md @@ -45,6 +45,12 @@ all connected components, and simplifies, subdivides, and projects each componen independently. A JSON transform is saved beside the training pair so decoded vertices can be restored to the input coordinate system. +The coarse-face budget specifies a target. If decimation removes every triangle +of a component, the builder increases that component's target until it retains +valid faces, keeping the original component as the final fallback. This can +exceed the requested budget. Transform metadata records each allocated and used +target, `actual_coarse_faces`, and `coarse_face_budget_overshoot`. + Run the complete basketball sequence with: ```bash diff --git a/open4d/codecs/qndf/build_dataset_open3d.py b/open4d/codecs/qndf/build_dataset_open3d.py index a931d96e..7ec9d156 100644 --- a/open4d/codecs/qndf/build_dataset_open3d.py +++ b/open4d/codecs/qndf/build_dataset_open3d.py @@ -107,6 +107,18 @@ def combine(meshes: list[o3d.geometry.TriangleMesh]) -> o3d.geometry.TriangleMes return result +def simplify_component(original: o3d.geometry.TriangleMesh, budget: int): + """Increase a collapsing target until the component retains valid faces.""" + target = int(budget) + original_faces = len(original.triangles) + while target < original_faces: + coarse = clean_mesh(original.simplify_quadric_decimation(target)) + if len(coarse.triangles): + return coarse, target + target = min(original_faces, target * 2) + return o3d.geometry.TriangleMesh(original), original_faces + + def build_pair(vertices, triangles, coarse_size: int, num_subdiv: int): """Create QNDF's coarse/subdivided input and projected training target.""" source = o3d.geometry.TriangleMesh() @@ -127,13 +139,7 @@ def build_pair(vertices, triangles, coarse_size: int, num_subdiv: int): budgets = allocate_face_budget(face_counts, coarse_size) inputs, targets, details = [], [], [] for index, (original, budget) in enumerate(zip(components, budgets)): - coarse = ( - original.simplify_quadric_decimation(int(budget)) - if budget < len(original.triangles) else o3d.geometry.TriangleMesh(original) - ) - coarse = clean_mesh(coarse) - if not len(coarse.triangles): - raise RuntimeError(f"component {index} vanished during simplification") + coarse, simplification_target = simplify_component(original, int(budget)) subdivided = o3d.geometry.TriangleMesh(coarse) for _ in range(num_subdiv): subdivided = subdivided.subdivide_midpoint(number_of_iterations=1) @@ -147,15 +153,19 @@ def build_pair(vertices, triangles, coarse_size: int, num_subdiv: int): "index": index, "original_vertices": len(original.vertices), "original_faces": len(original.triangles), "allocated_coarse_faces": int(budget), + "simplification_target_faces": simplification_target, "actual_coarse_vertices": len(coarse.vertices), "actual_coarse_faces": len(coarse.triangles), "training_vertices": len(subdivided.vertices), "training_faces": len(subdivided.triangles), }) low, target = combine(inputs), combine(targets) + actual_coarse_faces = sum(component["actual_coarse_faces"] for component in details) metadata = { "bbox_min": bbox_min.tolist(), "scale": scale, "component_count": len(components), "coarse_face_budget": coarse_size, + "actual_coarse_faces": actual_coarse_faces, + "coarse_face_budget_overshoot": max(0, actual_coarse_faces - coarse_size), "num_subdiv": num_subdiv, "components": details, } return ( diff --git a/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/ARAP.cs b/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/ARAP.cs index 25ec2356..bee2f99f 100644 --- a/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/ARAP.cs +++ b/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/ARAP.cs @@ -21,6 +21,8 @@ public static class ARAP public static Transform GetTransformUnweighted(Vector4[] pts1, Vector4[] pts2, List ind) { int n = ind.Count; + if (n == 0) + throw new ArgumentException("ARAP requires at least one point", nameof(ind)); Vector4[] a = new Vector4[n]; Vector4[] b = new Vector4[n]; @@ -94,6 +96,8 @@ public static Transform GetTransform(Vector4[] a, Vector4[] b, float[] weights) Vector4 cB = new Vector4(); float wSum = 0; int n = a.Length; + if (n == 0) + throw new ArgumentException("ARAP requires at least one point", nameof(a)); //Compute weighted centroids for (int i = 0; i < n; i++) @@ -103,6 +107,9 @@ public static Transform GetTransform(Vector4[] a, Vector4[] b, float[] weights) cB += weights[i] * b[i]; } + if (!float.IsFinite(wSum) || wSum <= 0) + throw new ArgumentException("ARAP requires a finite positive total weight", nameof(weights)); + cA /= wSum; cB /= wSum; diff --git a/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/KDTree.cs b/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/KDTree.cs index 784485ae..f5de3cb4 100644 --- a/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/KDTree.cs +++ b/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/KDTree.cs @@ -70,6 +70,10 @@ public KDTree(KDTree source) public KDTree(Vector4[] points) { + for (int i = 0; i < points.Length; i++) + if (!float.IsFinite(points[i].X) || !float.IsFinite(points[i].Y) || !float.IsFinite(points[i].Z)) + throw new ArgumentException("KD-tree points must have finite XYZ coordinates", nameof(points)); + this.pnts = points; List list = new List(); for (int i = 0; i < points.Length; i++) diff --git a/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/TransformDistance.cs b/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/TransformDistance.cs index 63358405..56fc0061 100644 --- a/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/TransformDistance.cs +++ b/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/TransformDistance.cs @@ -95,6 +95,9 @@ public static void TestTransformDistance() public float GetTransformDistance(Matrix R1, Matrix R2, Vector t1, Vector t2) { + if (R1.Equals(R2) && t1.Equals(t2)) + return 0; + var dif = t1 - t2; var greenTerm = 2 * (dif).DotProduct(R1 * qSum); var blueTerm = -2 * (dif).DotProduct(R2 * qSum); diff --git a/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs b/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs index 60c069f6..943c262a 100644 --- a/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs +++ b/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs @@ -47,10 +47,12 @@ public Vector3[] DeformCenters(Vector3[] centers, int[] centerIndices, Vector3[] } // Divide by sum of weights - weightedDifference /= weightSum; - // weightedDifference /= newPositions.Length; - // Normalize DQ - weightedDifference = weightedDifference.Normalize(); + if (!float.IsFinite(weightSum) || weightSum < 0) + throw new ArgumentException("Center deformation requires finite nonnegative affinity weights"); + if (weightSum == 0) + weightedDifference = DualQuaternion.Identity(); + else + weightedDifference = (weightedDifference / weightSum).Normalize(); // Apply transformation to original center newTransformations[i] = weightedDifference; diff --git a/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs b/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs index 8094623b..b89fa8bd 100644 --- a/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs +++ b/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs @@ -284,14 +284,16 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI { //return ComputeEmbeddedDeformationsWeightsForVertex(centersKdTree, vertices, vertex, affinity, affinityThreshold, oldCenters); // Find k-nearest center - var searchCount = Neighbors; + if (Neighbors <= 0 || centersKdTree.Count == 0) + throw new ArgumentException("Surface deformation requires centers and a positive neighbor count"); + var neighborCount = System.Math.Min(Neighbors, centersKdTree.Count); + var searchCount = neighborCount; var found = 0; KdTreeNode[] nearestCenters = null; int nearestCenter; do { nearestCenters = centersKdTree.GetNearestNeighbours(new float[] { vertex.X, vertex.Y, vertex.Z }, searchCount); - searchCount *= 2; nearestCenter = nearestCenters[0].Value; found = 0; for (var i = 0; i < nearestCenters.Length; i++) @@ -306,12 +308,16 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } } + if (found >= neighborCount || searchCount == centersKdTree.Count) + break; + searchCount = searchCount > centersKdTree.Count / 2 + ? centersKdTree.Count : searchCount * 2; if (searchCount > 200) affinityThreshold = 0; - } while (found < Neighbors); + } while (true); - var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(Neighbors).ToArray(); + var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(neighborCount).ToArray(); // var nearestCenter = kdTree.Nearest(new double[] { vertices[v].X, vertices[v].Y, vertices[v].Z }).Value; @@ -324,7 +330,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI var mostAffineCenters = centerAffinities.Select((a, i) => (a, i)).OrderByDescending(x => x.a).Take(Neighbors.Value).Select(x => x.i).ToArray();*/ // Compute distances - var distances = new float[Neighbors]; + var distances = new float[mostAffineCenters.Length]; var distMin = float.PositiveInfinity; for (var i = 0; i < distances.Length; i++) { @@ -334,7 +340,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } // Compute soft minimums vector - var softmin = new float[Neighbors]; + var softmin = new float[mostAffineCenters.Length]; var softminSum = 0f; var softminMin = float.PositiveInfinity; for (var i = 0; i < softmin.Length; i++) @@ -351,7 +357,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } } - var weights1 = new float[Neighbors]; + var weights1 = new float[mostAffineCenters.Length]; var weightsSum = 0f; for (var i = 0; i < weights1.Length; i++) { @@ -372,14 +378,16 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI private (int[], float[]) ComputeEmbeddedDeformationsWeightsForVertex(KdTree centersKdTree, Vector3[] vertices, Vector3 vertex, float[,] affinity, double affinityThreshold, Vector3[] oldCenters) { // Find k-nearest center - var searchCount = Neighbors; + if (Neighbors <= 0 || centersKdTree.Count == 0) + throw new ArgumentException("Surface deformation requires centers and a positive neighbor count"); + var neighborCount = System.Math.Min(Neighbors, centersKdTree.Count); + var searchCount = neighborCount; var found = 0; KdTreeNode[] nearestCenters = null; int nearestCenter; do { nearestCenters = centersKdTree.GetNearestNeighbours(new float[] { vertex.X, vertex.Y, vertex.Z }, searchCount); - searchCount *= 2; nearestCenter = nearestCenters[0].Value; found = 0; for (var i = 0; i < nearestCenters.Length; i++) @@ -394,15 +402,19 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } } + if (found >= neighborCount || searchCount == centersKdTree.Count) + break; + searchCount = searchCount > centersKdTree.Count / 2 + ? centersKdTree.Count : searchCount * 2; if (searchCount > 200) affinityThreshold = 0; - } while (found < Neighbors); + } while (true); - var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(Neighbors).ToArray(); + var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(neighborCount).ToArray(); // Compute distances - var distances = new float[Neighbors]; + var distances = new float[mostAffineCenters.Length]; var distMax = float.PositiveInfinity; for (var i = 0; i < distances.Length; i++) { @@ -411,7 +423,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI distMax = distances[i]; } - var weights1 = new float[Neighbors]; + var weights1 = new float[mostAffineCenters.Length]; var weightsSum = 0f; for (var i = 0; i < weights1.Length; i++) { diff --git a/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/ARAP.cs b/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/ARAP.cs index 25ec2356..bee2f99f 100644 --- a/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/ARAP.cs +++ b/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/ARAP.cs @@ -21,6 +21,8 @@ public static class ARAP public static Transform GetTransformUnweighted(Vector4[] pts1, Vector4[] pts2, List ind) { int n = ind.Count; + if (n == 0) + throw new ArgumentException("ARAP requires at least one point", nameof(ind)); Vector4[] a = new Vector4[n]; Vector4[] b = new Vector4[n]; @@ -94,6 +96,8 @@ public static Transform GetTransform(Vector4[] a, Vector4[] b, float[] weights) Vector4 cB = new Vector4(); float wSum = 0; int n = a.Length; + if (n == 0) + throw new ArgumentException("ARAP requires at least one point", nameof(a)); //Compute weighted centroids for (int i = 0; i < n; i++) @@ -103,6 +107,9 @@ public static Transform GetTransform(Vector4[] a, Vector4[] b, float[] weights) cB += weights[i] * b[i]; } + if (!float.IsFinite(wSum) || wSum <= 0) + throw new ArgumentException("ARAP requires a finite positive total weight", nameof(weights)); + cA /= wSum; cB /= wSum; diff --git a/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/KDTree.cs b/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/KDTree.cs index 784485ae..f5de3cb4 100644 --- a/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/KDTree.cs +++ b/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/KDTree.cs @@ -70,6 +70,10 @@ public KDTree(KDTree source) public KDTree(Vector4[] points) { + for (int i = 0; i < points.Length; i++) + if (!float.IsFinite(points[i].X) || !float.IsFinite(points[i].Y) || !float.IsFinite(points[i].Z)) + throw new ArgumentException("KD-tree points must have finite XYZ coordinates", nameof(points)); + this.pnts = points; List list = new List(); for (int i = 0; i < points.Length; i++) diff --git a/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/TransformDistance.cs b/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/TransformDistance.cs index 63358405..56fc0061 100644 --- a/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/TransformDistance.cs +++ b/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/TransformDistance.cs @@ -95,6 +95,9 @@ public static void TestTransformDistance() public float GetTransformDistance(Matrix R1, Matrix R2, Vector t1, Vector t2) { + if (R1.Equals(R2) && t1.Equals(t2)) + return 0; + var dif = t1 - t2; var greenTerm = 2 * (dif).DotProduct(R1 * qSum); var blueTerm = -2 * (dif).DotProduct(R2 * qSum); diff --git a/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs b/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs index 60c069f6..943c262a 100644 --- a/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs +++ b/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs @@ -47,10 +47,12 @@ public Vector3[] DeformCenters(Vector3[] centers, int[] centerIndices, Vector3[] } // Divide by sum of weights - weightedDifference /= weightSum; - // weightedDifference /= newPositions.Length; - // Normalize DQ - weightedDifference = weightedDifference.Normalize(); + if (!float.IsFinite(weightSum) || weightSum < 0) + throw new ArgumentException("Center deformation requires finite nonnegative affinity weights"); + if (weightSum == 0) + weightedDifference = DualQuaternion.Identity(); + else + weightedDifference = (weightedDifference / weightSum).Normalize(); // Apply transformation to original center newTransformations[i] = weightedDifference; diff --git a/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs b/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs index 8094623b..b89fa8bd 100644 --- a/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs +++ b/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs @@ -284,14 +284,16 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI { //return ComputeEmbeddedDeformationsWeightsForVertex(centersKdTree, vertices, vertex, affinity, affinityThreshold, oldCenters); // Find k-nearest center - var searchCount = Neighbors; + if (Neighbors <= 0 || centersKdTree.Count == 0) + throw new ArgumentException("Surface deformation requires centers and a positive neighbor count"); + var neighborCount = System.Math.Min(Neighbors, centersKdTree.Count); + var searchCount = neighborCount; var found = 0; KdTreeNode[] nearestCenters = null; int nearestCenter; do { nearestCenters = centersKdTree.GetNearestNeighbours(new float[] { vertex.X, vertex.Y, vertex.Z }, searchCount); - searchCount *= 2; nearestCenter = nearestCenters[0].Value; found = 0; for (var i = 0; i < nearestCenters.Length; i++) @@ -306,12 +308,16 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } } + if (found >= neighborCount || searchCount == centersKdTree.Count) + break; + searchCount = searchCount > centersKdTree.Count / 2 + ? centersKdTree.Count : searchCount * 2; if (searchCount > 200) affinityThreshold = 0; - } while (found < Neighbors); + } while (true); - var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(Neighbors).ToArray(); + var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(neighborCount).ToArray(); // var nearestCenter = kdTree.Nearest(new double[] { vertices[v].X, vertices[v].Y, vertices[v].Z }).Value; @@ -324,7 +330,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI var mostAffineCenters = centerAffinities.Select((a, i) => (a, i)).OrderByDescending(x => x.a).Take(Neighbors.Value).Select(x => x.i).ToArray();*/ // Compute distances - var distances = new float[Neighbors]; + var distances = new float[mostAffineCenters.Length]; var distMin = float.PositiveInfinity; for (var i = 0; i < distances.Length; i++) { @@ -334,7 +340,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } // Compute soft minimums vector - var softmin = new float[Neighbors]; + var softmin = new float[mostAffineCenters.Length]; var softminSum = 0f; var softminMin = float.PositiveInfinity; for (var i = 0; i < softmin.Length; i++) @@ -351,7 +357,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } } - var weights1 = new float[Neighbors]; + var weights1 = new float[mostAffineCenters.Length]; var weightsSum = 0f; for (var i = 0; i < weights1.Length; i++) { @@ -372,14 +378,16 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI private (int[], float[]) ComputeEmbeddedDeformationsWeightsForVertex(KdTree centersKdTree, Vector3[] vertices, Vector3 vertex, float[,] affinity, double affinityThreshold, Vector3[] oldCenters) { // Find k-nearest center - var searchCount = Neighbors; + if (Neighbors <= 0 || centersKdTree.Count == 0) + throw new ArgumentException("Surface deformation requires centers and a positive neighbor count"); + var neighborCount = System.Math.Min(Neighbors, centersKdTree.Count); + var searchCount = neighborCount; var found = 0; KdTreeNode[] nearestCenters = null; int nearestCenter; do { nearestCenters = centersKdTree.GetNearestNeighbours(new float[] { vertex.X, vertex.Y, vertex.Z }, searchCount); - searchCount *= 2; nearestCenter = nearestCenters[0].Value; found = 0; for (var i = 0; i < nearestCenters.Length; i++) @@ -394,15 +402,19 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } } + if (found >= neighborCount || searchCount == centersKdTree.Count) + break; + searchCount = searchCount > centersKdTree.Count / 2 + ? centersKdTree.Count : searchCount * 2; if (searchCount > 200) affinityThreshold = 0; - } while (found < Neighbors); + } while (true); - var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(Neighbors).ToArray(); + var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(neighborCount).ToArray(); // Compute distances - var distances = new float[Neighbors]; + var distances = new float[mostAffineCenters.Length]; var distMax = float.PositiveInfinity; for (var i = 0; i < distances.Length; i++) { @@ -411,7 +423,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI distMax = distances[i]; } - var weights1 = new float[Neighbors]; + var weights1 = new float[mostAffineCenters.Length]; var weightsSum = 0f; for (var i = 0; i < weights1.Length; i++) { diff --git a/open4d/core/tests/test_gaussians.py b/open4d/core/tests/test_gaussians.py index ae1fd985..12a65332 100644 --- a/open4d/core/tests/test_gaussians.py +++ b/open4d/core/tests/test_gaussians.py @@ -326,19 +326,40 @@ def test_real_vega_round_trip(tmp_path): def test_public_encode_decode_dispatch_to_vega(tmp_path, monkeypatch): import open4d - from types import SimpleNamespace + from open4d.codec import inspect_vmesh + from open4d.native import NativeSequence frames = [splats(), splats()] - output = tmp_path / "capture.vega" + output = tmp_path / "capture.vmesh" calls = [] + def encode(values, path, **options): calls.append((values, path, options)) - return SimpleNamespace(path=path) + native_bitstream(path) + return gaussians.VegaRun(path, tmp_path, sys.executable) + monkeypatch.setattr(gaussians, "encode_gaussians", encode) - monkeypatch.setattr(gaussians, "decode_gaussians", lambda path, **options: tuple(frames)) assert open4d.encode(frames, output, codec="vega", key_iterations=10) == output - assert calls == [(frames, output, {"key_iterations": 10})] - assert open4d.decode(output) == tuple(frames) + assert calls[0][0] is frames + assert calls[0][1].name == "native" + assert calls[0][2] == {"key_iterations": 10} + assert not calls[0][1].exists() + assert inspect_vmesh(output)["codec"] == "vega" + with open4d.decode(output) as restored: + assert isinstance(restored, NativeSequence) + assert restored.codec == "vega" + assert restored.frame_indices == (0, 1) + assert restored.timestamps == (0, 1 / 30) + + +@pytest.mark.parametrize("extension", ["vega", "o4d", ""]) +def test_public_vega_output_requires_vmesh(tmp_path, monkeypatch, extension): + import open4d + monkeypatch.setattr(gaussians, "encode_gaussians", lambda *args, **kwargs: pytest.fail("native encoder launched")) + output = tmp_path / (f"capture.{extension}" if extension else "capture") + with pytest.raises(ValueError, match=".vmesh"): + open4d.encode([splats(), splats()], output, codec="vega") + assert not output.exists() def test_public_reconstruction_dispatch(tmp_path, monkeypatch): diff --git a/open4d/core/tests/test_metrics.py b/open4d/core/tests/test_metrics.py index 47f15ef1..b1db17a5 100644 --- a/open4d/core/tests/test_metrics.py +++ b/open4d/core/tests/test_metrics.py @@ -89,7 +89,7 @@ def test_sequence_weights_frames_equally_and_uses_one_peak(): def test_comparison_after_codec_round_trip(tmp_path): with mesh_sequence(side=4, frames=3) as reference: - artifact = save(reference, tmp_path / "wave.o4d", codec=NumPyZipCodec()) + artifact = save(reference, tmp_path / "wave.vmesh", codec=NumPyZipCodec()) with load(artifact, codec=NumPyZipCodec()) as decoded: result = compare_sequences(reference, decoded, peak=10) assert result.symmetric_rms == 0 diff --git a/open4d/gaussians.py b/open4d/gaussians.py index 4f9640de..0450ed1a 100644 --- a/open4d/gaussians.py +++ b/open4d/gaussians.py @@ -8,15 +8,18 @@ import subprocess import sys import tempfile -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path -from typing import Sequence +from typing import TYPE_CHECKING, Sequence import numpy as np from numpy.typing import NDArray from ._files import publish_directory +if TYPE_CHECKING: + from .native import NativeSequence + @dataclass(frozen=True, eq=False) class _GaussianGeometry: @@ -373,6 +376,7 @@ class VegaRun: path: Path runtime: Path python: str + _owner: object = field(default=None, repr=False, compare=False) def __post_init__(self) -> None: object.__setattr__(self, "path", Path(self.path).expanduser().resolve()) @@ -466,17 +470,36 @@ def decode_gaussians( class _VegaCodec: id = "vega" - suffixes = (".vega",) + suffixes = (".vmesh",) representation = "gaussian_splats" backend = "research-subprocess" lossless = False preserves = ("positions", "scales", "rotations", "opacities", "neural_appearance") - def encode(self, sequence, destination: Path, **options) -> Path: - return encode_gaussians(sequence, destination, **options).path + def can_decode(self, source): + from .codec._vmesh_format import contains_codec + return contains_codec(source, self.id) - def decode(self, source: Path, **options) -> tuple[NeuralGaussianFrame, ...]: - return decode_gaussians(source, **options) + def encode(self, sequence, destination: Path, **options) -> Path: + from .codec._metadata import require_vmesh_destination + from .native import NativeSequence, import_native, save_native + from .codec._native_temporal import encode_native + destination = require_vmesh_destination(destination) + if isinstance(sequence, (NativeSequence, VegaRun, str, os.PathLike)): + return encode_native(sequence, destination, codec="vega", **options) + overwrite = options.pop("overwrite", False) + if destination.exists() and not overwrite: + raise FileExistsError(destination) + timeline = {key: options.pop(key) for key in ("fps", "timestamps", "frame_indices", "metadata", "frame_metadata") if key in options} + with tempfile.TemporaryDirectory(prefix="open4d-vega-encode-") as folder: + encoded = encode_gaussians(sequence, Path(folder) / "native", **options) + with import_native(encoded, **timeline) as native: + return save_native(native, destination, overwrite=overwrite) + + def decode(self, source: Path, **options) -> NativeSequence: + """Open the owned native state; call ``.decode()`` to evaluate frames.""" + from .codec._native_temporal import NativeTemporalCodec + return NativeTemporalCodec("vega").decode(source, **options) VEGA_CODEC = _VegaCodec() diff --git a/open4d/io/_api.py b/open4d/io/_api.py index a0b3c297..2037a429 100644 --- a/open4d/io/_api.py +++ b/open4d/io/_api.py @@ -18,7 +18,7 @@ import numpy as np -from open4d.core import Frame, Sequence, TopologyMode, TriangleMesh +from open4d.core import Frame, Representation, Sequence, TopologyMode, TriangleMesh from open4d._files import publish_directory, publish_file from . import _mesh, _usd @@ -491,10 +491,20 @@ def _write_frame( path: Path, frame: Frame, suffix: str, *, allow_lossy: bool = False ) -> Path: mesh = frame.geometry + if mesh.representation is Representation.GAUSSIANS: + raise UnsupportedFeatureError( + "Gaussian scales, rotations and opacities cannot be preserved by mesh export" + ) # Not every representation has connectivity: a PointCloud carries positions # and nothing to join them with, and `getattr` rather than attribute access # is what lets one writer serve both without asking which it has. triangles = getattr(mesh, "triangles", None) + if suffix in {".stl", ".glb", ".gltf"} and (triangles is None or not len(triangles)): + raise UnsupportedFeatureError(f"{suffix} export requires triangles") + if suffix == ".stl" and not allow_lossy: + raise UnsupportedFeatureError( + "STL drops unused vertices and vertex correspondence; pass allow_lossy=True" + ) present = {"positions"} if triangles is not None: present.add("triangles") @@ -611,9 +621,9 @@ def write_sequence( "geometry_kind": "triangle_mesh", "format": suffix.lstrip("."), "metadata": _json_value(sequence.metadata, "sequence"), - "topology": sequence.topology.value, - "has_constant_vertex_count": sequence.has_constant_vertex_count, - "has_vertex_correspondence": sequence.has_vertex_correspondence, + "topology": TopologyMode.UNKNOWN.value if suffix == ".stl" else sequence.topology.value, + "has_constant_vertex_count": None if suffix == ".stl" else sequence.has_constant_vertex_count, + "has_vertex_correspondence": False if suffix == ".stl" else sequence.has_vertex_correspondence, "allow_nonmonotonic_timestamps": sequence.allow_nonmonotonic_timestamps, "frames": [], } diff --git a/open4d/io/_native_usd.py b/open4d/io/_native_usd.py new file mode 100644 index 00000000..14175885 --- /dev/null +++ b/open4d/io/_native_usd.py @@ -0,0 +1,147 @@ +"""Self-contained VMESH native temporal data in USD, without executable loading. + +The authored VMESH prim is a custom data type, not a standard renderable +Gaussian or neural-field prim. Compressed bytes survive USDC round trips. +""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import tempfile + +import numpy as np + +from open4d._files import publish_file +from open4d.codec._protocol import CodecError +from open4d.codec._vmesh_format import _json, inspect_vmesh +from ._usd import _pxr + +_PRIM = "/VMESH" +_SCHEMA = "vmesh.usd/1" +_CHUNK = 1024 * 1024 + + +def _open(path): + Sdf, Usd, _, _ = _pxr() + from pxr import Tf + from ._errors import DecodeError + try: + layer = Sdf.Layer.OpenAsAnonymous(str(Path(path).absolute())) + except Tf.ErrorException as error: + raise DecodeError(f"cannot read USD layer {path}: {error}") from error + if layer is None: + raise CodecError(f"cannot read USD layer {path}") + stage = Usd.Stage.Open(layer) + return stage, stage.GetPrimAtPath(_PRIM) + + +def is_native_usd(path): + # Layer metadata first: ordinary mesh USD is rejected without parsing its + # scene description, so load() opens it once and reports its own errors. + Sdf, _, _, _ = _pxr() + from pxr import Tf + try: + layer = Sdf.Layer.OpenAsAnonymous(str(Path(path).absolute()), metadataOnly=True) + except Tf.ErrorException: + return False + if layer is None or layer.defaultPrim != _PRIM.lstrip("/"): + return False + _, prim = _open(path) + return bool(prim and prim.HasAttribute("vmesh:schema")) + + +def write_native_usd(sequence, destination, *, overwrite=False): + sequence._check_open() + destination = Path(destination).absolute() + if destination.suffix.lower() not in (".usd", ".usda", ".usdc"): + raise ValueError("native USD interchange supports .usd, .usda and .usdc") + if destination.exists() and not overwrite: + raise FileExistsError(destination) + Sdf, Usd, _, Vt = _pxr() + descriptor = inspect_vmesh(sequence.path) + if any(right <= left for left, right in zip(sequence.timestamps, sequence.timestamps[1:])): + raise CodecError("native USD export requires strictly increasing timestamps") + size = sequence.path.stat().st_size + count = (size + _CHUNK - 1) // _CHUNK + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f".{destination.name}-", dir=destination.parent) as directory: + temporary = Path(directory) / destination.name + stage = Usd.Stage.CreateNew(str(temporary)) + prim = stage.DefinePrim(_PRIM, "VMESH") + stage.SetDefaultPrim(prim) + stage.SetTimeCodesPerSecond(1.0) + stage.SetStartTimeCode(sequence.timestamps[0]) + stage.SetEndTimeCode(sequence.timestamps[-1]) + prim.CreateAttribute("vmesh:schema", Sdf.ValueTypeNames.String).Set(_SCHEMA) + frame = prim.CreateAttribute("vmesh:frameIndex", Sdf.ValueTypeNames.Int64) + for index, timestamp in zip(sequence.frame_indices, sequence.timestamps): + frame.Set(index, timestamp) + digest = hashlib.sha256() + with sequence.path.open("rb") as source: + for index in range(count): + data = source.read(_CHUNK) + digest.update(data) + prim.CreateAttribute(f"vmesh:payload:chunk{index:06d}", Sdf.ValueTypeNames.UCharArray).Set( + Vt.UCharArray.FromNumpy(np.frombuffer(data, dtype=np.uint8))) + if source.read(1): + raise CodecError("native stream changed while exporting USD") + header = dict(codec=sequence.codec, bytes=size, chunks=count, sha256=digest.hexdigest(), + representation=descriptor["representation"]) + prim.CreateAttribute("vmesh:payloadManifest", Sdf.ValueTypeNames.String).Set(json.dumps(header, separators=(",", ":"))) + stage.GetRootLayer().Save() + del stage + # Verify the USD actually contains the original stream before publishing. + with read_native_usd(temporary) as recovered: + if recovered.path.stat().st_size != size: + raise CodecError("USD native payload size changed") + publish_file(temporary, destination, overwrite=overwrite) + return destination + + +def read_native_usd(source, *, runtime=None, python=None): + from open4d.native import NativeSequence + + stage, prim = _open(source) + if not prim or prim.GetTypeName() != "VMESH" or prim.GetAttribute("vmesh:schema").Get() != _SCHEMA: + raise CodecError("unsupported VMESH USD schema") + raw = prim.GetAttribute("vmesh:payloadManifest").Get() + if not isinstance(raw, str) or len(raw) > 1024 * 1024: + raise CodecError("invalid native USD payload manifest") + header = _json(raw) + if not isinstance(header, dict): + raise CodecError("invalid native USD payload manifest") + count, size = header.get("chunks"), header.get("bytes") + if (type(count) is not int or not 1 <= count <= 65536 or type(size) is not int + or not (count - 1) * _CHUNK < size <= count * _CHUNK): + raise CodecError("invalid native USD payload bounds") + chunks = sorted(a.GetName() for a in prim.GetAttributes() if a.GetName().startswith("vmesh:payload:chunk")) + if chunks != [f"vmesh:payload:chunk{i:06d}" for i in range(count)]: + raise CodecError("missing or extra native USD chunks") + temporary = tempfile.TemporaryDirectory(prefix="open4d-native-usd-") + try: + path = Path(temporary.name) / "sequence.vmesh" + digest = hashlib.sha256() + with path.open("xb") as stream: + for index in range(count): + attribute = prim.GetAttribute(chunks[index]) + if str(attribute.GetTypeName()) != "uchar[]" or attribute.GetNumTimeSamples(): + raise CodecError("native USD chunks must be static uchar arrays") + data = bytes(attribute.Get()) + if len(data) != min(_CHUNK, size - index * _CHUNK): + raise CodecError("native USD chunk size mismatch") + digest.update(data) + stream.write(data) + if digest.hexdigest() != header.get("sha256"): + raise CodecError("native USD payload SHA-256 mismatch") + result = NativeSequence(path, temporary=temporary, runtime=runtime, python=python) + if result.codec != header.get("codec") or result.representation != header.get("representation"): + raise CodecError("native USD profile disagrees with carried stream") + frame = prim.GetAttribute("vmesh:frameIndex") + if (stage.GetTimeCodesPerSecond() != 1.0 or tuple(frame.GetTimeSamples()) != result.timestamps + or tuple(frame.Get(t) for t in result.timestamps) != result.frame_indices): + raise CodecError("USD timeline disagrees with native temporal payload") + return result + except BaseException: + temporary.cleanup() + raise diff --git a/open4d/io/_usd.py b/open4d/io/_usd.py index e015666e..6225a046 100644 --- a/open4d/io/_usd.py +++ b/open4d/io/_usd.py @@ -11,6 +11,7 @@ import tempfile from types import MappingProxyType from typing import Any +from uuid import uuid4 import numpy as np @@ -175,7 +176,13 @@ def __init__( self.path = Path(path).absolute() self._Sdf, self._Usd, self._UsdGeom, self._Vt = _pxr() try: - self._stage = self._Usd.Stage.Open(str(self.path)) + # File-format arguments are part of Sdf's layer identity. A unique + # reader key bypasses its path cache while retaining the real path + # for relative assets and leaving existing readers undisturbed. + layer = self._Sdf.Layer.FindOrOpen( + str(self.path), {"open4d_reader": uuid4().hex} + ) + self._stage = self._Usd.Stage.Open(layer) if layer is not None else None except Exception as error: raise DecodeError(f"USD could not open {self.path}: {error}") from error if self._stage is None: @@ -492,6 +499,8 @@ def _preflight(sequence: Sequence) -> tuple[dict[str, Any], list[dict[str, Any]] if frame.frame_index > np.iinfo(np.int64).max: raise EncodeError("USD frame indices must fit a signed 64-bit integer") mesh = frame.geometry + if not isinstance(mesh, TriangleMesh): + raise UnsupportedFeatureError("OpenUSD export currently requires triangle-mesh geometry") attributes = [] for name, array in mesh.attributes.items(): kind = _attribute_kind(array) diff --git a/open4d/io/tests/test_review_regressions.py b/open4d/io/tests/test_review_regressions.py new file mode 100644 index 00000000..8c7b06ff --- /dev/null +++ b/open4d/io/tests/test_review_regressions.py @@ -0,0 +1,85 @@ +import errno + +import numpy as np +import pytest + +from open4d import Frame, GaussianCloud, MemoryFrameProvider, PointCloud, Sequence, TriangleMesh +from open4d.demo import mesh_sequence +from open4d.io import UnsupportedFeatureError, open_sequence, write_sequence + + +def sequence(geometry): + return Sequence(MemoryFrameProvider([Frame(0, 0, geometry)], + has_vertex_correspondence=True)) + + +def test_gaussians_are_not_silently_exported_as_meshes(tmp_path): + cloud = GaussianCloud([[0., 0., 0.]], [[1., 1., 1.]], [[1., 0., 0., 0.]], [0.5]) + with pytest.raises(UnsupportedFeatureError, match="Gaussian"): + write_sequence(sequence(cloud), tmp_path / "frames") + assert not (tmp_path / "frames").exists() + + +def test_point_usd_export_has_supported_error(tmp_path): + with pytest.raises(UnsupportedFeatureError, match="triangle.mesh"): + write_sequence(sequence(PointCloud([[0., 0., 0.]])), tmp_path / "points.usda") + + +def test_stl_requires_lossy_opt_in_and_clears_correspondence(tmp_path): + pytest.importorskip("trimesh") + mesh = TriangleMesh([[0., 0, 0], [1, 0, 0], [1, 1, 0], [0, 1, 0], [2, 2, 2]], + [[0, 1, 2], [0, 2, 3]]) + with pytest.raises(UnsupportedFeatureError, match="allow_lossy"): + write_sequence(sequence(mesh), tmp_path / "strict", format="stl") + path = write_sequence(sequence(mesh), tmp_path / "lossy", format="stl", allow_lossy=True) + with open_sequence(path) as decoded: + assert decoded.has_vertex_correspondence is False + assert decoded.has_constant_vertex_count is None + + +@pytest.mark.parametrize("suffix", ["stl", "glb", "gltf"]) +def test_face_only_formats_reject_empty_connectivity(tmp_path, suffix): + mesh = TriangleMesh([[0., 0., 0.]], np.empty((0, 3), dtype=np.uint32)) + with pytest.raises(UnsupportedFeatureError, match="triangles"): + write_sequence(sequence(mesh), tmp_path / "frames", format=suffix, allow_lossy=True) + + +@pytest.mark.parametrize("suffix", ["usda", "usdc", "usdz"]) +def test_reopening_replaced_usd_reads_new_file_with_old_handle_open(tmp_path, suffix): + pytest.importorskip("pxr") + path = write_sequence(mesh_sequence(side=2, frames=2), tmp_path / f"take.{suffix}") + with open_sequence(path) as old: + write_sequence(mesh_sequence(side=2, frames=5), path, overwrite=True) + with open_sequence(path) as new: + assert len(new) == 5 + assert len(old) == 2 + assert old[1].frame_index == 1 + + +def test_publish_without_hard_links_preserves_existing_files(tmp_path, monkeypatch): + from open4d import _files + + def unsupported(*args): + raise OSError(errno.EOPNOTSUPP, "hard links unsupported") + + monkeypatch.setattr(_files.os, "link", unsupported) + temporary, destination = tmp_path / "temporary", tmp_path / "destination" + temporary.write_bytes(b"new") + _files.publish_file(temporary, destination) + assert destination.read_bytes() == b"new" + temporary.write_bytes(b"other") + with pytest.raises(FileExistsError): + _files.publish_file(temporary, destination) + assert destination.read_bytes() == b"new" + + +def test_usd_snapshot_resolves_relative_sublayers(tmp_path): + pytest.importorskip("pxr") + from pxr import Sdf + + write_sequence(mesh_sequence(side=2, frames=2), tmp_path / "geometry.usda") + root = Sdf.Layer.CreateNew(str(tmp_path / "root.usda")) + root.subLayerPaths = ["geometry.usda"] + root.Save() + with open_sequence(tmp_path / "root.usda") as decoded: + assert len(decoded) == 2 diff --git a/open4d/io/tests/test_unified_api.py b/open4d/io/tests/test_unified_api.py index 5af8e674..c39ce085 100644 --- a/open4d/io/tests/test_unified_api.py +++ b/open4d/io/tests/test_unified_api.py @@ -21,7 +21,7 @@ def sequence() -> Sequence: def test_top_level_codec_round_trip_and_unload(tmp_path): - artifact = open4d.save(sequence(), tmp_path / "capture.o4d", codec=NumPyZipCodec()) + artifact = open4d.save(sequence(), tmp_path / "capture.vmesh", codec=NumPyZipCodec()) loaded = open4d.load(artifact, codec=NumPyZipCodec()) np.testing.assert_array_equal(loaded[0].geometry.triangles, [[0, 1, 2]]) @@ -58,7 +58,7 @@ def test_top_level_load_rejects_conflicting_dispatch_overrides(tmp_path): open4d.load(path, format="obj", codec="npz") -def test_top_level_load_dispatches_raw_vmesh_to_the_read_only_vdmc_decoder( +def test_top_level_load_dispatches_vmesh_to_codec_detection( tmp_path, monkeypatch ): import open4d._api as implementation @@ -79,22 +79,23 @@ def decode(source, *, codec=None, **options): ) is expected assert received == { "source": path, - "codec": "vdmc", + "codec": None, "options": {"decoder": "/native/decoder", "fps": 24}, } @pytest.mark.parametrize( - "name", ("capture", "frame.obj", "capture.unknown", "capture.vmesh") + "name", ("capture", "frame.obj", "capture.unknown") ) def test_top_level_save_requires_a_sequence_file_extension(tmp_path, name): with pytest.raises(ValueError, match="sequence-file extension"): open4d.save(sequence(), tmp_path / name) -def test_top_level_save_requires_a_codec_for_ambiguous_codec_suffix(tmp_path): - with pytest.raises(ValueError, match=r"ambiguous.*\.v4d.*codec"): - open4d.save(sequence(), tmp_path / "capture.v4d") +@pytest.mark.parametrize("suffix", [".vmesh"]) +def test_top_level_save_requires_a_codec_for_ambiguous_codec_suffix(tmp_path, suffix): + with pytest.raises(ValueError, match="ambiguous.*codec"): + open4d.save(sequence(), tmp_path / f"capture{suffix}") def test_top_level_api_exports_visualize(): @@ -104,11 +105,11 @@ def test_top_level_api_exports_visualize(): @pytest.mark.parametrize("codec", ["draco", "raw", "npz", "temporal-delta"]) def test_utility_codecs_are_not_public_choices(tmp_path, codec): with pytest.raises(ValueError, match="unknown codec"): - open4d.encode(sequence(), tmp_path / "take.o4d", codec=codec) + open4d.encode(sequence(), tmp_path / "take.vmesh", codec=codec) def test_encoding_requires_an_explicit_codec(tmp_path): with pytest.raises(TypeError, match="codec"): - open4d.encode(sequence(), tmp_path / "take.o4d") - with pytest.raises(ValueError, match="sequence-file extension"): - open4d.save(sequence(), tmp_path / "take.o4d") + open4d.encode(sequence(), tmp_path / "take.vmesh") + with pytest.raises(ValueError, match="ambiguous.*codec"): + open4d.save(sequence(), tmp_path / "take.vmesh") diff --git a/open4d/io/tests/test_usd_sequence.py b/open4d/io/tests/test_usd_sequence.py index 5691dda4..30e8e6f6 100644 --- a/open4d/io/tests/test_usd_sequence.py +++ b/open4d/io/tests/test_usd_sequence.py @@ -217,6 +217,15 @@ def test_public_usd_load_is_lazy(tmp_path, monkeypatch): decoded.close() +def test_mesh_usd_load_does_not_fully_open_the_layer_twice(tmp_path, monkeypatch): + from open4d.io import _native_usd + + path = open4d.save(rich_sequence(), tmp_path / "capture.usdc") + monkeypatch.setattr(_native_usd, "_open", lambda path: pytest.fail("mesh USD opened as native")) + with open4d.load(path) as decoded: + assert len(decoded) == 2 + + def test_usd_empty_geometry_omits_extent_and_round_trips(tmp_path): empty = TriangleMesh( np.empty((0, 3), dtype=np.float32), @@ -408,7 +417,7 @@ def test_reader_accepts_the_prototype_open4d_usd_layout(tmp_path): "open4d": { "version": 1, "source": "prototype-capture", - "key_frame_indices": [0], + "key_frame_indices": Vt.IntArray([0]), } } stage.GetRootLayer().Save() diff --git a/open4d/native.py b/open4d/native.py new file mode 100644 index 00000000..a6fc3baf --- /dev/null +++ b/open4d/native.py @@ -0,0 +1,367 @@ +"""Portable native temporal runs and lossless USDC / VMESH interchange. + +Opening a run never executes its models. ``decode`` and ``render`` explicitly +invoke the selected research runtime. USD interchange retains the compressed +representation; it does not turn an image-trained field into a mesh codec. +""" +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +import json +import math +import os +from pathlib import Path +import shutil +import sys +import tempfile +from types import MappingProxyType + +import numpy as np + +from .codec._native import run +from .codec._native_profiles import NEURAL_CODECS, layout +from .codec._metadata import _json_value +from .codec._protocol import CodecError +from .codec._vmesh_format import _json, inspect_vmesh, pack_vmesh, unpack_vmesh +from ._files import publish_file + + +def _read_json(path): + path = Path(path) + if path.is_symlink() or not path.is_file() or path.stat().st_size > 16 * 1024 * 1024: + raise CodecError(f"missing, linked or oversized native configuration: {path}") + return _json(path.read_bytes()) + + +def _copy(source, destination): + source = Path(source) + if source.is_symlink() or not source.is_file() or not source.stat().st_size: + raise CodecError(f"missing, linked or empty native payload: {source}") + # Reject directory symlinks too: a temporal import has a closed payload set. + if any(parent.is_symlink() for parent in source.parents): + raise CodecError(f"native payload has a linked parent: {source}") + shutil.copyfile(source, destination) + + +def _latest_ply(root, *, added=False): + root = Path(root) + direct = root / ("added.ply" if added else "point_cloud.ply") + if direct.is_file(): + return direct + suffix = "added/point_cloud.ply" if added else "point_cloud.ply" + candidates = [p for p in (root / "point_cloud").glob("iteration_*") + if p.name[10:].isdigit() and (p / suffix).is_file()] + if not candidates: + raise CodecError(f"no {'added' if added else 'initial'} Gaussian payload in {root}") + return max(candidates, key=lambda p: int(p.name[10:])) / suffix + + +def _configuration(config): + if isinstance(config, Mapping): + return _json_value(dict(config), "native configuration") + if config is None: + return {} + path = Path(config) + if path.suffix.lower() in (".yaml", ".yml"): + try: + import yaml + except ImportError as error: + raise ImportError("Pass configuration as a dictionary, or install PyYAML to read YAML") from error + if path.is_symlink() or path.stat().st_size > 16 * 1024 * 1024: + raise CodecError("linked or oversized native configuration") + value = yaml.safe_load(path.read_text()) + else: + value = _read_json(path) + if not isinstance(value, dict): + raise CodecError("native configuration must be a JSON/YAML object") + return _json_value(value, "native configuration") + + +def _stage_native(source, destination, codec, config, initial_model, ntc_config): + """Copy only actual decoder dependencies, returning native indices/profile.""" + cfg = _configuration(config) + profile = {"profile": f"{codec}/1"} + if codec == "vega": + from ._gaussian_worker import _read_manifest + entries = _read_manifest(source) + indices = [e["frame_idx"] for e in entries] + for name in ["manifest.json", "color_model.pt", *(e["file"] for e in entries)]: + _copy(source / name, destination / name) + elif codec == "queen": + frames = sorted((p for p in (source / "frames").iterdir() if p.is_dir() and p.name.isdigit()), key=lambda p: int(p.name)) + indices = [int(p.name) for p in frames] + if len(frames) < 2 or indices != list(range(1, len(frames) + 1)): + raise CodecError("QUEEN requires initial frame 1 and contiguous compressed later frames") + model, quant = cfg.get("model_params", cfg), cfg.get("quantize_params", cfg) + if "sh_degree" not in model: + raise CodecError("QUEEN import needs config with sh_degree and gate settings used to encode the run") + gates = quant.get("gate_params", [quant.get(f"{name}_gate_params", "none") for name in ("xyz", "f_dc", "f_rest", "sc", "rot", "op", "flow")]) + profile.update(sh_degree=model["sh_degree"], gate_params=gates) + _copy(initial_model or _latest_ply(frames[0]), destination / "initial.ply") + for index, frame in enumerate(frames[1:], 1): + _copy(frame / "compressed/point_cloud.pkl", destination / f"frame_{index:06d}.pkl") + elif codec == "3dgstream": + if not cfg: + cfg = _read_json(source / "cfg_args.json") + frames = sorted((p for p in source.glob("frame[0-9]*") if p.is_dir() and p.name[5:].isdigit()), key=lambda p: int(p.name[5:])) + indices = [0] + [int(p.name[5:]) for p in frames] + if len(frames) < 1 or indices != list(range(len(frames) + 1)): + raise CodecError("3DGStream requires initial frame 0 and contiguous NTC frames starting at 1") + profile.update(sh_degree=cfg.get("sh_degree"), rotate_sh=cfg.get("rotate_sh"), only_mlp=cfg.get("only_mlp"), added=[False]) + _copy(initial_model or _latest_ply(source / "init"), destination / "initial.ply") + # Never follow arbitrary paths in cfg_args.json. The architecture is an explicit input. + architecture = _configuration(ntc_config or source / "ntc_config.json") + if not isinstance(architecture.get("network"), dict) or (not profile["only_mlp"] and not isinstance(architecture.get("encoding"), dict)): + raise CodecError("3DGStream needs the NTC network/encoding configuration used for training") + (destination / "ntc_config.json").write_text(json.dumps(architecture, allow_nan=False)) + stage_two = cfg.get("iterations_s2", 0) + if type(stage_two) is not int or stage_two < 0: + raise CodecError("invalid 3DGStream second-stage iteration count") + for index, frame in enumerate(frames, 1): + _copy(frame / "NTC.pth", destination / f"ntc_{index:06d}.pth") + added = stage_two > 0 or any((frame / "point_cloud").glob("iteration_*/added/point_cloud.ply")) + profile["added"].append(bool(added)) + if added: + _copy(_latest_ply(frame, added=True), destination / f"added_{index:06d}.ply") + elif codec == "rerf": + if not cfg: + cfg = _read_json(source / "decoder_config.json") + native = cfg.get("native", {}) + profile.update(group_size=native.get("group_size"), pca=native.get("pca"), + pca_channels=native.get("pca_channels", [7, 13]), frames=[]) + required = ("voxel_size", "fine_model_and_render", "data", "render") + if not all(key in cfg for key in required): + raise CodecError("ReRF needs a resolved decoder JSON configuration including render near/far; no training-directory fallback") + indices = sorted(int(p.stem[7:]) for p in source.glob("header_*.json") if p.stem[7:].isdigit()) + for index in indices: + header = _read_json(source / f"header_{index}.json") + headers = header.get("headers") if isinstance(header, dict) else None + if not isinstance(headers, list) or not headers or not isinstance(headers[0], dict): + raise CodecError("invalid ReRF frame header") + motion = (source / f"deform_{index}.npy").is_file() + if motion != (source / f"deform_mask_{index}.rerf").is_file(): + raise CodecError("incomplete ReRF motion payload") + channels = [] + quality = headers[0].get("quality") + for half, header_part in enumerate(headers): + size = header_part.get("size") if isinstance(header_part, dict) else None + if (not isinstance(size, list) or len(size) != 5 or type(quality) is not int + or header_part.get("quality") != quality - half): + raise CodecError("invalid ReRF entropy header shape/quality") + channels.append(size[1]) + profile["frames"].append(dict(id=index, quality=quality, motion=motion, channels=channels)) + # Layout validation detects missing keys, gaps, PCA halves and motion. + for name, _ in layout(codec, len(indices), profile)[1:]: + if name != "decoder_config.json": + _copy(source / name, destination / name) + (destination / "decoder_config.json").write_text(json.dumps(cfg, allow_nan=False)) + else: + raise CodecError(f"{codec!r} is not a native neural temporal profile") + layout(codec, len(indices), profile) + return indices, profile + + +@dataclass(frozen=True) +class NeuralFieldFrame: + """Decoded ReRF density/features; appearance remains in the native RGB model.""" + frame_index: int + timestamp: float + density: np.ndarray + features: np.ndarray + xyz_min: np.ndarray + xyz_max: np.ndarray + + +class NativeSequence: + """An owned temporal representation, preserving native compressed bytes. + + Use as a context manager. ``decode`` requires the method's configured + runtime, while inspection and USDC round trips require no CUDA runtime. + """ + def __init__(self, path, *, temporary=None, runtime=None, python=None): + self.path = Path(path).absolute() + self.manifest = inspect_vmesh(self.path) + self.codec = self.manifest["codec"] + self.representation = self.manifest["representation"] + self.metadata = MappingProxyType(self.manifest["sequence"].get("metadata", {})) + self.timestamps = tuple(f["timestamp"] for f in self.manifest["sequence"]["frames"]) + self.frame_indices = tuple(f["frame_index"] for f in self.manifest["sequence"]["frames"]) + self._temporary, self._extracted = temporary, None + self._runtime, self._python = runtime, python + self._closed = False + + def __len__(self): + return len(self.timestamps) + + def __enter__(self): + self._check_open() + return self + + def __exit__(self, *_): + self.close() + + def _check_open(self): + if self._closed: + raise ValueError("native sequence is closed") + + def close(self): + if not self._closed: + if self._extracted is not None: + self._extracted.cleanup() + if self._temporary is not None: + self._temporary.cleanup() + self._closed = True + + def unpack(self, destination): + self._check_open() + return unpack_vmesh(self.path, destination) + + def _native_directory(self): + self._check_open() + if self._extracted is None: + temporary = tempfile.TemporaryDirectory(prefix=f"open4d-{self.codec}-native-") + try: + unpack_vmesh(self.path, Path(temporary.name) / "native") + except BaseException: + temporary.cleanup() + raise + self._extracted = temporary + return Path(self._extracted.name) / "native" + + def _settings(self, runtime, python): + variable = "OPEN4D_" + self.codec.upper() + root = Path(runtime or self._runtime or os.environ.get(variable + "_ROOT") or + Path(__file__).parent / "reconstruction" / self.codec).absolute() + executable = str(python or self._python or os.environ.get(variable + "_PYTHON") or sys.executable) + return root, executable + + def decode(self, *, runtime=None, python=None): + """Evaluate actual temporal models, returning Gaussian or field frames. + + Native QUEEN pickles and ReRF checkpoints are executable research + formats. Call this only for trusted native runs, in their configured + runtime. Generic inspection/load/save never deserializes these models. + """ + if self.codec == "frames": + raise NotImplementedError("Use unpack() or browser playback for point, Gaussian and image frame payloads") + source = self._native_directory() + root, executable = self._settings(runtime, python) + if self.codec == "vega": + from .gaussians import VegaRun + # Frame appearances retain this owner so its extraction outlives decode. + run = VegaRun(source, root, executable, _owner=self) + return run.decode() + if self.codec not in NEURAL_CODECS: + from .codec import decode_sequence + return decode_sequence(self.path) + with tempfile.TemporaryDirectory(prefix=f"open4d-{self.codec}-decode-") as directory: + work = Path(directory) + self._worker("decode", source, work, root, executable) + paths = sorted(work.glob("frame_*.npz")) + if [p.name for p in paths] != [f"frame_{i:06d}.npz" for i in range(len(self))]: + raise CodecError(f"{self.codec} did not decode every native frame") + result = [] + for index, path in enumerate(paths): + with np.load(path, allow_pickle=False) as data: + if self.codec == "rerf": + values = {name: data[name] for name in ("density", "features", "xyz_min", "xyz_max")} + if not all(np.isfinite(v).all() for v in values.values()): + raise CodecError("ReRF decoded nonfinite field values") + result.append(NeuralFieldFrame(self.frame_indices[index], self.timestamps[index], **values)) + else: + from .gaussians import GaussianSplats + result.append(GaussianSplats(**{name: data[name] for name in + ("positions", "scales", "rotations", "opacities", "spherical_harmonics")})) + return tuple(result) + + def _worker(self, operation, source, work, root, executable, **options): + request = dict(operation=operation, codec=self.codec, source=str(source), output=str(work), runtime=str(root), **options) + path = work / "request.json" + path.write_text(json.dumps(request, allow_nan=False)) + run([executable, str(Path(__file__).with_name("_native_worker.py")), str(path)], + f"{self.codec} {operation}", cwd=work) + + def render(self, camera, *, runtime=None, python=None): + """Render a ReRF sequence from an explicit calibrated camera dictionary.""" + if self.codec != "rerf": + raise TypeError("NativeSequence.render currently accepts ReRF; Gaussian decode returns splat arrays") + root, executable = self._settings(runtime, python) + with tempfile.TemporaryDirectory(prefix="open4d-rerf-render-") as directory: + work = Path(directory) + self._worker("render", self._native_directory(), work, root, executable, + camera=_json_value(camera, "camera")) + paths = sorted(work.glob("image_*.npy")) + if len(paths) != len(self): + raise CodecError("ReRF did not render every frame") + return tuple(np.load(p, allow_pickle=False) for p in paths) + + +def import_native(source, *, codec=None, config=None, initial_model=None, ntc_config=None, + fps=30.0, timestamps=None, frame_indices=None, metadata=None, + frame_metadata=None, runtime=None, python=None): + """Import Vega, QUEEN, 3DGStream or ReRF native output, excluding dense exports. + + ``source`` can also be a GaussianRun/VegaRun. All dependencies are copied + into an owned .vmesh, so the returned object is independent of the run. + """ + if hasattr(source, "path"): + codec = codec or getattr(source, "method", "vega") + config = config if config is not None else getattr(source, "config", None) + initial_model = initial_model or getattr(source, "initial_model", None) + python = python or getattr(source, "python", None) + source_runtime = getattr(source, "runtime", None) + if source_runtime is not None: + runtime = runtime or (source_runtime if codec == "vega" else source_runtime.parent / codec) + source = source.path + if codec not in NEURAL_CODECS: + raise ValueError(f"codec must be one of {sorted(NEURAL_CODECS)}") + source = Path(source).absolute() + if isinstance(fps, bool) or not isinstance(fps, (float, int)) or not math.isfinite(fps) or fps <= 0: + raise ValueError("fps must be finite and positive") + temporary = tempfile.TemporaryDirectory(prefix=f"open4d-import-{codec}-") + try: + root = Path(temporary.name) + native = root / "native" + native.mkdir() + indices, profile = _stage_native(source, native, codec, config, initial_model, ntc_config) + count = len(indices) + times = list(timestamps) if timestamps is not None else [i / fps for i in range(count)] + ids = list(frame_indices) if frame_indices is not None else indices + metas = list(frame_metadata) if frame_metadata is not None else [{} for _ in indices] + if not len(times) == len(ids) == len(metas) == count: + raise ValueError("timestamps, frame_indices and frame_metadata must match the native frame count") + record = dict(version=1, codec=codec, native=profile, metadata=_json_value(metadata or {}, "sequence"), + frames=[dict(frame_index=i, timestamp=t, metadata=_json_value(m, "frame")) for i, t, m in zip(ids, times, metas)], + allow_nonmonotonic_timestamps=False) + (native / "metadata.json").write_text(json.dumps(record, allow_nan=False)) + artifact = pack_vmesh(native, root / "sequence.vmesh") + shutil.rmtree(native) + return NativeSequence(artifact, temporary=temporary, runtime=runtime, python=python) + except BaseException: + temporary.cleanup() + raise + + +def save_native(sequence, destination, *, overwrite=False): + """Save native temporal state as standalone .vmesh or a self-contained VMESH USD.""" + if not isinstance(sequence, NativeSequence): + raise TypeError("sequence must be a NativeSequence") + if not isinstance(overwrite, bool): + raise TypeError("overwrite must be bool") + sequence._check_open() + path = Path(destination).absolute() + if path.suffix.lower() in (".usd", ".usda", ".usdc"): + from .io._native_usd import write_native_usd + return write_native_usd(sequence, path, overwrite=overwrite) + if path.suffix.lower() != ".vmesh": + raise ValueError("native output requires .vmesh, .usd, .usda or .usdc") + if path.exists() and not overwrite: + raise FileExistsError(path) + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f".{path.name}-", dir=path.parent) as folder: + staged = Path(folder) / "sequence.vmesh" + shutil.copyfile(sequence.path, staged) + inspect_vmesh(staged) + publish_file(staged, path, overwrite=overwrite) + return path diff --git a/open4d/reconstruction/3dgstream/train_frames.py b/open4d/reconstruction/3dgstream/train_frames.py index 7cc1f3c6..d72b23d0 100644 --- a/open4d/reconstruction/3dgstream/train_frames.py +++ b/open4d/reconstruction/3dgstream/train_frames.py @@ -127,10 +127,10 @@ def training_one_frame(dataset, opt, pipe, load_iteration, testing_iterations, s torch.save((gaussians.capture(), iteration), scene.output_path + "/chkpnt" + str(iteration) + ".pth") s1_end_time=time.time() + # The temporal representation is required even when densification is off. + scene.dump_NTC() # Densify if(opt.iterations_s2>0): - # Dump the NTC - scene.dump_NTC() # Update Gaussians by NTC gaussians.update_by_ntc() # Prune, Clone and setting up diff --git a/open4d/reconstruction/queen/train.py b/open4d/reconstruction/queen/train.py index 52e2fd0f..ea05cd2b 100644 --- a/open4d/reconstruction/queen/train.py +++ b/open4d/reconstruction/queen/train.py @@ -74,6 +74,23 @@ except ImportError: WANDB_FOUND = False +def _save_frame_outputs(scene, gaussians, dataset, frame_idx, iteration, qp): + if not (dataset.log_ply or dataset.log_compressed): + return + gate = gaussians.gate_atts + was_training = gate.training if gate is not None else False + if gate is not None: + gate.eval() + try: + if dataset.log_ply or frame_idx == 1: + scene.save(iteration, save_point_cloud=True) + if dataset.log_compressed and frame_idx > 1: + scene.save_compressed(-1, qp) + finally: + if gate is not None: + gate.train(was_training) + + def training(dataset: ModelParams, opt: OptimizationParams, pipe: PipelineParams, qp:QuantizeParams, testing_iterations: list, saving_iterations: list, checkpoint_iterations, checkpoint: str, debug_from, args): """Main training function for QUEEN compressed Gaussian splatting.""" @@ -800,13 +817,6 @@ def training(dataset: ModelParams, opt: OptimizationParams, pipe: PipelineParams if dataset.log_images: save_image(gt_image,os.path.join(scene.model_path, "gt.png")) - if dataset.log_ply: - scene.save(iteration, save_point_cloud=True) - - if dataset.log_compressed: - if frame_idx == 1: - scene.save(frame_idx, save_point_cloud=True) - if frame_idx>1 and (dataset.adaptive_render and dataset.adaptive_update_period>0.0) and dataset.update_mask!="none": torchvision.utils.save_image(train_cameras[cam_idx].mask.unsqueeze(0)*gt_image, os.path.join(scene.model_path, "mask.png")) @@ -876,19 +886,7 @@ def training(dataset: ModelParams, opt: OptimizationParams, pipe: PipelineParams else: raise ValueError(f"Invalid save format {args.save_format}") - if dataset.log_compressed: - if frame_idx == 1: - scene.save(frame_idx, save_point_cloud=True) - else: - gate = gaussians.gate_atts - was_training = gate.training if gate is not None else False - if gate is not None: - gate.eval() - try: - scene.save_compressed(-1, qp) - finally: - if gate is not None: - gate.train(was_training) + _save_frame_outputs(scene, gaussians, dataset, frame_idx, opt.iterations, qp) # Update previous frame's attributes and latents for next frame's residual encoding diff --git a/open4d/reconstruction/rerf/rerf_stream/bitstream.py b/open4d/reconstruction/rerf/rerf_stream/bitstream.py index 5b4b5004..47f372c6 100644 --- a/open4d/reconstruction/rerf/rerf_stream/bitstream.py +++ b/open4d/reconstruction/rerf/rerf_stream/bitstream.py @@ -46,6 +46,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Iterator, List, Optional, Sequence +from collections.abc import Mapping import numpy as np @@ -98,7 +99,15 @@ def __init__( pca_channels: Sequence[int] = DEFAULT_PCA_CHANNELS, group_size: Optional[int] = None, device: str = "cuda", + render_bounds=None, ) -> None: + self.path = Path(compression_path).expanduser().resolve() + self.frames = self._available_frames() + if not self.frames: + raise FileNotFoundError( + f"{self.path} holds no ReRF bitstream: no header_.json. " + "Encode one with upstream's codec/compress.py." + ) env.activate() with env.upstream_cwd(): import mmcv @@ -132,9 +141,8 @@ def __init__( } self.device = torch.device(device) - self.config_path = Path(config_path).expanduser().resolve() - self.path = Path(compression_path).expanduser().resolve() - self.cfg = mmcv.Config.fromfile(str(self.config_path)) + self.config_path = None if isinstance(config_path, Mapping) else Path(config_path).expanduser().resolve() + self.cfg = mmcv.Config(dict(config_path)) if self.config_path is None else mmcv.Config.fromfile(str(self.config_path)) self.corpus_dir = Path(self.cfg.data["datadir"]).expanduser() self.pca = bool(pca) self.pca_channels = tuple(int(channel) for channel in pca_channels) @@ -142,12 +150,6 @@ def __init__( self.n_channel = int(self.cfg.fine_model_and_render["rgbnet_dim"]) + 1 self.voxel_size = int(self.cfg.voxel_size) - self.frames = self._available_frames() - if not self.frames: - raise FileNotFoundError( - f"{self.path} holds no ReRF bitstream: no header_.json. " - "Encode one with upstream's codec/compress.py." - ) # Upstream: `group_size = args.group_size if given else args.frame_num`. # Defaulting to the number of frames present plays a single-group # bitstream correctly without having to be told its own structure. @@ -173,7 +175,13 @@ def __init__( ) self.model.k0.eval() - near, far = inward_near_far(self.corpus_dir) + if render_bounds is None: + near, far = inward_near_far(self.corpus_dir) + else: + near, far = render_bounds["near"], render_bounds["far"] + if (isinstance(near, bool) or isinstance(far, bool) or + not np.isfinite([near, far]).all() or not 0 <= near < far): + raise ValueError("render bounds must be finite with 0 <= near < far") self.render_kwargs = { "near": near, "far": far, diff --git a/open4d/reconstruction/rerf/rerf_stream/env.py b/open4d/reconstruction/rerf/rerf_stream/env.py index 032fd685..4599299c 100644 --- a/open4d/reconstruction/rerf/rerf_stream/env.py +++ b/open4d/reconstruction/rerf/rerf_stream/env.py @@ -89,7 +89,7 @@ def patch_dependencies() -> None: for name, builtin in (("bool", bool), ("object", object), ("int", int), ("float", float), ("complex", complex), ("str", str)): - if not hasattr(numpy, name): + if name not in vars(numpy): setattr(numpy, name, builtin) try: diff --git a/open4d/reconstruction/rerf/rerf_stream_tests/test_bitstream.py b/open4d/reconstruction/rerf/rerf_stream_tests/test_bitstream.py index 2d06e30b..6035daf0 100644 --- a/open4d/reconstruction/rerf/rerf_stream_tests/test_bitstream.py +++ b/open4d/reconstruction/rerf/rerf_stream_tests/test_bitstream.py @@ -10,6 +10,7 @@ from __future__ import annotations import json +import os from pathlib import Path import numpy as np @@ -55,7 +56,7 @@ def test_numpy_aliases_are_restored(): env.patch_dependencies() import numpy - assert numpy.bool is bool + assert numpy.dtype(numpy.bool) == numpy.dtype(bool) assert numpy.str is str @@ -239,11 +240,16 @@ def test_waiting_times_out_rather_than_blocking_forever(): # ------------------------------------------------- the decode, when it can run --- -RUN = Path("/media/frozzzen/LocalDisk/nevo_runs/g_basketball") +RUN = Path(os.environ.get("OPEN4D_RERF_RUN", str(Path.home() / "open4d-data/rerf/g_basketball"))) CONFIG, BITSTREAM = RUN / "config.py", RUN / "rerf" +try: + _has_bitstream = CONFIG.is_file() and (BITSTREAM / "header_0.json").is_file() +except OSError: + _has_bitstream = False + requires_bitstream = pytest.mark.skipif( - not (CONFIG.is_file() and (BITSTREAM / "header_0.json").is_file()), + not _has_bitstream, reason="needs an encoded ReRF bitstream on this machine", ) diff --git a/open4d/reconstruction/rerf/rerf_stream_tests/test_geometry.py b/open4d/reconstruction/rerf/rerf_stream_tests/test_geometry.py index 6fb93e33..16aa40cf 100644 --- a/open4d/reconstruction/rerf/rerf_stream_tests/test_geometry.py +++ b/open4d/reconstruction/rerf/rerf_stream_tests/test_geometry.py @@ -13,6 +13,7 @@ from __future__ import annotations import json +import os import struct from pathlib import Path @@ -27,11 +28,16 @@ # Same fixture and skip rule as `test_bitstream.py`: the decode needs a real # encoded bitstream and a GPU, and neither is present everywhere. -RUN = Path("/media/frozzzen/LocalDisk/nevo_runs/g_basketball") +RUN = Path(os.environ.get("OPEN4D_RERF_RUN", str(Path.home() / "open4d-data/rerf/g_basketball"))) CONFIG, BITSTREAM = RUN / "config.py", RUN / "rerf" +try: + _has_bitstream = CONFIG.is_file() and (BITSTREAM / "header_0.json").is_file() +except OSError: + _has_bitstream = False + requires_bitstream = pytest.mark.skipif( - not (CONFIG.is_file() and (BITSTREAM / "header_0.json").is_file()), + not _has_bitstream, reason="needs an encoded ReRF bitstream on this machine", ) diff --git a/open4d/streamer/README.md b/open4d/streamer/README.md index 59ba8b64..669f1c4f 100644 --- a/open4d/streamer/README.md +++ b/open4d/streamer/README.md @@ -290,7 +290,7 @@ from_source("captures/basketball_player", out, fps=10, frame_format="draco") The decoder is Google's, vendored under `client/vendor/draco` and served by the bundle server from this origin — never a CDN, which is what keeps the page free -of external dependencies. `open4d[draco]` is needed on the encoding side only; +of external dependencies. `DracoPy` is needed on the encoding side only; the client needs nothing installed. Lossy in two bounded ways, both in the clip's notes and measured in diff --git a/open4d/streamer/streamer/bundle.py b/open4d/streamer/streamer/bundle.py index 038a4f97..81f4033e 100644 --- a/open4d/streamer/streamer/bundle.py +++ b/open4d/streamer/streamer/bundle.py @@ -42,6 +42,7 @@ import dataclasses import json import time +import tempfile from dataclasses import dataclass, field from pathlib import Path from typing import Any, Mapping @@ -369,7 +370,16 @@ def write( if detail: index["detail"] = detail path = out_dir / INDEX_NAME - path.write_text(json.dumps(index, indent=2) + "\n") + temporary = None + try: + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=out_dir, + prefix=".view-", suffix=".json", delete=False) as output: + temporary = Path(output.name) + output.write(json.dumps(index, indent=2) + "\n") + temporary.replace(path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) return path diff --git a/open4d/streamer/streamer/client/viewer.html b/open4d/streamer/streamer/client/viewer.html index 5c9bee3f..1b98e14f 100644 --- a/open4d/streamer/streamer/client/viewer.html +++ b/open4d/streamer/streamer/client/viewer.html @@ -937,48 +937,173 @@

session

return dot < 0 ? "" : path.slice(dot).toLowerCase(); } -const SEQ_MAGIC = "O4DSEQ\0\0"; -//: magic(8) + version(4) + count(4) + suffix length(4) -const SEQ_PREAMBLE = 20; +const SEQ_MAGIC = "VMESH\0\x01\0"; +const SEQ_PREAMBLE = 26; +const VMESH_CHUNK = 1048576; + +/* Pure SHA-256 also works on LAN HTTP pages, where Web Crypto is unavailable. */ +function vmeshSHA256(data) { + const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); + const k = [0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2]; + const h = [0x6a09e667,0xbb67ae85,0x3c6ef372,0xa54ff53a,0x510e527f,0x9b05688c,0x1f83d9ab,0x5be0cd19]; + const rotate = (x, n) => (x >>> n) | (x << (32 - n)); + const blocks = Math.ceil((bytes.length + 9) / 64), words = new Uint32Array(64); + const bitLength = bytes.length * 8; + for (let block = 0; block < blocks; block++) { + words.fill(0); + for (let i = 0; i < 64; i++) { + const at = block * 64 + i; + let value = at < bytes.length ? bytes[at] : at === bytes.length ? 128 : 0; + if (at >= blocks * 64 - 8) { + const shift = (blocks * 64 - 1 - at) * 8; + value = shift >= 32 ? (Math.floor(bitLength / 4294967296) >>> (shift - 32)) & 255 + : (bitLength >>> shift) & 255; + } + words[i >>> 2] |= value << (24 - (i % 4) * 8); + } + for (let i = 16; i < 64; i++) { + const x = words[i - 15], y = words[i - 2]; + words[i] = words[i - 16] + (rotate(x,7)^rotate(x,18)^(x>>>3)) + + words[i - 7] + (rotate(y,17)^rotate(y,19)^(y>>>10)); + } + let [a,b,c,d,e,f,g,j] = h; + for (let i = 0; i < 64; i++) { + const t = (j + (rotate(e,6)^rotate(e,11)^rotate(e,25)) + ((e&f)^(~e&g)) + k[i] + words[i]) | 0; + const u = ((rotate(a,2)^rotate(a,13)^rotate(a,22)) + ((a&b)^(a&c)^(b&c))) | 0; + j=g; g=f; f=e; e=(d+t)|0; d=c; c=b; b=a; a=(t+u)|0; + } + [a,b,c,d,e,f,g,j].forEach((value, i) => { h[i] = (h[i] + value) >>> 0; }); + } + return h.map(value => value.toString(16).padStart(8, "0")).join(""); +} -/* The header of a container, from its first bytes. Parsed separately from the - * body so a download that has only started can already say how many frames are - * coming, which is what a progress count needs. */ -function readSequenceHeader(buffer) { - const bytes = new Uint8Array(buffer); - if (bytes.length < SEQ_PREAMBLE) throw new Error("sequence header truncated"); - let magic = ""; - for (let i = 0; i < 8; i++) magic += String.fromCharCode(bytes[i]); - if (magic !== SEQ_MAGIC) throw new Error("not a sequence container"); +function vmeshJSON(text) { + const value = JSON.parse(text), stack = []; + for (const token of text.match(/"(?:[^"\\]|\\.)*"|[{}\[\],:]/g) || []) { + if (token === "{") stack.push({keys:new Set(), key:true}); + else if (token === "[") stack.push(null); + else if (token === "}" || token === "]") stack.pop(); + else if (token === ":") stack[stack.length - 1].key = false; + else if (token === ",") { + const current = stack[stack.length - 1]; + if (current) current.key = true; + } else { + const current = stack[stack.length - 1]; + if (current && current.key) { + const key = JSON.parse(token); + if (current.keys.has(key)) throw new Error("duplicate VMESH JSON key"); + current.keys.add(key); + } + } + } + return value; +} + +function vmeshRecord(buffer, at, kind, fileId, offset, length) { + if (at < 0 || at + 18 + length > buffer.byteLength) throw new Error("VMESH record truncated"); const view = new DataView(buffer); - const version = view.getUint32(8, true); - if (version !== 1) throw new Error(`sequence version ${version} is not 1`); - const count = view.getUint32(12, true); - const suffixLength = view.getUint32(16, true); - let suffix = ""; - for (let i = 0; i < suffixLength; i++) { - suffix += String.fromCharCode(bytes[SEQ_PREAMBLE + i]); - } - const table = SEQ_PREAMBLE + suffixLength; - if (bytes.length < table + 8 * count) throw new Error("sequence table truncated"); + const position = Number(view.getBigUint64(at + 10)); + if (!Number.isSafeInteger(position) || view.getUint32(at) !== 14 + length + || view.getUint8(at + 4) !== 1 || view.getUint8(at + 5) !== kind + || view.getUint32(at + 6) !== fileId || position !== offset) { + throw new Error("invalid VMESH record version, kind, file ID, offset or length"); + } +} + +/* The manifest is enough to calculate frame record ranges before body arrival. */ +function readSequenceHeader(buffer) { + const bytes = new Uint8Array(buffer), view = new DataView(buffer); + if (bytes.length < SEQ_PREAMBLE) throw new Error("VMESH manifest truncated"); + if (String.fromCharCode(...bytes.subarray(0, 8)) !== SEQ_MAGIC) throw new Error("not a VMESH container"); + const length = view.getUint32(8) - 14; + if (length <= 0 || length > 16777216) throw new Error("VMESH manifest exceeds limits"); + vmeshRecord(buffer, 8, 0, 0, 0, length); + const manifestBytes = bytes.subarray(SEQ_PREAMBLE, SEQ_PREAMBLE + length); + let manifest; + try { manifest = vmeshJSON(new TextDecoder("utf-8", {fatal:true}).decode(manifestBytes)); } + catch (_) { throw new Error("invalid VMESH manifest JSON"); } + const object = value => value !== null && typeof value === "object" && !Array.isArray(value); + const finiteJSON = value => { + if (typeof value === "number" && !Number.isFinite(value)) throw new Error("non-finite VMESH metadata"); + if (value !== null && typeof value === "object") Object.values(value).forEach(finiteJSON); + }; + finiteJSON(manifest); + if (!object(manifest)) throw new Error("VMESH manifest must be an object"); + const native = manifest.native, count = manifest.frame_count, sequence = manifest.sequence; + const suffixes = ["ply", "drc", "splat", "jpg", "jpeg", "png"]; + if (!object(manifest) || manifest.schema !== "vmesh/1" || manifest.codec !== "frames" + || manifest.native_version !== 1 || manifest.representation !== "frame_payloads" + || manifest.dependency_mode !== "independent" || !Number.isInteger(count) || count < 1 || count > 65536 + || !object(native) || native.profile !== "frames/1" || !suffixes.includes(native.suffix) + || Object.keys(native).sort().join(",") !== "profile,representation,suffix" + || !({mesh:["ply","drc"], points:["ply","drc"], gaussians:["ply","splat"], pixels:["jpg","jpeg","png"]}[native.representation] || []).includes(native.suffix) + || !Array.isArray(manifest.files) || manifest.files.length !== count + || !object(sequence) || Object.hasOwn(sequence, "schema") || sequence.version !== 1 || sequence.codec !== "frames" + || !object(sequence.native) || Object.keys(sequence.native).sort().join(",") !== "profile,representation,suffix" + || sequence.native.profile !== native.profile || sequence.native.suffix !== native.suffix + || sequence.native.representation !== native.representation + || !Array.isArray(sequence.frames) || sequence.frames.length !== count + || (sequence.allow_nonmonotonic_timestamps !== undefined && typeof sequence.allow_nonmonotonic_timestamps !== "boolean") + || (sequence.metadata !== undefined && !object(sequence.metadata)) + || ["has_constant_vertex_count", "has_vertex_correspondence"].some(name => + sequence[name] !== undefined && sequence[name] !== null && typeof sequence[name] !== "boolean")) { + throw new Error("unsupported VMESH browser frame profile"); + } const entries = []; + let offset = SEQ_PREAMBLE + length, previous = null; for (let i = 0; i < count; i++) { - entries.push({ - offset: view.getUint32(table + 8 * i, true), - length: view.getUint32(table + 8 * i + 4, true), - }); + const file = manifest.files[i], frame = sequence.frames[i]; + if (!object(file) || file.id !== i || file.name !== `frame_${String(i).padStart(6,"0")}.${native.suffix}` + || file.role !== "frame-payload" || !Number.isSafeInteger(file.size) || file.size <= 0 + || typeof file.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(file.sha256) + || !object(frame) || !Number.isSafeInteger(frame.frame_index) || frame.frame_index < 0 + || typeof frame.timestamp !== "number" || !Number.isFinite(frame.timestamp) + || (previous !== null && !sequence.allow_nonmonotonic_timestamps && frame.timestamp < previous) + || (frame.metadata !== undefined && !object(frame.metadata))) { + throw new Error("invalid VMESH frame metadata or payload descriptor"); + } + entries.push({offset, length:file.size, sha256:file.sha256}); + offset += file.size + 18 * Math.ceil(file.size / VMESH_CHUNK); + if (!Number.isSafeInteger(offset + 50)) throw new Error("VMESH offsets exceed browser integer limit"); + previous = frame.timestamp; } - return { suffix, entries }; + return {suffix:native.suffix, representation:native.representation, entries, endOffset:offset, + manifestHash:vmeshSHA256(manifestBytes)}; } -/* One frame's bytes, as its own ArrayBuffer. - * - * Sliced rather than viewed: the decode worker takes ownership of what it is - * given, and handing it a view into the container would transfer the whole - * container on the first frame and leave the rest unreachable. */ +function validateSequenceEnd(buffer, header) { + if (buffer.byteLength !== header.endOffset + 50) throw new Error("VMESH is truncated or has trailing data"); + vmeshRecord(buffer, header.endOffset, 2, 0, 0, 32); + const digest = new Uint8Array(buffer, header.endOffset + 18, 32); + const hex = Array.from(digest, byte => byte.toString(16).padStart(2,"0")).join(""); + if (hex !== header.manifestHash) throw new Error("VMESH end manifest SHA-256 mismatch"); +} + +/* Assemble chunk records into an owned buffer, then verify before decoding. */ function sequenceFrame(buffer, header, index) { + if (!Number.isInteger(index) || index < 0 || index >= header.entries.length) throw new Error("VMESH frame index out of range"); const entry = header.entries[index]; - return buffer.slice(entry.offset, entry.offset + entry.length); + if (entry.offset + entry.length + 18 * Math.ceil(entry.length / VMESH_CHUNK) > buffer.byteLength) { + throw new Error("VMESH frame payload truncated"); + } + const output = new Uint8Array(entry.length); + let at = entry.offset, copied = 0; + while (copied < entry.length) { + const length = Math.min(VMESH_CHUNK, entry.length - copied); + vmeshRecord(buffer, at, 1, index, copied, length); + output.set(new Uint8Array(buffer, at + 18, length), copied); + at += 18 + length; + copied += length; + } + if (vmeshSHA256(output) !== entry.sha256) throw new Error("VMESH frame SHA-256 mismatch"); + return output.buffer; } /* ------------------------------------------------------------- adaptation --- @@ -1445,18 +1570,22 @@

session

const fetched = performance.now(); this.stats.bytes += buffer.byteLength; const header = readSequenceHeader(buffer); + validateSequenceEnd(buffer, header); if (header.entries.length !== total) { throw new Error(`${url}: holds ${header.entries.length} frames,` + ` clip lists ${total}`); } // The frame list survives packing, and this is one reason it has to: a // frame is decoded by the codec its *own* name selects. Handing the - // container's name to the decode instead is a `.seq` suffix in a table of + // container's name to the decode instead is a `.vmesh` suffix in a table of // `.ply` and `.splat`, which is how every Gaussian pane went blank -- and // silently, because pixels are decoded from the media type on the main // thread and were unaffected, so it looked like one method being broken // rather than one code path. const names = this.frames; + if (this.clip.representation && this.clip.representation !== header.representation) { + throw new Error(`${url}: VMESH representation disagrees with clip`); + } if (suffixOf(names[0]) !== `.${header.suffix}`) { throw new Error(`${url}: holds .${header.suffix} frames, clip lists` + ` ${suffixOf(names[0])} -- one of them would be decoded wrongly`); @@ -1489,7 +1618,9 @@

session

*/ async _readPacked(response, expected, total, onProgress) { if (!response.body || !response.body.getReader) { - return response.arrayBuffer(); + const buffer = await response.arrayBuffer(); + if (buffer.byteLength > this.byteBudget) throw new Error("VMESH download exceeds byte budget"); + return buffer; } const reader = response.body.getReader(); const chunks = []; @@ -1498,8 +1629,12 @@

session

const { done, value } = await reader.read(); if (done) break; if (this.abandoned) { reader.cancel().catch(() => {}); return null; } - chunks.push(value); got += value.byteLength; + if (got > this.byteBudget) { + reader.cancel().catch(() => {}); + throw new Error("VMESH download exceeds byte budget"); + } + chunks.push(value); if (onProgress && expected) { // Bytes shown as frames, because frames are the unit the caller counts // and the two phases have to share one scale to make a bar that only @@ -2775,7 +2910,7 @@

session

* rejection: the pane stayed blank, nothing was logged where anyone * would look, and the page went on insisting it was fine. That is how a * decoder error reads as "this method is broken" -- which is what - * happened when every Gaussian frame was handed the container's `.seq` + * happened when every Gaussian frame was handed the container's `.vmesh` * name instead of its own. A pane that cannot show its content has to * say why. */ if (token !== app.downloadToken) return; diff --git a/open4d/streamer/streamer/codecs.py b/open4d/streamer/streamer/codecs.py index 4c6fc5b6..858a5ec4 100644 --- a/open4d/streamer/streamer/codecs.py +++ b/open4d/streamer/streamer/codecs.py @@ -5,12 +5,11 @@ at all: in what format do its frames travel, and can the client turn them back into that representation? -Until this existed the two questions were conflated, and the cost was concrete. -Open4D's codecs produce ``.d4d``, ``.v4d``, ``.q4d`` and six more; the client -could decode ``.ply``, ``.splat``, ``.jpg`` and ``.png``. The two sets did not -intersect at all, and nothing in the code said so — a producer found out by -watching a pane stay blank. A registry that names the wire format and where it -decodes makes that a lookup instead of a discovery. +Research sequences use ``.vmesh``; the manifest identifies their codec and +native payload dependencies. The browser decodes whole-frame ``.ply``, +``.drc``, ``.splat``, ``.jpg`` and ``.png`` payloads, including those carried +by VMESH's ``frames/1`` profile. Other research profiles need their native +server runtime. The registry makes that distinction explicit. **The key is (representation, suffix), not suffix.** ``.ply`` is claimed by three representations here and needs two different parsers: a 3DGS PLY and a mesh PLY @@ -29,8 +28,8 @@ ReRF's entropy coder exists only as a CPython 3.8 binary. Those are decoded and rendered where the GPU is, and what reaches the client is ``pixels``. That is the codec's output representation as the platform sees it, which is - why ``.rerf`` is registered as producing ``pixels`` rather than a volume it - never delivers. + why ReRF VMESH is registered for server-side rendering into ``pixels``. + Its field payload is selected by the VMESH manifest. """ from __future__ import annotations @@ -257,7 +256,7 @@ def media_types() -> dict[str, str]: )) register(CodecSpec( name="rerf", - suffix=".rerf", + suffix=".vmesh", representation=Representation.PIXELS, decodes="server", lossy=True, diff --git a/open4d/streamer/streamer/export.py b/open4d/streamer/streamer/export.py index 76738d1c..d9791155 100644 --- a/open4d/streamer/streamer/export.py +++ b/open4d/streamer/streamer/export.py @@ -62,16 +62,14 @@ def representation_of(sequence: Sequence) -> Representation: def _write_draco(sequence: Sequence, frames_at: Path, bits: int) -> list[Path]: """One ``.drc`` per frame, encoded with the same Draco this repository vendors. - Per frame rather than one container for the sequence, because a streaming - client fetches frames: `open4d.save(..., codec="draco")` writes a `.d4d` - holding the whole sequence, which is the right shape for an archive and the - wrong one for a wire. + Per frame because a streaming client fetches frames. DracoPy is an optional + dependency of this exporter, independent of the public codec registry. """ try: import DracoPy except ImportError as error: # pragma: no cover - depends on the environment raise RuntimeError( - "Draco frames need the DracoPy binding: pip install 'open4d[draco]'" + "Draco frames need the DracoPy binding: pip install 'DracoPy'" ) from error written: list[Path] = [] @@ -198,6 +196,7 @@ def from_source( out_dir = Path(out_dir).expanduser().resolve() declared = inspect_sequence(source).timing_source with open4d.load(source, fps=fps if declared == "default" else None) as sequence: + playback_fps = sequence.fps or 30.0 clip = from_sequence( sequence, out_dir, @@ -216,6 +215,6 @@ def from_source( title=f"{clip.representation} — {clip.name}", source=str(source), clips=[clip], - fps=int(fps) if fps else 30, + fps=playback_fps, ) return out_dir diff --git a/open4d/streamer/streamer/sequence.py b/open4d/streamer/streamer/sequence.py index 8bc6c829..e6b5f759 100644 --- a/open4d/streamer/streamer/sequence.py +++ b/open4d/streamer/streamer/sequence.py @@ -1,61 +1,51 @@ -"""A whole clip as one file. - -A clip is a directory of frames, and fetching it a frame at a time is how this -started: thirty requests for thirty frames. On loopback that is free, which is -why it survived. Over a link it is thirty round trips before anything plays -- -600 ms of pure latency on a 20 ms connection -- and it is thirty chances for -one frame to arrive late and stall a clip that was otherwise complete. - -For **on demand** none of that buys anything. A viewer with a free camera has -to hold the whole sequence in memory anyway, because a frame it has thrown away -cannot be redrawn from a new angle. So the unit of transfer should be the -sequence, and there is no reason to have asked for it in pieces. - -Hence a container: a header naming the frames, then their bytes end to end. One -request, one response, one progress bar. Deliberately not a zip or a tar -- -both would need a decoder in the client and neither buys anything here, since -the frames are already compressed and an archive's own compression would only -spend CPU to save nothing. - - header magic "O4DSEQ\\0\\0" | version | frame count | suffix - then per frame: offset, length -- both uint32 little-endian - body each frame's bytes, in playback order, unpadded - -The offsets are absolute within the file, so a client that has the header can -slice any frame out without walking the ones before it -- which is what lets a -partially arrived download start playing from the beginning while the rest -lands. +"""Whole browser clips in standalone VMESH with native frame payloads. + +The ``frames/1`` profile carries PLY, Draco, splat or image bytes verbatim. +VMESH owns the manifest, timestamps, record framing and SHA-256 integrity. +A manifest prefix is enough to locate any frame's chunk records, so downloads +and HTTP byte ranges retain stable offsets without a second private format. """ from __future__ import annotations import dataclasses +import hashlib +from io import BytesIO +import json +import math import struct +import tempfile from dataclasses import dataclass from pathlib import Path -from typing import Sequence +from typing import Sequence as TypingSequence +from open4d._files import publish_directory, publish_file +from open4d.codec import _vmesh_format as vmesh +from open4d.codec._protocol import CodecError from . import representations -MAGIC = b"O4DSEQ\x00\x00" +MAGIC = vmesh._MAGIC VERSION = 1 -#: ``magic + version + count + suffix length``, before the suffix itself. -_PREAMBLE = struct.calcsize("<8sIII") +_SUFFIXES = frozenset(("ply", "drc", "splat", "jpg", "jpeg", "png")) +_MAX_SAFE_INTEGER = 2**53 - 1 @dataclass(frozen=True) class Entry: - """Where one frame sits inside the container.""" + """First VMESH payload record, native byte count and native SHA-256.""" offset: int length: int + sha256: str @dataclass(frozen=True) class Sequence: - """A container's header: what is in it and where.""" + """A validated VMESH frame manifest, usable before its payload arrives.""" suffix: str entries: tuple[Entry, ...] + end_offset: int + manifest_hash: bytes @property def frames(self) -> int: @@ -66,99 +56,221 @@ def bytes(self) -> int: return sum(entry.length for entry in self.entries) -def pack(frames: Sequence[Path | str], destination: Path | str) -> Sequence: - """Write ``frames`` into one container at ``destination``. +def _fps(value): + if type(value) not in (int, float) or not math.isfinite(value) or value <= 0: + raise ValueError("fps must be positive and finite") + return value - Every frame must share a suffix. A container holding two formats would need - the client to switch decoders mid-sequence, and a clip whose frames are not - all one codec is a clip that should have been two. - """ - paths = [Path(f) for f in frames] + +def _header(descriptor, manifest_hash, start): + if descriptor["codec"] != "frames": + raise ValueError("VMESH is not a browser frame-payload profile") + entries, offset = [], start + for record in descriptor["files"]: + length = record["size"] + entries.append(Entry(offset, length, record["sha256"])) + offset += length + 18 * ((length + vmesh._CHUNK - 1) // vmesh._CHUNK) + if offset + 50 > _MAX_SAFE_INTEGER: + raise ValueError("VMESH frame offsets exceed the browser integer limit") + return Sequence(descriptor["native"]["suffix"], tuple(entries), offset, manifest_hash) + + +def pack(frames: TypingSequence[Path | str], destination: Path | str, *, + fps: float = 30, representation: str | None = None, + overwrite: bool = False) -> Sequence: + """Write native whole frames into one .vmesh without recompression.""" + destination = Path(destination) + if destination.suffix.lower() != ".vmesh": + raise ValueError("destination must have a .vmesh extension") + if destination.exists() and not overwrite: + raise FileExistsError(destination) + if not isinstance(overwrite, bool): + raise TypeError("overwrite must be bool") + fps = _fps(fps) + paths = [Path(path) for path in frames] if not paths: raise ValueError("a sequence needs at least one frame") + if len(paths) > 65536: + raise ValueError("VMESH frame count exceeds limits") suffixes = {path.suffix.lower().lstrip(".") for path in paths} if len(suffixes) != 1: - raise ValueError( - f"frames must share one suffix; got {', '.join(sorted(suffixes))}" - ) + raise ValueError(f"frames must share one suffix; got {', '.join(sorted(suffixes))}") suffix = suffixes.pop() - missing = [str(path) for path in paths if not path.is_file()] - if missing: - raise FileNotFoundError(f"{len(missing)} frame(s) missing, e.g. {missing[0]}") - - encoded = suffix.encode() - header = _PREAMBLE + len(encoded) + 8 * len(paths) - entries, offset = [], header - for path in paths: - length = path.stat().st_size - entries.append(Entry(offset=offset, length=length)) - offset += length - - destination = Path(destination) + if suffix not in _SUFFIXES: + raise ValueError(f"unsupported browser frame suffix {suffix!r}") + if representation is None: + representation = "pixels" if suffix in ("jpg", "jpeg", "png") else "gaussians" if suffix == "splat" else "mesh" + native = dict(profile="frames/1", suffix=suffix, representation=representation) + metadata = dict(version=1, codec="frames", native=native, metadata={}, + frames=[dict(frame_index=i, timestamp=i / fps, metadata={}) + for i in range(len(paths))]) + descriptor = dict(schema="vmesh/1", codec="frames", native_version=1, + representation="frame_payloads", dependency_mode="independent", + frame_count=len(paths), native=native, sequence=metadata, files=[]) + for index, path in enumerate(paths): + if path.is_symlink() or not path.is_file(): + raise FileNotFoundError(f"missing or linked frame: {path}") + digest, size = hashlib.sha256(), 0 + with path.open("rb") as stream: + while chunk := stream.read(vmesh._CHUNK): + digest.update(chunk) + size += len(chunk) + descriptor["files"].append(dict(id=index, name=f"frame_{index:06d}.{suffix}", + role="frame-payload", size=size, + sha256=digest.hexdigest())) + manifest = json.dumps(descriptor, separators=(",", ":"), allow_nan=False).encode() + try: + vmesh._descriptor(manifest) + except CodecError as error: + raise ValueError(str(error)) from error + if len(manifest) > vmesh._MAX_JSON: + raise ValueError("VMESH manifest exceeds size limit") + manifest_hash = hashlib.sha256(manifest).digest() + header = _header(descriptor, manifest_hash, 26 + len(manifest)) destination.parent.mkdir(parents=True, exist_ok=True) - with open(destination, "wb") as out: - out.write(struct.pack("<8sIII", MAGIC, VERSION, len(paths), len(encoded))) - out.write(encoded) - for entry in entries: - out.write(struct.pack(" Sequence: - """The header of a container, from its first bytes. - - Takes bytes rather than a path so a client that has only the start of a - download can already know what is coming -- which is what a progress bar - counting frames needs. - """ - if len(data) < _PREAMBLE: - raise ValueError("not enough bytes for a sequence header") - magic, version, count, suffix_length = struct.unpack_from("<8sIII", data) - if magic != MAGIC: - raise ValueError(f"not a sequence container: magic is {magic!r}") - if version != VERSION: - raise ValueError(f"sequence version {version} is not {VERSION}") - at = _PREAMBLE - suffix = data[at:at + suffix_length].decode() - at += suffix_length - needed = at + 8 * count - if len(data) < needed: - raise ValueError(f"header needs {needed} bytes, got {len(data)}") - entries = tuple( - Entry(*struct.unpack_from(" bytes: - """One frame's bytes out of a container.""" + """Recover and hash-check one frame's VMESH chunk records.""" header = header or read_header(data) + if type(index) is not int or not 0 <= index < header.frames: + raise IndexError("frame index out of range") entry = header.entries[index] - return data[entry.offset:entry.offset + entry.length] + stream, output, digest = BytesIO(data), bytearray(), hashlib.sha256() + stream.seek(entry.offset) + try: + while len(output) < entry.length: + kind, file_id, offset, chunk = vmesh._read_record(stream) + length = min(vmesh._CHUNK, entry.length - len(output)) + if (kind, file_id, offset, len(chunk)) != (1, index, len(output), length): + raise ValueError("invalid VMESH frame chunk order or length") + digest.update(chunk) + output.extend(chunk) + except CodecError as error: + raise ValueError(str(error)) from error + if digest.hexdigest() != entry.sha256: + raise ValueError("VMESH frame SHA-256 mismatch") + return bytes(output) def unpack(path: Path | str, destination: Path | str) -> list[Path]: - """Write a container back out as numbered frame files. - - The inverse, for checking a container round-trips and for anyone who wants - the frames on disk again. - """ - data = Path(path).read_bytes() - header = read_header(data) + """Validate the complete VMESH before publishing recovered native frames.""" destination = Path(destination) - destination.mkdir(parents=True, exist_ok=True) - written = [] - for index in range(header.frames): - target = destination / f"frame_{index:04d}.{header.suffix}" - target.write_bytes(frame(data, index, header)) - written.append(target) - return written - - -def pack_clip(bundle_dir, clip, *, keep_frames: bool = False) -> dict: + if destination.exists(): + raise FileExistsError(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + try: + descriptor = vmesh.inspect_vmesh(path) + if descriptor["codec"] != "frames": + raise ValueError("VMESH is not a browser frame-payload profile") + with tempfile.TemporaryDirectory(prefix=f".{destination.name}-", dir=destination.parent) as directory: + native, output = Path(directory) / "native", Path(directory) / "frames" + vmesh.unpack_vmesh(path, native) + output.mkdir() + names = [record["name"] for record in descriptor["files"]] + for name in names: + (native / name).replace(output / name) + publish_directory(output, destination) + return [destination / name for name in names] + except CodecError as error: + raise ValueError(str(error)) from error + + +def convert_legacy(source: Path | str, destination: Path | str, *, + fps: float = 30, representation: str | None = None, + overwrite: bool = False) -> Path: + """Explicitly migrate a bounded retired O4DSEQ clip to real VMESH records.""" + source, destination = Path(source), Path(destination) + if destination.suffix.lower() != ".vmesh": + raise ValueError("destination must have a .vmesh extension") + if not isinstance(overwrite, bool): + raise TypeError("overwrite must be bool") + if destination.exists() and not overwrite: + raise FileExistsError(destination) + fps = _fps(fps) + with source.open("rb") as stream: + preamble = stream.read(20) + if len(preamble) != 20: + raise ValueError("truncated legacy sequence header") + magic, version, count, suffix_length = struct.unpack("<8sIII", preamble) + if magic != b"O4DSEQ\x00\x00" or version != 1: + raise ValueError("unsupported legacy sequence magic/version") + if not 0 < count <= 65536 or not 0 < suffix_length <= 8: + raise ValueError("legacy sequence header exceeds limits") + try: + suffix = stream.read(suffix_length).decode("ascii") + except UnicodeError as error: + raise ValueError("invalid legacy sequence suffix") from error + if suffix not in _SUFFIXES: + raise ValueError("unsupported legacy frame suffix") + table = stream.read(8 * count) + if len(table) != 8 * count: + raise ValueError("truncated legacy sequence table") + offset = 20 + suffix_length + 8 * count + records = [] + for index in range(count): + position, length = struct.unpack_from(" dict: """Pack one clip's frames into a container beside the bundle root. Returns the mapping a `bundle.Clip` records as its ``sequence``. The frame @@ -167,8 +279,12 @@ def pack_clip(bundle_dir, clip, *, keep_frames: bool = False) -> dict: pieces. """ root = Path(bundle_dir) - relative = f"{clip.name}.seq" - header = pack([root / frame for frame in clip.frames], root / relative) + if not clip.name or clip.name in (".", "..") or any(c in clip.name for c in ("/", "\\", "\x00")): + raise ValueError("clip name must be a safe filename") + paths = _clip_paths(root, clip.frames) + relative = f"{clip.name}.vmesh" + header = pack(paths, root / relative, fps=fps, representation=clip.representation, + overwrite=True) if not keep_frames: directories = {Path(frame).parts[0] for frame in clip.frames} for frame in clip.frames: @@ -196,7 +312,7 @@ def pack_clip(bundle_dir, clip, *, keep_frames: bool = False) -> dict: def pack_bundle( bundle_dir: Path | str, *, - names: Sequence[str] | None = None, + names: TypingSequence[str] | None = None, keep_frames: bool = False, ) -> list[tuple[str, dict]]: """Pack a bundle's clips into containers and rewrite its manifest. @@ -204,7 +320,9 @@ def pack_bundle( Live clips are skipped rather than refused: a bundle is usually a mix, and a whole-bundle command that failed on the first stream would be unusable on exactly the bundles that have both. Clips already packed are skipped too, - so running this twice is not an error. + so running this twice is not an error. Retired packed clips are explicitly + converted and their URLs updated; originals are removed after the manifest + commits unless ``keep_frames`` is set. """ from . import bundle as _bundle @@ -220,18 +338,44 @@ def pack_bundle( if missing: raise KeyError(f"no clip named {', '.join(sorted(missing))}") - packed, changed = [], [] + packed, changed, retired = [], [], [] for clip in clips: skip = ( (wanted is not None and clip.name not in wanted) or clip.stream is not None - or clip.sequence is not None + or (clip.sequence is not None and not str(clip.sequence.get("url", "")).endswith(".seq")) or not clip.frames ) if skip: changed.append(clip) continue - entry = pack_clip(root, clip, keep_frames=keep_frames) + # Do not remove inputs until every clip is packed and the manifest + # has committed. A later codec or filesystem failure must be retryable. + if clip.sequence is not None: + _clip_paths(root, clip.frames) + old = Path(clip.sequence["url"]) + if old.is_absolute() or ".." in old.parts or "\\" in str(old): + raise ValueError("legacy sequence must use a safe relative path") + source = root / old + if source.is_symlink() or not source.resolve().is_relative_to(root.resolve()): + raise ValueError("legacy sequence must stay inside the bundle") + relative = str(old.with_suffix(".vmesh")) + target = root / relative + convert_legacy(source, target, fps=index.get("fps", 30), + representation=clip.representation, overwrite=True) + with target.open("rb") as stream: + descriptor, digest = vmesh._start(stream) + header = _header(descriptor, digest, stream.tell()) + if header.frames != len(clip.frames): + raise ValueError("legacy sequence disagrees with its logical frame list") + entry = dict(clip.sequence, url=relative, frames=header.frames, + bytes=target.stat().st_size, suffix=header.suffix) + media_type = representations.media_types().get(f".{header.suffix}") + if media_type: + entry["media_type"] = media_type + retired.append(source) + else: + entry = pack_clip(root, clip, keep_frames=True, fps=index.get("fps", 30)) changed.append(dataclasses.replace(clip, sequence=entry)) packed.append((clip.name, entry)) @@ -245,6 +389,21 @@ def pack_bundle( scenes=index.get("scenes") or {}, detail=index.get("detail"), ) + if not keep_frames: + from .transfer import frame_paths + + retained = set(frame_paths({"clips": [dataclasses.asdict(c) for c in changed]})) + packed_names = {name for name, _ in packed} + removable = {path for clip in clips if clip.name in packed_names + for path in clip.frames} - retained + for path in removable: + (root / path).unlink(missing_ok=True) + for path in set(retired): + if path.relative_to(root).as_posix() not in retained: + path.unlink() + for directory in {root / Path(path).parts[0] for path in removable}: + if directory.is_dir() and not any(directory.iterdir()): + directory.rmdir() return packed @@ -279,5 +438,68 @@ def main(argv=None) -> int: return 0 +class _MeshFrames: + """Keep validated native payloads alive for lazy canonical mesh decoding.""" + + def __init__(self, native, manifest, temporary): + from types import MappingProxyType + from open4d.core import TopologyMode + self.native, self.manifest, self.temporary = native, manifest, temporary + self.records = manifest["sequence"]["frames"] + self.metadata = MappingProxyType(manifest["sequence"].get("metadata", {})) + self.topology = TopologyMode(manifest["sequence"].get("topology", "unknown")) + self.has_constant_vertex_count = manifest["sequence"].get("has_constant_vertex_count") + self.has_vertex_correspondence = manifest["sequence"].get("has_vertex_correspondence") + self.allow_nonmonotonic_timestamps = manifest["sequence"].get("allow_nonmonotonic_timestamps", False) + + @property + def frame_count(self): + return len(self.records) + + @property + def timestamps(self): + return tuple(record["timestamp"] for record in self.records) + + def get_frame(self, index): + from open4d.core import Frame, TriangleMesh + if not 0 <= index < self.frame_count: + raise IndexError("frame index out of range") + record = self.records[index] + path = self.native / self.manifest["files"][index]["name"] + if self.manifest["native"]["suffix"] == "ply": + from open4d.io._mesh import read_ply + positions, triangles, colors = read_ply(path) + geometry = TriangleMesh(positions, triangles, colors=colors) + else: + from open4d.codec._draco import _backend + decoded = _backend().decode(path.read_bytes()) + colors = decoded.colors.astype("float32") / 255 if decoded.colors is not None else None + geometry = TriangleMesh(decoded.points, decoded.faces, colors=colors, + normals=decoded.normals, texture_coordinates=decoded.tex_coord) + return Frame(record["frame_index"], record["timestamp"], geometry, record.get("metadata", {})) + + def close(self): + self.temporary.cleanup() + + +def open_frames(path: Path | str): + """Open mesh payloads lazily, or retain other browser payloads natively.""" + from open4d.core import Sequence as MeshSequence + from open4d.native import NativeSequence + manifest = vmesh.inspect_vmesh(path) + if manifest["codec"] != "frames": + raise CodecError("VMESH is not a browser frame-payload profile") + if manifest["native"]["representation"] != "mesh": + return NativeSequence(path) + temporary = tempfile.TemporaryDirectory(prefix="vmesh-browser-mesh-") + try: + native = Path(temporary.name) / "native" + vmesh.unpack_vmesh(path, native) + return MeshSequence(_MeshFrames(native, manifest, temporary)) + except BaseException: + temporary.cleanup() + raise + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/open4d/streamer/streamer/server/__init__.py b/open4d/streamer/streamer/server/__init__.py index 19949e78..66c2b7f8 100644 --- a/open4d/streamer/streamer/server/__init__.py +++ b/open4d/streamer/streamer/server/__init__.py @@ -218,7 +218,7 @@ class _Handler(http.server.SimpleHTTPRequestHandler): # A packed clip is opaque: the frames inside keep their own type, which # the manifest names, so guessing one for the container would only be # wrong. See streamer/sequence.py. - ".seq": "application/octet-stream", + ".vmesh": "application/octet-stream", } def do_GET(self): # noqa: N802 - the base class names it diff --git a/open4d/streamer/streamer/session.py b/open4d/streamer/streamer/session.py index 0f739769..28deaa94 100644 --- a/open4d/streamer/streamer/session.py +++ b/open4d/streamer/streamer/session.py @@ -110,14 +110,15 @@ def __init__( *, title: str | None = None, source: Path | str | None = None, - fps: int = 30, + fps: float | None = None, scenes: dict[str, Any] | None = None, detail: dict[str, Any] | None = None, ) -> None: self.out_dir = Path(out_dir).expanduser().resolve() self.title = title or self.out_dir.name self.source = source - self.fps = fps + self.fps = 30.0 if fps is None else fps + self._infer_fps = fps is None self.scenes = scenes self.detail = detail self.clips: list[bundle.Clip] = [] @@ -157,6 +158,8 @@ def add( because which rendition should be the default is a delivery decision (interchange? cheapest? middle?) and not one a byte count settles. """ + if self._infer_fps and not self.clips: + self.fps = sequence.fps or 30.0 parsed = [parse_rung(rung) for rung in rungs] if not parsed: raise ValueError(f"{name}: needs at least one rung") diff --git a/open4d/streamer/streamer/transfer.py b/open4d/streamer/streamer/transfer.py index b43bacbd..5a06fd72 100644 --- a/open4d/streamer/streamer/transfer.py +++ b/open4d/streamer/streamer/transfer.py @@ -29,7 +29,7 @@ import urllib.parse import urllib.request from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import Callable, Iterable from . import bundle @@ -91,6 +91,10 @@ def _download(url: str, destination: Path, timeout: float) -> int: written = 0 with urllib.request.urlopen(request, timeout=timeout) as response: resuming = have > 0 and response.status == 206 + if resuming: + content_range = response.headers.get("Content-Range", "") + if not content_range.startswith(f"bytes {have}-"): + raise ValueError("invalid Content-Range for resumed download") if have and not resuming: # The server sent the whole file despite the range. Start over # rather than append: the alternative is a corrupt file that is @@ -122,11 +126,28 @@ def frame_paths(index: dict) -> tuple[str, ...]: """Every frame path a manifest names, in order, without duplicates.""" seen: dict[str, None] = {} for clip in index.get("clips", []): - for path in clip.get("frames", []): + packed = clip.get("sequence") + paths = [packed["url"]] if packed else clip.get("frames", []) + for variant in clip.get("variants", []): + paths = [*paths, *variant.get("frames", [])] + for path in paths: seen.setdefault(path, None) return tuple(seen) +def _safe_target(root: Path, path: str) -> Path: + if (not isinstance(path, str) or not path or "\\" in path or "\x00" in path + or PurePosixPath(path).is_absolute() or PureWindowsPath(path).drive + or any(part in {"", ".", ".."} for part in path.split("/")) + or path == bundle.INDEX_NAME or path.endswith(".partial")): + raise ValueError(f"unsafe bundle path: {path!r}") + target = root / path + for candidate in (target, target.with_name(target.name + ".partial")): + if root not in candidate.resolve().parents or candidate.is_symlink(): + raise ValueError(f"unsafe bundle path: {path!r}") + return target + + def fetch( url: str, destination: Path | str, @@ -157,17 +178,16 @@ def fetch( if missing: raise KeyError(f"no clip named {', '.join(sorted(missing))} in {base}") - # Written first and last: first so the directory is recognisable as a bundle - # while frames arrive, last so a `--only` subset's manifest is the filtered - # one rather than the whole thing. - (root / bundle.INDEX_NAME).write_text(json.dumps(index, indent=2) + "\n") - paths = frame_paths(index) + targets = {path: _safe_target(root, path) for path in paths} + index_path = root / bundle.INDEX_NAME + if index_path.is_symlink(): + raise ValueError("unsafe bundle manifest path") fetched: list[str] = [] skipped: list[str] = [] total_bytes = 0 for position, path in enumerate(paths, start=1): - target = root / path + target = targets[path] source = base + urllib.parse.quote(path) expected = _remote_size(source, timeout) if target.is_file() else None if target.is_file() and expected is not None and target.stat().st_size == expected: @@ -184,6 +204,9 @@ def fetch( if progress is not None: progress(path, position, len(paths)) + # Advertise the new bundle only once every referenced file is available. + index_path.write_text(json.dumps(index, indent=2) + "\n") + return FetchResult( root=root, fetched=tuple(fetched), diff --git a/open4d/streamer/streamer_tests/test_codecs.py b/open4d/streamer/streamer_tests/test_codecs.py index d931a7ee..08cce749 100644 --- a/open4d/streamer/streamer_tests/test_codecs.py +++ b/open4d/streamer/streamer_tests/test_codecs.py @@ -61,7 +61,7 @@ def test_by_name_finds_a_codec(): def test_rerf_is_registered_as_server_decoded(): """Not an omission: its entropy coder is a sourceless CPython 3.8 binary.""" - spec = codecs.for_frame("pixels", ".rerf") + spec = codecs.for_frame("pixels", ".vmesh") assert spec.decodes == "server" assert "browser" in spec.cost @@ -176,14 +176,14 @@ def test_adding_a_codec_is_one_registration(monkeypatch): codecs.register( CodecSpec( name="mesh-vdmc", - suffix=".v4d", + suffix=".vmesh", representation=Representation.MESH, lossy=True, cost="V-DMC is lossy at any rate worth using", ) ) - assert codecs.for_frame("mesh", ".v4d").name == "mesh-vdmc" - assert ".v4d" in codecs.media_types() + assert codecs.for_frame("mesh", ".vmesh").name == "mesh-vdmc" + assert ".vmesh" in codecs.media_types() assert "mesh-vdmc" in {spec.name for spec in codecs.client_decodable("mesh")} diff --git a/open4d/streamer/streamer_tests/test_draco.py b/open4d/streamer/streamer_tests/test_draco.py index a691072c..1eea4152 100644 --- a/open4d/streamer/streamer_tests/test_draco.py +++ b/open4d/streamer/streamer_tests/test_draco.py @@ -38,7 +38,7 @@ def _source() -> str: NODE = shutil.which("node") requires_node = pytest.mark.skipif(NODE is None, reason="node is not installed") -DracoPy = pytest.importorskip("DracoPy", reason="Draco frames need open4d[draco]") +DracoPy = pytest.importorskip("DracoPy", reason="Draco frames need DracoPy") CLIENT = viewer_path().parent VENDOR = CLIENT / "vendor" / "draco" diff --git a/open4d/streamer/streamer_tests/test_export.py b/open4d/streamer/streamer_tests/test_export.py index 3c578c5e..9a10c80e 100644 --- a/open4d/streamer/streamer_tests/test_export.py +++ b/open4d/streamer/streamer_tests/test_export.py @@ -149,5 +149,14 @@ def test_from_source_round_trips_through_open4d_load(tmp_path): assert len(index["clips"]) == 1 assert index["clips"][0]["representation"] == "mesh" assert len(index["clips"][0]["frames"]) == 2 - assert index["fps"] == 10 + assert index["fps"] == 30 # The source manifest's timing takes precedence. assert any("open4d.load" in note for note in index["clips"][0]["notes"]) + + +def test_export_keeps_fractional_source_rate(tmp_path): + from open4d.io import write_sequence + + source = Sequence(MemoryFrameProvider([Frame(i, i / 23.976, mesh()) for i in range(3)])) + path = write_sequence(source, tmp_path / "frames") + out = export.from_source(path, tmp_path / "bundle", fps=10) + assert bundle.read(out)["fps"] == pytest.approx(23.976) diff --git a/open4d/streamer/streamer_tests/test_open4d_stream.py b/open4d/streamer/streamer_tests/test_open4d_stream.py index b941479c..00bed082 100644 --- a/open4d/streamer/streamer_tests/test_open4d_stream.py +++ b/open4d/streamer/streamer_tests/test_open4d_stream.py @@ -20,13 +20,13 @@ pytestmark = pytest.mark.cpu -def sequence_of(count: int = 3) -> Sequence: +def sequence_of(count: int = 3, fps: float = 30) -> Sequence: geometry = TriangleMesh( np.asarray([[0, 0, 0], [1, 0, 0], [0, 1, 0]], dtype=np.float32), np.asarray([[0, 1, 2]], dtype=np.uint32), ) return Sequence( - MemoryFrameProvider([Frame(i, i / 30, geometry) for i in range(count)]) + MemoryFrameProvider([Frame(i, i / fps, geometry) for i in range(count)]) ) @@ -50,6 +50,21 @@ def test_a_sequence_is_exported_and_served(tmp_path): server.shutdown() +@pytest.mark.parametrize("as_path", [False, True]) +def test_browser_stream_uses_source_timestamps(tmp_path, as_path): + source = sequence_of(fps=23.976) + if as_path: + from open4d.io import write_sequence + source = write_sequence(source, tmp_path / "source") + server = open4d.stream(source, out_dir=tmp_path / "bundle", name="timed", + fps=10, open_browser=False, block=False) + try: + index = json.loads((server.bundle_dir / "view.json").read_text()) + assert index["fps"] == pytest.approx(23.976) + finally: + server.shutdown() + + def test_a_ladder_becomes_variants(tmp_path): server = open4d.stream( sequence_of(), diff --git a/open4d/streamer/streamer_tests/test_representations.py b/open4d/streamer/streamer_tests/test_representations.py index 530eb129..db77d107 100644 --- a/open4d/streamer/streamer_tests/test_representations.py +++ b/open4d/streamer/streamer_tests/test_representations.py @@ -75,11 +75,11 @@ def test_registering_a_codec_changes_what_a_representation_serves(monkeypatch): monkeypatch.setattr(codecs, "_REGISTRY", dict(codecs._REGISTRY)) codecs.register( codecs.CodecSpec( - name="mesh-vdmc", suffix=".v4d", representation=Representation.MESH + name="mesh-vdmc", suffix=".vmesh", representation=Representation.MESH ) ) - assert ".v4d" in representations.spec("mesh").media_types - assert ".v4d" in representations.media_types() + assert ".vmesh" in representations.spec("mesh").media_types + assert ".vmesh" in representations.media_types() def test_a_representation_exposes_its_codecs(): diff --git a/open4d/streamer/streamer_tests/test_sequence.py b/open4d/streamer/streamer_tests/test_sequence.py index afeb459b..5e3fdacf 100644 --- a/open4d/streamer/streamer_tests/test_sequence.py +++ b/open4d/streamer/streamer_tests/test_sequence.py @@ -54,53 +54,51 @@ def _frames(directory: Path, sizes: list[int], suffix: str = "splat") -> list[Pa def test_a_packed_container_round_trips_every_frame(tmp_path): frames = _frames(tmp_path / "clip", [10, 300, 7, 4096]) - header = sequence.pack(frames, tmp_path / "clip.seq") + header = sequence.pack(frames, tmp_path / "clip.vmesh") assert header.frames == 4 assert header.suffix == "splat" assert header.bytes == 10 + 300 + 7 + 4096 - data = (tmp_path / "clip.seq").read_bytes() + data = (tmp_path / "clip.vmesh").read_bytes() for index, path in enumerate(frames): assert sequence.frame(data, index, header) == path.read_bytes() -def test_offsets_are_absolute_and_leave_no_gaps(tmp_path): +def test_offsets_name_real_vmesh_records(tmp_path): frames = _frames(tmp_path / "clip", [10, 300, 7]) - header = sequence.pack(frames, tmp_path / "clip.seq") - size = (tmp_path / "clip.seq").stat().st_size - - # Absolute, so a client with the header can slice any frame without having - # walked the ones before it -- which is what lets a partial download play - # from the beginning. + header = sequence.pack(frames, tmp_path / "clip.vmesh") + data = (tmp_path / "clip.vmesh").read_bytes() + assert data[:8] == b"VMESH\x00\x01\x00" first = header.entries[0] - assert first.offset == size - header.bytes + assert first.offset == 26 + struct.unpack_from(">I", data, 8)[0] - 14 for before, after in zip(header.entries, header.entries[1:]): - assert before.offset + before.length == after.offset + assert before.offset + before.length + 18 == after.offset last = header.entries[-1] - assert last.offset + last.length == size + assert last.offset + last.length + 18 == header.end_offset + assert header.end_offset + 50 == len(data) def test_a_header_read_back_matches_the_one_written(tmp_path): frames = _frames(tmp_path / "clip", [10, 300, 7]) - written = sequence.pack(frames, tmp_path / "clip.seq") - read = sequence.read_header((tmp_path / "clip.seq").read_bytes()) + written = sequence.pack(frames, tmp_path / "clip.vmesh") + read = sequence.read_header((tmp_path / "clip.vmesh").read_bytes()) assert read == written def test_the_header_can_be_read_before_the_body_arrives(tmp_path): frames = _frames(tmp_path / "clip", [4096, 4096, 4096]) - written = sequence.pack(frames, tmp_path / "clip.seq") + written = sequence.pack(frames, tmp_path / "clip.vmesh") # The point of a fixed header: a progress count knows how many frames are # coming from the first few hundred bytes, not the last. - prefix = (tmp_path / "clip.seq").read_bytes()[: written.entries[0].offset] + prefix = (tmp_path / "clip.vmesh").read_bytes()[: written.entries[0].offset] assert sequence.read_header(prefix) == written def test_unpacking_restores_the_frames(tmp_path): frames = _frames(tmp_path / "clip", [10, 300, 7]) - sequence.pack(frames, tmp_path / "clip.seq") - restored = sequence.unpack(tmp_path / "clip.seq", tmp_path / "out") + sequence.pack(frames, tmp_path / "clip.vmesh") + restored = sequence.unpack(tmp_path / "clip.vmesh", tmp_path / "out") assert [path.read_bytes() for path in restored] == [ path.read_bytes() for path in frames ] @@ -111,7 +109,7 @@ def test_unpacking_restores_the_frames(tmp_path): def test_a_container_needs_a_frame(tmp_path): with pytest.raises(ValueError, match="at least one frame"): - sequence.pack([], tmp_path / "clip.seq") + sequence.pack([], tmp_path / "clip.vmesh") def test_frames_must_share_a_suffix(tmp_path): @@ -121,30 +119,30 @@ def test_frames_must_share_a_suffix(tmp_path): # Two formats in one container would need the client to switch decoders # mid-clip. A clip whose frames are not one codec is two clips. with pytest.raises(ValueError, match="share one suffix"): - sequence.pack(frames + [odd], tmp_path / "clip.seq") + sequence.pack(frames + [odd], tmp_path / "clip.vmesh") def test_a_missing_frame_is_named(tmp_path): frames = _frames(tmp_path / "clip", [10, 10]) with pytest.raises(FileNotFoundError, match="frame_0009"): sequence.pack( - frames + [tmp_path / "clip" / "frame_0009.splat"], tmp_path / "clip.seq" + frames + [tmp_path / "clip" / "frame_0009.splat"], tmp_path / "clip.vmesh" ) @pytest.mark.parametrize( "mangle, complaint", [ - (lambda data: b"NOTASEQ\x00" + data[8:], "not a sequence container"), - (lambda data: data[:8] + struct.pack(" str: return "\n".join(chunks) -READER = ("SEQ_MAGIC", "SEQ_PREAMBLE", "readSequenceHeader", "sequenceFrame") +READER = ("SEQ_MAGIC", "SEQ_PREAMBLE", "VMESH_CHUNK", "vmeshSHA256", + "vmeshJSON", "vmeshRecord", "readSequenceHeader", "validateSequenceEnd", "sequenceFrame") def run_js(body: str, tmp_path: Path, names=READER, name: str = "q.mjs") -> object: @@ -278,19 +277,20 @@ def run_js(body: str, tmp_path: Path, names=READER, name: str = "q.mjs") -> obje @requires_node def test_the_client_reads_a_header_python_wrote(tmp_path): frames = _frames(tmp_path / "clip", [10, 300, 7, 4096]) - header = sequence.pack(frames, tmp_path / "clip.seq") + header = sequence.pack(frames, tmp_path / "clip.vmesh") result = run_js( LOAD + f""" - const header = readSequenceHeader(load({str(tmp_path / "clip.seq")!r})); + const header = readSequenceHeader(load({str(tmp_path / "clip.vmesh")!r})); process.stdout.write(JSON.stringify(header)); """, tmp_path, ) assert result["suffix"] == header.suffix assert result["entries"] == [ - {"offset": entry.offset, "length": entry.length} for entry in header.entries + {"offset": entry.offset, "length": entry.length, "sha256": entry.sha256} + for entry in header.entries ] @@ -299,12 +299,12 @@ def test_the_client_slices_the_same_bytes(tmp_path): # Sizes chosen unequal and unaligned: equal-length frames would let an # off-by-one in the offset table pass every frame but the last. frames = _frames(tmp_path / "clip", [10, 300, 7, 4096, 33]) - sequence.pack(frames, tmp_path / "clip.seq") + sequence.pack(frames, tmp_path / "clip.vmesh") result = run_js( LOAD + f""" - const buffer = load({str(tmp_path / "clip.seq")!r}); + const buffer = load({str(tmp_path / "clip.vmesh")!r}); const header = readSequenceHeader(buffer); const out = []; for (let i = 0; i < header.entries.length; i++) {{ @@ -325,11 +325,11 @@ def test_the_client_slices_the_same_bytes(tmp_path): @requires_node def test_a_sliced_frame_does_not_alias_the_container(tmp_path): - sequence.pack(_frames(tmp_path / "clip", [64, 64]), tmp_path / "clip.seq") + sequence.pack(_frames(tmp_path / "clip", [64, 64]), tmp_path / "clip.vmesh") result = run_js( LOAD + f""" - const buffer = load({str(tmp_path / "clip.seq")!r}); + const buffer = load({str(tmp_path / "clip.vmesh")!r}); const header = readSequenceHeader(buffer); const first = sequenceFrame(buffer, header, 0); new Uint8Array(first).fill(0xff); @@ -338,7 +338,7 @@ def test_a_sliced_frame_does_not_alias_the_container(tmp_path): // the first frame and leave every later one unreachable. const container = new Uint8Array(buffer); process.stdout.write(JSON.stringify({{ - inContainer: container[header.entries[0].offset], + inContainer: container[header.entries[0].offset + 18], inFrame: new Uint8Array(first)[0], }})); """, @@ -351,8 +351,7 @@ def test_a_sliced_frame_does_not_alias_the_container(tmp_path): # The container's whole point is the request count, so that is what these # assert on: one request for a clip, not one per frame. -SCHEDULER = ("INDEPENDENT", "dependencyOf", "chain", "suffixOf", "SEQ_MAGIC", - "SEQ_PREAMBLE", "readSequenceHeader", "sequenceFrame", "Scheduler") +SCHEDULER = ("INDEPENDENT", "dependencyOf", "chain", "suffixOf", *READER, "Scheduler") HARNESS = """ import {readFileSync} from "node:fs"; @@ -382,15 +381,15 @@ def test_a_sliced_frame_does_not_alias_the_container(tmp_path): @requires_node def test_a_packed_clip_is_fetched_in_one_request(tmp_path): frames = _frames(tmp_path / "clip", [10, 300, 7, 4096]) - sequence.pack(frames, tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 4, "bytes": - (tmp_path / "clip.seq").stat().st_size, + sequence.pack(frames, tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 4, "bytes": + (tmp_path / "clip.vmesh").stat().st_size, "media_type": "application/octet-stream"} result = run_js( HARNESS + f""" - globalThis.serve({str(tmp_path / "clip.seq")!r}); + globalThis.serve({str(tmp_path / "clip.vmesh")!r}); const s = new Scheduler(packedClip(4, {json.dumps(packed)}), "", decode, {{cacheSize: 1}}); (async () => {{ @@ -406,7 +405,7 @@ def test_a_packed_clip_is_fetched_in_one_request(tmp_path): names=SCHEDULER, ) - assert result["requests"] == ["clip.seq"] + assert result["requests"] == ["clip.vmesh"] assert result["held"] is True assert [frame["bytes"] for frame in result["seen"]] == [10, 300, 7, 4096] # Each frame's bytes are all its own index, so this catches a slice that @@ -426,14 +425,14 @@ def test_a_packed_clip_is_fetched_in_one_request(tmp_path): @requires_node def test_progress_climbs_to_the_frame_count(tmp_path): - sequence.pack(_frames(tmp_path / "clip", [4096] * 6), tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 6, - "bytes": (tmp_path / "clip.seq").stat().st_size} + sequence.pack(_frames(tmp_path / "clip", [4096] * 6), tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 6, + "bytes": (tmp_path / "clip.vmesh").stat().st_size} result = run_js( HARNESS + f""" - globalThis.serve({str(tmp_path / "clip.seq")!r}); + globalThis.serve({str(tmp_path / "clip.vmesh")!r}); const s = new Scheduler(packedClip(6, {json.dumps(packed)}), "", decode, {{}}); (async () => {{ const seen = []; @@ -456,13 +455,13 @@ def test_progress_climbs_to_the_frame_count(tmp_path): @requires_node def test_a_container_too_big_to_hold_is_refused_unfetched(tmp_path): - sequence.pack(_frames(tmp_path / "clip", [4096] * 4), tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 4, "bytes": 900e6} + sequence.pack(_frames(tmp_path / "clip", [4096] * 4), tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 4, "bytes": 900e6} result = run_js( HARNESS + f""" - globalThis.serve({str(tmp_path / "clip.seq")!r}); + globalThis.serve({str(tmp_path / "clip.vmesh")!r}); const s = new Scheduler(packedClip(4, {json.dumps(packed)}), "", decode, {{}}); (async () => {{ const held = await s.downloadAll(null); @@ -486,14 +485,14 @@ def test_a_container_too_big_to_hold_is_refused_unfetched(tmp_path): @requires_node def test_a_container_that_disagrees_with_the_clip_is_an_error(tmp_path): - sequence.pack(_frames(tmp_path / "clip", [4096] * 4), tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 5, - "bytes": (tmp_path / "clip.seq").stat().st_size} + sequence.pack(_frames(tmp_path / "clip", [4096] * 4), tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 5, + "bytes": (tmp_path / "clip.vmesh").stat().st_size} result = run_js( HARNESS + f""" - globalThis.serve({str(tmp_path / "clip.seq")!r}); + globalThis.serve({str(tmp_path / "clip.vmesh")!r}); const s = new Scheduler(packedClip(5, {json.dumps(packed)}), "", decode, {{}}); (async () => {{ let message = null; @@ -512,14 +511,14 @@ def test_a_container_that_disagrees_with_the_clip_is_an_error(tmp_path): @requires_node def test_seeking_a_packed_clip_pulls_the_container_once(tmp_path): - sequence.pack(_frames(tmp_path / "clip", [4096] * 4), tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 4, - "bytes": (tmp_path / "clip.seq").stat().st_size} + sequence.pack(_frames(tmp_path / "clip", [4096] * 4), tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 4, + "bytes": (tmp_path / "clip.vmesh").stat().st_size} result = run_js( HARNESS + f""" - globalThis.serve({str(tmp_path / "clip.seq")!r}); + globalThis.serve({str(tmp_path / "clip.vmesh")!r}); const s = new Scheduler(packedClip(4, {json.dumps(packed)}), "", decode, {{}}); (async () => {{ // Two seeks started together, before either could finish. Packing @@ -534,7 +533,7 @@ def test_seeking_a_packed_clip_pulls_the_container_once(tmp_path): tmp_path, names=SCHEDULER, ) - assert result["requests"] == ["clip.seq"] + assert result["requests"] == ["clip.vmesh"] assert result["firsts"] == [3, 4] @@ -565,9 +564,9 @@ def test_an_unpacked_clip_still_fetches_frame_by_frame(tmp_path): @requires_node def test_a_rung_without_a_container_is_not_served_the_base_one(tmp_path): - sequence.pack(_frames(tmp_path / "clip", [4096] * 3), tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 3, - "bytes": (tmp_path / "clip.seq").stat().st_size} + sequence.pack(_frames(tmp_path / "clip", [4096] * 3), tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 3, + "bytes": (tmp_path / "clip.vmesh").stat().st_size} result = run_js( HARNESS @@ -596,7 +595,7 @@ def test_a_rung_without_a_container_is_not_served_the_base_one(tmp_path): # ------------------------------------------------ decoded under its own name --- # Reported as "why is vega blank". A frame is decoded by the codec its *name* # selects (`worker.js` sniffs the suffix), so handing the decode the -# container's name means a `.seq` in a table of `.ply` and `.splat`. Every +# container's name means a `.vmesh` in a table of `.ply` and `.splat`. Every # Gaussian pane went blank; the pixel panes did not, because an image is # decoded from its media type on the main thread -- so it looked like one # method being broken rather than one code path. @@ -604,14 +603,14 @@ def test_a_rung_without_a_container_is_not_served_the_base_one(tmp_path): @requires_node def test_a_packed_frame_is_decoded_under_its_own_name(tmp_path): - sequence.pack(_frames(tmp_path / "clip", [64] * 3), tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 3, - "bytes": (tmp_path / "clip.seq").stat().st_size} + sequence.pack(_frames(tmp_path / "clip", [64] * 3), tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 3, + "bytes": (tmp_path / "clip.vmesh").stat().st_size} result = run_js( HARNESS + f""" - globalThis.serve({str(tmp_path / "clip.seq")!r}); + globalThis.serve({str(tmp_path / "clip.vmesh")!r}); // The real rule, from worker.js: the codec comes from the suffix. const CODECS = {{gaussians: {{".ply": 1, ".splat": 1}}}}; const pick = (url) => {{ @@ -648,14 +647,14 @@ def test_a_container_whose_frames_are_a_different_format_is_refused(tmp_path): # plausible files and the length check passes, so without this the frames # would be handed to the Gaussian parser and come out as noise. sequence.pack(_frames(tmp_path / "clip", [64] * 3, suffix="jpg"), - tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 3, - "bytes": (tmp_path / "clip.seq").stat().st_size} + tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 3, + "bytes": (tmp_path / "clip.vmesh").stat().st_size} result = run_js( HARNESS + f""" - globalThis.serve({str(tmp_path / "clip.seq")!r}); + globalThis.serve({str(tmp_path / "clip.vmesh")!r}); const s = new Scheduler(packedClip(3, {json.dumps(packed)}), "./", decode, {{}}); (async () => {{ let message = null; @@ -691,7 +690,7 @@ def cut(source: str) -> str: + cut(worker).replace("function suffixOf(", "function original(") + """ const cases = ["a/b.splat", "a/b.SPLAT", "a/b.ply?v=2", "b.tar.gz", - "nodot", "a.b/c", "", "a/b.seq"]; + "nodot", "a.b/c", "", "a/b.vmesh"]; const rows = cases.map((c) => [c, suffixOf(c), original(c)]); process.stdout.write(JSON.stringify(rows)); """ @@ -704,7 +703,7 @@ def cut(source: str) -> str: disagree = [row for row in rows if row[1] != row[2]] assert not disagree, f"suffixOf disagrees with the worker on {disagree}" # And the case that started this: a container is not a frame format. - assert dict((row[0], row[1]) for row in rows)["a/b.seq"] == ".seq" + assert dict((row[0], row[1]) for row in rows)["a/b.vmesh"] == ".vmesh" def test_a_failed_download_is_explained_in_the_pane(): @@ -724,3 +723,317 @@ def test_a_failed_download_is_explained_in_the_pane(): assert "_explain(" in body # Carries on rather than abandoning the remaining panes. assert "continue;" in body +def test_failed_bundle_pack_keeps_original_frames_and_manifest(tmp_path, monkeypatch): + clips = [] + for name in ("first", "second"): + paths = _frames(tmp_path / name, [12, 12]) + clips.append(bundle.Clip(name=name, representation="gaussians", + frames=[str(p.relative_to(tmp_path)) for p in paths])) + bundle.write(tmp_path, title="test", source="test", clips=clips) + before = (tmp_path / "view.json").read_bytes() + original = sequence.pack_clip + + def fail_second(root, clip, **kwargs): + if clip.name == "second": + raise OSError("simulated packing failure") + return original(root, clip, **kwargs) + + monkeypatch.setattr(sequence, "pack_clip", fail_second) + with pytest.raises(OSError): + sequence.pack_bundle(tmp_path) + assert (tmp_path / "view.json").read_bytes() == before + assert all((tmp_path / path).is_file() for clip in clips for path in clip.frames) + + +def test_packing_keeps_frames_referenced_by_variants_or_unpacked_clips(tmp_path): + paths = _frames(tmp_path / "shared", [12, 12]) + frames = [str(p.relative_to(tmp_path)) for p in paths] + clips = [bundle.Clip(name="first", representation="gaussians", frames=frames, + variants=[{"name": "original", "frames": frames}]), + bundle.Clip(name="second", representation="gaussians", frames=frames)] + bundle.write(tmp_path, title="test", source="test", clips=clips) + sequence.pack_bundle(tmp_path, names=["first"]) + assert all(p.is_file() for p in paths) + + +def test_browser_vmesh_is_validated_by_shared_codec_inspector(tmp_path): + from open4d.codec import inspect_vmesh + paths = _frames(tmp_path / "input", [13, 1_048_581], suffix="drc") + target = tmp_path / "clip.vmesh" + header = sequence.pack(paths, target, fps=24) + descriptor = inspect_vmesh(target) + assert descriptor["codec"] == "frames" + assert descriptor["native"] == {"profile": "frames/1", "suffix": "drc", "representation": "mesh"} + assert descriptor["sequence"]["frames"][1]["timestamp"] == 1 / 24 + assert all(sequence.frame(target.read_bytes(), i, header) == path.read_bytes() + for i, path in enumerate(paths)) + + +@pytest.mark.parametrize("suffix", ["seq", "o4d", "bin"]) +def test_only_vmesh_is_a_public_sequence_output(tmp_path, suffix): + with pytest.raises(ValueError, match=".vmesh extension"): + sequence.pack(_frames(tmp_path / "input", [12]), tmp_path / f"clip.{suffix}") + + +@pytest.mark.parametrize("fps", [0, -1, float("nan"), float("inf"), True]) +def test_sequence_timing_must_be_finite_positive(tmp_path, fps): + with pytest.raises(ValueError, match="fps"): + sequence.pack(_frames(tmp_path / "input", [12]), tmp_path / "clip.vmesh", fps=fps) + + +def _legacy_bytes(payloads=(b"abc", b"defg"), suffix="splat"): + header = struct.pack("<8sIII", b"O4DSEQ\x00\x00", 1, len(payloads), len(suffix)) + at = 20 + len(suffix) + 8 * len(payloads) + table = bytearray() + for payload in payloads: + table.extend(struct.pack(" {{ + const bytes = new Uint8Array(sequenceFrame(buffer, header, i)); + return [bytes.length, bytes[0], bytes[bytes.length - 1]]; + }}))); + """, tmp_path) + assert result == [[1_048_581, 1, 1], [17, 2, 2]] + + +@requires_node +@pytest.mark.parametrize("corruption", ["payload", "record", "end", "trailing", "legacy"]) +def test_browser_refuses_corrupt_vmesh_before_decoder(tmp_path, corruption): + target = tmp_path / "clip.vmesh" + header = sequence.pack(_frames(tmp_path / "input", [64, 64]), target) + data = bytearray(target.read_bytes()) + if corruption == "payload": + data[header.entries[0].offset + 18] ^= 1 + elif corruption == "record": + data[header.entries[0].offset + 5] = 2 + elif corruption == "end": + data[-1] ^= 1 + elif corruption == "trailing": + data += b"extra" + else: + data = _legacy_bytes() + target.write_bytes(data) + result = run_js(LOAD + f""" + let message = null; + try {{ + const buffer = load({str(target)!r}), header = readSequenceHeader(buffer); + validateSequenceEnd(buffer, header); + sequenceFrame(buffer, header, 0); + }} catch (error) {{ message = error.message; }} + process.stdout.write(JSON.stringify(message)); + """, tmp_path) + assert result + + +def test_public_frame_loader_preserves_mesh_geometry_and_timestamps(tmp_path): + import numpy as np + from open4d.io._mesh import write_ply + paths = [] + positions = np.array([[0, 0, 0], [1, 0, 0], [0, 1, 0]], dtype=np.float32) + triangles = np.array([[0, 1, 2]], dtype=np.uint32) + for index in range(2): + path = tmp_path / f"frame_{index}.ply" + write_ply(path, positions + index, triangles) + paths.append(path) + target = tmp_path / "mesh.vmesh" + sequence.pack(paths, target, fps=12, representation="mesh") + with sequence.open_frames(target) as restored: + assert restored.timestamps == (0, 1 / 12) + assert [frame.frame_index for frame in restored] == [0, 1] + np.testing.assert_array_equal(restored[1].geometry.positions, positions + 1) + np.testing.assert_array_equal(restored[0].geometry.triangles, triangles) + + +def test_public_frame_loader_preserves_images_without_deserialization(tmp_path): + from open4d.native import NativeSequence + target = tmp_path / "pixels.vmesh" + sequence.pack(_frames(tmp_path / "input", [23, 35], suffix="png"), target) + with sequence.open_frames(target) as restored: + assert isinstance(restored, NativeSequence) + assert restored.codec == "frames" + assert restored.manifest["native"]["representation"] == "pixels" + assert len(restored) == 2 + + +def test_packing_a_bundle_uses_its_real_fps_and_representation(tmp_path): + from open4d.codec import inspect_vmesh + clip = _clip(tmp_path, [23, 35]) + bundle.write(tmp_path, title="test", source="test", clips=[clip], fps=24) + sequence.pack_bundle(tmp_path) + manifest = inspect_vmesh(tmp_path / "clip.vmesh") + assert manifest["sequence"]["frames"][1]["timestamp"] == 1 / 24 + assert manifest["native"]["representation"] == "gaussians" + + +@pytest.mark.parametrize("representation", ["unknown", "pixels", "points"]) +def test_frame_suffix_must_match_its_representation(tmp_path, representation): + with pytest.raises(ValueError, match="representation"): + sequence.pack(_frames(tmp_path / "input", [12]), tmp_path / "bad.vmesh", representation=representation) + + +@requires_node +@pytest.mark.parametrize("text", [ + '{"codec":"frames","codec":"frames"}', + '{"nested":{"name":1,"na\\u006de":2}}', + '{"a":[{"x":1,"x":2}]}', +]) +def test_browser_refuses_duplicate_json_keys(tmp_path, text): + result = run_js(f""" + let message = null; + try {{ vmeshJSON({json.dumps(text)}); }} catch (error) {{ message = error.message; }} + process.stdout.write(JSON.stringify(message)); + """, tmp_path) + assert "duplicate" in result + + +@requires_node +def test_browser_partial_download_can_recover_first_complete_frame(tmp_path): + target = tmp_path / "clip.vmesh" + header = sequence.pack(_frames(tmp_path / "input", [71, 103]), target) + partial = tmp_path / "partial.bin" + partial.write_bytes(target.read_bytes()[:header.entries[1].offset]) + result = run_js(LOAD + f""" + const buffer = load({str(partial)!r}), header = readSequenceHeader(buffer); + const frame = new Uint8Array(sequenceFrame(buffer, header, 0)); + let unavailable = false; + try {{ sequenceFrame(buffer, header, 1); }} catch (_) {{ unavailable = true; }} + process.stdout.write(JSON.stringify([frame.length, frame[0], unavailable])); + """, tmp_path) + assert result == [71, 1, True] + + +def test_existing_packed_bundle_migrates_and_rewrites_its_url(tmp_path): + from open4d.codec import inspect_vmesh + old = tmp_path / "capture.seq" + old.write_bytes(_legacy_bytes()) + clip = bundle.Clip(name="capture", representation="gaussians", + frames=["capture/frame_0000.splat", "capture/frame_0001.splat"], + sequence={"url": "capture.seq", "frames": 2, "suffix": "splat", "bytes": old.stat().st_size}) + bundle.write(tmp_path, title="test", source="test", clips=[clip], fps=12) + converted = sequence.pack_bundle(tmp_path) + assert converted[0][1]["url"] == "capture.vmesh" + assert bundle.read(tmp_path)["clips"][0]["sequence"]["url"] == "capture.vmesh" + assert not old.exists() + assert inspect_vmesh(tmp_path / "capture.vmesh")["sequence"]["frames"][1]["timestamp"] == 1 / 12 + assert sequence.pack_bundle(tmp_path) == [] + + +def test_failed_bundle_migration_preserves_original_and_manifest(tmp_path, monkeypatch): + old = tmp_path / "capture.seq" + data = _legacy_bytes() + old.write_bytes(data) + clip = bundle.Clip(name="capture", representation="gaussians", + frames=["f0.splat", "f1.splat"], + sequence={"url": "capture.seq", "frames": 2}) + bundle.write(tmp_path, title="test", source="test", clips=[clip]) + original = (tmp_path / "view.json").read_bytes() + monkeypatch.setattr(bundle, "write", lambda *args, **kwargs: (_ for _ in ()).throw(OSError("manifest write failed"))) + with pytest.raises(OSError, match="manifest write failed"): + sequence.pack_bundle(tmp_path) + assert old.read_bytes() == data + assert (tmp_path / "view.json").read_bytes() == original + + +def test_public_open4d_load_supports_self_describing_browser_vmesh(tmp_path): + import open4d + import numpy as np + from open4d.io._mesh import write_ply + path = tmp_path / "frame.ply" + positions = np.array([[0, 0, 0], [1, 0, 0], [0, 1, 0]], dtype=np.float32) + triangles = np.array([[0, 1, 2]], dtype=np.uint32) + write_ply(path, positions, triangles) + target = tmp_path / "mesh.vmesh" + sequence.pack([path], target, representation="mesh") + with open4d.load(target) as restored: + assert restored.timestamps == (0,) + np.testing.assert_array_equal(restored[0].geometry.positions, positions) + + +def test_migration_keeps_a_legacy_container_referenced_by_an_unselected_clip(tmp_path): + old = tmp_path / "shared.seq" + old.write_bytes(_legacy_bytes()) + clips = [bundle.Clip(name=name, representation="gaussians", frames=["f0.splat", "f1.splat"], + sequence={"url": "shared.seq", "frames": 2}) for name in ("one", "two")] + bundle.write(tmp_path, title="test", source="test", clips=clips) + sequence.pack_bundle(tmp_path, names=["one"]) + assert old.is_file() + index = bundle.read(tmp_path) + assert index["clips"][0]["sequence"]["url"] == "shared.vmesh" + assert index["clips"][1]["sequence"]["url"] == "shared.seq" + + +def test_legacy_bundle_migration_never_removes_paths_outside_the_bundle(tmp_path): + root = tmp_path / "bundle" + root.mkdir() + sentinel = tmp_path / "keep.splat" + sentinel.write_bytes(b"keep") + old = root / "capture.seq" + old.write_bytes(_legacy_bytes()) + clip = bundle.Clip(name="capture", representation="gaussians", frames=["../keep.splat", "f1.splat"], + sequence={"url": "capture.seq", "frames": 2}) + bundle.write(root, title="test", source="test", clips=[clip]) + with pytest.raises(ValueError, match="safe relative paths"): + sequence.pack_bundle(root) + assert sentinel.read_bytes() == b"keep" + assert old.is_file() + assert not (root / "capture.vmesh").exists() diff --git a/open4d/streamer/streamer_tests/test_server_and_transfer.py b/open4d/streamer/streamer_tests/test_server_and_transfer.py index 61f2e135..603d5c7a 100644 --- a/open4d/streamer/streamer_tests/test_server_and_transfer.py +++ b/open4d/streamer/streamer_tests/test_server_and_transfer.py @@ -17,6 +17,33 @@ pytestmark = pytest.mark.cpu +@pytest.mark.parametrize("path", ["../escaped", "/tmp/escaped", "C:/escaped", "dir/../../escaped", "dir\\escaped", "view.json", "frame.partial"]) +def test_fetch_rejects_unsafe_paths_before_writing_manifest(tmp_path, monkeypatch, path): + monkeypatch.setattr(transfer, "_get", lambda *args: json.dumps({"clips": [{"frames": [path]}]}).encode()) + with pytest.raises(ValueError, match="path"): + transfer.fetch("https://example.invalid/", tmp_path / "copy") + assert not (tmp_path / "copy" / "view.json").exists() + + +@pytest.mark.parametrize("partial", [False, True]) +def test_fetch_rejects_symlink_targets(tmp_path, monkeypatch, partial): + root = tmp_path / "copy" + root.mkdir() + outside = tmp_path / "outside" + outside.write_bytes(b"keep") + (root / ("frame.ply.partial" if partial else "frame.ply")).symlink_to(outside) + monkeypatch.setattr(transfer, "_get", lambda *args: b'{"clips":[{"frames":["frame.ply"]}]}') + with pytest.raises(ValueError, match="path"): + transfer.fetch("https://example.invalid/", root) + assert outside.read_bytes() == b"keep" + + +def test_fetch_paths_include_variants_and_packed_sequences(): + index = {"clips": [{"frames": ["logical.ply"], "sequence": {"url": "clip.vmesh"}, + "variants": [{"frames": ["low.ply"]}, {"frames": ["low.ply"]}]}]} + assert transfer.frame_paths(index) == ("clip.vmesh", "low.ply") + + def make_bundle(root, *, frames: int = 3): """A two-clip bundle: one geometry clip, one pixel clip.""" written = {"gaussians": [], "pixels": []} @@ -646,10 +673,10 @@ def test_a_frame_container_is_never_compressed(tmp_path): something the client did not ask for, which is what the header prefetch relies on.""" root = _repetitive_bundle(tmp_path) - (root / "clip.seq").write_bytes(b"O4DSEQ\x00\x00" + bytes(4096)) + (root / "clip.vmesh").write_bytes(b"VMESH\x00\x01\x00" + bytes(4096)) with serving(root) as base: request = urllib.request.Request( - f"{base}/clip.seq", headers={"Accept-Encoding": "gzip"}) + f"{base}/clip.vmesh", headers={"Accept-Encoding": "gzip"}) with urllib.request.urlopen(request) as response: body = response.read() headers = dict(response.headers) diff --git a/open4d/streaming/README.md b/open4d/streaming/README.md index 05b5474a..bdf1a722 100644 --- a/open4d/streaming/README.md +++ b/open4d/streaming/README.md @@ -43,6 +43,9 @@ source .venv/bin/activate python -m pip install numpy opencv-python zstandard ``` +From an Open4D checkout, `python -m pip install -e '.[capture]'` installs the +Python capture dependencies, including Open3D and Zstandard. + Install a CUDA-enabled Open3D build in that environment. The standard pip wheel may not include CUDA. Verify the installation: diff --git a/open4d/streaming/_transport.py b/open4d/streaming/_transport.py index 724d72f5..e51c44c7 100644 --- a/open4d/streaming/_transport.py +++ b/open4d/streaming/_transport.py @@ -98,7 +98,7 @@ def _decode(header, payload): raise ValueError(f"invalid mesh stream frame: {error}") from error -def send(frames, host="127.0.0.1", port=7000, *, realtime=True, timeout=30.0, +def send(frames, host="127.0.0.1", port=47004, *, realtime=True, timeout=30.0, max_frame_bytes=_DEFAULT_LIMIT) -> int: """Send a Sequence or iterable of Frames to a running receiver. @@ -194,7 +194,7 @@ def close(self): setattr(self, name, None) -def receive(host="127.0.0.1", port=7000, *, timeout=30.0, +def receive(host="127.0.0.1", port=47004, *, timeout=30.0, max_frame_bytes=_DEFAULT_LIMIT) -> Receiver: """Listen for one sender and yield Frames; use `with receive() as frames:`. diff --git a/open4d/streaming/python/live_two_camera_webrtc.py b/open4d/streaming/python/live_two_camera_webrtc.py index 41da34b7..68a2184a 100644 --- a/open4d/streaming/python/live_two_camera_webrtc.py +++ b/open4d/streaming/python/live_two_camera_webrtc.py @@ -282,12 +282,12 @@ def _process_loop( def run(self) -> None: self.args.output_dir.mkdir(parents=True, exist_ok=True) - self.server.start() - print( - "Browser viewer ready through Ubuntu 127.0.0.1:8888", - flush=True, - ) try: + self.server.start() + print( + "Browser viewer ready through Ubuntu 127.0.0.1:8888", + flush=True, + ) o3d.visualization.webrtc_server.enable_webrtc() o3d.visualization.draw( [], @@ -305,9 +305,13 @@ def run(self) -> None: self.stop = True if self.processor is not None: self.processor.join(timeout=10.0) - self.save() - self.server.close() - self.mesh_worker.close() + try: + self.save() + finally: + try: + self.server.close() + finally: + self.mesh_worker.close() report = { "status": "completed", "viewer": "Open3D WebRTC", diff --git a/open4d/streaming/tests/test_capture.py b/open4d/streaming/tests/test_capture.py index 26ad54a7..fa8bfa97 100644 --- a/open4d/streaming/tests/test_capture.py +++ b/open4d/streaming/tests/test_capture.py @@ -5,11 +5,38 @@ import sys import time import zlib +from types import SimpleNamespace from pathlib import Path import pytest +@pytest.mark.parametrize("save_fails", [False, True]) +def test_browser_start_failure_closes_receiver_and_mesh_worker(fusion, tmp_path, save_fails): + module = importlib.import_module("live_two_camera_webrtc") + app = module.BrowserFusion.__new__(module.BrowserFusion) + closed = [] + + def start(): + raise RuntimeError("port already in use") + + def save(): + if save_fails: + raise RuntimeError("save failed") + + app.args = SimpleNamespace(output_dir=tmp_path) + app.server = SimpleNamespace(start=start, close=lambda: closed.append("server"), report=lambda: {}) + app.mesh_worker = SimpleNamespace(close=lambda: closed.append("worker"), device="cpu", + fusion_mode="shared-tsdf", merge_mode="concatenate") + app.processor = None + app.save = save + app.processed = app.latest_pair = app.mesh_updates = 0 + app.processor_error = None + with pytest.raises(RuntimeError, match="save failed" if save_fails else "port already in use"): + app.run() + assert closed == ["server", "worker"] + + @pytest.fixture def protocol(monkeypatch): root = Path(__file__).resolve().parents[1] / "python" diff --git a/open4d/visualization/_api.py b/open4d/visualization/_api.py index fc3e00cc..9a1aef59 100644 --- a/open4d/visualization/_api.py +++ b/open4d/visualization/_api.py @@ -76,14 +76,14 @@ def _open_source(source: Sequence | str | os.PathLike[str]) -> tuple[Sequence, b if isinstance(source, (str, os.PathLike)): from open4d import _api as public_api - from open4d.codec import available_codecs + from open4d.codec._vmesh_format import is_mesh_profile suffix = Path(source).suffix.lower() - if any(suffix in info.suffixes and info.representation != "triangle_mesh" - for info in available_codecs()): + if suffix == ".vmesh" and not is_mesh_profile(source): raise TypeError("the Qt viewer takes mesh sequences; Gaussian runs need their native renderer") sequence = public_api.load(source) if not isinstance(sequence, Sequence): + sequence.close() raise TypeError("the Qt viewer takes an open4d.Sequence") return sequence, True raise TypeError("source must be an open4d.Sequence or path-like sequence source") diff --git a/open4d/visualization/tests/test_visualization.py b/open4d/visualization/tests/test_visualization.py index d41a13b1..c846163f 100644 --- a/open4d/visualization/tests/test_visualization.py +++ b/open4d/visualization/tests/test_visualization.py @@ -136,13 +136,14 @@ def test_visualize_path_owns_and_closes_loaded_sequence(monkeypatch): monkeypatch.setattr( _api, "load", lambda source: value ) + monkeypatch.setattr("open4d.codec._vmesh_format.is_mesh_profile", lambda source: True) monkeypatch.setattr(_qt, "check_available", lambda **options: None) monkeypatch.setattr(_qt, "play", lambda frames, options: frames[0]) monkeypatch.setattr( value, "close", lambda: (close_calls.append(True), real_close())[1] ) - visualize("capture.o4d") + visualize("capture.vmesh") assert close_calls == [True] assert value.closed is True @@ -166,13 +167,14 @@ def test_visualize_path_closes_after_renderer_failure(monkeypatch): from open4d.visualization import _qt monkeypatch.setattr(_api, "load", lambda source: value) + monkeypatch.setattr("open4d.codec._vmesh_format.is_mesh_profile", lambda source: True) monkeypatch.setattr(_qt, "check_available", lambda **options: None) monkeypatch.setattr( _qt, "play", lambda frames, options: (_ for _ in ()).throw(RuntimeError("boom")) ) with pytest.raises(RuntimeError, match="boom"): - visualize("capture.o4d") + visualize("capture.vmesh") assert value.closed is True @@ -264,13 +266,24 @@ def test_gaussian_paths_are_rejected_before_native_decode(tmp_path, monkeypatch, from open4d.visualization import _api, _qt import open4d._api as public + import json + from open4d.codec import pack_vmesh + native = tmp_path / "native" + native.mkdir() + (native / "metadata.json").write_text(json.dumps({ + "version": 1, "codec": "vega", "native": {"profile": "vega/1"}, + "frames": [{"frame_index": i, "timestamp": i / 30} for i in range(2)], + })) + for name in ("manifest.json", "color_model.pt", "frame_0000.pt", "frame_0001.pt"): + (native / name).write_bytes(b"opaque native fixture") + source = pack_vmesh(native, tmp_path / "capture.vmesh") monkeypatch.setattr(_qt, "check_available", lambda **options: None) monkeypatch.setattr(public, "load", lambda *args: pytest.fail("must not decode Gaussian data")) with pytest.raises(TypeError, match="native renderer"): if render: - _api.render_gif(tmp_path / "capture.vega", tmp_path / "capture.gif") + _api.render_gif(source, tmp_path / "capture.gif") else: - _api.visualize(tmp_path / "capture.vega") + _api.visualize(source) @pytest.mark.parametrize("scale", [1e-10, 1e10, 1e20]) @@ -429,3 +442,30 @@ def fail_save(self, path, **options): assert output.read_bytes() == b"original" assert list(tmp_path.iterdir()) == [output] assert gif_scene == [True] + + +def test_mesh_vmesh_path_reaches_player_and_releases_owned_payloads(tmp_path, monkeypatch): + from pathlib import Path + from open4d.io._mesh import write_ply + from open4d.visualization import _qt + + monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / "streamer")) + from streamer.sequence import pack + + source = tmp_path / "frame.ply" + mesh = sequence()[0].geometry + write_ply(source, mesh.positions, mesh.triangles) + target = tmp_path / "capture.vmesh" + pack([source], target, representation="mesh") + seen = {} + monkeypatch.setattr(_qt, "check_available", lambda **options: None) + + def play(frames, options): + seen["sequence"] = frames.sequence + seen["positions"] = frames[0].positions.copy() + assert not frames.sequence.closed + + monkeypatch.setattr(_qt, "play", play) + visualize(target) + np.testing.assert_array_equal(seen["positions"], mesh.positions) + assert seen["sequence"].closed diff --git a/open4d/webclients/README.md b/open4d/webclients/README.md index 5df9331d..f78eef41 100644 --- a/open4d/webclients/README.md +++ b/open4d/webclients/README.md @@ -39,6 +39,14 @@ servers (`baselines/`) and the corpora live in `4DVideoStreaming`, so `run_web_demo.sh` expects a checkout of it. Without one you can still build the clients and run the tests. +The Node control server binds to `127.0.0.1` by default. Set `HOST` explicitly +when remote clients need access; it provides no authentication. Browser requests +must use this server's origin, or an origin explicitly listed in the comma-separated +`VS4D_ALLOWED_ORIGINS`. IP addresses and `localhost` are accepted as request hosts; +list additional DNS names in `VS4D_ALLOWED_HOSTS` (comma-separated), or bind using +that name with `HOST`. Host checks prevent an unconfigured DNS name from rebinding +to this local control server. + ## Adding a method A method takes part in the study once it has a page that can run a study pass diff --git a/open4d/webclients/system/Server/path-safety.js b/open4d/webclients/system/Server/path-safety.js new file mode 100644 index 00000000..65417e23 --- /dev/null +++ b/open4d/webclients/system/Server/path-safety.js @@ -0,0 +1,26 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +function validDirectoryId(id, maxLength = 180) { + return typeof id === 'string' && id.length <= maxLength + && /^[A-Za-z0-9._-]+$/.test(id) && id !== '.' && id !== '..'; +} + +/** Resolve one directory component, rejecting traversal and existing symlinks. */ +function childDirectory(root, id, label = 'directory id', maxLength = 180) { + const invalid = () => Object.assign(new Error(`invalid ${label}`), { statusCode: 400 }); + if (!validDirectoryId(id, maxLength)) throw invalid(); + const base = path.resolve(root); + const target = path.resolve(base, id); + if (!target.startsWith(base + path.sep)) throw invalid(); + try { + if (fs.lstatSync(target).isSymbolicLink()) throw invalid(); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + } + return target; +} + +module.exports = { childDirectory, validDirectoryId }; diff --git a/open4d/webclients/system/Server/server.js b/open4d/webclients/system/Server/server.js index e82e1080..e4855e01 100644 --- a/open4d/webclients/system/Server/server.js +++ b/open4d/webclients/system/Server/server.js @@ -6,6 +6,7 @@ const net = require("net"); const path = require("path"); const { spawn, execFile } = require("child_process"); const readline = require("readline"); // add here +const { childDirectory, validDirectoryId } = require("./path-safety"); // -------------------- Logging -------------------- // One line per event: [YYYY-MM-DD HH:MM:SS.mmm][LEVEL][COMPONENT] message @@ -22,7 +23,26 @@ const logWarn = (component, message) => log("WARN", component, message); const logError = (component, message) => log("ERROR", component, message); const app = express(); -app.use(cors({ origin: "*" })); +// Native/headset requests have no Origin. Browser requests must come from +// this server or an explicitly configured companion UI. +const allowedOrigins = new Set((process.env.VS4D_ALLOWED_ORIGINS || "") + .split(",").map(value => value.trim()).filter(Boolean)); +const allowedHostnames = new Set(["localhost", process.env.HOST, + ...(process.env.VS4D_ALLOWED_HOSTS || "").split(",")] + .filter(Boolean).map(value => value.trim().toLowerCase())); +app.use((req, res, next) => { + // A same-origin check alone trusts attacker-controlled DNS names that can + // rebind to loopback. Accept IP literals and explicitly configured names. + const hostname = req.hostname?.replace(/^\[|\]$/g, "").toLowerCase(); + if (!hostname || (!net.isIP(hostname) && !allowedHostnames.has(hostname))) + return res.status(403).json({ error: "request hostname is not allowed" }); + const origin = req.get("Origin"); + if (!origin) return next(); + const sameOrigin = origin === `${req.protocol}://${req.get("Host")}`; + if (!sameOrigin && !allowedOrigins.has(origin)) + return res.status(403).json({ error: "browser origin is not allowed" }); + return cors({ origin })(req, res, next); +}); app.use(express.json({ limit: "200mb" })); // -------------------- Paths -------------------- @@ -286,6 +306,7 @@ app.get("/api/geometry-bundle", async (req, res) => { }); const PORT = process.env.PORT || 3000; +const HOST = process.env.HOST || "127.0.0.1"; // Point-cloud bridges this server spawns (via /api/pointcloud/start) inherit // the environment; this is how they find the study trace to follow, so a ViVo // or NAVA trial is shaped exactly as a mesh trial is. See study/follower.js. @@ -516,7 +537,7 @@ app.use("/api/study", createStudyRouter({ // live in server_results// so each run stays self-contained. function broadcastResultsDir(broadcastId) { const id = broadcastId || currentBroadcastId || "no-broadcast"; - const dir = path.join(SERVER_RESULTS_DIR, id); + const dir = childDirectory(SERVER_RESULTS_DIR, id, "broadcast id"); ensureDir(dir); return dir; } @@ -1007,7 +1028,7 @@ const SHAPED_INTERFACE = process.env.VS4D_SHAPED_INTERFACE || "eth0"; // so unlike every other system here, choosing objects means replacing the // process. That is why this exists rather than a query parameter. // -// This endpoint spawns processes and the server listens on 0.0.0.0, so the +// This endpoint spawns processes and can be bound to a network interface, so the // input is constrained hard rather than trusted: // * the baseline id must be one of the two configured ones, never a module // name from the request; @@ -1334,14 +1355,13 @@ app.get("/api/systems", async (req, res) => { app.get("/api/viewpoint-index", (req, res) => { const requested = String(req.query.dir || ""); - if (requested && !/^[A-Za-z0-9._-]+$/.test(requested)) { + if (requested && !validDirectoryId(requested)) { return res.status(400).json({ error: "invalid viewpoint directory" }); } const root = path.join(SYSTEM_ROOT_CLIENT, "viewpoints"); - const directory = requested ? path.join(root, requested) : root; - if (!path.resolve(directory).startsWith(path.resolve(root))) { - return res.status(400).json({ error: "viewpoint path escapes the client dir" }); - } + let directory; + try { directory = requested ? childDirectory(root, requested, "viewpoint directory") : root; } + catch (error) { return res.status(400).json({ error: error.message }); } if (!fs.existsSync(directory)) { return res.status(404).json({ error: `no viewpoint directory ${directory}` }); } @@ -2039,7 +2059,7 @@ function sceneRelativeTrajectoryPayload(value, preferredBroadcastId = "") { // older in-session runs fall back to playback-aligned Unity poses in the CSV. app.get("/api/offline-trajectory/:broadcastId", (req, res) => { const broadcastId = String(req.params.broadcastId || ""); - if (!/^[A-Za-z0-9._-]+$/.test(broadcastId)) + if (!validDirectoryId(broadcastId)) return res.status(400).json({ error: "invalid trajectory broadcast id" }); const filepath = path.join(SERVER_RESULTS_DIR, broadcastId, "viewport_trace.csv"); if (!fs.existsSync(filepath)) @@ -2156,6 +2176,9 @@ app.post("/api/client-log", (req, res) => { message: redactIpAddresses(entry.message), stack: entry.stack ? redactIpAddresses(entry.stack) : entry.stack, })); + if (broadcastId !== undefined && broadcastId !== null && broadcastId !== "" + && !validDirectoryId(broadcastId)) + return res.status(400).json({ error: "invalid broadcast id" }); const logFile = path.join(broadcastResultsDir(broadcastId), "client_log.jsonl"); fs.appendFileSync(logFile, safeEntries.map((entry) => JSON.stringify(entry)).join("\n") + "\n"); for (const entry of safeEntries) { @@ -2382,8 +2405,9 @@ app.get("/api/results", (req, res) => { }); function validViewportBroadcastId(value) { - return typeof value === "string" && /^[a-zA-Z0-9._-]+$/.test(value) - && fs.existsSync(path.join(SERVER_RESULTS_DIR, value)); + if (!validDirectoryId(value)) return false; + try { return fs.existsSync(childDirectory(SERVER_RESULTS_DIR, value, "broadcast id")); } + catch { return false; } } function latestViewportBroadcastId() { @@ -2585,7 +2609,7 @@ if (require.main === module) { process.exit(1); } if (COMPRESSED_ROOT) logInfo("SERVER", `corpus ${COMPRESSED_ROOT}`); - app.listen(PORT, "0.0.0.0", () => { + app.listen(PORT, HOST, () => { bootstrap().catch((e) => { logError("SERVER", `Bootstrap failed: ${e.message}`); }); diff --git a/open4d/webclients/system/Server/study/routes.js b/open4d/webclients/system/Server/study/routes.js index 4fad5dbd..daf9818e 100644 --- a/open4d/webclients/system/Server/study/routes.js +++ b/open4d/webclients/system/Server/study/routes.js @@ -153,6 +153,9 @@ function createStudyRouter({ resultsRoot, shaper, tracesDir, layoutFile }) { })); router.post('/sessions/:id/trials/:position/finish', wrap(req => { + if (!live || live.session !== req.params.id || live.position !== Number(req.params.position)) { + throw Object.assign(new Error('this trial does not own the live link'), { statusCode: 409 }); + } const shaping = shaper.status(); const result = store.finishTrial(req.params.id, req.params.position, { ...(req.body || {}), shaping diff --git a/open4d/webclients/system/Server/study/store.js b/open4d/webclients/system/Server/study/store.js index 6cc04fe5..bbde920e 100644 --- a/open4d/webclients/system/Server/study/store.js +++ b/open4d/webclients/system/Server/study/store.js @@ -18,6 +18,7 @@ const crypto = require('crypto'); const fs = require('fs'); const path = require('path'); +const { childDirectory } = require('../path-safety'); const questionnaire = require('./questionnaire'); const { participantIndex, trialOrder } = require('./order'); @@ -25,7 +26,6 @@ const { parseTrace, windowTrace, summarize } = require('./trace'); const SCHEMA_VERSION = 1; const PARTICIPANT = /^[A-Za-z0-9._-]{1,40}$/; -const SESSION_ID = /^[A-Za-z0-9._-]{1,80}$/; /** * Every method the comparison page knows, and whether it can follow a camera @@ -116,10 +116,7 @@ class StudyStore { } _dir(sessionId) { - if (!SESSION_ID.test(String(sessionId || ''))) { - throw Object.assign(new Error('invalid session id'), { statusCode: 400 }); - } - return path.join(this.root, sessionId); + return childDirectory(this.root, sessionId, 'session id', 80); } _read(sessionId, name) { @@ -182,7 +179,7 @@ class StudyStore { // always gets the same row whatever order the methods were ticked in. const canonical = Object.keys(METHODS).filter(id => methods.includes(id)); const { row, order } = trialOrder(canonical, index); - const id = `${participant}-${timestamp()}`; + const id = `${participant}-${timestamp()}-${crypto.randomBytes(8).toString("hex")}`; const trajectory = spec.trajectory ? validateTrajectory(spec.trajectory) : null; const session = { @@ -209,6 +206,7 @@ class StudyStore { : null, trials: order.map(entry => ({ ...entry, status: 'pending' })) }; + fs.mkdirSync(this._dir(id)); this._write(id, 'session.json', session); fs.writeFileSync(path.join(this._dir(id), 'trace.csv'), spec.trace.text); if (trajectory) this._write(id, 'trajectory.json', trajectory); @@ -342,8 +340,11 @@ class StudyStore { ]; const cell = value => { if (value === null || value === undefined) return ''; - const text = String(value); - return /[",\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; + // Quoting CSV syntax alone does not prevent spreadsheet formulas. + const raw = String(value); + const text = typeof value === 'string' && /^[=+@\-\t\r\n]/.test(raw) + ? "'" + raw : raw; + return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; }; const rows = records.map(record => [ record.session, record.participant, record.position + 1, record.label, record.method, @@ -353,7 +354,7 @@ class StudyStore { record.questionnaire.timing.totalSeconds, ...metricKeys.map(key => record.metrics[key]) ].map(cell).join(',')); - return `${columns.join(',')}\n${rows.join('\n')}${rows.length ? '\n' : ''}`; + return `${columns.map(cell).join(',')}\n${rows.join('\n')}${rows.length ? '\n' : ''}`; } } diff --git a/open4d/webclients/tests/test_server_security.js b/open4d/webclients/tests/test_server_security.js new file mode 100644 index 00000000..978097b2 --- /dev/null +++ b/open4d/webclients/tests/test_server_security.js @@ -0,0 +1,63 @@ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const http = require('node:http'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { app } = require('../system/Server/server'); + +const results = path.resolve(__dirname, '../system/Server/server_results'); + +test('client log cannot write outside the results root', async t => { + const outside = fs.mkdtempSync(path.join(os.tmpdir(), 'open4d-log-security-')); + t.after(() => fs.rmSync(outside, { recursive: true, force: true })); + const server = await new Promise(resolve => { + const s = app.listen(0, '127.0.0.1', () => resolve(s)); + }); + t.after(() => new Promise(resolve => server.close(resolve))); + const response = await fetch(`http://127.0.0.1:${server.address().port}/api/client-log`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ broadcastId: path.relative(results, outside), + entries: [{ message: 'fixture' }] }) + }); + assert.equal(response.status, 400); + assert.equal(fs.existsSync(path.join(outside, 'client_log.jsonl')), false); +}); + +test('foreign websites cannot invoke the local control API', async t => { + const server = await new Promise(resolve => { + const s = app.listen(0, '127.0.0.1', () => resolve(s)); + }); + t.after(() => new Promise(resolve => server.close(resolve))); + const response = await fetch(`http://127.0.0.1:${server.address().port}/api/health`, { + headers: { Origin: 'https://untrusted.example' } + }); + assert.equal(response.status, 403); + assert.equal(response.headers.get('access-control-allow-origin'), null); +}); + +test('a forged hostname cannot bypass the browser origin check', async t => { + const server = await new Promise(resolve => { + const s = app.listen(0, '127.0.0.1', () => resolve(s)); + }); + t.after(() => new Promise(resolve => server.close(resolve))); + const status = await new Promise((resolve, reject) => { + const request = http.get({ host: '127.0.0.1', port: server.address().port, + path: '/api/health', headers: { Host: 'untrusted.example', Origin: 'http://untrusted.example' } + }, response => { response.resume(); resolve(response.statusCode); }); + request.on('error', reject); + }); + assert.equal(status, 403); +}); + +test('the same-origin UI can still read the API', async t => { + const server = await new Promise(resolve => { + const s = app.listen(0, '127.0.0.1', () => resolve(s)); + }); + t.after(() => new Promise(resolve => server.close(resolve))); + const base = `http://127.0.0.1:${server.address().port}`; + const response = await fetch(`${base}/api/health`, { headers: { Origin: base } }); + assert.equal(response.status, 200); + assert.equal(response.headers.get('access-control-allow-origin'), base); +}); diff --git a/open4d/webclients/tests/test_study_session.js b/open4d/webclients/tests/test_study_session.js index d31d7462..02018e14 100644 --- a/open4d/webclients/tests/test_study_session.js +++ b/open4d/webclients/tests/test_study_session.js @@ -291,3 +291,54 @@ test('abort frees the link without recording a result', { skip: NO_EXPRESS }, as server.close(); } }); + + +test('session ids cannot name the root or its parent', () => { + const store = new StudyStore(tempDir()); + for (const id of ['.', '..', '../outside', '/outside', ['p01']]) { + assert.throws(() => store._dir(id), /invalid session id/); + } +}); + +test('two sessions created in the same second never overwrite each other', () => { + const store = new StudyStore(tempDir()); + const spec = { participant: 'p03', methods: ['mesh'], durationSeconds: 20, + trace: { text: TRACE } }; + const sessions = Array.from({ length: 10 }, () => store.createSession(spec)); + assert.strictEqual(new Set(sessions.map(s => s.id)).size, sessions.length); + assert.strictEqual(store.listSessions().length, sessions.length); +}); + +test('finishing an abandoned trial cannot disarm another session', { skip: NO_EXPRESS }, async () => { + const { server, call, shaper } = await serve(); + try { + const make = async participant => (await call('POST', '/api/study/sessions', { + participant, methods: ['mesh'], durationSeconds: 20, + trace: { text: TRACE }, trajectory: trajectory() + })).json; + const first = await make('p10'); + const second = await make('p11'); + await call('POST', `/api/study/sessions/${first.id}/trials/0/start`); + await call('POST', '/api/study/abort'); + await call('POST', `/api/study/sessions/${second.id}/trials/0/start`); + const response = await call('POST', `/api/study/sessions/${first.id}/trials/0/finish`, + { metrics: {}, questionnaire: RESPONSE }); + assert.strictEqual(response.status, 409); + assert.strictEqual(shaper.armed, true); + assert.strictEqual((await call('GET', '/api/study/shaping')).json.live.session, second.id); + } finally { shaper.disarm(); server.close(); } +}); + +test('CSV exports keep uploaded formula-like strings as text', () => { + const store = new StudyStore(tempDir()); + const session = store.createSession({ participant: 'p12', methods: ['mesh'], + durationSeconds: 20, trace: { name: '=1+1', text: TRACE }, trajectory: trajectory() }); + store.startTrial(session.id, 0); + store.finishTrial(session.id, 0, { metrics: { note: '@SUM(1)', loss: -1, 'bad,key': 'safe' }, + questionnaire: RESPONSE }); + const csv = store.exportCsv(session.id); + assert.ok(csv.includes("'=1+1")); + assert.ok(csv.includes("'@SUM(1)")); + assert.ok(csv.includes(',-1,')); + assert.ok(csv.includes('"m_bad,key"')); +}); diff --git a/open4d/webclients/tests/test_tile_ladder.py b/open4d/webclients/tests/test_tile_ladder.py index 3d2208a9..d2426be7 100644 --- a/open4d/webclients/tests/test_tile_ladder.py +++ b/open4d/webclients/tests/test_tile_ladder.py @@ -8,7 +8,7 @@ import pytest -from baselines.ViVo.orbitvivo.tile_ladder import ( +from open4d.webclients.tile_ladder import ( SourceCorpusUnavailable, TileCatalogLadder, ) diff --git a/open4d/webclients/tests/test_v4ds_protocol.js b/open4d/webclients/tests/test_v4ds_protocol.js index 87e6beb4..7069c9ec 100644 --- a/open4d/webclients/tests/test_v4ds_protocol.js +++ b/open4d/webclients/tests/test_v4ds_protocol.js @@ -339,7 +339,7 @@ test('an empty-but-present selection list is distinguishable from absent', () => 'a u16 count of zero is still written'); }); -test('feedback round-trips through the Python decoder', () => { +test('feedback round-trips through the Python decoder', (t) => { // Encoding matching bytes proves the layout; this proves the Python side // actually accepts what we produce. const { execFileSync } = require('child_process'); @@ -353,8 +353,14 @@ test('feedback round-trips through the Python decoder', () => { }); const python = process.env.VS4D_TEST_PYTHON || '/home/ryan/miniconda3/envs/open4d/bin/python'; + const reference = path.join(__dirname, '..', + 'baselines/DeltaStream/orbitstream/protocol.py'); + if (!fs.existsSync(reference)) { + t.skip('upstream DeltaStream Python decoder is not vendored'); + return; + } if (!fs.existsSync(python)) { - console.log('skipping Python round-trip: no interpreter at', python); + t.skip(`Python reference interpreter is unavailable: ${python}`); return; } const script = ` diff --git a/open4d/webclients/tests/test_vgs_format.js b/open4d/webclients/tests/test_vgs_format.js index 9f3bfa87..864dad39 100644 --- a/open4d/webclients/tests/test_vgs_format.js +++ b/open4d/webclients/tests/test_vgs_format.js @@ -41,7 +41,7 @@ const haveAsset = () => GOLDEN && fs.existsSync( fs.existsSync(GOLDEN.file) ? GOLDEN.file : path.join(__dirname, '..', GOLDEN.file)); -test('the golden fixture and its source asset are present', () => { +test('the golden fixture and its source asset are present', { skip: !haveAsset() }, () => { assert.ok(GOLDEN, 'fixtures_vgs_golden.json is missing; run the generator'); assert.ok(haveAsset(), `asset ${GOLDEN.file} is missing; re-export with orbitvega.export_quest`); diff --git a/open4d/webclients/tile_ladder.py b/open4d/webclients/tile_ladder.py index 5b3e06bc..fb8dc492 100644 --- a/open4d/webclients/tile_ladder.py +++ b/open4d/webclients/tile_ladder.py @@ -50,8 +50,6 @@ from dataclasses import dataclass from pathlib import Path -from baselines.ViVo.orbitvivo.ladder import Representation - CATALOG_NAME = "catalog.json" SUPPORTED_CATALOG_VERSION = 2 @@ -60,6 +58,18 @@ class SourceCorpusUnavailable(RuntimeError): """Raised when a caller needs the RGB-D source this ladder does not have.""" +@dataclass(frozen=True) +class Representation: + """One prepared level; self-contained metadata for the tile catalogue.""" + + representation_id: int + name: str + root: Path + manifest_path: Path + manifest: TileManifest + point_ratio: float + + @dataclass(frozen=True) class TileObjectSequence: """The object fields a tile-served baseline uses, and no others. diff --git a/pyproject.toml b/pyproject.toml index 2360eddd..d74cf24f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,12 +28,13 @@ player = ["PyQt6", "pyqtgraph", "PyOpenGL", "pillow>=9", "scipy"] open3d = ["open3d>=0.19,<0.20", "pillow>=9"] usd = ["usd-core>=24.3"] torch = ["torch"] -klt = ["torch", "trimesh", "zstd", "tqdm", "point-cloud-utils"] +klt = ["torch", "trimesh", "zstd", "zstandard", "tqdm", "point-cloud-utils"] n4mc = ["torch", "trimesh", "scikit-image", "point-cloud-utils"] qndf = ["torch", "open3d>=0.19,<0.20", "trimesh", "tqdm"] gaussians = ["plyfile"] +capture = ["opencv-python", "zstandard", "open3d>=0.19,<0.20", "pillow>=9"] dev = ["build>=1.2", "pytest>=8", "scipy", "tomli>=2; python_version < '3.11'"] -all = ["plyfile", "trimesh", "PyQt6", "pyqtgraph", "PyOpenGL", "open3d>=0.19,<0.20", "pillow>=9", "usd-core>=24.3", "scipy", "torch", "zstd", "tqdm", "point-cloud-utils", "scikit-image"] +all = ["plyfile", "trimesh", "PyQt6", "pyqtgraph", "PyOpenGL", "open3d>=0.19,<0.20", "pillow>=9", "usd-core>=24.3", "scipy", "torch", "zstd", "tqdm", "point-cloud-utils", "scikit-image", "opencv-python", "zstandard"] [tool.setuptools] include-package-data = false diff --git a/scripts/benchmark_codec.py b/scripts/benchmark_codec.py index 2fb17979..27466854 100644 --- a/scripts/benchmark_codec.py +++ b/scripts/benchmark_codec.py @@ -162,7 +162,7 @@ def decode_and_validate(sequence): assert len(actual.geometry.positions) and len(actual.geometry.triangles) exact &= _geometry_exact(actual.geometry, expected.geometry) if (actual.geometry.positions.shape == expected.geometry.positions.shape - and (info.lossless or sequence.has_vertex_correspondence is True)): + and (info.lossless or decoded.has_vertex_correspondence is True)): error = (actual.geometry.positions.astype(np.float64) - expected.geometry.positions.astype(np.float64)) squared_error += float(np.square(error).sum()) @@ -185,14 +185,17 @@ def decode_and_validate(sequence): surface_rms, surface_maximum, ) - result, validate_s = timed(lambda: decode_and_validate(decoded)) + frames, consume_s = timed(lambda: tuple(decoded)) + validation_started = time.perf_counter() + result = decode_and_validate(frames) + validate_s = time.perf_counter() - validation_started ( vertices, triangles, exact, rms_error, maximum_error, surface_rms_error, surface_max_error, ) = result finally: decoded.close() - decode_s = open_s + validate_s + decode_s = open_s + consume_s def decode_consume_close(): measured_sequence = decode_sequence(artifact, codec=selected, **(decode_options or {})) diff --git a/scripts/smoke_installed_io.py b/scripts/smoke_installed_io.py index 50a5eb2b..6884c4a3 100644 --- a/scripts/smoke_installed_io.py +++ b/scripts/smoke_installed_io.py @@ -72,9 +72,11 @@ np.testing.assert_array_equal(frame.geometry.triangles, [[0, 1, 2]]) assert all(callable(getattr(open4d, name)) for name in ("encode", "decode", "visualize", "reconstruct", "stream", "receive")) - installed = {info.id for info in available_codecs()} - assert {"klt", "n4mc", "qndf", "qndf-int8", "vdmc", "faster_vdmc", "tvmc", "tsmc", "vega"} <= installed - assert not installed & {"npz", "raw", "draco", "temporal-delta", "temporal-pca"} + codecs = available_codecs() + installed = {info.id for info in codecs} + assert installed == {"klt", "n4mc", "qndf", "qndf-int8", "vdmc", "faster_vdmc", + "tvmc", "tsmc", "vega", "queen", "3dgstream", "rerf"} + assert all(info.suffixes == (".vmesh",) for info in codecs) from open4d.codec._research import research_module os.environ.pop("OPEN4D_RESEARCH_ROOT", None) try: @@ -85,9 +87,20 @@ raise AssertionError("wheel unexpectedly includes KLT research source") from concurrent.futures import ThreadPoolExecutor + from open4d.codec import inspect_vmesh + from open4d.codec._npz import REFERENCE_CODECS from open4d.demo import mesh_sequence with mesh_sequence(side=3, frames=2) as source: + reference = next(codec for codec in REFERENCE_CODECS if codec.id == "npz") + artifact = open4d.encode(source, Path(directory) / "reference.vmesh", codec=reference) + assert artifact.read_bytes().startswith(b"VMESH\x00\x01\x00") + assert inspect_vmesh(artifact)["codec"] == "npz" + with open4d.decode(artifact) as decoded: + assert decoded.timestamps == source.timestamps + for expected, actual in zip(source, decoded): + np.testing.assert_array_equal(actual.geometry.positions, expected.geometry.positions) + np.testing.assert_array_equal(actual.geometry.triangles, expected.geometry.triangles) with open4d.receive(port=0, timeout=5) as receiver, ThreadPoolExecutor(1) as pool: transfer = pool.submit(open4d.stream, source, *receiver.address, realtime=False) restored = list(receiver) @@ -95,4 +108,4 @@ for expected, actual in zip(source, restored): np.testing.assert_array_equal(actual.geometry.positions, expected.geometry.positions) assert actual.timestamp == expected.timestamp - print("Installed I/O, CLI, codec discovery and mesh streaming passed") + print("Installed I/O, CLI, VMESH round trip, codec discovery and mesh streaming passed") diff --git a/scripts/tests/test_benchmark_codec.py b/scripts/tests/test_benchmark_codec.py index 6ad65098..24878802 100644 --- a/scripts/tests/test_benchmark_codec.py +++ b/scripts/tests/test_benchmark_codec.py @@ -72,14 +72,37 @@ def fixed_timed(function): monkeypatch.setattr(benchmark_codec, "peak_bytes", lambda function, cleanup=None: 0) source = benchmark_codec.synthetic(3, 2) - result = benchmark_codec.run(source, Path(tmp_path) / "take.o4d") + result = benchmark_codec.run(source, Path(tmp_path) / "take.vmesh") assert result["decode_open_ms"] == 2000 - assert result["decode_validate_s"] == 3 + assert result["decode_validate_s"] >= 0 assert result["decode_all_s"] == 5 assert result["decode_frames_per_s"] == pytest.approx(2 / 5) +def test_decode_timing_excludes_surface_validation(tmp_path, monkeypatch): + measuring = False + real_timed = benchmark_codec.timed + real_surface = benchmark_codec._surface_errors + + def marked_timed(function): + nonlocal measuring + measuring = True + try: + return real_timed(function) + finally: + measuring = False + + # Validation has its own explicit timer, outside timed decode consumption. + def check(left, right): + assert not measuring + return real_surface(left, right) + + monkeypatch.setattr(benchmark_codec, "timed", marked_timed) + monkeypatch.setattr(benchmark_codec, "_surface_errors", check) + benchmark_codec.run(benchmark_codec.synthetic(3, 2), tmp_path / "test.vmesh") + + def test_decode_peak_memory_excludes_surface_validation(tmp_path, monkeypatch): inside_peak_measurement = False real_peak_bytes = benchmark_codec.peak_bytes @@ -101,7 +124,7 @@ def checked_surface_errors(left, right): monkeypatch.setattr(benchmark_codec, "_surface_errors", checked_surface_errors) result = benchmark_codec.run( - benchmark_codec.synthetic(3, 2), Path(tmp_path) / "take.o4d" + benchmark_codec.synthetic(3, 2), Path(tmp_path) / "take.vmesh" ) assert result["decode_all_peak_bytes"] > 0 @@ -127,7 +150,7 @@ def decode(*args, **kwargs): has_vertex_correspondence=False)) monkeypatch.setattr(benchmark_codec, "decode_sequence", decode) - result = benchmark_codec.run(source, tmp_path / "test.o4d", codec="reordered") + result = benchmark_codec.run(source, tmp_path / "test.vmesh", codec="reordered") assert result["position_rms_error"] is None assert result["position_max_error"] is None assert result["surface_rms_error"] == 0 @@ -146,7 +169,7 @@ def decode(*args, **kwargs): monkeypatch.setattr(benchmark_codec, "decode_sequence", decode) with pytest.raises(AssertionError): - benchmark_codec.run(source, tmp_path / "test.o4d") + benchmark_codec.run(source, tmp_path / "test.vmesh") assert all(value.closed for value in opened) assert not source.closed diff --git a/scripts/tests/test_cli.py b/scripts/tests/test_cli.py index 57476655..0f061c6b 100644 --- a/scripts/tests/test_cli.py +++ b/scripts/tests/test_cli.py @@ -56,9 +56,9 @@ def test_inspect_uses_registered_codec_loader(tmp_path, capsys, monkeypatch): from open4d.codec import _api from open4d.codec._npz import NumPyZipCodec - monkeypatch.setitem(_api._CODECS, "test-fixture", NumPyZipCodec()) + monkeypatch.setitem(_api._CODECS, "npz", NumPyZipCodec()) with mesh_sequence(side=3, frames=2) as sequence: - path = save(sequence, tmp_path / "existing.o4d", codec="test-fixture") + path = save(sequence, tmp_path / "existing.vmesh", codec="npz") assert main(["inspect", str(path), "--json"]) == 0 info = json.loads(capsys.readouterr().out) assert info["frame_count"] == 2 @@ -100,7 +100,7 @@ def view(sequence, **options): def test_missing_player_is_reported_before_decoding(tmp_path, monkeypatch, capsys): from open4d.visualization import VisualizationDependencyError, _qt - path = tmp_path / "take.o4d" + path = tmp_path / "take.vmesh" path.touch() def unavailable():