From 8a20a21c2a24a3043b4fa513fd7e4e2bbf6b80f5 Mon Sep 17 00:00:00 2001 From: cicm Date: Fri, 25 Sep 2026 16:06:16 -0400 Subject: [PATCH 1/4] Add native research codec vmesh integration and examples Carry native codec payloads in the Open4D V3C profile, preserve them through USDC round trips, and expose native sequence decoding. Add concise notebooks and regression coverage alongside the tested codec, I/O, and streaming fixes. --- .github/workflows/ci.yml | 2 + README.md | 29 +- THIRD_PARTY.md | 5 + docs/api.md | 64 ++- examples/streaming_demo.py | 2 +- examples/visualization/compare_frames.py | 10 +- examples/visualization/compare_sequences.py | 2 +- examples/visualization/tests/test_compare.py | 21 + examples/vmesh/01_container_and_usdc.ipynb | 107 +++++ examples/vmesh/02_mesh_codecs.ipynb | 107 +++++ examples/vmesh/03_gaussian_codecs.ipynb | 149 +++++++ examples/vmesh/04_rerf.ipynb | 120 ++++++ integrations/open3d/open4d_open3d.py | 10 + .../open3d/tests/test_open4d_open3d.py | 8 + open4d/__init__.py | 4 + open4d/_api.py | 35 +- open4d/_files.py | 18 +- open4d/_native_worker.py | 132 +++++++ open4d/_streamer.py | 1 - open4d/codec/__init__.py | 4 + open4d/codec/_api.py | 32 +- open4d/codec/_klt.py | 4 + open4d/codec/_n4mc.py | 75 +++- open4d/codec/_native.py | 39 ++ open4d/codec/_native_profiles.py | 115 ++++++ open4d/codec/_native_temporal.py | 57 +++ open4d/codec/_tracked.py | 38 +- open4d/codec/_v3c.py | 346 +++++++++++++++++ open4d/codec/_vmesh.py | 44 ++- open4d/codec/tests/test_codec.py | 18 +- open4d/codec/tests/test_codec_review.py | 49 +++ open4d/codec/tests/test_n4mc_vmesh.py | 146 +++++++ open4d/codec/tests/test_native_temporal.py | 227 +++++++++++ open4d/codec/tests/test_research_codecs.py | 1 + open4d/codec/tests/test_tracked.py | 26 +- open4d/codec/tests/test_v3c.py | 342 ++++++++++++++++ open4d/codecs/n4mc/README.md | 19 + open4d/codecs/n4mc/losses/tsdf_losses.py | 9 +- open4d/codecs/n4mc/tests/test_losses.py | 4 +- open4d/gaussians.py | 30 +- open4d/io/_api.py | 18 +- open4d/io/_native_usd.py | 137 +++++++ open4d/io/_usd.py | 11 +- open4d/io/tests/test_review_regressions.py | 85 ++++ open4d/io/tests/test_unified_api.py | 13 +- open4d/io/tests/test_usd_sequence.py | 2 +- open4d/native.py | 365 ++++++++++++++++++ .../reconstruction/3dgstream/train_frames.py | 4 +- .../rerf/rerf_stream/bitstream.py | 28 +- open4d/reconstruction/rerf/rerf_stream/env.py | 2 +- .../rerf/rerf_stream_tests/test_bitstream.py | 12 +- .../rerf/rerf_stream_tests/test_geometry.py | 10 +- open4d/streamer/README.md | 2 +- open4d/streamer/streamer/bundle.py | 12 +- open4d/streamer/streamer/export.py | 11 +- open4d/streamer/streamer/sequence.py | 16 +- open4d/streamer/streamer/session.py | 7 +- open4d/streamer/streamer/transfer.py | 39 +- open4d/streamer/streamer_tests/test_draco.py | 2 +- open4d/streamer/streamer_tests/test_export.py | 11 +- .../streamer_tests/test_open4d_stream.py | 19 +- .../streamer/streamer_tests/test_sequence.py | 31 ++ .../test_server_and_transfer.py | 27 ++ open4d/streaming/README.md | 3 + open4d/streaming/_transport.py | 4 +- .../python/live_two_camera_webrtc.py | 20 +- open4d/streaming/tests/test_capture.py | 27 ++ open4d/webclients/README.md | 3 + open4d/webclients/system/Server/server.js | 3 +- open4d/webclients/tests/test_tile_ladder.py | 2 +- open4d/webclients/tests/test_vgs_format.js | 2 +- open4d/webclients/tile_ladder.py | 14 +- pyproject.toml | 3 +- scripts/benchmark_codec.py | 9 +- scripts/tests/test_benchmark_codec.py | 25 +- 75 files changed, 3249 insertions(+), 181 deletions(-) create mode 100644 examples/vmesh/01_container_and_usdc.ipynb create mode 100644 examples/vmesh/02_mesh_codecs.ipynb create mode 100644 examples/vmesh/03_gaussian_codecs.ipynb create mode 100644 examples/vmesh/04_rerf.ipynb create mode 100644 open4d/_native_worker.py create mode 100644 open4d/codec/_native.py create mode 100644 open4d/codec/_native_profiles.py create mode 100644 open4d/codec/_native_temporal.py create mode 100644 open4d/codec/_v3c.py create mode 100644 open4d/codec/tests/test_codec_review.py create mode 100644 open4d/codec/tests/test_n4mc_vmesh.py create mode 100644 open4d/codec/tests/test_native_temporal.py create mode 100644 open4d/codec/tests/test_v3c.py create mode 100644 open4d/io/_native_usd.py create mode 100644 open4d/io/tests/test_review_regressions.py create mode 100644 open4d/native.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 300f6d67..ff744889 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,9 @@ jobs: open4d/codec/tests/test_research_cpu.py open4d/codec/tests/test_qndf_preprocess.py open4d/codec/tests/test_qndf_entrypoints.py + open4d/codec/tests/test_codec_review.py open4d/codecs/n4mc/tests/test_codec.py + open4d/codecs/n4mc/tests/test_losses.py io-tools: name: I/O / real optional formats and benchmark smoke diff --git a/README.md b/README.md index 0431e039..c8d87401 100644 --- a/README.md +++ b/README.md @@ -73,20 +73,35 @@ with open4d.decode("wave.v4d") as decoded: The [notebook](examples/open4d_sequence_codec.ipynb) walks through these calls, reconstruction and streaming in separate short cells. +Short `.vmesh` examples: + +- [Preserve compressed state through USDC](examples/vmesh/01_container_and_usdc.ipynb) +- [TVMC, TSMC, V-DMC, faster V-DMC and N4MC](examples/vmesh/02_mesh_codecs.ipynb) +- [Vega, QUEEN and 3DGStream](examples/vmesh/03_gaussian_codecs.ipynb) +- [ReRF](examples/vmesh/04_rerf.ipynb) | Codec | Input | Output | Backend setup | | --- | --- | --- | --- | -| `vdmc` | Mesh sequence | `.v4d` | Build the V-DMC submodule and configure its encoder and decoder | -| `faster_vdmc` | Mesh sequence | `.v4d` | Build the faster V-DMC submodule and configure its encoder and decoder | -| `tvmc` | Mesh sequence | `.tvmc` directory | [TVMC setup](open4d/codecs/tvmc/README.md) | -| `tsmc` | Mesh sequence | `.tsmc` directory | [TSMC setup](open4d/codecs/tsmc/README.md) | +| `vdmc` | Mesh sequence | `.vmesh` or `.v4d` | Build the V-DMC submodule and configure its encoder and decoder | +| `faster_vdmc` | Mesh sequence | `.vmesh` or `.v4d` | Build the faster V-DMC submodule and configure its encoder and decoder | +| `tvmc` | Mesh sequence | `.vmesh` file or `.tvmc` directory | [TVMC setup](open4d/codecs/tvmc/README.md) | +| `tsmc` | Mesh sequence | `.vmesh` file or `.tsmc` directory | [TSMC setup](open4d/codecs/tsmc/README.md) | | `klt` | Mesh sequence converted to TSDF volumes | `.k4d` | Research source and `.[klt]` | -| `n4mc` | Mesh sequence converted to TSDF volumes | `.n4d` | Research source and `.[n4mc]` | +| `n4mc` | Mesh sequence converted to TSDF volumes | `.vmesh` or `.n4d` | Research source and `.[n4mc]` | | `qndf`, `qndf-int8` | Mesh frames | `.q4d`, `.qi4d` | Research source and `.[qndf]` | -| `vega` | Gaussian splat frames | `.vega` directory | [Vega CUDA environment](open4d/reconstruction/vega/README.md) | +| `vega` | Gaussian splat frames or native run | `.vmesh` or `.vega` directory | [Vega CUDA environment](open4d/reconstruction/vega/README.md) | +| `queen`, `3dgstream`, `rerf` | Native temporal research output | `.vmesh`, with native USDC interchange | Method-specific CUDA runtime for evaluation | QNDF-int8 now writes version 2 artifacts. Version 1 artifacts must be encoded again. +TVMC, TSMC, V-DMC, Vega, QUEEN, 3DGStream, ReRF and N4MC support +[native `.vmesh` carriage](examples/vmesh/01_container_and_usdc.ipynb). Mesh codecs support mesh USDC +import/export; neural methods preserve their compressed temporal state in an +O4D native USDC schema. Existing native outputs can be packed without recompression. +N4MC carries a shared model and independent frame latents as an explicit +exception to the temporal-dependency requirement. +This experimental O4D V3C application format requires O4D decoding support. + KLT, N4MC and QNDF run in Python. TVMC, TSMC, V-DMC and Gaussian methods use separate research runtimes. N4MC and QNDF currently process frames independently; they remain available as research methods. Mesh codecs currently encode geometry @@ -163,7 +178,7 @@ send(mesh_sequence(frames=30)) ``` This sends decoded mesh arrays over TCP, at their recorded frame timing. It is -not a compression method. Both calls default to this computer on port 7000. +not a compression method. Both calls default to this computer on port 47004. Pass `host=` and `port=` for another address. Remote transport needs a trusted network or SSH tunnel; this protocol has no authentication or encryption. Use `realtime=False` to transfer a recorded sequence as fast as possible. diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index bd55a776..53ac65b5 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -82,6 +82,11 @@ boundaries; they do not resolve any release block above: frame on longer clips, use the last available frame on short clips, and remove unused alignment work that also accessed an unavailable sixth frame. +Additional local patches (2026-09-23): N4MC restores SSIM loss; ReRF validates +inputs before native loading and makes test data configurable; webclients use +a local tile record and default to loopback. Licenses and release boundaries +are unchanged. + ## Required record for new material Any copied source, submodule, binary, model, dataset, paper, image, archive, or diff --git a/docs/api.md b/docs/api.md index e2e7ace2..b5ba8084 100644 --- a/docs/api.md +++ b/docs/api.md @@ -7,15 +7,15 @@ sequences independently of their storage format: import open4d with open4d.load("capture.usdc") as sequence: - open4d.save(sequence, "capture.o4d") + open4d.save(sequence, "capture-copy.usdc") open4d.visualize(sequence) # A path can go straight to the lazy viewer; it is closed when the window exits. -open4d.visualize("capture.o4d") +open4d.visualize("capture-copy.usdc") ``` `.usd`, `.usda`, `.usdc`, and `.usdz` are OpenUSD interchange containers. -`.o4d` and the registered codec suffixes are codec artifacts. Both carry whole +The registered codec suffixes identify codec artifacts. Both carry whole sequences and use the same `Sequence` interface. `open4d.unload(sequence)` is an explicit, idempotent alternative to the context manager. @@ -55,6 +55,11 @@ Single mesh-file exports require `allow_lossy=True` because that storage cannot preserve sequence timing, metadata, or topology declarations. Trimesh-backed OFF/GLB/glTF color export also requires that opt-in because OFF drops vertex color and GLB/glTF quantize canonical float colors to eight bits. +STL also requires `allow_lossy=True`: it discards unused vertices and vertex +identity, so exported manifests clear correspondence and topology guarantees. +STL/GLB/glTF reject geometry without triangles. Gaussian clouds cannot be +exported through these mesh writers, and USD writing currently accepts only +triangle meshes. OpenUSD is the public interchange container. `--pack-usd out.usdc` packs any source into one compressed `.usdc` file carrying the frame rate, the key-frame @@ -77,6 +82,8 @@ For a loaded sequence, pass browser options such as `name="capture"` and decoded-mesh TCP transport and pairs with `open4d.receive`. Existing `open4d.stream(sequence, host, port)` calls, and frame iterables without browser options, retain that TCP behavior without requiring `open4d-streamer`. +The TCP default is `127.0.0.1:47004`; pass port 7000 explicitly when talking to +an older Open4D receiver. `rungs` is the quality ladder. The first is the rendition a client plays by default and the rest are what it can switch to mid-playback, so a one-entry @@ -103,18 +110,54 @@ other. It is not this API and shares no code with it. ## Codecs -Five lossless, in-process reference codecs are included: `raw`, `deflate`, -`bzip2`, `lzma`, and byte-level `rle` (`npz` remains the default DEFLATE alias). -They share a safe NumPy-array container so they compare storage strategies, not -research geometry models. +TVMC and TSMC can encode directly to a single `.vmesh` file: -Source checkouts register in-process adapters for `klt`, `n4mc`, `qndf`, and -`qndf-int8`; the lightweight wheel omits them until their provenance review is -complete. Open4D's separate `temporal-delta` and `temporal-pca` experiments are +```python +open4d.encode("capture.usdc", "capture.vmesh", codec="tvmc") +with open4d.load("capture.vmesh") as sequence: + open4d.save(sequence, "decoded.usdc") +``` + +The embedded codec is detected on load. The experimental O4D V3C application +format preserves native compressed payloads and timing; existing `.tvmc` and +`.tsmc` directories remain supported. See the [native `.vmesh` notebook](../examples/vmesh/01_container_and_usdc.ipynb) +for inspection, packing, extraction, and an exact compressed-state USDC round trip. +The [mesh example](../examples/vmesh/02_mesh_codecs.ipynb) demonstrates encoding +and saving decoded geometry as USDC. + +Vega, QUEEN, 3DGStream and ReRF also support native `.vmesh` output. Import a +research run with `open4d.import_native(run, codec="queen", config=...)`, or +use `open4d.encode(run, "capture.vmesh", codec="queen", config=...)`. +Loading returns a `NativeSequence`; `native.decode(runtime=..., python=...)` +evaluates the actual temporal models. Saving it to USDC preserves the exact +compressed representation using O4D's native USD schema. This route does not +train a neural scene from an arbitrary mesh USD. See the linked guide for +required model/configuration files and representation-specific outputs. + +N4MC also accepts `.vmesh` output from mesh sequences or mesh USDC using +`open4d.encode(..., codec="n4mc")`. Loading reconstructs a mesh `Sequence`. +`open4d.codec.pack_vmesh("old.n4d", "new.vmesh")` migrates existing O4D N4MC +artifacts without retraining. Its profile explicitly identifies independent +quantized TSDF latents sharing one model; it does not claim temporal prediction. +The guide also documents exact compressed-state preservation in native USDC. + +`available_codecs()` lists the public research adapters: `klt`, `n4mc`, `qndf`, +`qndf-int8`, `vdmc`, `faster_vdmc`, `tvmc`, `tsmc`, `vega`, `queen`, +`3dgstream`, and `rerf`. The NumPy +reference codecs and temporal experiments are internal benchmark baselines; +`npz` and `.o4d` are not supported public save defaults. Use USD for interchange +or select a registered codec and its suffix. + +The lightweight wheel includes the adapters but excludes their research +implementations. KLT, N4MC and QNDF need a source checkout and their optional +dependencies; set `OPEN4D_RESEARCH_ROOT` to that checkout when using an installed +wheel. Open4D's separate `temporal-delta` and `temporal-pca` experiments are not the repository's TVMC or TSMC pipelines. The V-DMC adapters do not execute shell scripts, but they do invoke configured native encoder and decoder processes once per sequence. Callers can also register another `open4d.codec.Codec`. +Native mesh processes default to a one-hour timeout; set +`OPEN4D_NATIVE_TIMEOUT` to a positive number of seconds for longer jobs. For an all-registered-codec attempt using `4d_files/Rafa_Approves_hd_4k`, open [`examples/open4d_sequence_codec.ipynb`](../examples/open4d_sequence_codec.ipynb). @@ -128,6 +171,7 @@ The N4MC and QNDF adapters accept `device="auto"` (CUDA, then Apple Metal/MPS, then CPU), or an explicit `"cuda"`, `"mps"`, or `"cpu"`. QNDF-int8 can train on CUDA or Metal, but its quantized decoder remains CPU-only. Override the notebook selection with `OPEN4D_NOTEBOOK_DEVICE=mps` when needed. +KLT decoding also accepts `device="auto"`. ### Test coverage diff --git a/examples/streaming_demo.py b/examples/streaming_demo.py index bdeeb73a..a8848553 100644 --- a/examples/streaming_demo.py +++ b/examples/streaming_demo.py @@ -7,7 +7,7 @@ python examples/streaming_demo.py Needs three things beyond the base install: the optional `open4d-streamer` -package (``pip install -e open4d/streamer``), `open4d[draco]` for the Draco +package (``pip install -e open4d/streamer``), `DracoPy` for the Draco rungs, and SciPy for the nearest-neighbour search the quality column uses. Output goes to ``examples/out/``, which the repository ignores. diff --git a/examples/visualization/compare_frames.py b/examples/visualization/compare_frames.py index f51192ba..b9e01e90 100644 --- a/examples/visualization/compare_frames.py +++ b/examples/visualization/compare_frames.py @@ -77,6 +77,9 @@ def all_distances(self) -> np.ndarray: def summary(self) -> "DirectionalSummary": """Aggregate the per-frame figures over the whole comparison.""" + aggregate = mesh_metrics.SequenceComparison( + tuple(frame.error for frame in self.frames), (), self.peak, self.metric + ) return DirectionalSummary( symmetric_rms=float( np.sqrt(np.mean([f.error.symmetric_rms ** 2 for f in self.frames])) @@ -85,9 +88,7 @@ def summary(self) -> "DirectionalSummary": worst_frame=int( np.argmax([f.error.symmetric_rms for f in self.frames]) ), - mean_psnr_db=float( - np.mean([f.error.symmetric_psnr_db for f in self.frames]) - ), + mean_psnr_db=aggregate.symmetric_psnr_db, ) @@ -203,7 +204,8 @@ def resolve_clamp( return 0.0 if percentile is None: return float(np.max(distances)) - return float(np.percentile(distances, percentile)) + clamp = float(np.percentile(distances, percentile)) + return clamp if clamp > 0 else float(np.max(distances)) def diffuse_intensity(frame: RenderFrame) -> np.ndarray: diff --git a/examples/visualization/compare_sequences.py b/examples/visualization/compare_sequences.py index 207fd37a..8dda7ea1 100644 --- a/examples/visualization/compare_sequences.py +++ b/examples/visualization/compare_sequences.py @@ -153,7 +153,7 @@ def report_error(comparison) -> None: summary = comparison.summary() print(f"\n sequence symmetric RMS : {summary.symmetric_rms:.6g}") print(f" sequence Hausdorff : {summary.hausdorff:.6g}") - print(f" mean symmetric PSNR : {summary.mean_psnr_db:.2f} dB") + print(f" sequence symmetric PSNR: {summary.mean_psnr_db:.2f} dB") print(f" worst frame : {summary.worst_frame}") diff --git a/examples/visualization/tests/test_compare.py b/examples/visualization/tests/test_compare.py index ed9b0020..9a15155f 100644 --- a/examples/visualization/tests/test_compare.py +++ b/examples/visualization/tests/test_compare.py @@ -521,3 +521,24 @@ def test_comparison_stride_keeps_playback_speed(folders): args = cli.build_parser().parse_args([str(folders[0]), str(folders[1]), "--stride", "2", "--info"]) cli.run(args) assert args.fps == 15 +def test_sparse_errors_do_not_collapse_the_color_scale(): + from types import SimpleNamespace + from compare_frames import resolve_clamp + + distances = np.zeros(1000) + distances[-1] = 1 + frames = [SimpleNamespace(decoded_distances=distances, reference_distances=distances)] + assert resolve_clamp(frames, None, 99) > 0 + + +def test_summary_psnr_uses_aggregate_error(reference_sequence, shifted_sequence): + from compare_frames import compare_sequences, Comparison + from open4d.metrics import SequenceComparison + + exact = compare_sequences(reference_sequence, reference_sequence) + shifted = compare_sequences(reference_sequence, shifted_sequence) + frames = [exact.frames[0], shifted.frames[0]] + combined = Comparison(frames, "point", shifted.peak, shifted.clamp, 99, None) + expected = SequenceComparison(tuple(f.error for f in frames), (0., 1.), shifted.peak, "point") + assert np.isfinite(combined.summary().mean_psnr_db) + assert combined.summary().mean_psnr_db == pytest.approx(expected.symmetric_psnr_db) diff --git a/examples/vmesh/01_container_and_usdc.ipynb b/examples/vmesh/01_container_and_usdc.ipynb new file mode 100644 index 00000000..90ff7b12 --- /dev/null +++ b/examples/vmesh/01_container_and_usdc.ipynb @@ -0,0 +1,107 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "01-00", + "metadata": {}, + "source": [ + "# Preserve a .vmesh through USDC\n", + "\n", + "Set the paths below. This works with every supported codec and needs OpenUSD, but no codec runtime.\n", + "The USDC stores compressed native state in O4D's custom schema; it is not a renderable mesh export." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01-01", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "import hashlib\n", + "from pathlib import Path\n", + "import open4d\n", + "from open4d.codec import inspect_vmesh, pack_vmesh, unpack_vmesh\n", + "\n", + "source = Path(\"motion.vmesh\")\n", + "output = Path(\"output/native-usdc\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01-02", + "metadata": {}, + "outputs": [], + "source": [ + "info = inspect_vmesh(source)\n", + "print(info[\"codec\"], info[\"frame_count\"], \"frames\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01-03", + "metadata": {}, + "outputs": [], + "source": [ + "with open4d.NativeSequence(source) as native:\n", + " open4d.save(native, output / \"native.usdc\", overwrite=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01-04", + "metadata": {}, + "outputs": [], + "source": [ + "with open4d.load(output / \"native.usdc\") as native:\n", + " restored = open4d.save(native, output / \"restored.vmesh\", overwrite=True)\n", + "\n", + "with source.open(\"rb\") as a, restored.open(\"rb\") as b:\n", + " assert hashlib.file_digest(a, \"sha256\").digest() == hashlib.file_digest(b, \"sha256\").digest()\n", + "print(\"Identical compressed bytes\")" + ] + }, + { + "cell_type": "markdown", + "id": "01-05", + "metadata": {}, + "source": [ + "To inspect or migrate native files without decoding:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01-06", + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "\n", + "with tempfile.TemporaryDirectory() as folder:\n", + " native_files = unpack_vmesh(source, Path(folder) / \"native\")\n", + " repacked = pack_vmesh(native_files, output / \"repacked.vmesh\", overwrite=True)\n", + "print(inspect_vmesh(repacked)[\"codec\"])\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/vmesh/02_mesh_codecs.ipynb b/examples/vmesh/02_mesh_codecs.ipynb new file mode 100644 index 00000000..79a7bbb0 --- /dev/null +++ b/examples/vmesh/02_mesh_codecs.ipynb @@ -0,0 +1,107 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "02-00", + "metadata": {}, + "source": [ + "# Mesh USDC → .vmesh → mesh USDC\n", + "\n", + "Set your input and choose `tvmc`, `tsmc`, `vdmc`, `faster_vdmc` or `n4mc`.\n", + "Configure its [native backend](../../README.md) first. These examples encode geometry only." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02-01", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import open4d\n", + "from open4d.codec import inspect_vmesh\n", + "\n", + "source = Path(\"input.usdc\")\n", + "output = Path(\"output/mesh\")\n", + "codec = \"tvmc\"\n", + "encode_options = {}\n", + "decode_options = {}" + ] + }, + { + "cell_type": "markdown", + "id": "02-02", + "metadata": {}, + "source": [ + "For N4MC on CPU, set `codec=\"n4mc\"` and both option dictionaries to `{\"device\": \"cpu\"}`.\n", + "Add `resolution` and `epochs` to `encode_options` to control training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02-03", + "metadata": {}, + "outputs": [], + "source": [ + "encoded = open4d.encode(\n", + " source, output / \"motion.vmesh\", codec=codec, overwrite=True, **encode_options)\n", + "print(inspect_vmesh(encoded)[\"frame_count\"], \"frames encoded\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02-04", + "metadata": {}, + "outputs": [], + "source": [ + "with open4d.load(encoded, options=decode_options) as decoded:\n", + " open4d.save(decoded, output / \"reconstructed.usdc\", overwrite=True)\n", + " print(len(decoded), \"frames decoded\")" + ] + }, + { + "cell_type": "markdown", + "id": "02-05", + "metadata": {}, + "source": [ + "Existing O4D `.tvmc` / `.tsmc` directories or `.n4d` files can be migrated without re-encoding.\n", + "Run the next cell only when you have such an artifact." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02-06", + "metadata": { + "tags": [ + "migration" + ] + }, + "outputs": [], + "source": [ + "from open4d.codec import pack_vmesh\n", + "\n", + "pack_vmesh(\"existing.tvmc\", output / \"migrated.vmesh\", overwrite=True)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/vmesh/03_gaussian_codecs.ipynb b/examples/vmesh/03_gaussian_codecs.ipynb new file mode 100644 index 00000000..e6140df8 --- /dev/null +++ b/examples/vmesh/03_gaussian_codecs.ipynb @@ -0,0 +1,149 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "03-00", + "metadata": {}, + "source": [ + "# Native Gaussian runs → .vmesh\n", + "\n", + "Run the section for your codec after setting its paths. Use the configuration that trained\n", + "the run. Packing needs no CUDA; the final decoding cell requires the codec's configured runtime." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03-01", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import open4d\n", + "\n", + "output = Path(\"output/gaussians\")" + ] + }, + { + "cell_type": "markdown", + "id": "03-02", + "metadata": {}, + "source": [ + "Vega" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03-03", + "metadata": { + "tags": [ + "vega" + ] + }, + "outputs": [], + "source": [ + "with open4d.import_native(\"runs/capture.vega\", codec=\"vega\") as native:\n", + " open4d.save(native, output / \"vega.vmesh\", overwrite=True)\n", + " print(len(native), \"Vega frames packed\")" + ] + }, + { + "cell_type": "markdown", + "id": "03-04", + "metadata": {}, + "source": [ + "QUEEN" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03-05", + "metadata": { + "tags": [ + "queen" + ] + }, + "outputs": [], + "source": [ + "with open4d.import_native(\"runs/queen\", codec=\"queen\", config=\"runs/queen/config.yaml\") as native:\n", + " open4d.save(native, output / \"queen.vmesh\", overwrite=True)\n", + " print(len(native), \"QUEEN frames packed\")" + ] + }, + { + "cell_type": "markdown", + "id": "03-06", + "metadata": {}, + "source": [ + "3DGStream. The run must contain its initial model, NTC checkpoints and architecture configuration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03-07", + "metadata": { + "tags": [ + "3dgstream" + ] + }, + "outputs": [], + "source": [ + "with open4d.import_native(\"runs/3dgstream\", codec=\"3dgstream\",\n", + " ntc_config=\"runs/3dgstream/ntc_config.json\") as native:\n", + " open4d.save(native, output / \"3dgstream.vmesh\", overwrite=True)\n", + " print(len(native), \"3DGStream frames packed\")" + ] + }, + { + "cell_type": "markdown", + "id": "03-08", + "metadata": {}, + "source": [ + "Decode one saved artifact. Set `OPEN4D__ROOT` and `OPEN4D__PYTHON` for its native CUDA runtime." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03-09", + "metadata": { + "tags": [ + "native-cuda" + ] + }, + "outputs": [], + "source": [ + "with open4d.load(output / \"queen.vmesh\") as native:\n", + " frames = native.decode()\n", + " print(len(frames), \"frames;\", len(frames[0].positions), \"Gaussians in the first frame\")" + ] + }, + { + "cell_type": "markdown", + "id": "03-10", + "metadata": {}, + "source": [ + "Use the [USDC example](01_container_and_usdc.ipynb) to preserve any of these compressed artifacts through USDC." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/vmesh/04_rerf.ipynb b/examples/vmesh/04_rerf.ipynb new file mode 100644 index 00000000..091deafc --- /dev/null +++ b/examples/vmesh/04_rerf.ipynb @@ -0,0 +1,120 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "04-00", + "metadata": {}, + "source": [ + "# Native ReRF run → .vmesh\n", + "\n", + "Set the compressed run path. It must contain its entropy files, motion, shared RGB network\n", + "and resolved `decoder_config.json`, including rendering bounds. Packing needs no CUDA." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04-01", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "import json\n", + "from pathlib import Path\n", + "import open4d\n", + "from open4d.codec import inspect_vmesh\n", + "\n", + "source = Path(\"runs/rerf\")\n", + "output = Path(\"output/rerf\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04-02", + "metadata": {}, + "outputs": [], + "source": [ + "with open4d.import_native(source, codec=\"rerf\", config=source / \"decoder_config.json\") as native:\n", + " artifact = open4d.save(native, output / \"rerf.vmesh\", overwrite=True)\n", + "print(inspect_vmesh(artifact)[\"frame_count\"], \"frames packed\")" + ] + }, + { + "cell_type": "markdown", + "id": "04-03", + "metadata": {}, + "source": [ + "Decode using `OPEN4D_RERF_ROOT` and `OPEN4D_RERF_PYTHON` for the native CUDA runtime." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04-04", + "metadata": { + "tags": [ + "native-cuda" + ] + }, + "outputs": [], + "source": [ + "with open4d.load(artifact) as native:\n", + " fields = native.decode()\n", + "print(fields[0].density.shape, fields[0].features.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "04-05", + "metadata": {}, + "source": [ + "Render a calibrated camera. `camera.json` contains `camera_id`, `width`, `height`,\n", + "`fx`, `fy`, `cx`, `cy` and a 4×4 `c2w` matrix for this run." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04-06", + "metadata": { + "tags": [ + "native-cuda" + ] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "camera = json.loads(Path(\"camera.json\").read_text())\n", + "with open4d.load(artifact) as native:\n", + " images = native.render(camera)\n", + "plt.imshow(images[0])\n", + "plt.axis(\"off\");" + ] + }, + { + "cell_type": "markdown", + "id": "04-07", + "metadata": {}, + "source": [ + "Use the [USDC example](01_container_and_usdc.ipynb) for an exact compressed-state round trip." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/integrations/open3d/open4d_open3d.py b/integrations/open3d/open4d_open3d.py index 8c530590..1e96b923 100644 --- a/integrations/open3d/open4d_open3d.py +++ b/integrations/open3d/open4d_open3d.py @@ -10,6 +10,7 @@ from collections.abc import Mapping, Sequence from typing import Any +import warnings import numpy as np from numpy.typing import ArrayLike, NDArray @@ -106,6 +107,15 @@ def _vertex_attribute( def _colors(value: ArrayLike, vertex_count: int) -> NDArray[np.float64]: + raw = np.asarray(value) + if raw.shape == (vertex_count, 4): + alpha = raw[:, 3] + maximum = 255 if np.issubdtype(raw.dtype, np.integer) else 1 + if not np.isfinite(alpha).all() or np.any(alpha < 0) or np.any(alpha > maximum): + raise ValueError("Color alpha must be finite and within the color range") + if np.any(alpha != maximum): + warnings.warn("Open3D legacy geometry does not preserve color alpha", UserWarning, stacklevel=3) + value = raw[:, :3] array = _vertex_attribute(value, vertex_count, "Colors") if np.issubdtype(np.asarray(value).dtype, np.integer): if np.any(array < 0) or np.any(array > 255): diff --git a/integrations/open3d/tests/test_open4d_open3d.py b/integrations/open3d/tests/test_open4d_open3d.py index 402617c6..1909298c 100644 --- a/integrations/open3d/tests/test_open4d_open3d.py +++ b/integrations/open3d/tests/test_open4d_open3d.py @@ -17,6 +17,14 @@ TRIANGLES = np.array([[0, 1, 2]], dtype=np.uint32) +def test_rgba_colors_are_accepted_with_explicit_alpha_warning(): + colors = np.tile([1., 0.5, 0., 0.4], (3, 1)) + frame = Frame(0, 0., TriangleMesh(VERTICES, TRIANGLES, colors=colors)) + with pytest.warns(UserWarning, match="alpha"): + mesh = frame_to_open3d(frame) + np.testing.assert_allclose(np.asarray(mesh.vertex_colors), colors[:, :3]) + + def test_core_frame_conversion() -> None: frame = Frame(7, 0.25, TriangleMesh(VERTICES, TRIANGLES)) diff --git a/open4d/__init__.py b/open4d/__init__.py index 895cf6b7..4ff80fe3 100644 --- a/open4d/__init__.py +++ b/open4d/__init__.py @@ -30,6 +30,7 @@ from .gaussians import GaussianSplats, GaussianRun, NeuralGaussianFrame, load_gaussians from .metrics import compare_meshes, compare_sequences from .visualization import visualize +from .native import NativeSequence, NeuralFieldFrame, import_native __all__ = [ "ATTRIBUTE_FLOAT_DTYPE", @@ -41,6 +42,9 @@ "GaussianSplats", "GaussianRun", "NeuralGaussianFrame", + "NativeSequence", + "NeuralFieldFrame", + "import_native", "FrameProvider", "GaussianCloud", "Geometry", diff --git a/open4d/_api.py b/open4d/_api.py index 27d40704..42bbae44 100644 --- a/open4d/_api.py +++ b/open4d/_api.py @@ -10,9 +10,10 @@ from .core import Sequence from .gaussians import NeuralGaussianFrame from .io import open_sequence, write_sequence +from .native import NativeSequence, save_native _USD_SUFFIXES = frozenset((".usd", ".usda", ".usdc", ".usdz")) -_RAW_VMESH_SUFFIX = ".vmesh" +_VMESH_SUFFIX = ".vmesh" def _options(value: Mapping[str, object] | None) -> dict[str, object]: @@ -46,33 +47,39 @@ def load( codec: str | Codec | None = None, fps: float | None = None, options: Mapping[str, object] | None = None, -) -> Sequence | tuple[NeuralGaussianFrame, ...]: - """Open a sequence artifact, raw V-DMC bitstream, or geometry source.""" +) -> Sequence | NativeSequence | tuple[NeuralGaussianFrame, ...]: + """Open a sequence artifact, V3C .vmesh bitstream, or geometry source.""" if format is not None and codec is not None: raise TypeError("format and codec are mutually exclusive") values = _options(options) path = Path(source) if codec is not None: - if fps is not None and path.suffix.lower() != _RAW_VMESH_SUFFIX: + if fps is not None and path.suffix.lower() != _VMESH_SUFFIX: raise TypeError("fps applies to I/O sources, not codec artifacts") _set_raw_fps(values, fps) return decode_sequence(path, codec=codec, **values) - if not path.is_dir() and path.suffix.lower() == _RAW_VMESH_SUFFIX: + if not path.is_dir() and path.suffix.lower() == _VMESH_SUFFIX: if format is not None: - raise TypeError("format cannot select a raw V-DMC bitstream") + raise TypeError("format cannot select a .vmesh bitstream") _set_raw_fps(values, fps) - return decode_sequence(path, codec="vdmc", **values) + return decode_sequence(path, **values) if path.suffix.lower() in _codec_suffixes(): if format is not None: raise TypeError("format cannot select a codec artifact") if fps is not None: raise TypeError("fps applies to I/O sources, not codec artifacts") return decode_sequence(path, **values) + if path.suffix.lower() in (".usd", ".usda", ".usdc"): + from .io._native_usd import is_native_usd, read_native_usd + if is_native_usd(path): + if fps is not None or format is not None: + raise TypeError("native USD has its own representation and timestamps") + return read_native_usd(path, **values) return open_sequence(path, format=format, fps=fps, options=values) def save( - sequence: Sequence, + sequence: Sequence | NativeSequence, destination: str | os.PathLike[str], *, codec: str | Codec | None = None, @@ -82,8 +89,14 @@ def save( options: Mapping[str, object] | None = None, ) -> Path: """Write a sequence to an OpenUSD file or a research codec artifact.""" + if isinstance(sequence, NativeSequence): + if fps is not None or up_axis is not None or options: + raise TypeError("native repacking preserves its configuration/timeline and accepts no geometry options") + if codec is not None and (codec if isinstance(codec, str) else codec.id) != sequence.codec: + raise ValueError("codec does not match the native representation") + return save_native(sequence, destination, overwrite=overwrite) if not isinstance(sequence, Sequence): - raise TypeError("sequence must be an open4d.Sequence") + raise TypeError("sequence must be an open4d.Sequence or NativeSequence") if not isinstance(overwrite, bool): raise TypeError("overwrite must be bool") path = Path(destination) @@ -139,8 +152,8 @@ def save( def unload(sequence: Sequence) -> None: """Release resources owned by a loaded sequence.""" - if not isinstance(sequence, Sequence): - raise TypeError("sequence must be an open4d.Sequence") + if not isinstance(sequence, (Sequence, NativeSequence)): + raise TypeError("sequence must be an open4d.Sequence or NativeSequence") sequence.close() diff --git a/open4d/_files.py b/open4d/_files.py index 89954261..ef7e2002 100644 --- a/open4d/_files.py +++ b/open4d/_files.py @@ -1,6 +1,8 @@ """Publish completed outputs while preserving existing destinations.""" +import errno import os +import shutil from pathlib import Path @@ -9,7 +11,21 @@ def publish_file(temporary: Path, destination: Path, *, overwrite: bool = False) if overwrite: temporary.replace(destination) else: - os.link(temporary, destination) + try: + os.link(temporary, destination) + except OSError as error: + if error.errno not in {errno.EPERM, errno.EOPNOTSUPP, errno.ENOSYS, errno.EXDEV}: + raise + # Filesystems such as exFAT have no hard links. Exclusive creation + # retains no-clobber semantics, though this fallback is not atomic + # for readers observing the destination while it is being copied. + with temporary.open("rb") as source: + with destination.open("xb") as target: + try: + shutil.copyfileobj(source, target) + except BaseException: + destination.unlink(missing_ok=True) + raise temporary.unlink() diff --git a/open4d/_native_worker.py b/open4d/_native_worker.py new file mode 100644 index 00000000..c27907fe --- /dev/null +++ b/open4d/_native_worker.py @@ -0,0 +1,132 @@ +"""Native temporal evaluation in isolated, method-specific Python runtimes. + +Keep this entry point compatible with ReRF's required Python 3.8. It does not +import the public package (whose base interpreter may be newer). +""" +from __future__ import annotations + +from collections import OrderedDict +import json +from pathlib import Path +import sys +from types import SimpleNamespace + +import numpy as np + + +def _save_gaussians(model, path): + def array(value): + return value.detach().float().cpu().numpy() + np.savez(path, positions=array(model.get_xyz), scales=array(model.get_scaling), + rotations=array(model.get_rotation), opacities=array(model.get_opacity).reshape(-1), + spherical_harmonics=array(model.get_features)) + + +def _queen(source, output, metadata): + import torch + from scene.gaussian_model import GaussianModel + from scene.decoders import DecoderIdentity + + profile = metadata["native"] + names = ["xyz", "f_dc", "f_rest", "sc", "rot", "op", "flow"] + # Initial PLY uses identity decoders. Later payloads carry their complete + # quantized decoder architecture and weights; no training setup is needed. + latent = SimpleNamespace(param_names=names, seed=0, gate_params=profile["gate_params"], + quant_type=["none"] * len(names)) + arguments = SimpleNamespace(gate_gamma=-0.5, gate_eta=1.01, gate_lr=0.1, gate_temp=0.3) + model = GaussianModel(profile["sh_degree"], latent, arguments) + model.latent_decoders = OrderedDict((name, DecoderIdentity()) for name in names) + model.gate_params = dict.fromkeys(names, False) + model.load_ply(str(source / "initial.ply")) + with torch.no_grad(): + for index in range(len(metadata["frames"])): + model.frame_idx = index + 1 + if index: + model.load_compressed_pkl(str(source / ("frame_%06d.pkl" % index))) + _save_gaussians(model, output / ("frame_%06d.npz" % index)) + # This is the native compressed renderer's state update, including + # decoded attributes and latent state, after each residual frame. + for name in model.get_atts: + model.prev_atts[name] = model.get_decoded_atts[name].detach().clone() + model.prev_latents[name] = model.get_atts[name].detach().clone() + + +def _gstream(source, output, metadata): + import torch + import tinycudann as tcnn + from scene.gaussian_model import GaussianModel + from ntc import NeuralTransformationCache + + profile = metadata["native"] + architecture = json.loads((source / "ntc_config.json").read_text()) + model = GaussianModel(profile["sh_degree"], profile["rotate_sh"]) + model.load_ply(str(source / "initial.ply"), 1.0) + fields = ("_xyz", "_features_dc", "_features_rest", "_opacity", "_scaling", "_rotation") + with torch.no_grad(): + _save_gaussians(model, output / "frame_000000.npz") + for index in range(1, len(metadata["frames"])): + state = torch.load(source / ("ntc_%06d.pth" % index), weights_only=True, map_location="cuda") + options = dict(n_input_dims=3, n_output_dims=8, network_config=architecture["network"]) + network = (tcnn.Network(**options) if profile["only_mlp"] else + tcnn.NetworkWithInputEncoding(**options, encoding_config=architecture["encoding"])) + model.ntc = NeuralTransformationCache(network.cuda(), state["xyz_bound_min"], state["xyz_bound_max"]) + model.ntc.load_state_dict(state) + model.query_ntc() + model.update_by_ntc() + recurrent = {name: getattr(model, name) for name in fields} + if profile["added"][index]: + added = GaussianModel(profile["sh_degree"]) + added.load_ply(str(source / ("added_%06d.ply" % index)), 1.0) + for name in fields: + setattr(model, name, torch.cat((getattr(model, name), getattr(added, name))).detach()) + _save_gaussians(model, output / ("frame_%06d.npz" % index)) + # Stage-two additions render this frame but are not NTC state. + for name, value in recurrent.items(): + setattr(model, name, value) + + +def _rerf(request, source, output, metadata): + from rerf_stream.bitstream import BitstreamPlayer + from rerf_stream.cameras import Camera + + config = json.loads((source / "decoder_config.json").read_text()) + profile = metadata["native"] + player = BitstreamPlayer(config, source, pca=profile["pca"], + pca_channels=profile["pca_channels"], group_size=profile["group_size"], + render_bounds=config["render"]) + if player.frames != [f["id"] for f in profile["frames"]]: + raise ValueError("ReRF decoded frame index disagrees with profile") + camera = None + if request["operation"] == "render": + camera = Camera(**dict(request["camera"], c2w=np.asarray(request["camera"]["c2w"]))) + for index, frame in enumerate(player.play(loop=False)): + if camera is not None: + np.save(output / ("image_%06d.npy" % index), player.render(camera)) + else: + np.savez(output / ("frame_%06d.npz" % index), + density=frame.model.density.detach().cpu().numpy(), + features=frame.model.k0.k0.detach().cpu().numpy(), + xyz_min=np.asarray(player.model_kwargs["xyz_min"]), + xyz_max=np.asarray(player.model_kwargs["xyz_max"])) + + +def main(): + request = json.loads(Path(sys.argv[1]).read_text()) + root = Path(request["runtime"]) + if not root.is_dir(): + raise FileNotFoundError("research runtime is missing: " + str(root)) + sys.path.insert(0, str(root)) + source, output = Path(request["source"]), Path(request["output"]) + metadata = json.loads((source / "metadata.json").read_text()) + if request["codec"] != metadata["codec"]: + raise ValueError("native codec disagrees with worker request") + if request["operation"] == "decode" and request["codec"] in ("queen", "3dgstream"): + (_queen if request["codec"] == "queen" else _gstream)(source, output, metadata) + elif request["operation"] in ("decode", "render") and request["codec"] == "rerf": + _rerf(request, source, output, metadata) + else: + raise ValueError("unsupported native operation") + + +if __name__ == "__main__": + main() diff --git a/open4d/_streamer.py b/open4d/_streamer.py index 9f4e4b55..601e32af 100644 --- a/open4d/_streamer.py +++ b/open4d/_streamer.py @@ -118,7 +118,6 @@ def stream( out_dir, title=title, source=None if loaded else source, - fps=int(fps) if fps else 30, ) with session: if loaded: diff --git a/open4d/codec/__init__.py b/open4d/codec/__init__.py index 65814ace..9c177cc8 100644 --- a/open4d/codec/__init__.py +++ b/open4d/codec/__init__.py @@ -13,6 +13,7 @@ from ._qndf import QNDFCodec from ._vmesh import VMeshCodec from ._tracked import TrackedMeshCodec +from ._v3c import inspect_vmesh, pack_vmesh, unpack_vmesh __all__ = [ "Codec", @@ -27,4 +28,7 @@ "decode_sequence", "encode_sequence", "register_codec", + "inspect_vmesh", + "pack_vmesh", + "unpack_vmesh", ] diff --git a/open4d/codec/_api.py b/open4d/codec/_api.py index b72dcacc..396053f6 100644 --- a/open4d/codec/_api.py +++ b/open4d/codec/_api.py @@ -5,6 +5,7 @@ from dataclasses import dataclass import os from pathlib import Path +from typing import TYPE_CHECKING from open4d.core import Sequence from open4d.gaussians import GaussianSplats, NeuralGaussianFrame, VEGA_CODEC @@ -16,6 +17,11 @@ from ._qndf import QNDF_CODEC, QNDF_INT8_CODEC from ._tracked import TVMC_CODEC, TSMC_CODEC from ._vmesh import FASTER_VDMC_CODEC, VDMC_CODEC +from ._v3c import probe_codec +from ._native_temporal import QUEEN_CODEC, GSTREAM_CODEC, RERF_CODEC + +if TYPE_CHECKING: + from open4d.native import NativeSequence @dataclass(frozen=True) @@ -32,6 +38,7 @@ class CodecInfo: codec.id: codec for codec in ( KLT_CODEC, N4MC_CODEC, QNDF_CODEC, QNDF_INT8_CODEC, VDMC_CODEC, FASTER_VDMC_CODEC, TVMC_CODEC, TSMC_CODEC, VEGA_CODEC, + QUEEN_CODEC, GSTREAM_CODEC, RERF_CODEC, ) } @@ -95,14 +102,16 @@ def encode_sequence( fps: float | None = None, **options, ) -> Path: - """Encode meshes (Sequence or path), or GaussianSplats frames with Vega.""" + """Encode mesh/Gaussian data, or carry native temporal neural outputs.""" if "overwrite" in options and not isinstance(options["overwrite"], bool): raise TypeError("overwrite must be bool") path = Path(destination) implementation = _codec(codec, path) - if getattr(implementation, "representation", "triangle_mesh") == "gaussian_splats": - if input_format is not None or fps is not None: - raise TypeError("input_format and fps apply only to mesh path inputs") + if getattr(implementation, "representation", "triangle_mesh") != "triangle_mesh": + if input_format is not None: + raise TypeError("input_format applies only to mesh path inputs") + if fps is not None: + options["fps"] = fps return implementation.encode(sequence, path, **options) if isinstance(sequence, Sequence): if input_format is not None or fps is not None: @@ -118,7 +127,18 @@ def encode_sequence( def decode_sequence( source: str | Path, *, codec: str | Codec | None = None, **options -) -> Sequence | tuple[NeuralGaussianFrame, ...]: - """Decode a mesh Sequence or Vega frames using a named or inferred codec.""" +) -> Sequence | NativeSequence | tuple[NeuralGaussianFrame, ...]: + """Open mesh reconstructions, native temporal state, or legacy Vega frames.""" path = Path(source) + if path.suffix.lower() == ".vmesh" and not path.is_dir(): + detected = probe_codec(path) + if detected is not None: + selected = codec if isinstance(codec, str) else getattr(codec, "id", None) + if selected is not None and selected != detected: + raise CodecError(f".vmesh contains {detected}, but codec={selected!r} was requested") + if "fps" in options: + raise TypeError("O4D .vmesh stores frame timestamps; fps cannot override them") + codec = detected if codec is None else codec + elif codec is None: + codec = "vdmc" return _codec(codec, path).decode(path, **options) diff --git a/open4d/codec/_klt.py b/open4d/codec/_klt.py index 041b6b12..84c75e8b 100644 --- a/open4d/codec/_klt.py +++ b/open4d/codec/_klt.py @@ -3,6 +3,7 @@ from __future__ import annotations from contextlib import redirect_stderr, redirect_stdout +from importlib import import_module import json import os from pathlib import Path @@ -20,6 +21,7 @@ from ._protocol import CodecError from ._research import research_module from ._tsdf import write_tsdf_sequence +from ._torch import torch_device _SCHEMA = "open4d.klt-sequence/v1" @@ -155,6 +157,8 @@ def encode( return destination def decode(self, source: Path, *, device=None) -> Sequence: + if device == "auto": + device = str(torch_device(import_module("torch"), device)) source = Path(source).absolute() temporary = tempfile.TemporaryDirectory(prefix="open4d-klt-decode-") work = Path(temporary.name) diff --git a/open4d/codec/_n4mc.py b/open4d/codec/_n4mc.py index 982f486e..b4f84a5e 100644 --- a/open4d/codec/_n4mc.py +++ b/open4d/codec/_n4mc.py @@ -6,6 +6,7 @@ import json from numbers import Integral from pathlib import Path +import shutil import tempfile import warnings from zipfile import BadZipFile, ZIP_DEFLATED, ZipFile @@ -21,10 +22,40 @@ from ._research import research_module from ._torch import torch_device from ._tsdf import write_tsdf_sequence +from ._v3c import pack_vmesh, probe_codec, unpack_vmesh _SCHEMA = "open4d.n4mc-sequence/v1" +def _extract_n4d(source: Path, destination: Path) -> dict: + """Read the legacy adapter's fixed payloads without loading a model.""" + try: + with ZipFile(source) as archive: + names = archive.namelist() + if len(names) != len(set(names)): + raise CodecError("duplicate N4MC archive member") + if archive.getinfo("manifest.json").file_size > 16 * 1024 * 1024: + raise CodecError("oversized N4MC manifest") + manifest = json.loads(archive.read("manifest.json")) + _validate_manifest(manifest, schema=_SCHEMA, codec="n4mc") + _normalization(manifest) + required = ["checkpoint.pt", *(f"frame_{i:06d}.npz" for i in range(len(manifest["frames"])))] + for name in required: + if archive.getinfo(name).file_size == 0: + raise CodecError(f"empty N4MC payload: {name}") + with archive.open(name) as payload, (destination / name).open("xb") as output: + shutil.copyfileobj(payload, output) + return manifest + except (BadZipFile, KeyError, ValueError, TypeError) as error: + raise CodecError(f"invalid N4MC archive: {error}") from error + + +def _write_native_metadata(manifest: dict, destination: Path) -> None: + metadata = {key: value for key, value in manifest.items() if key != "schema"} + metadata.update(version=1, native={"profile": "n4mc/1"}) + (destination / "metadata.json").write_text(json.dumps(metadata, allow_nan=False), encoding="utf-8") + + def _device(torch, requested): target = torch_device(torch, requested) if target.type != "mps": @@ -89,14 +120,24 @@ def _filter_components(mesh, min_component_faces: int | None): return import_module("trimesh").util.concatenate(kept) +def _reconstruct_mesh(volume, metrics): + mesh = metrics.reconstruct_mesh_from_tsdf(volume) + if mesh is not None: + # The research helper returns [-1, 1]; our TSDF samples [-1.1, 1.1]. + mesh.vertices *= 1.1 + return mesh + + class N4MCCodec: id = "n4mc" - suffixes = (".n4d",) + suffixes = (".n4d", ".vmesh") backend = "python-in-process" lossless = False preserves = ("positions", "triangles") def can_decode(self, source: Path) -> bool: + if Path(source).suffix.lower() == ".vmesh": + return probe_codec(source) == self.id try: with ZipFile(source) as archive: manifest = json.loads(archive.read("manifest.json")) @@ -113,6 +154,8 @@ def encode( destination = Path(destination).absolute() if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") + if destination.suffix.lower() == ".vmesh" and len(sequence) < 2: + raise CodecError("N4MC .vmesh carriage requires at least two frames") for frame in sequence: mesh = frame.geometry if any((mesh.colors is not None, mesh.normals is not None, @@ -179,6 +222,9 @@ def encode( bottleneck_shape=encoded["bottleneck_shape"].cpu().numpy(), ) packs.append(pack) + if destination.suffix.lower() == ".vmesh": + _write_native_metadata(manifest, work) + return pack_vmesh(work, destination, overwrite=overwrite) with tempfile.NamedTemporaryFile( prefix=f".{destination.name}.", suffix=".tmp", dir=destination.parent, delete=False, @@ -200,21 +246,26 @@ def decode( self, source: Path, *, device: str | None = None, min_component_faces: int | None = None, ) -> Sequence: - torch, models, _, metrics = _backend() - target_device = _device(torch, device) + source = Path(source).absolute() temporary = tempfile.TemporaryDirectory(prefix="open4d-n4mc-decode-") work = Path(temporary.name) decoded = None try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - _validate_manifest(manifest, schema=_SCHEMA, codec=self.id) + if source.suffix.lower() == ".vmesh": + detected = probe_codec(source) + if detected != self.id: + raise CodecError(f".vmesh contains {detected!r}, not {self.id}") + native = unpack_vmesh(source, work / "native") + manifest = json.loads((native / "metadata.json").read_text()) + _validate_manifest(manifest, schema=None, codec=self.id) _normalization(manifest) - archive.extract("checkpoint.pt", work) - for ordinal in range(len(manifest["frames"])): - archive.extract(f"frame_{ordinal:06d}.npz", work) + else: + native = work + manifest = _extract_n4d(source, native) + torch, models, _, metrics = _backend() + target_device = _device(torch, device) checkpoint = torch.load( - work / "checkpoint.pt", map_location=target_device, weights_only=True + native / "checkpoint.pt", map_location=target_device, weights_only=True ) model = models.TSDFCompressionAutoencoder( **checkpoint["model_config"] @@ -225,12 +276,12 @@ def decode( output.mkdir() with torch.inference_mode(): for ordinal in range(len(manifest["frames"])): - with np.load(work / f"frame_{ordinal:06d}.npz", allow_pickle=False) as pack: + with np.load(native / f"frame_{ordinal:06d}.npz", allow_pickle=False) as pack: latent = torch.from_numpy(pack["quantized_latent"]).unsqueeze(0).to(target_device) volume = model.decode_quantized_latent( latent, pack["bottleneck_shape"], pack["original_shape"] )[0] - mesh = metrics.reconstruct_mesh_from_tsdf(volume) + mesh = _reconstruct_mesh(volume, metrics) if mesh is None: raise CodecError(f"N4MC frame {ordinal} has no decoded surface") mesh = _filter_components(mesh, min_component_faces) diff --git a/open4d/codec/_native.py b/open4d/codec/_native.py new file mode 100644 index 00000000..44c48b8b --- /dev/null +++ b/open4d/codec/_native.py @@ -0,0 +1,39 @@ +"""Bounded native-tool execution with robust diagnostics.""" + +from collections import deque +import math +import os +import signal +import subprocess +import tempfile + +from ._protocol import CodecError + + +def run(command, label, *, cwd=None): + timeout = float(os.environ.get("OPEN4D_NATIVE_TIMEOUT", "3600")) + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError("OPEN4D_NATIVE_TIMEOUT must be finite and positive") + with tempfile.TemporaryFile(mode="w+", encoding="utf-8", errors="replace") as log: + with subprocess.Popen( + command, cwd=cwd, stdout=log, stderr=subprocess.STDOUT, + start_new_session=os.name != "nt", + ) as process: + try: + process.wait(timeout=timeout) + except BaseException as error: + if os.name != "nt": + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + else: + process.kill() + process.wait() + if isinstance(error, subprocess.TimeoutExpired): + raise CodecError(f"{label} timed out after {timeout:g} seconds") from error + raise + if process.returncode: + log.seek(0) + detail = "".join(deque(log, maxlen=16)).strip() + raise CodecError(f"{label} exited {process.returncode}: {detail or 'no diagnostic output'}") diff --git a/open4d/codec/_native_profiles.py b/open4d/codec/_native_profiles.py new file mode 100644 index 00000000..e5ff1785 --- /dev/null +++ b/open4d/codec/_native_profiles.py @@ -0,0 +1,115 @@ +"""Closed payload layouts for native research representations. + +This module inspects JSON only. It never imports or deserializes a native model. +All paths are generated here, rather than taken from an archive's filenames. +""" +from __future__ import annotations + +from ._protocol import CodecError + +PROFILES = { + "tvmc": ("triangle_mesh", "shared-reference"), + "tsmc": ("triangle_mesh", "whole-group"), + "vega": ("neural_gaussians", "shared-reference"), + "queen": ("gaussian_splats", "previous-frame"), + "3dgstream": ("gaussian_splats", "previous-frame"), + "rerf": ("neural_field", "previous-frame"), + "vdmc": ("triangle_mesh", "native-sequence"), + "faster_vdmc": ("triangle_mesh", "native-sequence"), + # Explicit scope exception: a shared spatial model, with no temporal + # prediction or residual dependency between the quantized frame latents. + "n4mc": ("neural_tsdf", "shared-model-independent-frames"), +} +NEURAL_CODECS = frozenset(("vega", "queen", "3dgstream", "rerf")) + + +def _require(condition, message): + if not condition: + raise CodecError(f"invalid native profile: {message}") + + +def layout(codec, count, native=None): + """Validate a profile and return its exact ordered filename/role pairs.""" + _require(codec in PROFILES, f"unsupported codec {codec!r}") + _require(type(count) is int and count > 0, "invalid frame count") + files = [("metadata.json", "sequence-metadata")] + if codec in ("tvmc", "tsmc"): + _require(native is None, "unexpected tracked-mesh profile") + files.append(("reference.drc", "reference-mesh")) + if codec == "tvmc": + files += [(f"displacement_{i:06d}.{suffix}", role) for i in range(count) + for suffix, role in (("drc", "frame-displacement"), ("npy", "vertex-order"))] + else: + files += [("B_matrix.txt", "trajectory-basis"), ("T_matrix.txt", "trajectory-offset"), + ("delta_trajectories_encoded.npy", "entropy-coded-trajectories"), + ("entropy_model.npz", "entropy-model")] + return files + _require(isinstance(native, dict) and native.get("profile") == f"{codec}/1", "unsupported profile version") + if codec in NEURAL_CODECS or codec == "n4mc": + _require(count >= 2, "native sequence profiles require at least two frames") + if codec == "n4mc": + _require(set(native) == {"profile"}, "unexpected N4MC configuration") + files.append(("checkpoint.pt", "shared-neural-model")) + files += [(f"frame_{i:06d}.npz", "quantized-tsdf-latent") for i in range(count)] + elif codec == "vega": + _require(set(native) == {"profile"}, "unexpected Vega configuration") + files += [("manifest.json", "native-frame-index"), ("color_model.pt", "shared-neural-appearance")] + files += [(f"frame_{i:04d}.pt", "key-state" if i == 0 else "residual-state") for i in range(count)] + elif codec == "queen": + _require(set(native) == {"profile", "sh_degree", "gate_params"}, "unexpected QUEEN configuration") + _require(type(native["sh_degree"]) is int and 0 <= native["sh_degree"] <= 3, "invalid SH degree") + gates = native["gate_params"] + _require(isinstance(gates, list) and len(gates) == 7 and all(g in ("on", "none") for g in gates), "invalid QUEEN gates") + _require(all(g == "none" for g in gates[1:]), "only QUEEN position gating is supported by its native serializer") + files.append(("initial.ply", "initial-gaussians")) + files += [(f"frame_{i:06d}.pkl", "compressed-attribute-residual") for i in range(1, count)] + elif codec == "3dgstream": + _require(set(native) == {"profile", "sh_degree", "rotate_sh", "only_mlp", "added"}, "unexpected 3DGStream configuration") + _require(type(native["sh_degree"]) is int and 0 <= native["sh_degree"] <= 3, "invalid SH degree") + _require(type(native["rotate_sh"]) is bool and type(native["only_mlp"]) is bool, "invalid NTC flags") + _require(not native["rotate_sh"] or native["sh_degree"] == 1, "the native SH rotation requires degree 1") + added = native["added"] + _require(isinstance(added, list) and len(added) == count and all(type(v) is bool for v in added) and not added[0], "invalid added-Gaussian index") + files += [("initial.ply", "initial-gaussians"), ("ntc_config.json", "neural-transform-architecture")] + for i in range(1, count): + files.append((f"ntc_{i:06d}.pth", "neural-transform")) + if added[i]: + files.append((f"added_{i:06d}.ply", "added-gaussians")) + elif codec == "rerf": + _require(set(native) == {"profile", "group_size", "pca", "pca_channels", "frames"}, "unexpected ReRF configuration") + group, pca, frames = native["group_size"], native["pca"], native["frames"] + _require(type(group) is int and group > 0 and type(pca) is bool, "invalid ReRF group/PCA mode") + channels = native["pca_channels"] + _require(isinstance(channels, list) and len(channels) == 2 and all(type(c) is int and c > 0 for c in channels), "invalid PCA channels") + _require(isinstance(frames, list) and len(frames) == count, "invalid ReRF frame index") + files += [("decoder_config.json", "field-configuration"), ("model_kwargs.json", "field-model"), ("rgb_net.tar", "shared-rgb-network")] + previous = None + for frame in frames: + _require(isinstance(frame, dict) and set(frame) == {"id", "quality", "motion", "channels"}, "invalid ReRF frame record") + index, quality, motion = frame["id"], frame["quality"], frame["motion"] + _require(type(index) is int and index >= 0 and (previous is None or index == previous + 1), "ReRF frames must be contiguous") + _require(previous is not None or index % group == 0, "ReRF sequence must start at a group key") + _require(type(quality) is int and 0 <= quality <= 100 and type(motion) is bool, "invalid ReRF quality/motion") + key = index % group == 0 + _require(motion != key, "ReRF residual frames require both motion payloads") + halves = frame["channels"] + _require(isinstance(halves, list) and len(halves) == (2 if not key and pca else 1) + and all(type(n) is int and 0 < n <= 4096 for n in halves), "invalid ReRF entropy channels") + files += [(f"header_{index}.json", "field-frame-header"), (f"mask_{index}.rerf", "occupancy-mask")] + # The native ac_dc_encode2 writes one file per channel; .rerf is + # a prefix, not an aggregate entropy file. + for half, channels_in_half in enumerate(halves): + files += [(f"feature_{index}_{quality-half}.rerf{channel}", "key-features" if key else "residual-features") + for channel in range(channels_in_half)] + if not key and pca: + _require(quality > 0, "PCA residual quality must exceed zero") + files.append((f"pca_m_{index}.npy", "pca-basis")) + if motion: + files += [(f"deform_{index}.npy", "motion-field"), (f"deform_mask_{index}.rerf", "motion-mask")] + previous = index + else: + _require(set(native) == {"profile", "decoder_config"} and type(native["decoder_config"]) is bool, "invalid V-DMC configuration") + files.append(("sequence.vmesh", "native-v3c-sequence")) + if native["decoder_config"]: + files.append(("decoder.cfg", "native-decoder-configuration")) + return files diff --git a/open4d/codec/_native_temporal.py b/open4d/codec/_native_temporal.py new file mode 100644 index 00000000..1e79ca17 --- /dev/null +++ b/open4d/codec/_native_temporal.py @@ -0,0 +1,57 @@ +"""Public codec adapters for native temporal neural research artifacts.""" +from pathlib import Path +import os + +from ._native_profiles import PROFILES +from ._protocol import CodecError +from ._v3c import probe_codec + + +def encode_native(source, destination, *, codec, overwrite=False, **options): + from open4d.native import NativeSequence, import_native, save_native + if isinstance(source, NativeSequence): + if options: + raise TypeError("native repacking does not accept training/import options") + if source.codec != codec: + raise CodecError(f"native source contains {source.codec}, not {codec}") + return save_native(source, destination, overwrite=overwrite) + if isinstance(source, (str, os.PathLike)) and Path(source).suffix.lower() in (".usd", ".usdc", ".usda", ".vmesh"): + from open4d import load + if options: + raise TypeError("native repacking does not accept training/import options") + with load(source) as opened: + if not isinstance(opened, NativeSequence): + raise TypeError(f"{codec} needs a native temporal USD representation; generic mesh USD is not an encoder input") + return encode_native(opened, destination, codec=codec, overwrite=overwrite) + with import_native(source, codec=codec, **options) as imported: + return save_native(imported, destination, overwrite=overwrite) + + +class NativeTemporalCodec: + suffixes = (".vmesh",) + backend = "research-subprocess" + lossless = False + preserves = ("native_temporal_representation", "timestamps", "metadata") + + def __init__(self, identifier): + self.id = identifier + self.representation = PROFILES[identifier][0] + + def can_decode(self, path): + return probe_codec(path) == self.id + + def encode(self, sequence, destination, **options): + return encode_native(sequence, destination, codec=self.id, **options) + + def decode(self, source, *, runtime=None, python=None): + from open4d.native import NativeSequence + result = NativeSequence(source, runtime=runtime, python=python) + if result.codec != self.id: + result.close() + raise CodecError(f"native stream does not contain {self.id}") + return result + + +QUEEN_CODEC = NativeTemporalCodec("queen") +GSTREAM_CODEC = NativeTemporalCodec("3dgstream") +RERF_CODEC = NativeTemporalCodec("rerf") diff --git a/open4d/codec/_tracked.py b/open4d/codec/_tracked.py index c0445ba0..84f586be 100644 --- a/open4d/codec/_tracked.py +++ b/open4d/codec/_tracked.py @@ -2,12 +2,10 @@ from __future__ import annotations -from collections import deque import json import os from pathlib import Path import shutil -import subprocess import sys import tempfile from types import MappingProxyType @@ -19,6 +17,8 @@ from ._npz import _json_value, _validate_manifest from ._protocol import CodecError +from ._native import run as _run_native +from ._v3c import pack_vmesh, probe_codec, unpack_vmesh def _executable(value, label: str) -> str: @@ -36,15 +36,7 @@ def _positive_integer(value, name: str) -> int: def _run(python: str, action: str, request: Path) -> None: worker = Path(__file__).with_name("_tracked_worker.py") - with tempfile.TemporaryFile(mode="w+", encoding="utf-8") as log: - result = subprocess.run( - [python, str(worker), action, str(request)], cwd=request.parent, - stdout=log, stderr=subprocess.STDOUT, text=True, check=False, - ) - if result.returncode: - log.seek(0) - detail = "".join(deque(log, maxlen=16)).strip() - raise CodecError(f"{action} failed (exit {result.returncode}):\n{detail}") + _run_native([python, str(worker), action, str(request)], action, cwd=request.parent) def _manifest(source: Path, codec: str) -> dict: @@ -103,9 +95,12 @@ class TrackedMeshCodec: def __init__(self, identifier): self.id = identifier - self.suffixes = (f".{identifier}",) + self.suffixes = (f".{identifier}", ".vmesh") def can_decode(self, source: Path) -> bool: + source = Path(source) + if source.is_file() and source.suffix.lower() == ".vmesh": + return probe_codec(source) == self.id try: _manifest(source, self.id) except CodecError: @@ -148,11 +143,14 @@ def encode( grid_resolution=512, key_frame=None, components=None, quantization=None, overwrite=False, ) -> Path: - """Encode geometry into a directory of native payloads and timing metadata.""" + """Encode native payloads to a .vmesh file or legacy codec directory.""" destination = Path(destination).absolute() + container = destination.suffix.lower() == ".vmesh" if destination.exists() and not overwrite: raise FileExistsError(f"destination already exists: {destination}") - if destination.exists() and not destination.is_dir(): + if container and destination.is_dir(): + raise IsADirectoryError(destination) + if not container and destination.exists() and not destination.is_dir(): raise NotADirectoryError(destination) if len(sequence) < 2: raise CodecError(f"{self.id} requires at least two frames") @@ -228,6 +226,8 @@ def encode( if not (result / name).is_file() or not (result / name).stat().st_size: raise CodecError(f"{self.id} encoder produced no {name}") (result / "metadata.json").write_text(json.dumps(manifest), encoding="utf-8") + if container: + return pack_vmesh(result, destination, overwrite=overwrite) previous = None if destination.exists(): if not overwrite: @@ -259,14 +259,18 @@ def encode( return destination def decode(self, source: Path, *, backend=None, python=None, decoder=None) -> Sequence: - """Reconstruct frames using only the encoded directory.""" + """Reconstruct frames from a .vmesh file or native codec directory.""" source = Path(source).absolute() - manifest = _manifest(source, self.id) - settings = self._settings(backend, python, None, decoder, encoding=False) temporary = tempfile.TemporaryDirectory(prefix=f"open4d-{self.id}-decode-") decoded = None try: work = Path(temporary.name) + if source.is_file() and source.suffix.lower() == ".vmesh": + if probe_codec(source) != self.id: + raise CodecError(f".vmesh does not contain {self.id} payloads") + source = unpack_vmesh(source, work / "native") + manifest = _manifest(source, self.id) + settings = self._settings(backend, python, None, decoder, encoding=False) output = work / "decoded" output.mkdir() settings.update(input=str(source), output=str(output), frames=len(manifest["frames"])) diff --git a/open4d/codec/_v3c.py b/open4d/codec/_v3c.py new file mode 100644 index 00000000..1eb779b1 --- /dev/null +++ b/open4d/codec/_v3c.py @@ -0,0 +1,346 @@ +"""Experimental O4D native-codec carriage in V3C sample streams. + +This deliberately implements one bounded application format, not a general +V3C parser or an ISO-conforming V-DMC geometry decoder. Usage examples are +in examples/vmesh/. +Native temporal payloads are carried; independent decoded frame sequences are +not an admitted profile. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import struct +import tempfile +import uuid + +from open4d._files import publish_directory, publish_file + +from ._npz import _validate_manifest +from ._protocol import CodecError +from ._native_profiles import PROFILES, layout as _layout + +_UUID = uuid.UUID("e23b8c47-9135-4e34-90ad-421c6c57b63a").bytes +_SCHEMA = "open4d.vmesh.native/1" +_CHUNK = 1024 * 1024 +_MAX_JSON = 16 * 1024 * 1024 +_MAX_UNIT = _MAX_JSON + _MAX_JSON // 255 + 128 +_RECORD = struct.Struct(">BBIQ") # version, kind, file ID, byte offset +_U32 = struct.Struct(">I") +_AD = b"\x08\x00\x00\x00" +_NAL = b"\x56\x01" # atlas PREFIX_NSEI (43), layer 0, temporal_id_plus1 1 + + +def _bootstrap() -> bytes: + """VPS syntax from MPEG V-DMC v14, with an unspecified (type 0) extension.""" + bits = [] + + def put(value, width): + bits.append(f"{value:0{width}b}") + + def ue(value): + encoded = f"{value + 1:b}" + bits.append("0" * (len(encoded) - 1) + encoded) + + # PTL: codec group 1, toolset 0, unconstrained reconstruction. These are + # syntax fields, not a claim of conformance to a standardized toolset. + for value, width in ((0, 1), (1, 7), (0, 8), (255, 8), (0, 1), (0, 8), + (0, 7), (15, 4), (0xfff, 12), (0, 8), (0, 6), + (0, 1), (0, 1)): + put(value, width) + for value, width in ((0, 4), (0, 8), (0, 6), (0, 6)): + put(value, width) # VPS 0, reserved, one atlas, atlas ID 0 + ue(1) # placeholder width/height; no atlas pictures or video components + ue(1) + put(0, 4) # one map + put(0, 4) # auxiliary, occupancy, geometry, attribute video absent + put(1, 1) + put(1, 8) # one extension + extension = _UUID + b"O4D\x01" + ue(3 + len(extension) - 1) + put(0, 8) # VPS_EXT_UNSPECIFIED + put(len(extension), 16) + for byte in extension: + put(byte, 8) + bits.append("1") + binary = "".join(bits) + binary += "0" * (-len(binary) % 8) + vps = b"\x00" * 4 + int(binary, 2).to_bytes(len(binary) // 8, "big") + return b"\x60" + _U32.pack(len(vps)) + vps + + +_BOOTSTRAP = _bootstrap() + + +def _error(message): + return CodecError(f"invalid O4D .vmesh: {message}") + + +def _read(stream, size): + data = stream.read(size) + if len(data) != size: + raise _error("truncated stream") + return data + + +def _json(data): + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError(f"duplicate JSON key {key!r}") + result[key] = value + return result + + def constant(value): + raise ValueError(f"non-finite JSON value {value}") + + try: + return json.loads(data, object_pairs_hook=pairs, parse_constant=constant) + except (ValueError, UnicodeError, RecursionError) as error: + raise _error(f"invalid JSON: {error}") from error + + +def _sequence_metadata(data, codec): + value = _json(data) + try: + if not isinstance(value, dict) or type(value.get("version")) is not int or value["version"] != 1: + raise ValueError("unsupported native metadata version") + _validate_manifest(value, schema=None, codec=codec) + if not value["frames"]: + raise ValueError("empty frame list") + _layout(codec, len(value["frames"]), value.get("native")) + except (ValueError, TypeError, KeyError) as error: + raise _error(f"invalid sequence metadata: {error}") from error + return value + + +def _descriptor(data): + value = _json(data) + if not isinstance(value, dict) or value.get("schema") != _SCHEMA: + raise _error("unsupported application schema/version") + codec, count, files = value.get("codec"), value.get("frame_count"), value.get("files") + if not isinstance(codec, str) or codec not in PROFILES: + raise _error(f"unsupported native codec {codec!r}") + if (value.get("native_version") != 1 or type(value.get("native_version")) is not int + or value.get("representation") != PROFILES[codec][0] + or value.get("dependency_mode") != PROFILES[codec][1]): + raise _error("unsupported native representation/dependencies") + if type(count) is not int or not 1 <= count <= _MAX_JSON // 16: + raise _error("invalid frame count") + expected = _layout(codec, count, value.get("native")) + if not isinstance(files, list) or len(files) != len(expected): + raise _error("invalid native file list") + for index, (record, (name, role)) in enumerate(zip(files, expected)): + if (not isinstance(record, dict) or type(record.get("id")) is not int + or record["id"] != index or record.get("name") != name or record.get("role") != role): + raise _error("unexpected native filename, role or file ID") + size, digest = record.get("size"), record.get("sha256") + if type(size) is not int or not 0 < size < 2**64: + raise _error("invalid native payload size") + if not isinstance(digest, str) or len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest): + raise _error("invalid SHA-256 digest") + if files[0]["size"] > _MAX_JSON: + raise _error("sequence metadata exceeds size limit") + return value + + +def _write_record(stream, kind, file_id, offset, data): + payload = _UUID + _RECORD.pack(1, kind, file_id, offset) + data + size = len(payload) + # Atlas SEI user_data_unregistered is payload type 4 (not video SEI 5). + sei = b"\x04" + b"\xff" * (size // 255) + bytes([size % 255]) + payload + b"\x80" + nal = _NAL + sei + unit = _AD + b"\x60" + _U32.pack(len(nal)) + nal + stream.write(_U32.pack(len(unit))) + stream.write(unit) + + +def _read_record(stream): + size = _U32.unpack(_read(stream, 4))[0] + if not 45 <= size <= _MAX_UNIT: + raise _error("V3C unit size outside limits") + unit = _read(stream, size) + if unit[:5] != _AD + b"\x60" or _U32.unpack(unit[5:9])[0] != size - 9: + raise _error("invalid atlas unit or NAL sample length") + if unit[9:12] != _NAL + b"\x04": + raise _error("expected atlas prefix NSEI user_data_unregistered") + pos, payload_size = 12, 0 + while pos < size: + byte = unit[pos] + payload_size += byte + pos += 1 + if byte != 255: + break + if payload_size < 16 + _RECORD.size or pos + payload_size + 1 != size or unit[-1] != 0x80: + raise _error("invalid SEI payload length or trailing bits") + if unit[pos:pos + 16] != _UUID: + raise _error("unexpected application UUID") + version, kind, file_id, offset = _RECORD.unpack_from(unit, pos + 16) + if version != 1 or kind not in (0, 1, 2): + raise _error("unsupported record version/type") + return kind, file_id, offset, unit[pos + 16 + _RECORD.size:-1] + + +def _start(stream): + if _read(stream, len(_BOOTSTRAP)) != _BOOTSTRAP: + raise _error("unsupported VPS/bootstrap") + kind, file_id, offset, data = _read_record(stream) + if (kind, file_id, offset) != (0, 0, 0) or len(data) > _MAX_JSON: + raise _error("missing or oversized application manifest") + return _descriptor(data), hashlib.sha256(data).digest() + + +def probe_codec(source: str | Path) -> str | None: + """Identify this application format without loading native libraries. + + None means an ordinary/other V3C stream; recognized malformed O4D streams + raise CodecError. This probe does not validate the native payload hashes. + """ + with Path(source).open("rb") as stream: + prefix = stream.read(len(_BOOTSTRAP)) + if prefix != _BOOTSTRAP: + if _UUID in prefix or b"O4D\x01" in prefix: + raise _error("damaged VPS/bootstrap") + return None + stream.seek(0) + descriptor, _ = _start(stream) + return descriptor["codec"] + + +def _consume(source, destination=None): + with Path(source).open("rb") as stream: + descriptor, manifest_hash = _start(stream) + sequence_data = bytearray() + for record in descriptor["files"]: + digest, offset = hashlib.sha256(), 0 + target = (destination / record["name"]).open("xb") if destination is not None else None + try: + while offset < record["size"]: + kind, file_id, position, data = _read_record(stream) + if ((kind, file_id, position) != (1, record["id"], offset) + or not 0 < len(data) <= min(_CHUNK, record["size"] - offset)): + raise _error("missing, reordered or oversized native payload chunk") + digest.update(data) + if target is not None: + target.write(data) + if record["id"] == 0: + sequence_data.extend(data) + offset += len(data) + finally: + if target is not None: + target.close() + if digest.hexdigest() != record["sha256"]: + raise _error(f"SHA-256 mismatch for {record['name']}") + kind, file_id, offset, data = _read_record(stream) + if (kind, file_id, offset, data) != (2, 0, 0, manifest_hash) or stream.read(1): + raise _error("invalid end record or trailing data") + sequence = _sequence_metadata(sequence_data, descriptor["codec"]) + if len(sequence["frames"]) != descriptor["frame_count"]: + raise _error("frame count disagrees with sequence metadata") + if sequence.get("native") != descriptor.get("native"): + raise _error("native profile disagrees with sequence metadata") + return {**descriptor, "sequence": sequence} + + +def inspect_vmesh(source: str | Path) -> dict: + """Validate an O4D .vmesh and return its manifest and sequence metadata. + + Streams and verifies all payload hashes without extracting/deserializing + native data or requiring the research codecs' optional dependencies. + """ + return _consume(source) + + +def pack_vmesh(source: str | Path, destination: str | Path, *, overwrite: bool = False) -> Path: + """Pack an O4D native directory or legacy N4MC .n4d into one .vmesh. + + Required native files are preserved byte for byte. Scratch files and + decoded geometry are excluded. This does not recompress the native data. + """ + if not isinstance(overwrite, bool): + raise TypeError("overwrite must be bool") + source, destination = Path(source).absolute(), Path(destination).absolute() + if destination.suffix.lower() != ".vmesh": + raise ValueError("destination must have a .vmesh extension") + if destination.exists() and not overwrite: + raise FileExistsError(destination) + if destination.is_dir(): + raise IsADirectoryError(destination) + if source.suffix.lower() == ".n4d" and source.is_file(): + if source.is_symlink(): + raise _error("linked N4MC source archive") + from ._n4mc import _extract_n4d, _write_native_metadata + with tempfile.TemporaryDirectory(prefix="open4d-n4mc-repack-") as directory: + native = Path(directory) + metadata = _extract_n4d(source, native) + _write_native_metadata(metadata, native) + return pack_vmesh(native, destination, overwrite=overwrite) + metadata = source / "metadata.json" + if metadata.is_symlink() or not metadata.is_file() or metadata.stat().st_size > _MAX_JSON: + raise _error("missing, linked or oversized metadata.json") + data = metadata.read_bytes() + parsed = _json(data) + codec = parsed.get("codec") if isinstance(parsed, dict) else None + if not isinstance(codec, str) or codec not in PROFILES: + raise _error(f"unsupported native codec {codec!r}") + sequence = _sequence_metadata(data, codec) + descriptor = dict(schema=_SCHEMA, codec=codec, native_version=1, + representation=PROFILES[codec][0], frame_count=len(sequence["frames"]), + dependency_mode=PROFILES[codec][1], files=[]) + if "native" in sequence: + descriptor["native"] = sequence["native"] + for index, (name, role) in enumerate(_layout(codec, len(sequence["frames"]), sequence.get("native"))): + path = source / name + if path.is_symlink() or not path.is_file(): + raise _error(f"missing or linked native payload {name}") + digest, size = hashlib.sha256(), 0 + with path.open("rb") as stream: + while chunk := stream.read(_CHUNK): + digest.update(chunk) + size += len(chunk) + # Preserve the exact metadata we validated even if the source changed. + if index == 0 and digest.digest() != hashlib.sha256(data).digest(): + raise _error("metadata changed during packing") + descriptor["files"].append(dict(id=index, name=name, role=role, size=size, sha256=digest.hexdigest())) + manifest = json.dumps(descriptor, separators=(",", ":"), allow_nan=False).encode("utf-8") + if len(manifest) > _MAX_JSON: + raise _error("application manifest exceeds size limit") + _descriptor(manifest) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f".{destination.name}-", dir=destination.parent) as directory: + temporary = Path(directory) / "stream" + with temporary.open("xb") as output: + output.write(_BOOTSTRAP) + _write_record(output, 0, 0, 0, manifest) + for record in descriptor["files"]: + digest, offset = hashlib.sha256(), 0 + with (source / record["name"]).open("rb") as stream: + while chunk := stream.read(_CHUNK): + digest.update(chunk) + _write_record(output, 1, record["id"], offset, chunk) + offset += len(chunk) + if offset != record["size"] or digest.hexdigest() != record["sha256"]: + raise _error(f"{record['name']} changed during packing") + _write_record(output, 2, 0, 0, hashlib.sha256(manifest).digest()) + publish_file(temporary, destination, overwrite=overwrite) + return destination + + +def unpack_vmesh(source: str | Path, destination: str | Path) -> Path: + """Validate and recover a native directory without decoding geometry. + + The destination must not exist. Partial or corrupt input is never published. + """ + destination = Path(destination).absolute() + if destination.exists(): + raise FileExistsError(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f".{destination.name}-", dir=destination.parent) as directory: + result = Path(directory) / "native" + result.mkdir() + _consume(source, result) + publish_directory(result, destination) + return destination diff --git a/open4d/codec/_vmesh.py b/open4d/codec/_vmesh.py index 2de8feba..a1c627e9 100644 --- a/open4d/codec/_vmesh.py +++ b/open4d/codec/_vmesh.py @@ -8,7 +8,6 @@ from numbers import Real import os from pathlib import Path -import subprocess import tempfile from types import MappingProxyType from zipfile import BadZipFile, ZIP_STORED, ZipFile @@ -21,6 +20,8 @@ from ._npz import _json_value, _publish_file, _validate_manifest from ._protocol import CodecError +from ._native import run as _run +from ._v3c import pack_vmesh, probe_codec, unpack_vmesh _SCHEMA = "open4d.vmesh-sequence/v1" _POSITION_BIT_DEPTH = 12 @@ -74,18 +75,6 @@ def _position_normalization(manifest): return bounds, (1 << bits) - 1 -def _run(command: list[str], label: str) -> None: - result = subprocess.run( - command, shell=False, check=False, capture_output=True, text=True - ) - if result.returncode: - detail = (result.stderr or result.stdout).strip().splitlines() - raise CodecError( - f"{label} exited {result.returncode}: " - f"{detail[-1] if detail else 'no diagnostic output'}" - ) - - class _DecodedProvider: def __init__(self, temporary: tempfile.TemporaryDirectory, decoded: Sequence, manifest: dict): self.temporary = temporary @@ -175,7 +164,7 @@ def close(self) -> None: class VMeshCodec: """Invoke one external V-Mesh process per sequence direction.""" - suffixes = (".v4d",) + suffixes = (".v4d", ".vmesh") backend = "native-sequence" lossless = False preserves = ("positions", "triangles") @@ -185,6 +174,8 @@ def __init__(self, identifier: str) -> None: self._environment = identifier.upper() def can_decode(self, source: Path) -> bool: + if Path(source).suffix.lower() == ".vmesh": + return probe_codec(source) == self.id try: with ZipFile(source) as archive: manifest = json.loads(archive.read("manifest.json")) @@ -270,6 +261,16 @@ def encode( _run(command, f"{self.id} encoder") if not stream.is_file() or not stream.stat().st_size: raise CodecError(f"{self.id} encoder produced no bitstream") + if destination.suffix.lower() == ".vmesh": + # Preserve native V3C bytes plus the timing and normalization + # previously available only in the .v4d ZIP wrapper. + native_manifest = {k: v for k, v in manifest.items() if k != "schema"} + native_manifest.update(version=1, native={"profile": f"{self.id}/1", "decoder_config": bool(decoder_config)}) + (work / "metadata.json").write_text(json.dumps(native_manifest), encoding="utf-8") + if decoder_config: + import shutil + shutil.copyfile(Path(decoder_config).absolute(), work / "decoder.cfg") + return pack_vmesh(work, destination, overwrite=overwrite) with tempfile.NamedTemporaryFile( prefix=f".{destination.name}.", suffix=".tmp", dir=destination.parent, delete=False, @@ -296,7 +297,10 @@ def decode( fps: float | None = None, ) -> Sequence: source = Path(source).absolute() - raw = source.suffix.lower() == ".vmesh" + carried = probe_codec(source) if source.suffix.lower() == ".vmesh" else None + if carried is not None and carried != self.id: + raise CodecError(f".vmesh contains {carried}, not {self.id}") + raw = source.suffix.lower() == ".vmesh" and carried is None if not raw and fps is not None: raise TypeError("fps applies only to manifest-free .vmesh bitstreams") raw_rate = _raw_fps(fps) if raw else None @@ -311,7 +315,13 @@ def decode( work = Path(temporary.name) decoded = None try: - if raw: + if carried: + native = unpack_vmesh(source, work / "native") + manifest = json.loads((native / "metadata.json").read_text()) + _validate_manifest(manifest, schema=None, codec=self.id) + _position_normalization(manifest) + stream = native / "sequence.vmesh" + elif raw: stream = source manifest = None else: @@ -327,7 +337,7 @@ def decode( output.mkdir() # The pinned V-DMC decoder parses decTex even with zero attributes. command = [str(executable)] - embedded_config = work / "decoder.cfg" + embedded_config = (work / "native" if carried else work) / "decoder.cfg" config = configured or ( embedded_config if embedded_config.is_file() diff --git a/open4d/codec/tests/test_codec.py b/open4d/codec/tests/test_codec.py index 2a5b20c3..01ebb9f7 100644 --- a/open4d/codec/tests/test_codec.py +++ b/open4d/codec/tests/test_codec.py @@ -315,7 +315,8 @@ def test_missing_research_source_has_an_actionable_error(tmp_path, monkeypatch): research_module("klt.klt") -def test_vmesh_uses_one_native_call_per_sequence_direction(tmp_path, monkeypatch): +@pytest.mark.parametrize("identifier,suffix", [("native-test", ".v4d"), ("vdmc", ".vmesh"), ("faster_vdmc", ".vmesh")]) +def test_vmesh_uses_one_native_call_per_sequence_direction(tmp_path, monkeypatch, identifier, suffix): import open4d.codec._vmesh as implementation mesh = TriangleMesh([[-2.0, 3, 4], [2, 3, 4], [-2, 7, 4]], [[0, 1, 2]]) @@ -346,9 +347,9 @@ def native_call(command, label): ) monkeypatch.setattr(implementation, "_run", native_call) - codec = VMeshCodec("native-test") + codec = VMeshCodec(identifier) artifact = codec.encode( - clean, tmp_path / "sequence.v4d", encoder=executable, + clean, tmp_path / ("sequence" + suffix), encoder=executable, ) decoded = codec.decode(artifact, decoder=executable) @@ -360,7 +361,7 @@ def native_call(command, label): assert decoded.has_vertex_correspondence is None assert decoded.timestamps == clean.timestamps assert [label for _, label in calls] == [ - "native-test encoder", "native-test decoder" + f"{identifier} encoder", f"{identifier} decoder" ] assert all(isinstance(command, list) for command, _ in calls) assert "--encodeDisplacements=1" in calls[0][0] @@ -420,7 +421,8 @@ def native_call(command, label): assert not decoded_directory.exists() -def test_klt_artifact_fresh_decode_uses_saved_payload(tmp_path, monkeypatch): +@pytest.mark.parametrize("requested", ["cpu", "auto"]) +def test_klt_artifact_fresh_decode_uses_saved_payload(tmp_path, monkeypatch, requested): import open4d.codec._klt as implementation source = Sequence(MemoryFrameProvider([ @@ -449,11 +451,15 @@ def decompress(source, destination, device): ) monkeypatch.setattr(implementation, "write_tsdf_sequence", prepare) + if requested == "auto": + torch = pytest.importorskip("torch") + monkeypatch.setattr(torch.cuda, "is_available", lambda: False) + monkeypatch.setattr(torch.backends.mps, "is_available", lambda: False) monkeypatch.setattr(implementation, "_backend", lambda: SimpleNamespace( run_compression=compress, decode_compressed=decompress, )) artifact = encode_sequence(source, tmp_path / "take.k4d", codec="klt") - decoded = decode_sequence(artifact, device="cpu") + decoded = decode_sequence(artifact, device=requested) assert calls == [ ("prepare", 1, 63), ("encode", 1, False), diff --git a/open4d/codec/tests/test_codec_review.py b/open4d/codec/tests/test_codec_review.py new file mode 100644 index 00000000..c5918bc3 --- /dev/null +++ b/open4d/codec/tests/test_codec_review.py @@ -0,0 +1,49 @@ +import numpy as np +import pytest + + +def test_native_non_utf8_diagnostic_is_a_codec_error(): + import sys + from open4d.codec import CodecError, _vmesh + + with pytest.raises(CodecError, match="exited 2"): + _vmesh._run([sys.executable, "-c", "import os; os.write(2, b'bad\\xff'); exit(2)"], "native") + + +def test_native_timeout_is_bounded_and_explained(monkeypatch): + import sys + from open4d.codec import CodecError, _vmesh + + monkeypatch.setenv("OPEN4D_NATIVE_TIMEOUT", "0.1") + with pytest.raises(CodecError, match="timed out"): + _vmesh._run([sys.executable, "-c", "import time; time.sleep(1)"], "native") + + +def test_n4mc_reconstruction_uses_tsdf_sampling_extent(): + pytest.importorskip("torch") + pytest.importorskip("skimage") + pytest.importorskip("trimesh") + from open4d.codec import _n4mc + + axis = np.linspace(-1.1, 1.1, 48) + grid = np.stack(np.meshgrid(axis, axis, axis, indexing="ij"), axis=-1) + volume = np.linalg.norm(grid, axis=-1) - 0.8 + mesh = _n4mc._reconstruct_mesh(volume, _n4mc._backend()[3]) + radii = np.linalg.norm(mesh.vertices, axis=1) + np.testing.assert_allclose(radii, 0.8, atol=0.002) + + +def test_n4mc_ssim_weight_changes_loss_and_gradients(): + torch = pytest.importorskip("torch") + from open4d.codec._research import research_module + + losses = research_module("n4mc.losses") + prediction = torch.full((1, 1, 8, 8, 8), 0.5, requires_grad=True) + outputs = {"reconstruction": prediction, "rate_bpv": torch.tensor(0.)} + config = dict(lambda_rec=0., lambda_rate=0., lambda_band=0., lambda_sign=0.) + zero, _ = losses.compute_rd_loss(outputs, torch.zeros_like(prediction), {**config, "lambda_ssim": 0.}) + weighted, terms = losses.compute_rd_loss(outputs, torch.zeros_like(prediction), {**config, "lambda_ssim": 0.5}) + assert weighted > zero + assert weighted.item() == pytest.approx(0.5 * terms["ssim_loss"].item()) + weighted.backward() + assert prediction.grad.abs().sum() > 0 diff --git a/open4d/codec/tests/test_n4mc_vmesh.py b/open4d/codec/tests/test_n4mc_vmesh.py new file mode 100644 index 00000000..a37f123f --- /dev/null +++ b/open4d/codec/tests/test_n4mc_vmesh.py @@ -0,0 +1,146 @@ +"""N4MC carries a shared model and independent latents, not temporal residuals.""" +import json +from zipfile import ZipFile + +import numpy as np +import pytest + +import open4d +from open4d.codec import CodecError, inspect_vmesh, pack_vmesh, unpack_vmesh +from open4d.codec import _n4mc + +pytestmark = pytest.mark.cpu + + +def legacy_archive(path, *, omit=None, duplicate=False, normalization=None): + manifest = dict(schema="open4d.n4mc-sequence/v1", codec="n4mc", + normalization=normalization or dict(center=[1, 2, 3], scale=2.), + metadata={"scene": "fixture"}, allow_nonmonotonic_timestamps=False, + frames=[dict(frame_index=7, timestamp=.125, metadata={"key": "first"}), + dict(frame_index=19, timestamp=.875, metadata={"key": "last"})]) + # These bytes only test carriage, never neural reconstruction. + payloads = {"checkpoint.pt": b"opaque shared model", "frame_000000.npz": b"opaque latent 0", + "frame_000001.npz": b"opaque latent 1"} + with ZipFile(path, "w") as archive: + archive.writestr("manifest.json", json.dumps(manifest)) + for name, data in payloads.items(): + if name != omit: + archive.writestr(name, data) + if duplicate: + with pytest.warns(UserWarning, match="Duplicate"): + archive.writestr("checkpoint.pt", b"ambiguous") + archive.writestr("scratch.obj", b"never carry dense meshes") + return manifest, payloads + + +def test_n4d_migration_preserves_native_payloads_without_loading_models(tmp_path, monkeypatch): + source = tmp_path / "legacy.n4d" + manifest, payloads = legacy_archive(source) + monkeypatch.setattr(_n4mc, "_backend", lambda: pytest.fail("repacking must not load a model")) + artifact = pack_vmesh(source, tmp_path / "motion.vmesh") + source.unlink() + assert artifact.read_bytes()[0] == 0x60 + info = inspect_vmesh(artifact) + assert info["codec"] == "n4mc" + assert info["representation"] == "neural_tsdf" + assert info["dependency_mode"] == "shared-model-independent-frames" + assert info["sequence"]["normalization"] == manifest["normalization"] + assert info["sequence"]["frames"] == manifest["frames"] + assert _n4mc.N4MC_CODEC.can_decode(artifact) + recovered = unpack_vmesh(artifact, tmp_path / "recovered") + assert sorted(p.name for p in recovered.iterdir()) == sorted(["metadata.json", *payloads]) + for name, data in payloads.items(): + assert (recovered / name).read_bytes() == data + repacked = pack_vmesh(recovered, tmp_path / "again.vmesh") + assert artifact.read_bytes() == repacked.read_bytes() + with pytest.raises(CodecError, match="contains n4mc"): + open4d.load(artifact, codec="tsmc") + + +@pytest.mark.parametrize("options", [dict(omit="checkpoint.pt"), dict(omit="frame_000001.npz"), + dict(duplicate=True), dict(normalization={"center": [0, 0, 0], "scale": 0})]) +def test_invalid_n4d_never_publishes_vmesh(tmp_path, options): + source = tmp_path / "bad.n4d" + legacy_archive(source, **options) + destination = tmp_path / "result.vmesh" + with pytest.raises(CodecError): + pack_vmesh(source, destination) + assert not destination.exists() + + +def test_n4mc_native_usdc_preserves_compressed_state_without_decoding(tmp_path, monkeypatch): + pytest.importorskip("pxr.Usd") + source = tmp_path / "legacy.n4d" + legacy_archive(source) + artifact = pack_vmesh(source, tmp_path / "motion.vmesh") + monkeypatch.setattr(_n4mc, "_backend", lambda: pytest.fail("native USD must not decode N4MC")) + with open4d.NativeSequence(artifact) as native: + usd = open4d.save(native, tmp_path / "native.usdc") + source.unlink() + expected = artifact.read_bytes() + artifact.unlink() + with open4d.load(usd) as restored: + assert restored.codec == "n4mc" + assert restored.timestamps == (.125, .875) + assert restored.frame_indices == (7, 19) + assert restored.path.read_bytes() == expected + repacked = open4d.save(restored, tmp_path / "restored.vmesh") + assert repacked.read_bytes() == expected + + +def test_corrupt_n4mc_vmesh_fails_before_loading_model(tmp_path, monkeypatch): + source = tmp_path / "original.n4d" + legacy_archive(source) + artifact = pack_vmesh(source, tmp_path / "original.vmesh") + data = artifact.read_bytes() + assert b"opaque shared model" in data + artifact.write_bytes(data.replace(b"opaque shared model", b"broken shared model")) + monkeypatch.setattr(_n4mc, "_backend", lambda: pytest.fail("corrupt carriage must not load a model")) + with pytest.raises(CodecError, match="SHA-256"): + open4d.load(artifact) + + +@pytest.mark.torch +@pytest.mark.slow +def test_real_n4mc_mesh_usdc_vmesh_decode_matches_legacy_latents(tmp_path): + pytest.importorskip("pxr.Usd") + pytest.importorskip("torch") + pytest.importorskip("trimesh") + pytest.importorskip("point_cloud_utils") + pytest.importorskip("skimage") + from open4d.codec.tests.test_research_cpu import moving_cube + + with moving_cube() as source: + usd = open4d.save(source, tmp_path / "source.usdc") + artifact = open4d.encode(usd, tmp_path / "cube.vmesh", codec="n4mc", device="cpu", + resolution=7, epochs=30, hidden_channels=(4, 8), + latent_channels=4, learning_rate=3e-3) + native = unpack_vmesh(artifact, tmp_path / "native") + info = inspect_vmesh(artifact) + # Reconstruct the legacy packaging from exactly the same native model and + # latents, proving the new carrier does not change neural reconstruction. + legacy = tmp_path / "same-latents.n4d" + metadata = info["sequence"] + manifest = {k: v for k, v in metadata.items() if k not in ("version", "native")} + manifest["schema"] = "open4d.n4mc-sequence/v1" + with ZipFile(legacy, "w") as archive: + archive.writestr("manifest.json", json.dumps(manifest)) + for record in info["files"][1:]: + archive.write(native / record["name"], record["name"]) + with open4d.load(artifact, options={"device": "cpu"}) as decoded, open4d.load(legacy, options={"device": "cpu"}) as original: + assert len(decoded) == len(original) == 2 + assert decoded.timestamps == original.timestamps == (1.25, 2.75) + assert decoded.metadata == original.metadata + assert decoded.metadata["capture"] == "synthetic-moving-cube" + for left, right in zip(decoded, original): + assert left.frame_index == right.frame_index + assert left.metadata == right.metadata + assert len(left.geometry.positions) and len(left.geometry.triangles) + np.testing.assert_array_equal(left.geometry.positions, right.geometry.positions) + np.testing.assert_array_equal(left.geometry.triangles, right.geometry.triangles) + recovered = open4d.save(decoded, tmp_path / "decoded.usdc") + with open4d.load(recovered) as reopened: + assert reopened.timestamps == decoded.timestamps + for left, right in zip(reopened, decoded): + np.testing.assert_array_equal(left.geometry.positions, right.geometry.positions) + np.testing.assert_array_equal(left.geometry.triangles, right.geometry.triangles) diff --git a/open4d/codec/tests/test_native_temporal.py b/open4d/codec/tests/test_native_temporal.py new file mode 100644 index 00000000..3d7076da --- /dev/null +++ b/open4d/codec/tests/test_native_temporal.py @@ -0,0 +1,227 @@ +"""Carriage/interchange tests; opaque fixtures do not claim native codec fidelity.""" +import hashlib +import json +from pathlib import Path +import shutil + +import pytest + +import open4d +from open4d.codec import CodecError, inspect_vmesh, pack_vmesh, unpack_vmesh +from open4d.codec._native_profiles import NEURAL_CODECS, layout + +pytestmark = pytest.mark.cpu + + +def test_gstream_additions_are_visible_only_in_their_own_frame(tmp_path, monkeypatch): + import sys + from types import SimpleNamespace + import open4d._native_worker as worker + + torch = pytest.importorskip("torch") + fields = ("_xyz", "_features_dc", "_features_rest", "_opacity", "_scaling", "_rotation") + queried_counts, rendered = [], [] + + class Model: + def __init__(self, *args): + pass + + def load_ply(self, path, scale): + value = 100 + int(Path(path).stem[6:]) if "added_" in path else 0 + for name in fields: + setattr(self, name, torch.tensor([[float(value)]])) + + def query_ntc(self): + queried_counts.append(len(self._xyz)) + + def update_by_ntc(self): + self._xyz = self._xyz + 1 + + class Network: + def __init__(self, **kwargs): + pass + + def cuda(self): + return self + + class Cache: + def __init__(self, *args): + pass + + def load_state_dict(self, state): + pass + + monkeypatch.setitem(sys.modules, "scene.gaussian_model", SimpleNamespace(GaussianModel=Model)) + monkeypatch.setitem(sys.modules, "tinycudann", SimpleNamespace(NetworkWithInputEncoding=Network)) + monkeypatch.setitem(sys.modules, "ntc", SimpleNamespace(NeuralTransformationCache=Cache)) + monkeypatch.setattr(torch, "load", lambda *a, **k: {"xyz_bound_min": 0, "xyz_bound_max": 1}) + monkeypatch.setattr(worker, "_save_gaussians", lambda model, path: rendered.append(model._xyz.clone())) + (tmp_path / "ntc_config.json").write_text(json.dumps({"network": {}, "encoding": {}})) + metadata = {"frames": [{}, {}, {}], "native": { + "sh_degree": 1, "rotate_sh": False, "only_mlp": False, "added": [False, True, True], + }} + + worker._gstream(tmp_path, tmp_path, metadata) + + assert queried_counts == [1, 1] + assert [frame.flatten().tolist() for frame in rendered] == [[0], [1, 101], [2, 102]] + + +def native_run(tmp_path, codec): + root = tmp_path / codec + root.mkdir() + options = {} + if codec == "vega": + (root / "manifest.json").write_text(json.dumps({"frames": [dict(frame_idx=i, group_id=3, frame_type="key" if i == 0 else "residual", file=f"frame_{i:04d}.pt") for i in range(2)]})) + for name in ("color_model.pt", "frame_0000.pt", "frame_0001.pt"): + (root / name).write_bytes(bytes(range(256)) * 5000) + elif codec == "queen": + for i in (1, 2): + frame = root / f"frames/{i:04d}" + (frame / "compressed").mkdir(parents=True) + (frame / "point_cloud.ply").write_bytes(b"opaque dense fixture") + (frame / "compressed/point_cloud.pkl").write_bytes(b"opaque compressed fixture") + options["config"] = dict(sh_degree=1, gate_params=["on"] + ["none"] * 6) + elif codec == "3dgstream": + (root / "init").mkdir() + (root / "init/point_cloud.ply").write_bytes(b"opaque initial fixture") + frame = root / "frame000001" + frame.mkdir() + (frame / "NTC.pth").write_bytes(b"opaque transform fixture") + (frame / "point_cloud.ply").write_bytes(b"must not include this dense frame") + (root / "cfg_args.json").write_text(json.dumps(dict(sh_degree=1, rotate_sh=True, only_mlp=False, iterations_s2=0))) + (root / "ntc_config.json").write_text(json.dumps(dict(network={}, encoding={}))) + else: + config = dict(native=dict(group_size=2, pca=True), voxel_size=8, + fine_model_and_render=dict(rgbnet_dim=12, stepsize=.5), + data=dict(datadir="", white_bkgd=False, inverse_y=True, flip_x=False, flip_y=False), + render=dict(near=.1, far=10.0)) + (root / "decoder_config.json").write_text(json.dumps(config)) + (root / "model_kwargs.json").write_text('{}') + (root / "rgb_net.tar").write_bytes(b"opaque shared network") + for i in (0, 1): + headers = [dict(quality=8 - half, size=[1, 2, 8, 8, 8]) for half in range(i + 1)] + (root / f"header_{i}.json").write_text(json.dumps({"headers": headers})) + for name in (f"mask_{i}.rerf", f"feature_{i}_8.rerf0", f"feature_{i}_8.rerf1"): + (root / name).write_bytes(b"opaque features") + for name in ("feature_1_7.rerf0", "feature_1_7.rerf1", "pca_m_1.npy", "deform_1.npy", "deform_mask_1.rerf"): + (root / name).write_bytes(b"opaque residual dependencies") + return root, options + + +@pytest.mark.parametrize("codec", sorted(NEURAL_CODECS)) +def test_native_import_excludes_dense_exports_and_survives_source_removal(tmp_path, codec): + source, options = native_run(tmp_path, codec) + with open4d.import_native(source, codec=codec, timestamps=[.125, .875], frame_indices=[12, 40], + metadata={"units": "m"}, frame_metadata=[{"label": "key"}, {"label": "later"}], **options) as native: + expected = native.path.read_bytes() + shutil.rmtree(source) + artifact = open4d.save(native, tmp_path / "isolated.vmesh") + assert artifact.read_bytes() == expected + with open4d.load(artifact) as loaded: + assert isinstance(loaded, open4d.NativeSequence) + assert loaded.codec == codec and loaded.timestamps == (.125, .875) + assert loaded.frame_indices == (12, 40) and loaded.metadata == {"units": "m"} + manifest = loaded.manifest + expected_names = [name for name, _ in layout(codec, 2, manifest["native"])] + extracted = loaded.unpack(tmp_path / "native") + assert sorted(p.name for p in extracted.iterdir()) == sorted(expected_names) + assert not any(p.suffix == ".obj" for p in extracted.iterdir()) + assert not (extracted / "frame000001/point_cloud.ply").exists() + repacked = pack_vmesh(extracted, tmp_path / "repacked.vmesh") + assert repacked.read_bytes() == expected + with pytest.raises(ValueError, match="closed"): + loaded.decode() + + +@pytest.mark.parametrize("codec", sorted(NEURAL_CODECS)) +def test_self_contained_usdc_restores_identical_native_bytes_without_runtime(tmp_path, codec, monkeypatch): + pytest.importorskip("pxr.Usd") + import open4d.native as module + monkeypatch.setattr(module, "run", lambda *a, **k: pytest.fail("USD interchange must not execute native models")) + source, options = native_run(tmp_path, codec) + with open4d.import_native(source, codec=codec, timestamps=[1.125, 3.75], frame_indices=[5, 11], **options) as original: + expected = original.path.read_bytes() + usd = open4d.save(original, tmp_path / "take.usdc") + shutil.rmtree(source) + with open4d.load(usd) as restored: + assert restored.timestamps == (1.125, 3.75) + assert restored.frame_indices == (5, 11) + assert restored.path.read_bytes() == expected + output = open4d.save(restored, tmp_path / "again.vmesh") + assert output.read_bytes() == expected + output2 = open4d.encode(usd, tmp_path / "encode.vmesh", codec=codec) + assert output2.read_bytes() == expected + + +@pytest.mark.parametrize("codec", sorted(NEURAL_CODECS)) +def test_public_encode_accepts_native_research_outputs(tmp_path, codec): + source, options = native_run(tmp_path, codec) + result = open4d.encode(source, tmp_path / "encoded.vmesh", codec=codec, fps=12, **options) + assert inspect_vmesh(result)["sequence"]["frames"][1]["timestamp"] == 1 / 12 + with pytest.raises(CodecError, match="contains"): + open4d.load(result, codec="tsmc") + with pytest.raises(TypeError, match="timestamps"): + open4d.load(result, fps=24) + + +@pytest.mark.parametrize("codec,missing", [("queen", "frames/0002/compressed/point_cloud.pkl"), + ("3dgstream", "frame000001/NTC.pth"), + ("rerf", "deform_mask_1.rerf"), + ("rerf", "feature_1_7.rerf1"), + ("rerf", "pca_m_1.npy")]) +def test_dense_frames_cannot_substitute_for_native_dependencies(tmp_path, codec, missing): + source, options = native_run(tmp_path, codec) + (source / missing).unlink() + target = tmp_path / "result.vmesh" + with pytest.raises((CodecError, FileNotFoundError)): + open4d.encode(source, target, codec=codec, **options) + assert not target.exists() + + +def test_usd_rejects_tampered_payload_and_timeline(tmp_path): + Usd = pytest.importorskip("pxr.Usd") + source, options = native_run(tmp_path, "queen") + with open4d.import_native(source, codec="queen", **options) as native: + usd = open4d.save(native, tmp_path / "take.usdc") + stage = Usd.Stage.Open(str(usd)) + prim = stage.GetPrimAtPath("/Open4DNative") + prim.GetAttribute("open4d:frameIndex").Set(999, 0) + stage.GetRootLayer().Save() + with pytest.raises(CodecError, match="timeline"): + open4d.load(usd) + payload = prim.GetAttribute("open4d:payload:chunk000000") + values = payload.Get() + values[100] ^= 1 + payload.Set(values) + stage.GetRootLayer().Save() + with pytest.raises(CodecError, match="SHA-256"): + open4d.load(usd) + + +def test_unrecognized_independent_research_codecs_are_not_native_profiles(tmp_path): + for codec in ("klt", "qndf", "qndf_int8", "obj"): + with pytest.raises(ValueError, match="codec"): + open4d.import_native(tmp_path, codec=codec) + + +def test_native_overwrite_is_atomic_and_explicit(tmp_path): + source, options = native_run(tmp_path, "queen") + with open4d.import_native(source, codec="queen", **options) as native: + target = tmp_path / "keep.vmesh" + target.write_bytes(b"keep") + with pytest.raises(FileExistsError): + open4d.save(native, target) + assert target.read_bytes() == b"keep" + open4d.save(native, target, overwrite=True) + assert target.read_bytes() == native.path.read_bytes() + + +def test_native_usd_rejects_duplicate_times_before_publishing(tmp_path): + pytest.importorskip("pxr.Usd") + source, options = native_run(tmp_path, "queen") + destination = tmp_path / "duplicate.usdc" + with open4d.import_native(source, codec="queen", timestamps=[0, 0], **options) as native: + with pytest.raises(CodecError, match="strictly increasing"): + open4d.save(native, destination) + assert not destination.exists() diff --git a/open4d/codec/tests/test_research_codecs.py b/open4d/codec/tests/test_research_codecs.py index b5d02808..9083e429 100644 --- a/open4d/codec/tests/test_research_codecs.py +++ b/open4d/codec/tests/test_research_codecs.py @@ -96,6 +96,7 @@ def test_research_codecs_fresh_decode_quality_and_export_real_rafa( exported = write_sequence( first, tmp_path / f"{codec}-{input_format}-{output_format}", format=output_format, + allow_lossy=output_format == "stl", ) assert len(open_sequence(exported)) == 2 first.close() diff --git a/open4d/codec/tests/test_tracked.py b/open4d/codec/tests/test_tracked.py index 60421e56..8c0f9bc1 100644 --- a/open4d/codec/tests/test_tracked.py +++ b/open4d/codec/tests/test_tracked.py @@ -301,13 +301,12 @@ def test_tracking_xml_escapes_user_paths(tmp_path): def test_worker_failure_reports_native_diagnostic(tmp_path, monkeypatch): - def failed(command, **kwargs): + def failed(command, label, **kwargs): assert command[1].endswith("_tracked_worker.py") assert kwargs["cwd"] == tmp_path - kwargs["stdout"].write("missing native dependency\n") - return SimpleNamespace(returncode=2) + raise CodecError("missing native dependency") - monkeypatch.setattr(_tracked.subprocess, "run", failed) + monkeypatch.setattr(_tracked, "_run_native", failed) with pytest.raises(CodecError, match="missing native dependency"): _tracked._run(sys.executable, "encode", tmp_path / "request.json") @@ -375,7 +374,8 @@ def test_unrepresentable_entropy_input_is_rejected(tmp_path, delta): _tracked_worker._save_entropy_model(source, model) -def test_tvmc_native_payload_decodes_without_original_geometry(tmp_path): +@pytest.mark.parametrize("container", [False, True], ids=["directory", "vmesh"]) +def test_tvmc_native_payload_decodes_without_original_geometry(tmp_path, container): encoder = os.environ.get("OPEN4D_TEST_TVMC_ENCODER") decoder = os.environ.get("OPEN4D_TEST_TVMC_DECODER") if not encoder or not decoder: @@ -409,10 +409,24 @@ def test_tvmc_native_payload_decodes_without_original_geometry(tmp_path): reference.unlink() for path in tmp_path.glob("*.ply"): path.unlink() - with TVMC_CODEC.decode(encoded, python=sys.executable, decoder=decoder) as decoded: + if container: + from open4d.codec import pack_vmesh + + native = encoded + encoded = pack_vmesh(native, tmp_path / "take.vmesh") + shutil.rmtree(native) + import open4d + + with open4d.decode(encoded, python=sys.executable, decoder=decoder) as decoded: from scipy.spatial import cKDTree for index in range(2): distance, _ = cKDTree(vertices + offsets[index]).query(decoded[index].geometry.positions) assert distance.max() < 0.001 assert decoded.timestamps == (0.0, 1 / 24) + if container: + pytest.importorskip("pxr") + exported = open4d.save(decoded, tmp_path / "reconstructed.usdc") + with open4d.load(exported) as reopened: + assert reopened.timestamps == decoded.timestamps + np.testing.assert_array_equal(reopened[1].geometry.positions, decoded[1].geometry.positions) diff --git a/open4d/codec/tests/test_v3c.py b/open4d/codec/tests/test_v3c.py new file mode 100644 index 00000000..599a8013 --- /dev/null +++ b/open4d/codec/tests/test_v3c.py @@ -0,0 +1,342 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import shutil +import sys + +import numpy as np +import pytest + +import open4d +from open4d.codec import CodecError, decode_sequence, inspect_vmesh, pack_vmesh, unpack_vmesh +from open4d.codec import _api, _tracked, _v3c +from open4d.io._mesh import write_obj + +pytestmark = pytest.mark.cpu + + +def native_directory(root, codec, *, large=False): + source = root / f"native.{codec}" + source.mkdir() + metadata = { + "codec": codec, "version": 1, "metadata": {"subject": "motion", "units": "m"}, + "frames": [{"frame_index": 10, "timestamp": 0.125, "metadata": {"key": True}}, + {"frame_index": 42, "timestamp": 0.375, "metadata": {"key": False}}], + } + (source / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") + # Opaque bytes test carriage; native Draco/ANS validity is tested separately. + (source / "reference.drc").write_bytes(bytes(range(256)) * (5000 if large else 1)) + if codec == "tvmc": + for index in range(2): + (source / f"displacement_{index:06d}.drc").write_bytes(b"displacement" + bytes([index])) + np.save(source / f"displacement_{index:06d}.npy", np.arange(3, dtype=np.uint32)) + else: + np.savetxt(source / "B_matrix.txt", np.arange(12).reshape(2, 6)) + np.savetxt(source / "T_matrix.txt", np.zeros((1, 6))) + np.save(source / "delta_trajectories_encoded.npy", np.array([1234, 5678], dtype=np.uint32)) + np.savez(source / "entropy_model.npz", shape=[3, 2], means=[0., 0.], stds=[1., 1.]) + return source, metadata + + +def records(path): + with path.open("rb") as stream: + stream.seek(len(_v3c._BOOTSTRAP)) + result = [] + while stream.peek(1): + result.append(_v3c._read_record(stream)) + return result + + +def rewrite(path, items): + with path.open("wb") as stream: + stream.write(_v3c._BOOTSTRAP) + for item in items: + _v3c._write_record(stream, *item) + + +@pytest.mark.parametrize("codec", ["tvmc", "tsmc"]) +def test_native_bytes_and_irregular_timing_survive_single_file_carriage(tmp_path, codec): + source, metadata = native_directory(tmp_path, codec, large=True) + expected = {p.name: p.read_bytes() for p in source.iterdir()} + (source / "frame_000.obj").write_text("scratch OBJ must not be packaged") + path = pack_vmesh(source, tmp_path / "take.vmesh") + shutil.rmtree(source) + assert path.is_file() + assert _v3c.probe_codec(path) == codec + manifest = inspect_vmesh(path) + assert manifest["sequence"] == metadata + assert manifest["frame_count"] == 2 + assert manifest["dependency_mode"] == ("shared-reference" if codec == "tvmc" else "whole-group") + for record in manifest["files"]: + assert hashlib.sha256(expected[record["name"]]).hexdigest() == record["sha256"] + recovered = unpack_vmesh(path, tmp_path / "recovered") + assert {p.name: p.read_bytes() for p in recovered.iterdir()} == expected + with pytest.raises(FileExistsError): + unpack_vmesh(path, recovered) + + +def test_wire_is_v3c_atlas_sei_with_no_private_top_level_units(tmp_path): + source, _ = native_directory(tmp_path, "tvmc") + data = pack_vmesh(source, tmp_path / "take.vmesh").read_bytes() + # Independent framing walk: V3C four-byte lengths; VPS then AD; each AD + # has an atlas NAL sample stream containing user_data_unregistered SEI. + assert data[0] == 0x60 + pos, types, messages = 1, [], [] + while pos < len(data): + size = int.from_bytes(data[pos:pos + 4], "big") + unit = data[pos + 4:pos + 4 + size] + assert len(unit) == size + pos += 4 + size + kind = int.from_bytes(unit[:4], "big") >> 27 + types.append(kind) + if kind == 0: + assert len(types) == 1 + assert unit[:4] == bytes(4) + assert b"O4D\x01" in unit + continue + assert unit[:5] == b"\x08\x00\x00\x00\x60" + assert int.from_bytes(unit[5:9], "big") == len(unit) - 9 + assert unit[9:12] == b"\x56\x01\x04" + index, length = 12, 0 + while unit[index] == 255: + length += 255 + index += 1 + length += unit[index] + payload = unit[index + 1:-1] + assert len(payload) == length and unit[-1] == 128 + assert payload[:16].hex() == "e23b8c4791354e3490ad421c6c57b63a" + assert payload[16] == 1 + messages.append(payload[17]) + assert pos == len(data) and types == [0] + [1] * (len(types) - 1) + assert messages == [0] + [1] * 6 + [2] + assert b"PK\x03\x04" not in data # TVMC fixture contains no ZIP data + + +@pytest.mark.parametrize("length", [224, 225, 226, 480, 481, 1024 * 1024, 1024 * 1024 + 1]) +def test_sei_extended_lengths_and_chunk_boundaries(tmp_path, length): + source, _ = native_directory(tmp_path, "tvmc") + payload = bytes(range(256)) * (length // 256) + bytes(range(length % 256)) + (source / "reference.drc").write_bytes(payload) + path = pack_vmesh(source, tmp_path / "take.vmesh") + chunks = [item for item in records(path) if item[0] == 1 and item[1] == 1] + assert b"".join(item[3] for item in chunks) == payload + assert all(len(item[3]) <= 1024 * 1024 for item in chunks) + recovered = unpack_vmesh(path, tmp_path / "native") + assert (recovered / "reference.drc").read_bytes() == payload + + +@pytest.mark.parametrize("damage", ["truncated", "no-end", "trailing", "hash", "order", "offset", + "bootstrap", "huge-unit", "wrong-nal", "wrong-end"]) +def test_malformed_stream_never_publishes_or_launches_native_decode(tmp_path, monkeypatch, damage): + source, _ = native_directory(tmp_path, "tvmc") + path = pack_vmesh(source, tmp_path / "take.vmesh") + items = records(path) + if damage == "no-end": + rewrite(path, items[:-1]) + elif damage in ("hash", "order", "offset", "wrong-end"): + if damage == "hash": + kind, file_id, offset, blob = items[2] + items[2] = kind, file_id, offset, bytes([blob[0] ^ 1]) + blob[1:] + elif damage == "order": + items[1], items[2] = items[2], items[1] + elif damage == "offset": + kind, file_id, offset, blob = items[2] + items[2] = kind, file_id, offset + 1, blob + else: + items[-1] = (2, 0, 0, bytes(32)) + rewrite(path, items) + else: + data = bytearray(path.read_bytes()) + if damage == "truncated": + del data[-1] + elif damage == "trailing": + data.extend(b"junk") + elif damage == "bootstrap": + data[0] ^= 1 + elif damage == "huge-unit": + data[len(_v3c._BOOTSTRAP):len(_v3c._BOOTSTRAP) + 4] = b"\xff" * 4 + else: + data[len(_v3c._BOOTSTRAP) + 4 + 9] ^= 2 + path.write_bytes(data) + monkeypatch.setattr(_tracked, "_run", lambda *args: pytest.fail("native worker called on corrupt input")) + with pytest.raises(CodecError): + inspect_vmesh(path) + with pytest.raises(CodecError): + unpack_vmesh(path, tmp_path / "recovered") + with pytest.raises(CodecError): + decode_sequence(path, codec="tvmc") + assert not (tmp_path / "recovered").exists() + assert not list(tmp_path.glob(".recovered-*")) + + +@pytest.mark.parametrize("change", ["path", "duplicate", "size", "version", "codec", "count", "mode", "metadata"]) +def test_manifest_rejects_unsafe_or_inconsistent_native_descriptions(tmp_path, change): + source, _ = native_directory(tmp_path, "tvmc") + path = pack_vmesh(source, tmp_path / "take.vmesh") + items = records(path) + manifest = json.loads(items[0][3]) + if change == "path": + manifest["files"][1]["name"] = "../escaped.drc" + elif change == "duplicate": + manifest["files"][1] = manifest["files"][0] + elif change == "size": + manifest["files"][1]["size"] = -1 + elif change == "version": + manifest["schema"] = "open4d.vmesh.native/2" + elif change == "codec": + manifest["codec"] = "obj-sequence" + elif change == "count": + manifest["frame_count"] = True + elif change == "mode": + manifest["dependency_mode"] = "independent-frames" + else: + sequence = json.loads(items[1][3]) + sequence["frames"][1]["timestamp"] = float("nan") + payload = json.dumps(sequence).encode() + items[1] = (1, 0, 0, payload) + manifest["files"][0].update(size=len(payload), sha256=hashlib.sha256(payload).hexdigest()) + blob = json.dumps(manifest).encode() + items[0], items[-1] = (0, 0, 0, blob), (2, 0, 0, hashlib.sha256(blob).digest()) + rewrite(path, items) + with pytest.raises(CodecError): + unpack_vmesh(path, tmp_path / "recovered") + assert not (tmp_path / "escaped.drc").exists() + + +def test_pack_preserves_existing_file_on_failure_and_requires_explicit_overwrite(tmp_path, monkeypatch): + source, _ = native_directory(tmp_path, "tsmc") + path = tmp_path / "take.vmesh" + path.write_bytes(b"original") + with pytest.raises(FileExistsError): + pack_vmesh(source, path) + original_writer = _v3c._write_record + + def failing_writer(stream, kind, file_id, offset, data): + if kind == 1: + raise OSError("disk full") + original_writer(stream, kind, file_id, offset, data) + + monkeypatch.setattr(_v3c, "_write_record", failing_writer) + with pytest.raises(OSError, match="disk full"): + pack_vmesh(source, path, overwrite=True) + assert path.read_bytes() == b"original" + assert not list(tmp_path.glob(".take.vmesh-*")) + monkeypatch.setattr(_v3c, "_write_record", original_writer) + pack_vmesh(source, path, overwrite=True) + assert inspect_vmesh(path)["codec"] == "tsmc" + + +@pytest.mark.parametrize("case", ["missing", "symlink", "empty"]) +def test_pack_requires_self_contained_native_payload(tmp_path, case): + source, _ = native_directory(tmp_path, "tsmc") + path = source / "entropy_model.npz" + path.unlink() + if case == "symlink": + path.symlink_to(source / "reference.drc") + elif case == "empty": + path.touch() + with pytest.raises(CodecError): + pack_vmesh(source, tmp_path / "take.vmesh") + assert not (tmp_path / "take.vmesh").exists() + + +@pytest.mark.parametrize("codec", ["tvmc", "tsmc"]) +def test_dispatch_selects_embedded_codec_and_rejects_conflicts(tmp_path, monkeypatch, codec): + source, _ = native_directory(tmp_path, codec) + path = pack_vmesh(source, tmp_path / "take.vmesh") + calls = [] + implementation = _api._CODECS[codec] + monkeypatch.setattr(implementation, "decode", lambda source, **kw: calls.append((source, kw))) + open4d.load(path) + decode_sequence(path, codec=codec) + assert calls == [(path, {}), (path, {})] + with pytest.raises(CodecError, match="contains"): + decode_sequence(path, codec="vdmc") + with pytest.raises(TypeError, match="timestamps"): + open4d.load(path, fps=24) + + +def test_raw_vdmc_vmesh_keeps_existing_decoder_and_fps(tmp_path, monkeypatch): + path = tmp_path / "raw.vmesh" + path.write_bytes(b"ordinary VDMC stream (decoder stub)") + calls = [] + monkeypatch.setattr(_api.VDMC_CODEC, "decode", lambda source, **kw: calls.append((source, kw))) + open4d.load(path, fps=24) + assert calls == [(path, {"fps": 24})] + + +def test_tsmc_real_entropy_stream_decodes_after_packing_and_removing_source(tmp_path): + constriction = pytest.importorskip("constriction") + from open4d.codec import _tracked_worker + + source, _ = native_directory(tmp_path, "tsmc") + delta = np.array([[0.125, 4.0], [-0.75, 4.0], [0.33333, 4.0]]) + original = tmp_path / "original.npy" + np.save(original, delta) + model_path = source / "entropy_model.npz" + _tracked_worker._save_entropy_model(original, model_path) + expected = np.round(delta * 10000).astype(np.int32) + with np.load(model_path, allow_pickle=False) as model: + distribution = constriction.stream.model.QuantizedGaussian(int(model["minimum"]), int(model["maximum"])) + encoder = constriction.stream.stack.AnsCoder() + encoder.encode_reverse(expected.ravel(), distribution, + np.tile(model["means"], 3), np.tile(model["stds"], 3)) + np.save(source / "delta_trajectories_encoded.npy", encoder.get_compressed()) + path = pack_vmesh(source, tmp_path / "tsmc.vmesh") + original.unlink() + shutil.rmtree(source) + recovered = unpack_vmesh(path, tmp_path / "recovered") + actual = _tracked_worker._decode_entropy(recovered / "delta_trajectories_encoded.npy", + recovered / "entropy_model.npz") + np.testing.assert_array_equal(actual, expected / 10000) + + +@pytest.mark.parametrize("codec", ["tvmc", "tsmc"]) +def test_usdc_vmesh_usdc_api_round_trip_with_native_worker_stub(tmp_path, monkeypatch, codec): + pytest.importorskip("pxr.Usd") + source, metadata = native_directory(tmp_path, codec) + expected = {p.name: p.read_bytes() for p in source.iterdir() if p.name != "metadata.json"} + sequence = open4d.Sequence(open4d.MemoryFrameProvider([ + open4d.Frame(record["frame_index"], record["timestamp"], open4d.TriangleMesh( + [[float(i), 0, 0], [i + 1.0, 0, 0], [float(i), 1, 0]], [[0, 1, 2]]), record["metadata"]) + for i, record in enumerate(metadata["frames"]) + ], metadata=metadata["metadata"])) + usd = open4d.save(sequence, tmp_path / "input.usdc") + calls = [] + + def run(python, action, request): + settings = json.loads(request.read_text()) + calls.append(settings) + output = Path(settings["output"]) + if action == "encode": + for name, payload in expected.items(): + (output / name).write_bytes(payload) + else: + assert {p.name: p.read_bytes() for p in Path(settings["input"]).iterdir() + if p.name != "metadata.json"} == expected + for i in range(settings["frames"]): + write_obj(output / f"frame_{i:06d}.obj", sequence[i].geometry.positions, + sequence[i].geometry.triangles) + + monkeypatch.setattr(_tracked, "_run", run) + monkeypatch.setattr(_api._CODECS[codec], "_settings", lambda *a, **kw: {"codec": codec, "python": sys.executable}) + with open4d.load(usd) as opened: + path = open4d.save(opened, tmp_path / "take.vmesh", codec=codec, options={"dotnet": sys.executable}) + timestamps, indices = opened.timestamps, [f.frame_index for f in opened] + input_metadata = dict(opened.metadata) + shutil.rmtree(source) + usd.unlink() + with open4d.load(path) as decoded: + assert decoded.timestamps == timestamps + assert [f.frame_index for f in decoded] == indices + assert decoded.metadata == input_metadata + assert decoded[1].metadata == sequence[1].metadata + reconstructed = open4d.save(decoded, tmp_path / "output.usdc") + private = Path(calls[-1]["input"]).parent + assert private.exists() + assert not private.exists() + with open4d.load(reconstructed) as result: + assert result.timestamps == timestamps + np.testing.assert_allclose(result[1].geometry.positions, sequence[1].geometry.positions) diff --git a/open4d/codecs/n4mc/README.md b/open4d/codecs/n4mc/README.md index 9e764ce6..d05c01ae 100644 --- a/open4d/codecs/n4mc/README.md +++ b/open4d/codecs/n4mc/README.md @@ -128,3 +128,22 @@ python -m evaluation.restore_sequence \ --normalization datasets/basketball_normalized/normalization.npz \ --output-dir outputs/basketball_sequence_n4mc/original_scale ``` + +## O4D `.vmesh` carriage + +The public N4MC adapter can write `.vmesh` as well as `.n4d`: + +```python +import open4d +from open4d.codec import pack_vmesh + +open4d.encode("input.usdc", "output.vmesh", codec="n4mc", device="cpu") +with open4d.load("output.vmesh", options={"device": "cpu"}) as decoded: + open4d.save(decoded, "reconstructed.usdc") +pack_vmesh("existing.n4d", "migrated.vmesh") +``` + +This carries the shared checkpoint and independent quantized TSDF latents, +including normalization and timing. It does not add temporal prediction. +See the [mesh codec notebook](../../../examples/vmesh/02_mesh_codecs.ipynb) for the explicit native USDC +route that preserves compressed bytes without neural evaluation. diff --git a/open4d/codecs/n4mc/losses/tsdf_losses.py b/open4d/codecs/n4mc/losses/tsdf_losses.py index a950b4be..c6d797cd 100644 --- a/open4d/codecs/n4mc/losses/tsdf_losses.py +++ b/open4d/codecs/n4mc/losses/tsdf_losses.py @@ -119,14 +119,13 @@ def compute_rd_loss(outputs: dict, target: torch.Tensor, config: dict) -> tuple[ target=target, temperature=float(config.get("sign_temperature", 0.1)), ) - ''' + lambda_ssim = float(config.get("lambda_ssim", 0.1)) ssim = ssim_loss( prediction=prediction, target=target, window_size=int(config.get("ssim_window_size", 5)), sigma=float(config.get("ssim_sigma", 1.5)), - ) - ''' + ) if lambda_ssim else prediction.new_zeros(()) rate = outputs["rate_bpv"].float() total = ( @@ -134,7 +133,7 @@ def compute_rd_loss(outputs: dict, target: torch.Tensor, config: dict) -> tuple[ + float(config.get("lambda_rec", 1.0)) * rec + float(config.get("lambda_band", 1.0)) * band + float(config.get("lambda_sign", 0.1)) * sign - #+ float(config.get("lambda_ssim", 0.1)) * ssim + + lambda_ssim * ssim ) return total, { "total_loss": total.detach(), @@ -142,5 +141,5 @@ def compute_rd_loss(outputs: dict, target: torch.Tensor, config: dict) -> tuple[ "rec_loss": rec.detach(), "band_loss": band.detach(), "sign_loss": sign.detach(), - #"ssim_loss": ssim.detach(), + "ssim_loss": ssim.detach(), } diff --git a/open4d/codecs/n4mc/tests/test_losses.py b/open4d/codecs/n4mc/tests/test_losses.py index 145adad9..977ece7b 100644 --- a/open4d/codecs/n4mc/tests/test_losses.py +++ b/open4d/codecs/n4mc/tests/test_losses.py @@ -1,6 +1,8 @@ import torch -from losses.tsdf_losses import compute_rd_loss, narrow_band_loss, sign_consistency_loss, ssim_loss +from open4d.codecs.n4mc.losses.tsdf_losses import ( + compute_rd_loss, narrow_band_loss, sign_consistency_loss, ssim_loss, +) def test_narrow_band_loss_focuses_on_surface(): diff --git a/open4d/gaussians.py b/open4d/gaussians.py index 4f9640de..1d757484 100644 --- a/open4d/gaussians.py +++ b/open4d/gaussians.py @@ -8,7 +8,7 @@ import subprocess import sys import tempfile -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Sequence @@ -373,6 +373,7 @@ class VegaRun: path: Path runtime: Path python: str + _owner: object = field(default=None, repr=False, compare=False) def __post_init__(self) -> None: object.__setattr__(self, "path", Path(self.path).expanduser().resolve()) @@ -466,16 +467,41 @@ def decode_gaussians( class _VegaCodec: id = "vega" - suffixes = (".vega",) + suffixes = (".vega", ".vmesh") representation = "gaussian_splats" backend = "research-subprocess" lossless = False preserves = ("positions", "scales", "rotations", "opacities", "neural_appearance") + def can_decode(self, source): + if Path(source).suffix.lower() == ".vmesh": + from .codec._v3c import probe_codec + return probe_codec(source) == self.id + return Path(source).is_dir() and (Path(source) / "manifest.json").is_file() + def encode(self, sequence, destination: Path, **options) -> Path: + if destination.suffix.lower() == ".vmesh": + from .native import NativeSequence, import_native, save_native + from .codec._native_temporal import encode_native + if isinstance(sequence, (NativeSequence, VegaRun, str, os.PathLike)): + return encode_native(sequence, destination, codec="vega", **options) + overwrite = options.pop("overwrite", False) + if destination.exists() and not overwrite: + raise FileExistsError(destination) + timeline = {key: options.pop(key) for key in ("fps", "timestamps", "frame_indices", "metadata", "frame_metadata") if key in options} + with tempfile.TemporaryDirectory(prefix="open4d-vega-encode-") as folder: + encoded = encode_gaussians(sequence, Path(folder) / "native", **options) + with import_native(encoded, **timeline) as native: + return save_native(native, destination, overwrite=overwrite) + if "overwrite" in options: + if options.pop("overwrite"): + raise ValueError("Vega directory encoding does not support overwrite; use a new directory or .vmesh") return encode_gaussians(sequence, destination, **options).path def decode(self, source: Path, **options) -> tuple[NeuralGaussianFrame, ...]: + if source.suffix.lower() == ".vmesh": + from .codec._native_temporal import NativeTemporalCodec + return NativeTemporalCodec("vega").decode(source, **options) return decode_gaussians(source, **options) diff --git a/open4d/io/_api.py b/open4d/io/_api.py index a0b3c297..2037a429 100644 --- a/open4d/io/_api.py +++ b/open4d/io/_api.py @@ -18,7 +18,7 @@ import numpy as np -from open4d.core import Frame, Sequence, TopologyMode, TriangleMesh +from open4d.core import Frame, Representation, Sequence, TopologyMode, TriangleMesh from open4d._files import publish_directory, publish_file from . import _mesh, _usd @@ -491,10 +491,20 @@ def _write_frame( path: Path, frame: Frame, suffix: str, *, allow_lossy: bool = False ) -> Path: mesh = frame.geometry + if mesh.representation is Representation.GAUSSIANS: + raise UnsupportedFeatureError( + "Gaussian scales, rotations and opacities cannot be preserved by mesh export" + ) # Not every representation has connectivity: a PointCloud carries positions # and nothing to join them with, and `getattr` rather than attribute access # is what lets one writer serve both without asking which it has. triangles = getattr(mesh, "triangles", None) + if suffix in {".stl", ".glb", ".gltf"} and (triangles is None or not len(triangles)): + raise UnsupportedFeatureError(f"{suffix} export requires triangles") + if suffix == ".stl" and not allow_lossy: + raise UnsupportedFeatureError( + "STL drops unused vertices and vertex correspondence; pass allow_lossy=True" + ) present = {"positions"} if triangles is not None: present.add("triangles") @@ -611,9 +621,9 @@ def write_sequence( "geometry_kind": "triangle_mesh", "format": suffix.lstrip("."), "metadata": _json_value(sequence.metadata, "sequence"), - "topology": sequence.topology.value, - "has_constant_vertex_count": sequence.has_constant_vertex_count, - "has_vertex_correspondence": sequence.has_vertex_correspondence, + "topology": TopologyMode.UNKNOWN.value if suffix == ".stl" else sequence.topology.value, + "has_constant_vertex_count": None if suffix == ".stl" else sequence.has_constant_vertex_count, + "has_vertex_correspondence": False if suffix == ".stl" else sequence.has_vertex_correspondence, "allow_nonmonotonic_timestamps": sequence.allow_nonmonotonic_timestamps, "frames": [], } diff --git a/open4d/io/_native_usd.py b/open4d/io/_native_usd.py new file mode 100644 index 00000000..80f1face --- /dev/null +++ b/open4d/io/_native_usd.py @@ -0,0 +1,137 @@ +"""Self-contained O4D native temporal data in USD, without executable loading. + +The authored Xform is an O4D application schema, not a standard renderable +Gaussian or neural-field prim. Compressed bytes survive USDC round trips. +""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import tempfile + +import numpy as np + +from open4d._files import publish_file +from open4d.codec._protocol import CodecError +from open4d.codec._v3c import _json, inspect_vmesh +from ._usd import _pxr + +_PRIM = "/Open4DNative" +_SCHEMA = "open4d.usd-native-temporal/1" +_CHUNK = 1024 * 1024 + + +def _open(path): + Sdf, Usd, _, _ = _pxr() + from pxr import Tf + from ._errors import DecodeError + try: + layer = Sdf.Layer.OpenAsAnonymous(str(Path(path).absolute())) + except Tf.ErrorException as error: + raise DecodeError(f"cannot read USD layer {path}: {error}") from error + if layer is None: + raise CodecError(f"cannot read USD layer {path}") + stage = Usd.Stage.Open(layer) + return stage, stage.GetPrimAtPath(_PRIM) + + +def is_native_usd(path): + _, prim = _open(path) + return bool(prim and prim.HasAttribute("open4d:nativeSchema")) + + +def write_native_usd(sequence, destination, *, overwrite=False): + sequence._check_open() + destination = Path(destination).absolute() + if destination.suffix.lower() not in (".usd", ".usda", ".usdc"): + raise ValueError("native USD interchange supports .usd, .usda and .usdc") + if destination.exists() and not overwrite: + raise FileExistsError(destination) + Sdf, Usd, UsdGeom, Vt = _pxr() + descriptor = inspect_vmesh(sequence.path) + if any(right <= left for left, right in zip(sequence.timestamps, sequence.timestamps[1:])): + raise CodecError("native USD export requires strictly increasing timestamps") + size = sequence.path.stat().st_size + count = (size + _CHUNK - 1) // _CHUNK + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f".{destination.name}-", dir=destination.parent) as directory: + temporary = Path(directory) / destination.name + stage = Usd.Stage.CreateNew(str(temporary)) + prim = UsdGeom.Xform.Define(stage, _PRIM).GetPrim() + stage.SetDefaultPrim(prim) + stage.SetTimeCodesPerSecond(1.0) + stage.SetStartTimeCode(sequence.timestamps[0]) + stage.SetEndTimeCode(sequence.timestamps[-1]) + prim.CreateAttribute("open4d:nativeSchema", Sdf.ValueTypeNames.String).Set(_SCHEMA) + frame = prim.CreateAttribute("open4d:frameIndex", Sdf.ValueTypeNames.Int64) + for index, timestamp in zip(sequence.frame_indices, sequence.timestamps): + frame.Set(index, timestamp) + digest = hashlib.sha256() + with sequence.path.open("rb") as source: + for index in range(count): + data = source.read(_CHUNK) + digest.update(data) + prim.CreateAttribute(f"open4d:payload:chunk{index:06d}", Sdf.ValueTypeNames.UCharArray).Set( + Vt.UCharArray.FromNumpy(np.frombuffer(data, dtype=np.uint8))) + if source.read(1): + raise CodecError("native stream changed while exporting USD") + header = dict(codec=sequence.codec, bytes=size, chunks=count, sha256=digest.hexdigest(), + representation=descriptor["representation"]) + prim.CreateAttribute("open4d:payloadManifest", Sdf.ValueTypeNames.String).Set(json.dumps(header, separators=(",", ":"))) + stage.GetRootLayer().Save() + del stage + # Verify the USD actually contains the original stream before publishing. + with read_native_usd(temporary) as recovered: + if recovered.path.stat().st_size != size: + raise CodecError("USD native payload size changed") + publish_file(temporary, destination, overwrite=overwrite) + return destination + + +def read_native_usd(source, *, runtime=None, python=None): + from open4d.native import NativeSequence + + stage, prim = _open(source) + if not prim or prim.GetAttribute("open4d:nativeSchema").Get() != _SCHEMA: + raise CodecError("unsupported O4D native USD schema") + raw = prim.GetAttribute("open4d:payloadManifest").Get() + if not isinstance(raw, str) or len(raw) > 1024 * 1024: + raise CodecError("invalid native USD payload manifest") + header = _json(raw) + if not isinstance(header, dict): + raise CodecError("invalid native USD payload manifest") + count, size = header.get("chunks"), header.get("bytes") + if (type(count) is not int or not 1 <= count <= 65536 or type(size) is not int + or not (count - 1) * _CHUNK < size <= count * _CHUNK): + raise CodecError("invalid native USD payload bounds") + chunks = sorted(a.GetName() for a in prim.GetAttributes() if a.GetName().startswith("open4d:payload:chunk")) + if chunks != [f"open4d:payload:chunk{i:06d}" for i in range(count)]: + raise CodecError("missing or extra native USD chunks") + temporary = tempfile.TemporaryDirectory(prefix="open4d-native-usd-") + try: + path = Path(temporary.name) / "sequence.vmesh" + digest = hashlib.sha256() + with path.open("xb") as stream: + for index in range(count): + attribute = prim.GetAttribute(chunks[index]) + if str(attribute.GetTypeName()) != "uchar[]" or attribute.GetNumTimeSamples(): + raise CodecError("native USD chunks must be static uchar arrays") + data = bytes(attribute.Get()) + if len(data) != min(_CHUNK, size - index * _CHUNK): + raise CodecError("native USD chunk size mismatch") + digest.update(data) + stream.write(data) + if digest.hexdigest() != header.get("sha256"): + raise CodecError("native USD payload SHA-256 mismatch") + result = NativeSequence(path, temporary=temporary, runtime=runtime, python=python) + if result.codec != header.get("codec") or result.representation != header.get("representation"): + raise CodecError("native USD profile disagrees with carried stream") + frame = prim.GetAttribute("open4d:frameIndex") + if (stage.GetTimeCodesPerSecond() != 1.0 or tuple(frame.GetTimeSamples()) != result.timestamps + or tuple(frame.Get(t) for t in result.timestamps) != result.frame_indices): + raise CodecError("USD timeline disagrees with native temporal payload") + return result + except BaseException: + temporary.cleanup() + raise diff --git a/open4d/io/_usd.py b/open4d/io/_usd.py index e015666e..6225a046 100644 --- a/open4d/io/_usd.py +++ b/open4d/io/_usd.py @@ -11,6 +11,7 @@ import tempfile from types import MappingProxyType from typing import Any +from uuid import uuid4 import numpy as np @@ -175,7 +176,13 @@ def __init__( self.path = Path(path).absolute() self._Sdf, self._Usd, self._UsdGeom, self._Vt = _pxr() try: - self._stage = self._Usd.Stage.Open(str(self.path)) + # File-format arguments are part of Sdf's layer identity. A unique + # reader key bypasses its path cache while retaining the real path + # for relative assets and leaving existing readers undisturbed. + layer = self._Sdf.Layer.FindOrOpen( + str(self.path), {"open4d_reader": uuid4().hex} + ) + self._stage = self._Usd.Stage.Open(layer) if layer is not None else None except Exception as error: raise DecodeError(f"USD could not open {self.path}: {error}") from error if self._stage is None: @@ -492,6 +499,8 @@ def _preflight(sequence: Sequence) -> tuple[dict[str, Any], list[dict[str, Any]] if frame.frame_index > np.iinfo(np.int64).max: raise EncodeError("USD frame indices must fit a signed 64-bit integer") mesh = frame.geometry + if not isinstance(mesh, TriangleMesh): + raise UnsupportedFeatureError("OpenUSD export currently requires triangle-mesh geometry") attributes = [] for name, array in mesh.attributes.items(): kind = _attribute_kind(array) diff --git a/open4d/io/tests/test_review_regressions.py b/open4d/io/tests/test_review_regressions.py new file mode 100644 index 00000000..8c7b06ff --- /dev/null +++ b/open4d/io/tests/test_review_regressions.py @@ -0,0 +1,85 @@ +import errno + +import numpy as np +import pytest + +from open4d import Frame, GaussianCloud, MemoryFrameProvider, PointCloud, Sequence, TriangleMesh +from open4d.demo import mesh_sequence +from open4d.io import UnsupportedFeatureError, open_sequence, write_sequence + + +def sequence(geometry): + return Sequence(MemoryFrameProvider([Frame(0, 0, geometry)], + has_vertex_correspondence=True)) + + +def test_gaussians_are_not_silently_exported_as_meshes(tmp_path): + cloud = GaussianCloud([[0., 0., 0.]], [[1., 1., 1.]], [[1., 0., 0., 0.]], [0.5]) + with pytest.raises(UnsupportedFeatureError, match="Gaussian"): + write_sequence(sequence(cloud), tmp_path / "frames") + assert not (tmp_path / "frames").exists() + + +def test_point_usd_export_has_supported_error(tmp_path): + with pytest.raises(UnsupportedFeatureError, match="triangle.mesh"): + write_sequence(sequence(PointCloud([[0., 0., 0.]])), tmp_path / "points.usda") + + +def test_stl_requires_lossy_opt_in_and_clears_correspondence(tmp_path): + pytest.importorskip("trimesh") + mesh = TriangleMesh([[0., 0, 0], [1, 0, 0], [1, 1, 0], [0, 1, 0], [2, 2, 2]], + [[0, 1, 2], [0, 2, 3]]) + with pytest.raises(UnsupportedFeatureError, match="allow_lossy"): + write_sequence(sequence(mesh), tmp_path / "strict", format="stl") + path = write_sequence(sequence(mesh), tmp_path / "lossy", format="stl", allow_lossy=True) + with open_sequence(path) as decoded: + assert decoded.has_vertex_correspondence is False + assert decoded.has_constant_vertex_count is None + + +@pytest.mark.parametrize("suffix", ["stl", "glb", "gltf"]) +def test_face_only_formats_reject_empty_connectivity(tmp_path, suffix): + mesh = TriangleMesh([[0., 0., 0.]], np.empty((0, 3), dtype=np.uint32)) + with pytest.raises(UnsupportedFeatureError, match="triangles"): + write_sequence(sequence(mesh), tmp_path / "frames", format=suffix, allow_lossy=True) + + +@pytest.mark.parametrize("suffix", ["usda", "usdc", "usdz"]) +def test_reopening_replaced_usd_reads_new_file_with_old_handle_open(tmp_path, suffix): + pytest.importorskip("pxr") + path = write_sequence(mesh_sequence(side=2, frames=2), tmp_path / f"take.{suffix}") + with open_sequence(path) as old: + write_sequence(mesh_sequence(side=2, frames=5), path, overwrite=True) + with open_sequence(path) as new: + assert len(new) == 5 + assert len(old) == 2 + assert old[1].frame_index == 1 + + +def test_publish_without_hard_links_preserves_existing_files(tmp_path, monkeypatch): + from open4d import _files + + def unsupported(*args): + raise OSError(errno.EOPNOTSUPP, "hard links unsupported") + + monkeypatch.setattr(_files.os, "link", unsupported) + temporary, destination = tmp_path / "temporary", tmp_path / "destination" + temporary.write_bytes(b"new") + _files.publish_file(temporary, destination) + assert destination.read_bytes() == b"new" + temporary.write_bytes(b"other") + with pytest.raises(FileExistsError): + _files.publish_file(temporary, destination) + assert destination.read_bytes() == b"new" + + +def test_usd_snapshot_resolves_relative_sublayers(tmp_path): + pytest.importorskip("pxr") + from pxr import Sdf + + write_sequence(mesh_sequence(side=2, frames=2), tmp_path / "geometry.usda") + root = Sdf.Layer.CreateNew(str(tmp_path / "root.usda")) + root.subLayerPaths = ["geometry.usda"] + root.Save() + with open_sequence(tmp_path / "root.usda") as decoded: + assert len(decoded) == 2 diff --git a/open4d/io/tests/test_unified_api.py b/open4d/io/tests/test_unified_api.py index 5af8e674..424ccdb1 100644 --- a/open4d/io/tests/test_unified_api.py +++ b/open4d/io/tests/test_unified_api.py @@ -58,7 +58,7 @@ def test_top_level_load_rejects_conflicting_dispatch_overrides(tmp_path): open4d.load(path, format="obj", codec="npz") -def test_top_level_load_dispatches_raw_vmesh_to_the_read_only_vdmc_decoder( +def test_top_level_load_dispatches_vmesh_to_codec_detection( tmp_path, monkeypatch ): import open4d._api as implementation @@ -79,22 +79,23 @@ def decode(source, *, codec=None, **options): ) is expected assert received == { "source": path, - "codec": "vdmc", + "codec": None, "options": {"decoder": "/native/decoder", "fps": 24}, } @pytest.mark.parametrize( - "name", ("capture", "frame.obj", "capture.unknown", "capture.vmesh") + "name", ("capture", "frame.obj", "capture.unknown") ) def test_top_level_save_requires_a_sequence_file_extension(tmp_path, name): with pytest.raises(ValueError, match="sequence-file extension"): open4d.save(sequence(), tmp_path / name) -def test_top_level_save_requires_a_codec_for_ambiguous_codec_suffix(tmp_path): - with pytest.raises(ValueError, match=r"ambiguous.*\.v4d.*codec"): - open4d.save(sequence(), tmp_path / "capture.v4d") +@pytest.mark.parametrize("suffix", [".v4d", ".vmesh"]) +def test_top_level_save_requires_a_codec_for_ambiguous_codec_suffix(tmp_path, suffix): + with pytest.raises(ValueError, match="ambiguous.*codec"): + open4d.save(sequence(), tmp_path / f"capture{suffix}") def test_top_level_api_exports_visualize(): diff --git a/open4d/io/tests/test_usd_sequence.py b/open4d/io/tests/test_usd_sequence.py index 5691dda4..52891f6b 100644 --- a/open4d/io/tests/test_usd_sequence.py +++ b/open4d/io/tests/test_usd_sequence.py @@ -408,7 +408,7 @@ def test_reader_accepts_the_prototype_open4d_usd_layout(tmp_path): "open4d": { "version": 1, "source": "prototype-capture", - "key_frame_indices": [0], + "key_frame_indices": Vt.IntArray([0]), } } stage.GetRootLayer().Save() diff --git a/open4d/native.py b/open4d/native.py new file mode 100644 index 00000000..d1ac6707 --- /dev/null +++ b/open4d/native.py @@ -0,0 +1,365 @@ +"""Portable native temporal runs and lossless USDC / V3C interchange. + +Opening a run never executes its models. ``decode`` and ``render`` explicitly +invoke the selected research runtime. USD interchange retains the compressed +representation; it does not turn an image-trained field into a mesh codec. +""" +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +import json +import math +import os +from pathlib import Path +import shutil +import sys +import tempfile +from types import MappingProxyType + +import numpy as np + +from .codec._native import run +from .codec._native_profiles import NEURAL_CODECS, layout +from .codec._npz import _json_value +from .codec._protocol import CodecError +from .codec._v3c import _json, inspect_vmesh, pack_vmesh, unpack_vmesh +from ._files import publish_file + + +def _read_json(path): + path = Path(path) + if path.is_symlink() or not path.is_file() or path.stat().st_size > 16 * 1024 * 1024: + raise CodecError(f"missing, linked or oversized native configuration: {path}") + return _json(path.read_bytes()) + + +def _copy(source, destination): + source = Path(source) + if source.is_symlink() or not source.is_file() or not source.stat().st_size: + raise CodecError(f"missing, linked or empty native payload: {source}") + # Reject directory symlinks too: a temporal import has a closed payload set. + if any(parent.is_symlink() for parent in source.parents): + raise CodecError(f"native payload has a linked parent: {source}") + shutil.copyfile(source, destination) + + +def _latest_ply(root, *, added=False): + root = Path(root) + direct = root / ("added.ply" if added else "point_cloud.ply") + if direct.is_file(): + return direct + suffix = "added/point_cloud.ply" if added else "point_cloud.ply" + candidates = [p for p in (root / "point_cloud").glob("iteration_*") + if p.name[10:].isdigit() and (p / suffix).is_file()] + if not candidates: + raise CodecError(f"no {'added' if added else 'initial'} Gaussian payload in {root}") + return max(candidates, key=lambda p: int(p.name[10:])) / suffix + + +def _configuration(config): + if isinstance(config, Mapping): + return _json_value(dict(config), "native configuration") + if config is None: + return {} + path = Path(config) + if path.suffix.lower() in (".yaml", ".yml"): + try: + import yaml + except ImportError as error: + raise ImportError("Pass configuration as a dictionary, or install PyYAML to read YAML") from error + if path.is_symlink() or path.stat().st_size > 16 * 1024 * 1024: + raise CodecError("linked or oversized native configuration") + value = yaml.safe_load(path.read_text()) + else: + value = _read_json(path) + if not isinstance(value, dict): + raise CodecError("native configuration must be a JSON/YAML object") + return _json_value(value, "native configuration") + + +def _stage_native(source, destination, codec, config, initial_model, ntc_config): + """Copy only actual decoder dependencies, returning native indices/profile.""" + cfg = _configuration(config) + profile = {"profile": f"{codec}/1"} + if codec == "vega": + from ._gaussian_worker import _read_manifest + entries = _read_manifest(source) + indices = [e["frame_idx"] for e in entries] + for name in ["manifest.json", "color_model.pt", *(e["file"] for e in entries)]: + _copy(source / name, destination / name) + elif codec == "queen": + frames = sorted((p for p in (source / "frames").iterdir() if p.is_dir() and p.name.isdigit()), key=lambda p: int(p.name)) + indices = [int(p.name) for p in frames] + if len(frames) < 2 or indices != list(range(1, len(frames) + 1)): + raise CodecError("QUEEN requires initial frame 1 and contiguous compressed later frames") + model, quant = cfg.get("model_params", cfg), cfg.get("quantize_params", cfg) + if "sh_degree" not in model: + raise CodecError("QUEEN import needs config with sh_degree and gate settings used to encode the run") + gates = quant.get("gate_params", [quant.get(f"{name}_gate_params", "none") for name in ("xyz", "f_dc", "f_rest", "sc", "rot", "op", "flow")]) + profile.update(sh_degree=model["sh_degree"], gate_params=gates) + _copy(initial_model or _latest_ply(frames[0]), destination / "initial.ply") + for index, frame in enumerate(frames[1:], 1): + _copy(frame / "compressed/point_cloud.pkl", destination / f"frame_{index:06d}.pkl") + elif codec == "3dgstream": + if not cfg: + cfg = _read_json(source / "cfg_args.json") + frames = sorted((p for p in source.glob("frame[0-9]*") if p.is_dir() and p.name[5:].isdigit()), key=lambda p: int(p.name[5:])) + indices = [0] + [int(p.name[5:]) for p in frames] + if len(frames) < 1 or indices != list(range(len(frames) + 1)): + raise CodecError("3DGStream requires initial frame 0 and contiguous NTC frames starting at 1") + profile.update(sh_degree=cfg.get("sh_degree"), rotate_sh=cfg.get("rotate_sh"), only_mlp=cfg.get("only_mlp"), added=[False]) + _copy(initial_model or _latest_ply(source / "init"), destination / "initial.ply") + # Never follow arbitrary paths in cfg_args.json. The architecture is an explicit input. + architecture = _configuration(ntc_config or source / "ntc_config.json") + if not isinstance(architecture.get("network"), dict) or (not profile["only_mlp"] and not isinstance(architecture.get("encoding"), dict)): + raise CodecError("3DGStream needs the NTC network/encoding configuration used for training") + (destination / "ntc_config.json").write_text(json.dumps(architecture, allow_nan=False)) + stage_two = cfg.get("iterations_s2", 0) + if type(stage_two) is not int or stage_two < 0: + raise CodecError("invalid 3DGStream second-stage iteration count") + for index, frame in enumerate(frames, 1): + _copy(frame / "NTC.pth", destination / f"ntc_{index:06d}.pth") + added = stage_two > 0 or any((frame / "point_cloud").glob("iteration_*/added/point_cloud.ply")) + profile["added"].append(bool(added)) + if added: + _copy(_latest_ply(frame, added=True), destination / f"added_{index:06d}.ply") + elif codec == "rerf": + if not cfg: + cfg = _read_json(source / "decoder_config.json") + native = cfg.get("native", {}) + profile.update(group_size=native.get("group_size"), pca=native.get("pca"), + pca_channels=native.get("pca_channels", [7, 13]), frames=[]) + required = ("voxel_size", "fine_model_and_render", "data", "render") + if not all(key in cfg for key in required): + raise CodecError("ReRF needs a resolved decoder JSON configuration including render near/far; no training-directory fallback") + indices = sorted(int(p.stem[7:]) for p in source.glob("header_*.json") if p.stem[7:].isdigit()) + for index in indices: + header = _read_json(source / f"header_{index}.json") + headers = header.get("headers") if isinstance(header, dict) else None + if not isinstance(headers, list) or not headers or not isinstance(headers[0], dict): + raise CodecError("invalid ReRF frame header") + motion = (source / f"deform_{index}.npy").is_file() + if motion != (source / f"deform_mask_{index}.rerf").is_file(): + raise CodecError("incomplete ReRF motion payload") + channels = [] + quality = headers[0].get("quality") + for half, header_part in enumerate(headers): + size = header_part.get("size") if isinstance(header_part, dict) else None + if (not isinstance(size, list) or len(size) != 5 or type(quality) is not int + or header_part.get("quality") != quality - half): + raise CodecError("invalid ReRF entropy header shape/quality") + channels.append(size[1]) + profile["frames"].append(dict(id=index, quality=quality, motion=motion, channels=channels)) + # Layout validation detects missing keys, gaps, PCA halves and motion. + for name, _ in layout(codec, len(indices), profile)[1:]: + if name != "decoder_config.json": + _copy(source / name, destination / name) + (destination / "decoder_config.json").write_text(json.dumps(cfg, allow_nan=False)) + else: + raise CodecError(f"{codec!r} is not a native neural temporal profile") + layout(codec, len(indices), profile) + return indices, profile + + +@dataclass(frozen=True) +class NeuralFieldFrame: + """Decoded ReRF density/features; appearance remains in the native RGB model.""" + frame_index: int + timestamp: float + density: np.ndarray + features: np.ndarray + xyz_min: np.ndarray + xyz_max: np.ndarray + + +class NativeSequence: + """An owned temporal representation, preserving native compressed bytes. + + Use as a context manager. ``decode`` requires the method's configured + runtime, while inspection and USDC round trips require no CUDA runtime. + """ + def __init__(self, path, *, temporary=None, runtime=None, python=None): + self.path = Path(path).absolute() + self.manifest = inspect_vmesh(self.path) + self.codec = self.manifest["codec"] + self.representation = self.manifest["representation"] + self.metadata = MappingProxyType(self.manifest["sequence"].get("metadata", {})) + self.timestamps = tuple(f["timestamp"] for f in self.manifest["sequence"]["frames"]) + self.frame_indices = tuple(f["frame_index"] for f in self.manifest["sequence"]["frames"]) + self._temporary, self._extracted = temporary, None + self._runtime, self._python = runtime, python + self._closed = False + + def __len__(self): + return len(self.timestamps) + + def __enter__(self): + self._check_open() + return self + + def __exit__(self, *_): + self.close() + + def _check_open(self): + if self._closed: + raise ValueError("native sequence is closed") + + def close(self): + if not self._closed: + if self._extracted is not None: + self._extracted.cleanup() + if self._temporary is not None: + self._temporary.cleanup() + self._closed = True + + def unpack(self, destination): + self._check_open() + return unpack_vmesh(self.path, destination) + + def _native_directory(self): + self._check_open() + if self._extracted is None: + temporary = tempfile.TemporaryDirectory(prefix=f"open4d-{self.codec}-native-") + try: + unpack_vmesh(self.path, Path(temporary.name) / "native") + except BaseException: + temporary.cleanup() + raise + self._extracted = temporary + return Path(self._extracted.name) / "native" + + def _settings(self, runtime, python): + variable = "OPEN4D_" + self.codec.upper() + root = Path(runtime or self._runtime or os.environ.get(variable + "_ROOT") or + Path(__file__).parent / "reconstruction" / self.codec).absolute() + executable = str(python or self._python or os.environ.get(variable + "_PYTHON") or sys.executable) + return root, executable + + def decode(self, *, runtime=None, python=None): + """Evaluate actual temporal models, returning Gaussian or field frames. + + Native QUEEN pickles and ReRF checkpoints are executable research + formats. Call this only for trusted native runs, in their configured + runtime. Generic inspection/load/save never deserializes these models. + """ + source = self._native_directory() + root, executable = self._settings(runtime, python) + if self.codec == "vega": + from .gaussians import VegaRun + # Frame appearances retain this owner so its extraction outlives decode. + run = VegaRun(source, root, executable, _owner=self) + return run.decode() + if self.codec not in NEURAL_CODECS: + from .codec import decode_sequence + return decode_sequence(self.path) + with tempfile.TemporaryDirectory(prefix=f"open4d-{self.codec}-decode-") as directory: + work = Path(directory) + self._worker("decode", source, work, root, executable) + paths = sorted(work.glob("frame_*.npz")) + if [p.name for p in paths] != [f"frame_{i:06d}.npz" for i in range(len(self))]: + raise CodecError(f"{self.codec} did not decode every native frame") + result = [] + for index, path in enumerate(paths): + with np.load(path, allow_pickle=False) as data: + if self.codec == "rerf": + values = {name: data[name] for name in ("density", "features", "xyz_min", "xyz_max")} + if not all(np.isfinite(v).all() for v in values.values()): + raise CodecError("ReRF decoded nonfinite field values") + result.append(NeuralFieldFrame(self.frame_indices[index], self.timestamps[index], **values)) + else: + from .gaussians import GaussianSplats + result.append(GaussianSplats(**{name: data[name] for name in + ("positions", "scales", "rotations", "opacities", "spherical_harmonics")})) + return tuple(result) + + def _worker(self, operation, source, work, root, executable, **options): + request = dict(operation=operation, codec=self.codec, source=str(source), output=str(work), runtime=str(root), **options) + path = work / "request.json" + path.write_text(json.dumps(request, allow_nan=False)) + run([executable, str(Path(__file__).with_name("_native_worker.py")), str(path)], + f"{self.codec} {operation}", cwd=work) + + def render(self, camera, *, runtime=None, python=None): + """Render a ReRF sequence from an explicit calibrated camera dictionary.""" + if self.codec != "rerf": + raise TypeError("NativeSequence.render currently accepts ReRF; Gaussian decode returns splat arrays") + root, executable = self._settings(runtime, python) + with tempfile.TemporaryDirectory(prefix="open4d-rerf-render-") as directory: + work = Path(directory) + self._worker("render", self._native_directory(), work, root, executable, + camera=_json_value(camera, "camera")) + paths = sorted(work.glob("image_*.npy")) + if len(paths) != len(self): + raise CodecError("ReRF did not render every frame") + return tuple(np.load(p, allow_pickle=False) for p in paths) + + +def import_native(source, *, codec=None, config=None, initial_model=None, ntc_config=None, + fps=30.0, timestamps=None, frame_indices=None, metadata=None, + frame_metadata=None, runtime=None, python=None): + """Import Vega, QUEEN, 3DGStream or ReRF native output, excluding dense exports. + + ``source`` can also be a GaussianRun/VegaRun. All dependencies are copied + into an owned .vmesh, so the returned object is independent of the run. + """ + if hasattr(source, "path"): + codec = codec or getattr(source, "method", "vega") + config = config if config is not None else getattr(source, "config", None) + initial_model = initial_model or getattr(source, "initial_model", None) + python = python or getattr(source, "python", None) + source_runtime = getattr(source, "runtime", None) + if source_runtime is not None: + runtime = runtime or (source_runtime if codec == "vega" else source_runtime.parent / codec) + source = source.path + if codec not in NEURAL_CODECS: + raise ValueError(f"codec must be one of {sorted(NEURAL_CODECS)}") + source = Path(source).absolute() + if isinstance(fps, bool) or not isinstance(fps, (float, int)) or not math.isfinite(fps) or fps <= 0: + raise ValueError("fps must be finite and positive") + temporary = tempfile.TemporaryDirectory(prefix=f"open4d-import-{codec}-") + try: + root = Path(temporary.name) + native = root / "native" + native.mkdir() + indices, profile = _stage_native(source, native, codec, config, initial_model, ntc_config) + count = len(indices) + times = list(timestamps) if timestamps is not None else [i / fps for i in range(count)] + ids = list(frame_indices) if frame_indices is not None else indices + metas = list(frame_metadata) if frame_metadata is not None else [{} for _ in indices] + if not len(times) == len(ids) == len(metas) == count: + raise ValueError("timestamps, frame_indices and frame_metadata must match the native frame count") + record = dict(version=1, codec=codec, native=profile, metadata=_json_value(metadata or {}, "sequence"), + frames=[dict(frame_index=i, timestamp=t, metadata=_json_value(m, "frame")) for i, t, m in zip(ids, times, metas)], + allow_nonmonotonic_timestamps=False) + (native / "metadata.json").write_text(json.dumps(record, allow_nan=False)) + artifact = pack_vmesh(native, root / "sequence.vmesh") + shutil.rmtree(native) + return NativeSequence(artifact, temporary=temporary, runtime=runtime, python=python) + except BaseException: + temporary.cleanup() + raise + + +def save_native(sequence, destination, *, overwrite=False): + """Save native temporal state as V3C .vmesh or a self-contained O4D USD.""" + if not isinstance(sequence, NativeSequence): + raise TypeError("sequence must be a NativeSequence") + if not isinstance(overwrite, bool): + raise TypeError("overwrite must be bool") + sequence._check_open() + path = Path(destination).absolute() + if path.suffix.lower() in (".usd", ".usda", ".usdc"): + from .io._native_usd import write_native_usd + return write_native_usd(sequence, path, overwrite=overwrite) + if path.suffix.lower() != ".vmesh": + raise ValueError("native output requires .vmesh, .usd, .usda or .usdc") + if path.exists() and not overwrite: + raise FileExistsError(path) + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f".{path.name}-", dir=path.parent) as folder: + staged = Path(folder) / "sequence.vmesh" + shutil.copyfile(sequence.path, staged) + inspect_vmesh(staged) + publish_file(staged, path, overwrite=overwrite) + return path diff --git a/open4d/reconstruction/3dgstream/train_frames.py b/open4d/reconstruction/3dgstream/train_frames.py index 7cc1f3c6..d72b23d0 100644 --- a/open4d/reconstruction/3dgstream/train_frames.py +++ b/open4d/reconstruction/3dgstream/train_frames.py @@ -127,10 +127,10 @@ def training_one_frame(dataset, opt, pipe, load_iteration, testing_iterations, s torch.save((gaussians.capture(), iteration), scene.output_path + "/chkpnt" + str(iteration) + ".pth") s1_end_time=time.time() + # The temporal representation is required even when densification is off. + scene.dump_NTC() # Densify if(opt.iterations_s2>0): - # Dump the NTC - scene.dump_NTC() # Update Gaussians by NTC gaussians.update_by_ntc() # Prune, Clone and setting up diff --git a/open4d/reconstruction/rerf/rerf_stream/bitstream.py b/open4d/reconstruction/rerf/rerf_stream/bitstream.py index 5b4b5004..47f372c6 100644 --- a/open4d/reconstruction/rerf/rerf_stream/bitstream.py +++ b/open4d/reconstruction/rerf/rerf_stream/bitstream.py @@ -46,6 +46,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Iterator, List, Optional, Sequence +from collections.abc import Mapping import numpy as np @@ -98,7 +99,15 @@ def __init__( pca_channels: Sequence[int] = DEFAULT_PCA_CHANNELS, group_size: Optional[int] = None, device: str = "cuda", + render_bounds=None, ) -> None: + self.path = Path(compression_path).expanduser().resolve() + self.frames = self._available_frames() + if not self.frames: + raise FileNotFoundError( + f"{self.path} holds no ReRF bitstream: no header_.json. " + "Encode one with upstream's codec/compress.py." + ) env.activate() with env.upstream_cwd(): import mmcv @@ -132,9 +141,8 @@ def __init__( } self.device = torch.device(device) - self.config_path = Path(config_path).expanduser().resolve() - self.path = Path(compression_path).expanduser().resolve() - self.cfg = mmcv.Config.fromfile(str(self.config_path)) + self.config_path = None if isinstance(config_path, Mapping) else Path(config_path).expanduser().resolve() + self.cfg = mmcv.Config(dict(config_path)) if self.config_path is None else mmcv.Config.fromfile(str(self.config_path)) self.corpus_dir = Path(self.cfg.data["datadir"]).expanduser() self.pca = bool(pca) self.pca_channels = tuple(int(channel) for channel in pca_channels) @@ -142,12 +150,6 @@ def __init__( self.n_channel = int(self.cfg.fine_model_and_render["rgbnet_dim"]) + 1 self.voxel_size = int(self.cfg.voxel_size) - self.frames = self._available_frames() - if not self.frames: - raise FileNotFoundError( - f"{self.path} holds no ReRF bitstream: no header_.json. " - "Encode one with upstream's codec/compress.py." - ) # Upstream: `group_size = args.group_size if given else args.frame_num`. # Defaulting to the number of frames present plays a single-group # bitstream correctly without having to be told its own structure. @@ -173,7 +175,13 @@ def __init__( ) self.model.k0.eval() - near, far = inward_near_far(self.corpus_dir) + if render_bounds is None: + near, far = inward_near_far(self.corpus_dir) + else: + near, far = render_bounds["near"], render_bounds["far"] + if (isinstance(near, bool) or isinstance(far, bool) or + not np.isfinite([near, far]).all() or not 0 <= near < far): + raise ValueError("render bounds must be finite with 0 <= near < far") self.render_kwargs = { "near": near, "far": far, diff --git a/open4d/reconstruction/rerf/rerf_stream/env.py b/open4d/reconstruction/rerf/rerf_stream/env.py index 032fd685..4599299c 100644 --- a/open4d/reconstruction/rerf/rerf_stream/env.py +++ b/open4d/reconstruction/rerf/rerf_stream/env.py @@ -89,7 +89,7 @@ def patch_dependencies() -> None: for name, builtin in (("bool", bool), ("object", object), ("int", int), ("float", float), ("complex", complex), ("str", str)): - if not hasattr(numpy, name): + if name not in vars(numpy): setattr(numpy, name, builtin) try: diff --git a/open4d/reconstruction/rerf/rerf_stream_tests/test_bitstream.py b/open4d/reconstruction/rerf/rerf_stream_tests/test_bitstream.py index 2d06e30b..6035daf0 100644 --- a/open4d/reconstruction/rerf/rerf_stream_tests/test_bitstream.py +++ b/open4d/reconstruction/rerf/rerf_stream_tests/test_bitstream.py @@ -10,6 +10,7 @@ from __future__ import annotations import json +import os from pathlib import Path import numpy as np @@ -55,7 +56,7 @@ def test_numpy_aliases_are_restored(): env.patch_dependencies() import numpy - assert numpy.bool is bool + assert numpy.dtype(numpy.bool) == numpy.dtype(bool) assert numpy.str is str @@ -239,11 +240,16 @@ def test_waiting_times_out_rather_than_blocking_forever(): # ------------------------------------------------- the decode, when it can run --- -RUN = Path("/media/frozzzen/LocalDisk/nevo_runs/g_basketball") +RUN = Path(os.environ.get("OPEN4D_RERF_RUN", str(Path.home() / "open4d-data/rerf/g_basketball"))) CONFIG, BITSTREAM = RUN / "config.py", RUN / "rerf" +try: + _has_bitstream = CONFIG.is_file() and (BITSTREAM / "header_0.json").is_file() +except OSError: + _has_bitstream = False + requires_bitstream = pytest.mark.skipif( - not (CONFIG.is_file() and (BITSTREAM / "header_0.json").is_file()), + not _has_bitstream, reason="needs an encoded ReRF bitstream on this machine", ) diff --git a/open4d/reconstruction/rerf/rerf_stream_tests/test_geometry.py b/open4d/reconstruction/rerf/rerf_stream_tests/test_geometry.py index 6fb93e33..16aa40cf 100644 --- a/open4d/reconstruction/rerf/rerf_stream_tests/test_geometry.py +++ b/open4d/reconstruction/rerf/rerf_stream_tests/test_geometry.py @@ -13,6 +13,7 @@ from __future__ import annotations import json +import os import struct from pathlib import Path @@ -27,11 +28,16 @@ # Same fixture and skip rule as `test_bitstream.py`: the decode needs a real # encoded bitstream and a GPU, and neither is present everywhere. -RUN = Path("/media/frozzzen/LocalDisk/nevo_runs/g_basketball") +RUN = Path(os.environ.get("OPEN4D_RERF_RUN", str(Path.home() / "open4d-data/rerf/g_basketball"))) CONFIG, BITSTREAM = RUN / "config.py", RUN / "rerf" +try: + _has_bitstream = CONFIG.is_file() and (BITSTREAM / "header_0.json").is_file() +except OSError: + _has_bitstream = False + requires_bitstream = pytest.mark.skipif( - not (CONFIG.is_file() and (BITSTREAM / "header_0.json").is_file()), + not _has_bitstream, reason="needs an encoded ReRF bitstream on this machine", ) diff --git a/open4d/streamer/README.md b/open4d/streamer/README.md index 59ba8b64..669f1c4f 100644 --- a/open4d/streamer/README.md +++ b/open4d/streamer/README.md @@ -290,7 +290,7 @@ from_source("captures/basketball_player", out, fps=10, frame_format="draco") The decoder is Google's, vendored under `client/vendor/draco` and served by the bundle server from this origin — never a CDN, which is what keeps the page free -of external dependencies. `open4d[draco]` is needed on the encoding side only; +of external dependencies. `DracoPy` is needed on the encoding side only; the client needs nothing installed. Lossy in two bounded ways, both in the clip's notes and measured in diff --git a/open4d/streamer/streamer/bundle.py b/open4d/streamer/streamer/bundle.py index 038a4f97..81f4033e 100644 --- a/open4d/streamer/streamer/bundle.py +++ b/open4d/streamer/streamer/bundle.py @@ -42,6 +42,7 @@ import dataclasses import json import time +import tempfile from dataclasses import dataclass, field from pathlib import Path from typing import Any, Mapping @@ -369,7 +370,16 @@ def write( if detail: index["detail"] = detail path = out_dir / INDEX_NAME - path.write_text(json.dumps(index, indent=2) + "\n") + temporary = None + try: + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=out_dir, + prefix=".view-", suffix=".json", delete=False) as output: + temporary = Path(output.name) + output.write(json.dumps(index, indent=2) + "\n") + temporary.replace(path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) return path diff --git a/open4d/streamer/streamer/export.py b/open4d/streamer/streamer/export.py index 76738d1c..d9791155 100644 --- a/open4d/streamer/streamer/export.py +++ b/open4d/streamer/streamer/export.py @@ -62,16 +62,14 @@ def representation_of(sequence: Sequence) -> Representation: def _write_draco(sequence: Sequence, frames_at: Path, bits: int) -> list[Path]: """One ``.drc`` per frame, encoded with the same Draco this repository vendors. - Per frame rather than one container for the sequence, because a streaming - client fetches frames: `open4d.save(..., codec="draco")` writes a `.d4d` - holding the whole sequence, which is the right shape for an archive and the - wrong one for a wire. + Per frame because a streaming client fetches frames. DracoPy is an optional + dependency of this exporter, independent of the public codec registry. """ try: import DracoPy except ImportError as error: # pragma: no cover - depends on the environment raise RuntimeError( - "Draco frames need the DracoPy binding: pip install 'open4d[draco]'" + "Draco frames need the DracoPy binding: pip install 'DracoPy'" ) from error written: list[Path] = [] @@ -198,6 +196,7 @@ def from_source( out_dir = Path(out_dir).expanduser().resolve() declared = inspect_sequence(source).timing_source with open4d.load(source, fps=fps if declared == "default" else None) as sequence: + playback_fps = sequence.fps or 30.0 clip = from_sequence( sequence, out_dir, @@ -216,6 +215,6 @@ def from_source( title=f"{clip.representation} — {clip.name}", source=str(source), clips=[clip], - fps=int(fps) if fps else 30, + fps=playback_fps, ) return out_dir diff --git a/open4d/streamer/streamer/sequence.py b/open4d/streamer/streamer/sequence.py index 8bc6c829..53c4c19f 100644 --- a/open4d/streamer/streamer/sequence.py +++ b/open4d/streamer/streamer/sequence.py @@ -231,7 +231,9 @@ def pack_bundle( if skip: changed.append(clip) continue - entry = pack_clip(root, clip, keep_frames=keep_frames) + # Do not remove inputs until every clip is packed and the manifest + # has committed. A later codec or filesystem failure must be retryable. + entry = pack_clip(root, clip, keep_frames=True) changed.append(dataclasses.replace(clip, sequence=entry)) packed.append((clip.name, entry)) @@ -245,6 +247,18 @@ def pack_bundle( scenes=index.get("scenes") or {}, detail=index.get("detail"), ) + if not keep_frames: + from .transfer import frame_paths + + retained = set(frame_paths({"clips": [dataclasses.asdict(c) for c in changed]})) + packed_names = {name for name, _ in packed} + removable = {path for clip in clips if clip.name in packed_names + for path in clip.frames} - retained + for path in removable: + (root / path).unlink(missing_ok=True) + for directory in {root / Path(path).parts[0] for path in removable}: + if directory.is_dir() and not any(directory.iterdir()): + directory.rmdir() return packed diff --git a/open4d/streamer/streamer/session.py b/open4d/streamer/streamer/session.py index 0f739769..28deaa94 100644 --- a/open4d/streamer/streamer/session.py +++ b/open4d/streamer/streamer/session.py @@ -110,14 +110,15 @@ def __init__( *, title: str | None = None, source: Path | str | None = None, - fps: int = 30, + fps: float | None = None, scenes: dict[str, Any] | None = None, detail: dict[str, Any] | None = None, ) -> None: self.out_dir = Path(out_dir).expanduser().resolve() self.title = title or self.out_dir.name self.source = source - self.fps = fps + self.fps = 30.0 if fps is None else fps + self._infer_fps = fps is None self.scenes = scenes self.detail = detail self.clips: list[bundle.Clip] = [] @@ -157,6 +158,8 @@ def add( because which rendition should be the default is a delivery decision (interchange? cheapest? middle?) and not one a byte count settles. """ + if self._infer_fps and not self.clips: + self.fps = sequence.fps or 30.0 parsed = [parse_rung(rung) for rung in rungs] if not parsed: raise ValueError(f"{name}: needs at least one rung") diff --git a/open4d/streamer/streamer/transfer.py b/open4d/streamer/streamer/transfer.py index b43bacbd..5a06fd72 100644 --- a/open4d/streamer/streamer/transfer.py +++ b/open4d/streamer/streamer/transfer.py @@ -29,7 +29,7 @@ import urllib.parse import urllib.request from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import Callable, Iterable from . import bundle @@ -91,6 +91,10 @@ def _download(url: str, destination: Path, timeout: float) -> int: written = 0 with urllib.request.urlopen(request, timeout=timeout) as response: resuming = have > 0 and response.status == 206 + if resuming: + content_range = response.headers.get("Content-Range", "") + if not content_range.startswith(f"bytes {have}-"): + raise ValueError("invalid Content-Range for resumed download") if have and not resuming: # The server sent the whole file despite the range. Start over # rather than append: the alternative is a corrupt file that is @@ -122,11 +126,28 @@ def frame_paths(index: dict) -> tuple[str, ...]: """Every frame path a manifest names, in order, without duplicates.""" seen: dict[str, None] = {} for clip in index.get("clips", []): - for path in clip.get("frames", []): + packed = clip.get("sequence") + paths = [packed["url"]] if packed else clip.get("frames", []) + for variant in clip.get("variants", []): + paths = [*paths, *variant.get("frames", [])] + for path in paths: seen.setdefault(path, None) return tuple(seen) +def _safe_target(root: Path, path: str) -> Path: + if (not isinstance(path, str) or not path or "\\" in path or "\x00" in path + or PurePosixPath(path).is_absolute() or PureWindowsPath(path).drive + or any(part in {"", ".", ".."} for part in path.split("/")) + or path == bundle.INDEX_NAME or path.endswith(".partial")): + raise ValueError(f"unsafe bundle path: {path!r}") + target = root / path + for candidate in (target, target.with_name(target.name + ".partial")): + if root not in candidate.resolve().parents or candidate.is_symlink(): + raise ValueError(f"unsafe bundle path: {path!r}") + return target + + def fetch( url: str, destination: Path | str, @@ -157,17 +178,16 @@ def fetch( if missing: raise KeyError(f"no clip named {', '.join(sorted(missing))} in {base}") - # Written first and last: first so the directory is recognisable as a bundle - # while frames arrive, last so a `--only` subset's manifest is the filtered - # one rather than the whole thing. - (root / bundle.INDEX_NAME).write_text(json.dumps(index, indent=2) + "\n") - paths = frame_paths(index) + targets = {path: _safe_target(root, path) for path in paths} + index_path = root / bundle.INDEX_NAME + if index_path.is_symlink(): + raise ValueError("unsafe bundle manifest path") fetched: list[str] = [] skipped: list[str] = [] total_bytes = 0 for position, path in enumerate(paths, start=1): - target = root / path + target = targets[path] source = base + urllib.parse.quote(path) expected = _remote_size(source, timeout) if target.is_file() else None if target.is_file() and expected is not None and target.stat().st_size == expected: @@ -184,6 +204,9 @@ def fetch( if progress is not None: progress(path, position, len(paths)) + # Advertise the new bundle only once every referenced file is available. + index_path.write_text(json.dumps(index, indent=2) + "\n") + return FetchResult( root=root, fetched=tuple(fetched), diff --git a/open4d/streamer/streamer_tests/test_draco.py b/open4d/streamer/streamer_tests/test_draco.py index a691072c..1eea4152 100644 --- a/open4d/streamer/streamer_tests/test_draco.py +++ b/open4d/streamer/streamer_tests/test_draco.py @@ -38,7 +38,7 @@ def _source() -> str: NODE = shutil.which("node") requires_node = pytest.mark.skipif(NODE is None, reason="node is not installed") -DracoPy = pytest.importorskip("DracoPy", reason="Draco frames need open4d[draco]") +DracoPy = pytest.importorskip("DracoPy", reason="Draco frames need DracoPy") CLIENT = viewer_path().parent VENDOR = CLIENT / "vendor" / "draco" diff --git a/open4d/streamer/streamer_tests/test_export.py b/open4d/streamer/streamer_tests/test_export.py index 3c578c5e..9a10c80e 100644 --- a/open4d/streamer/streamer_tests/test_export.py +++ b/open4d/streamer/streamer_tests/test_export.py @@ -149,5 +149,14 @@ def test_from_source_round_trips_through_open4d_load(tmp_path): assert len(index["clips"]) == 1 assert index["clips"][0]["representation"] == "mesh" assert len(index["clips"][0]["frames"]) == 2 - assert index["fps"] == 10 + assert index["fps"] == 30 # The source manifest's timing takes precedence. assert any("open4d.load" in note for note in index["clips"][0]["notes"]) + + +def test_export_keeps_fractional_source_rate(tmp_path): + from open4d.io import write_sequence + + source = Sequence(MemoryFrameProvider([Frame(i, i / 23.976, mesh()) for i in range(3)])) + path = write_sequence(source, tmp_path / "frames") + out = export.from_source(path, tmp_path / "bundle", fps=10) + assert bundle.read(out)["fps"] == pytest.approx(23.976) diff --git a/open4d/streamer/streamer_tests/test_open4d_stream.py b/open4d/streamer/streamer_tests/test_open4d_stream.py index b941479c..00bed082 100644 --- a/open4d/streamer/streamer_tests/test_open4d_stream.py +++ b/open4d/streamer/streamer_tests/test_open4d_stream.py @@ -20,13 +20,13 @@ pytestmark = pytest.mark.cpu -def sequence_of(count: int = 3) -> Sequence: +def sequence_of(count: int = 3, fps: float = 30) -> Sequence: geometry = TriangleMesh( np.asarray([[0, 0, 0], [1, 0, 0], [0, 1, 0]], dtype=np.float32), np.asarray([[0, 1, 2]], dtype=np.uint32), ) return Sequence( - MemoryFrameProvider([Frame(i, i / 30, geometry) for i in range(count)]) + MemoryFrameProvider([Frame(i, i / fps, geometry) for i in range(count)]) ) @@ -50,6 +50,21 @@ def test_a_sequence_is_exported_and_served(tmp_path): server.shutdown() +@pytest.mark.parametrize("as_path", [False, True]) +def test_browser_stream_uses_source_timestamps(tmp_path, as_path): + source = sequence_of(fps=23.976) + if as_path: + from open4d.io import write_sequence + source = write_sequence(source, tmp_path / "source") + server = open4d.stream(source, out_dir=tmp_path / "bundle", name="timed", + fps=10, open_browser=False, block=False) + try: + index = json.loads((server.bundle_dir / "view.json").read_text()) + assert index["fps"] == pytest.approx(23.976) + finally: + server.shutdown() + + def test_a_ladder_becomes_variants(tmp_path): server = open4d.stream( sequence_of(), diff --git a/open4d/streamer/streamer_tests/test_sequence.py b/open4d/streamer/streamer_tests/test_sequence.py index afeb459b..512e5119 100644 --- a/open4d/streamer/streamer_tests/test_sequence.py +++ b/open4d/streamer/streamer_tests/test_sequence.py @@ -724,3 +724,34 @@ def test_a_failed_download_is_explained_in_the_pane(): assert "_explain(" in body # Carries on rather than abandoning the remaining panes. assert "continue;" in body +def test_failed_bundle_pack_keeps_original_frames_and_manifest(tmp_path, monkeypatch): + clips = [] + for name in ("first", "second"): + paths = _frames(tmp_path / name, [12, 12]) + clips.append(bundle.Clip(name=name, representation="gaussians", + frames=[str(p.relative_to(tmp_path)) for p in paths])) + bundle.write(tmp_path, title="test", source="test", clips=clips) + before = (tmp_path / "view.json").read_bytes() + original = sequence.pack_clip + + def fail_second(root, clip, **kwargs): + if clip.name == "second": + raise OSError("simulated packing failure") + return original(root, clip, **kwargs) + + monkeypatch.setattr(sequence, "pack_clip", fail_second) + with pytest.raises(OSError): + sequence.pack_bundle(tmp_path) + assert (tmp_path / "view.json").read_bytes() == before + assert all((tmp_path / path).is_file() for clip in clips for path in clip.frames) + + +def test_packing_keeps_frames_referenced_by_variants_or_unpacked_clips(tmp_path): + paths = _frames(tmp_path / "shared", [12, 12]) + frames = [str(p.relative_to(tmp_path)) for p in paths] + clips = [bundle.Clip(name="first", representation="gaussians", frames=frames, + variants=[{"name": "original", "frames": frames}]), + bundle.Clip(name="second", representation="gaussians", frames=frames)] + bundle.write(tmp_path, title="test", source="test", clips=clips) + sequence.pack_bundle(tmp_path, names=["first"]) + assert all(p.is_file() for p in paths) diff --git a/open4d/streamer/streamer_tests/test_server_and_transfer.py b/open4d/streamer/streamer_tests/test_server_and_transfer.py index 61f2e135..c9ae85ca 100644 --- a/open4d/streamer/streamer_tests/test_server_and_transfer.py +++ b/open4d/streamer/streamer_tests/test_server_and_transfer.py @@ -17,6 +17,33 @@ pytestmark = pytest.mark.cpu +@pytest.mark.parametrize("path", ["../escaped", "/tmp/escaped", "C:/escaped", "dir/../../escaped", "dir\\escaped", "view.json", "frame.partial"]) +def test_fetch_rejects_unsafe_paths_before_writing_manifest(tmp_path, monkeypatch, path): + monkeypatch.setattr(transfer, "_get", lambda *args: json.dumps({"clips": [{"frames": [path]}]}).encode()) + with pytest.raises(ValueError, match="path"): + transfer.fetch("https://example.invalid/", tmp_path / "copy") + assert not (tmp_path / "copy" / "view.json").exists() + + +@pytest.mark.parametrize("partial", [False, True]) +def test_fetch_rejects_symlink_targets(tmp_path, monkeypatch, partial): + root = tmp_path / "copy" + root.mkdir() + outside = tmp_path / "outside" + outside.write_bytes(b"keep") + (root / ("frame.ply.partial" if partial else "frame.ply")).symlink_to(outside) + monkeypatch.setattr(transfer, "_get", lambda *args: b'{"clips":[{"frames":["frame.ply"]}]}') + with pytest.raises(ValueError, match="path"): + transfer.fetch("https://example.invalid/", root) + assert outside.read_bytes() == b"keep" + + +def test_fetch_paths_include_variants_and_packed_sequences(): + index = {"clips": [{"frames": ["logical.ply"], "sequence": {"url": "clip.seq"}, + "variants": [{"frames": ["low.ply"]}, {"frames": ["low.ply"]}]}]} + assert transfer.frame_paths(index) == ("clip.seq", "low.ply") + + def make_bundle(root, *, frames: int = 3): """A two-clip bundle: one geometry clip, one pixel clip.""" written = {"gaussians": [], "pixels": []} diff --git a/open4d/streaming/README.md b/open4d/streaming/README.md index 05b5474a..bdf1a722 100644 --- a/open4d/streaming/README.md +++ b/open4d/streaming/README.md @@ -43,6 +43,9 @@ source .venv/bin/activate python -m pip install numpy opencv-python zstandard ``` +From an Open4D checkout, `python -m pip install -e '.[capture]'` installs the +Python capture dependencies, including Open3D and Zstandard. + Install a CUDA-enabled Open3D build in that environment. The standard pip wheel may not include CUDA. Verify the installation: diff --git a/open4d/streaming/_transport.py b/open4d/streaming/_transport.py index 724d72f5..e51c44c7 100644 --- a/open4d/streaming/_transport.py +++ b/open4d/streaming/_transport.py @@ -98,7 +98,7 @@ def _decode(header, payload): raise ValueError(f"invalid mesh stream frame: {error}") from error -def send(frames, host="127.0.0.1", port=7000, *, realtime=True, timeout=30.0, +def send(frames, host="127.0.0.1", port=47004, *, realtime=True, timeout=30.0, max_frame_bytes=_DEFAULT_LIMIT) -> int: """Send a Sequence or iterable of Frames to a running receiver. @@ -194,7 +194,7 @@ def close(self): setattr(self, name, None) -def receive(host="127.0.0.1", port=7000, *, timeout=30.0, +def receive(host="127.0.0.1", port=47004, *, timeout=30.0, max_frame_bytes=_DEFAULT_LIMIT) -> Receiver: """Listen for one sender and yield Frames; use `with receive() as frames:`. diff --git a/open4d/streaming/python/live_two_camera_webrtc.py b/open4d/streaming/python/live_two_camera_webrtc.py index 41da34b7..68a2184a 100644 --- a/open4d/streaming/python/live_two_camera_webrtc.py +++ b/open4d/streaming/python/live_two_camera_webrtc.py @@ -282,12 +282,12 @@ def _process_loop( def run(self) -> None: self.args.output_dir.mkdir(parents=True, exist_ok=True) - self.server.start() - print( - "Browser viewer ready through Ubuntu 127.0.0.1:8888", - flush=True, - ) try: + self.server.start() + print( + "Browser viewer ready through Ubuntu 127.0.0.1:8888", + flush=True, + ) o3d.visualization.webrtc_server.enable_webrtc() o3d.visualization.draw( [], @@ -305,9 +305,13 @@ def run(self) -> None: self.stop = True if self.processor is not None: self.processor.join(timeout=10.0) - self.save() - self.server.close() - self.mesh_worker.close() + try: + self.save() + finally: + try: + self.server.close() + finally: + self.mesh_worker.close() report = { "status": "completed", "viewer": "Open3D WebRTC", diff --git a/open4d/streaming/tests/test_capture.py b/open4d/streaming/tests/test_capture.py index 26ad54a7..fa8bfa97 100644 --- a/open4d/streaming/tests/test_capture.py +++ b/open4d/streaming/tests/test_capture.py @@ -5,11 +5,38 @@ import sys import time import zlib +from types import SimpleNamespace from pathlib import Path import pytest +@pytest.mark.parametrize("save_fails", [False, True]) +def test_browser_start_failure_closes_receiver_and_mesh_worker(fusion, tmp_path, save_fails): + module = importlib.import_module("live_two_camera_webrtc") + app = module.BrowserFusion.__new__(module.BrowserFusion) + closed = [] + + def start(): + raise RuntimeError("port already in use") + + def save(): + if save_fails: + raise RuntimeError("save failed") + + app.args = SimpleNamespace(output_dir=tmp_path) + app.server = SimpleNamespace(start=start, close=lambda: closed.append("server"), report=lambda: {}) + app.mesh_worker = SimpleNamespace(close=lambda: closed.append("worker"), device="cpu", + fusion_mode="shared-tsdf", merge_mode="concatenate") + app.processor = None + app.save = save + app.processed = app.latest_pair = app.mesh_updates = 0 + app.processor_error = None + with pytest.raises(RuntimeError, match="save failed" if save_fails else "port already in use"): + app.run() + assert closed == ["server", "worker"] + + @pytest.fixture def protocol(monkeypatch): root = Path(__file__).resolve().parents[1] / "python" diff --git a/open4d/webclients/README.md b/open4d/webclients/README.md index 50ad3765..63fba111 100644 --- a/open4d/webclients/README.md +++ b/open4d/webclients/README.md @@ -35,6 +35,9 @@ servers (`baselines/`) and the corpora live in `4DVideoStreaming`, so `run_web_demo.sh` expects a checkout of it. Without one you can still build the clients and run the tests. +The Node control server binds to `127.0.0.1` by default. Set `HOST` explicitly +when remote clients need access; it provides no authentication. + ## Adding a method Write a page, then add one line to `DESCRIPTION` in `src/chooser.js`, one entry diff --git a/open4d/webclients/system/Server/server.js b/open4d/webclients/system/Server/server.js index 980b9e11..4f9003b2 100644 --- a/open4d/webclients/system/Server/server.js +++ b/open4d/webclients/system/Server/server.js @@ -276,6 +276,7 @@ app.get("/api/geometry-bundle", async (req, res) => { }); const PORT = process.env.PORT || 3000; +const HOST = process.env.HOST || "127.0.0.1"; function numberFromEnv(name, fallback) { const value = Number(process.env[name]); @@ -2526,7 +2527,7 @@ if (require.main === module) { process.exit(1); } if (COMPRESSED_ROOT) logInfo("SERVER", `corpus ${COMPRESSED_ROOT}`); - app.listen(PORT, "0.0.0.0", () => { + app.listen(PORT, HOST, () => { bootstrap().catch((e) => { logError("SERVER", `Bootstrap failed: ${e.message}`); }); diff --git a/open4d/webclients/tests/test_tile_ladder.py b/open4d/webclients/tests/test_tile_ladder.py index 3d2208a9..d2426be7 100644 --- a/open4d/webclients/tests/test_tile_ladder.py +++ b/open4d/webclients/tests/test_tile_ladder.py @@ -8,7 +8,7 @@ import pytest -from baselines.ViVo.orbitvivo.tile_ladder import ( +from open4d.webclients.tile_ladder import ( SourceCorpusUnavailable, TileCatalogLadder, ) diff --git a/open4d/webclients/tests/test_vgs_format.js b/open4d/webclients/tests/test_vgs_format.js index 9f3bfa87..864dad39 100644 --- a/open4d/webclients/tests/test_vgs_format.js +++ b/open4d/webclients/tests/test_vgs_format.js @@ -41,7 +41,7 @@ const haveAsset = () => GOLDEN && fs.existsSync( fs.existsSync(GOLDEN.file) ? GOLDEN.file : path.join(__dirname, '..', GOLDEN.file)); -test('the golden fixture and its source asset are present', () => { +test('the golden fixture and its source asset are present', { skip: !haveAsset() }, () => { assert.ok(GOLDEN, 'fixtures_vgs_golden.json is missing; run the generator'); assert.ok(haveAsset(), `asset ${GOLDEN.file} is missing; re-export with orbitvega.export_quest`); diff --git a/open4d/webclients/tile_ladder.py b/open4d/webclients/tile_ladder.py index 5b3e06bc..fb8dc492 100644 --- a/open4d/webclients/tile_ladder.py +++ b/open4d/webclients/tile_ladder.py @@ -50,8 +50,6 @@ from dataclasses import dataclass from pathlib import Path -from baselines.ViVo.orbitvivo.ladder import Representation - CATALOG_NAME = "catalog.json" SUPPORTED_CATALOG_VERSION = 2 @@ -60,6 +58,18 @@ class SourceCorpusUnavailable(RuntimeError): """Raised when a caller needs the RGB-D source this ladder does not have.""" +@dataclass(frozen=True) +class Representation: + """One prepared level; self-contained metadata for the tile catalogue.""" + + representation_id: int + name: str + root: Path + manifest_path: Path + manifest: TileManifest + point_ratio: float + + @dataclass(frozen=True) class TileObjectSequence: """The object fields a tile-served baseline uses, and no others. diff --git a/pyproject.toml b/pyproject.toml index 2360eddd..b16562db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,8 +32,9 @@ klt = ["torch", "trimesh", "zstd", "tqdm", "point-cloud-utils"] n4mc = ["torch", "trimesh", "scikit-image", "point-cloud-utils"] qndf = ["torch", "open3d>=0.19,<0.20", "trimesh", "tqdm"] gaussians = ["plyfile"] +capture = ["opencv-python", "zstandard", "open3d>=0.19,<0.20", "pillow>=9"] dev = ["build>=1.2", "pytest>=8", "scipy", "tomli>=2; python_version < '3.11'"] -all = ["plyfile", "trimesh", "PyQt6", "pyqtgraph", "PyOpenGL", "open3d>=0.19,<0.20", "pillow>=9", "usd-core>=24.3", "scipy", "torch", "zstd", "tqdm", "point-cloud-utils", "scikit-image"] +all = ["plyfile", "trimesh", "PyQt6", "pyqtgraph", "PyOpenGL", "open3d>=0.19,<0.20", "pillow>=9", "usd-core>=24.3", "scipy", "torch", "zstd", "tqdm", "point-cloud-utils", "scikit-image", "opencv-python", "zstandard"] [tool.setuptools] include-package-data = false diff --git a/scripts/benchmark_codec.py b/scripts/benchmark_codec.py index 2fb17979..27466854 100644 --- a/scripts/benchmark_codec.py +++ b/scripts/benchmark_codec.py @@ -162,7 +162,7 @@ def decode_and_validate(sequence): assert len(actual.geometry.positions) and len(actual.geometry.triangles) exact &= _geometry_exact(actual.geometry, expected.geometry) if (actual.geometry.positions.shape == expected.geometry.positions.shape - and (info.lossless or sequence.has_vertex_correspondence is True)): + and (info.lossless or decoded.has_vertex_correspondence is True)): error = (actual.geometry.positions.astype(np.float64) - expected.geometry.positions.astype(np.float64)) squared_error += float(np.square(error).sum()) @@ -185,14 +185,17 @@ def decode_and_validate(sequence): surface_rms, surface_maximum, ) - result, validate_s = timed(lambda: decode_and_validate(decoded)) + frames, consume_s = timed(lambda: tuple(decoded)) + validation_started = time.perf_counter() + result = decode_and_validate(frames) + validate_s = time.perf_counter() - validation_started ( vertices, triangles, exact, rms_error, maximum_error, surface_rms_error, surface_max_error, ) = result finally: decoded.close() - decode_s = open_s + validate_s + decode_s = open_s + consume_s def decode_consume_close(): measured_sequence = decode_sequence(artifact, codec=selected, **(decode_options or {})) diff --git a/scripts/tests/test_benchmark_codec.py b/scripts/tests/test_benchmark_codec.py index 6ad65098..70a9312e 100644 --- a/scripts/tests/test_benchmark_codec.py +++ b/scripts/tests/test_benchmark_codec.py @@ -75,11 +75,34 @@ def fixed_timed(function): result = benchmark_codec.run(source, Path(tmp_path) / "take.o4d") assert result["decode_open_ms"] == 2000 - assert result["decode_validate_s"] == 3 + assert result["decode_validate_s"] >= 0 assert result["decode_all_s"] == 5 assert result["decode_frames_per_s"] == pytest.approx(2 / 5) +def test_decode_timing_excludes_surface_validation(tmp_path, monkeypatch): + measuring = False + real_timed = benchmark_codec.timed + real_surface = benchmark_codec._surface_errors + + def marked_timed(function): + nonlocal measuring + measuring = True + try: + return real_timed(function) + finally: + measuring = False + + # Validation has its own explicit timer, outside timed decode consumption. + def check(left, right): + assert not measuring + return real_surface(left, right) + + monkeypatch.setattr(benchmark_codec, "timed", marked_timed) + monkeypatch.setattr(benchmark_codec, "_surface_errors", check) + benchmark_codec.run(benchmark_codec.synthetic(3, 2), tmp_path / "test.o4d") + + def test_decode_peak_memory_excludes_surface_validation(tmp_path, monkeypatch): inside_peak_measurement = False real_peak_bytes = benchmark_codec.peak_bytes From 9b34180f357cbeac3ccd5246ad5e4dbd4f3122de Mon Sep 17 00:00:00 2001 From: cicm Date: Wed, 30 Sep 2026 15:47:36 -0400 Subject: [PATCH 2/4] Make VMESH the standalone format for every codec Replace private V3C carriage and retired compressed output types with validated VMESH profiles, explicit migration, and compressed USD interchange. Harden browser routes and codec numerical handling, and add regression and optional-dependency CI coverage. --- .agents.txt | 11 + .codex/config.toml | 1 + .github/workflows/ci.yml | 73 ++- README.md | 56 ++- SECURITY.md | 35 ++ docs/api.md | 154 ++++++- examples/open4d_sequence_codec.ipynb | 14 +- examples/visualization/README.md | 18 +- examples/visualization/compare_sequences.py | 4 +- examples/visualization/frame_sources.py | 15 +- .../visualization/tests/test_frame_sources.py | 12 +- .../tests/test_visualize_sequence.py | 6 +- examples/visualization/visualize_sequence.py | 8 +- examples/vmesh/01_container_and_usdc.ipynb | 6 +- examples/vmesh/02_mesh_codecs.ipynb | 11 +- examples/vmesh/03_gaussian_codecs.ipynb | 2 +- open4d/__init__.py | 3 +- open4d/_api.py | 2 +- open4d/_cli.py | 8 +- open4d/codec/__init__.py | 4 +- open4d/codec/_api.py | 25 +- open4d/codec/_draco.py | 146 +++--- open4d/codec/_klt.py | 113 +++-- open4d/codec/_metadata.py | 70 +++ open4d/codec/_migration.py | 108 +++++ open4d/codec/_n4mc.py | 76 ++- open4d/codec/_native_profiles.py | 62 ++- open4d/codec/_native_temporal.py | 8 +- open4d/codec/_npz.py | 232 ++++------ open4d/codec/_qndf.py | 235 ++++++---- open4d/codec/_temporal.py | 68 +-- open4d/codec/_tracked.py | 66 +-- open4d/codec/_vmesh.py | 71 +-- open4d/codec/{_v3c.py => _vmesh_format.py} | 201 ++++---- open4d/codec/tests/editor_numerics.cs | 140 ++++++ .../codec/tests/test_artifact_validation.py | 99 ++-- open4d/codec/tests/test_codec.py | 64 +-- open4d/codec/tests/test_draco.py | 4 +- open4d/codec/tests/test_editor_numerics.py | 54 +++ open4d/codec/tests/test_klt_quantization.py | 88 ++++ open4d/codec/tests/test_n4mc_determinism.py | 62 +++ open4d/codec/tests/test_n4mc_vmesh.py | 44 +- open4d/codec/tests/test_native_temporal.py | 13 +- open4d/codec/tests/test_qndf_preprocess.py | 35 ++ .../tests/test_queen_training_exports.py | 68 +++ open4d/codec/tests/test_real_dataset.py | 4 +- open4d/codec/tests/test_reference_vmesh.py | 60 +++ open4d/codec/tests/test_research_codecs.py | 4 +- open4d/codec/tests/test_research_cpu.py | 16 +- open4d/codec/tests/test_research_vmesh.py | 206 +++++++++ open4d/codec/tests/test_tracked.py | 169 +++---- open4d/codec/tests/test_tracker_numerics.py | 58 +++ .../{test_v3c.py => test_vmesh_format.py} | 155 +++++-- open4d/codec/tests/test_vmesh_only.py | 129 ++++++ open4d/codec/tests/tracker_numerics.cs | 102 ++++ open4d/codecs/klt/klt.py | 153 +++++- open4d/codecs/qndf/README.md | 6 + open4d/codecs/qndf/build_dataset_open3d.py | 24 +- .../Framework/Util/ARAP.cs | 7 + .../Framework/Util/KDTree.cs | 4 + .../Framework/Util/TransformDistance.cs | 3 + .../AffinityCenterDeformation.cs | 10 +- .../CustomSurfaceDeformation.cs | 38 +- .../Framework/Util/ARAP.cs | 7 + .../Framework/Util/KDTree.cs | 4 + .../Framework/Util/TransformDistance.cs | 3 + .../AffinityCenterDeformation.cs | 10 +- .../CustomSurfaceDeformation.cs | 38 +- open4d/core/tests/test_gaussians.py | 33 +- open4d/core/tests/test_metrics.py | 2 +- open4d/gaussians.py | 49 +- open4d/io/_native_usd.py | 36 +- open4d/io/tests/test_unified_api.py | 12 +- open4d/native.py | 10 +- open4d/reconstruction/queen/train.py | 38 +- open4d/streamer/streamer/client/viewer.html | 209 +++++++-- open4d/streamer/streamer/codecs.py | 17 +- open4d/streamer/streamer/sequence.py | 436 +++++++++++++----- open4d/streamer/streamer/server/__init__.py | 2 +- open4d/streamer/streamer_tests/test_codecs.py | 8 +- .../streamer_tests/test_representations.py | 6 +- .../streamer/streamer_tests/test_sequence.py | 436 ++++++++++++++---- .../test_server_and_transfer.py | 8 +- open4d/visualization/_api.py | 6 +- .../visualization/tests/test_visualization.py | 44 +- open4d/webclients/README.md | 7 +- .../webclients/system/Server/path-safety.js | 26 ++ open4d/webclients/system/Server/server.js | 45 +- .../webclients/system/Server/study/routes.js | 3 + .../webclients/system/Server/study/store.js | 19 +- .../webclients/tests/test_server_security.js | 63 +++ open4d/webclients/tests/test_study_session.js | 51 ++ open4d/webclients/tests/test_v4ds_protocol.js | 10 +- pyproject.toml | 2 +- scripts/smoke_installed_io.py | 21 +- scripts/tests/test_benchmark_codec.py | 10 +- scripts/tests/test_cli.py | 6 +- 97 files changed, 3955 insertions(+), 1465 deletions(-) create mode 100644 .agents.txt create mode 100644 .codex/config.toml create mode 100644 open4d/codec/_metadata.py create mode 100644 open4d/codec/_migration.py rename open4d/codec/{_v3c.py => _vmesh_format.py} (61%) create mode 100644 open4d/codec/tests/editor_numerics.cs create mode 100644 open4d/codec/tests/test_editor_numerics.py create mode 100644 open4d/codec/tests/test_klt_quantization.py create mode 100644 open4d/codec/tests/test_n4mc_determinism.py create mode 100644 open4d/codec/tests/test_queen_training_exports.py create mode 100644 open4d/codec/tests/test_reference_vmesh.py create mode 100644 open4d/codec/tests/test_research_vmesh.py create mode 100644 open4d/codec/tests/test_tracker_numerics.py rename open4d/codec/tests/{test_v3c.py => test_vmesh_format.py} (70%) create mode 100644 open4d/codec/tests/test_vmesh_only.py create mode 100644 open4d/codec/tests/tracker_numerics.cs create mode 100644 open4d/webclients/system/Server/path-safety.js create mode 100644 open4d/webclients/tests/test_server_security.js diff --git a/.agents.txt b/.agents.txt new file mode 100644 index 00000000..a6b5857e --- /dev/null +++ b/.agents.txt @@ -0,0 +1,11 @@ +Repository instructions for all agents + +Never create random or unsolicited Markdown (.md) files in this repository. +Create a new Markdown file only when the user explicitly requests that file. +Update the relevant existing documentation when documenting code changes. +Keep task plans, audit reports, progress notes, summaries, and scratch notes in +the conversation unless the user explicitly requests a saved artifact. +Do not evade this rule by creating unsolicited reports with another extension. +Keep temporary files outside the repository and remove your temporary artifacts. +Preserve existing useful documentation, licenses, and vendored documentation. +Follow these instructions throughout the repository, including subdirectories. diff --git a/.codex/config.toml b/.codex/config.toml new file mode 100644 index 00000000..00809464 --- /dev/null +++ b/.codex/config.toml @@ -0,0 +1 @@ +project_doc_fallback_filenames = [".agents.txt"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff744889..39cc173c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,17 +84,47 @@ jobs: python-version: "3.12" cache: pip - name: Install research codec dependencies - run: python -m pip install -e '.[dev,klt,n4mc,qndf]' - - name: Run real in-process encode and fresh-decode contracts on CPU + run: python -m pip install -e '.[dev,klt,n4mc,qndf,usd,gaussians]' constriction 'DracoPy>=2,<3' + - name: Run default suite with research dependencies + run: python -m pytest + - name: Run native N4MC model and loss contracts run: >- python -m pytest - open4d/codec/tests/test_research_cpu.py - open4d/codec/tests/test_qndf_preprocess.py - open4d/codec/tests/test_qndf_entrypoints.py - open4d/codec/tests/test_codec_review.py open4d/codecs/n4mc/tests/test_codec.py open4d/codecs/n4mc/tests/test_losses.py + native-numerics: + name: Native tracker and editor / .NET 10 + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 15 + env: + DOTNET_NOLOGO: "1" + DOTNET_CLI_TELEMETRY_OPTOUT: "1" + OPEN4D_TEST_DOTNET: dotnet + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "10.0.x" + - name: Install native regression test dependencies + run: python -m pip install -e '.[dev]' + - name: Build and execute native numerical contracts + run: | + dotnet --info + python -m pytest open4d/codec/tests/test_tracker_numerics.py open4d/codec/tests/test_editor_numerics.py --junitxml="${RUNNER_TEMP}/native-numerics.xml" + python - <<'PY' + import os + from pathlib import Path + from xml.etree import ElementTree + report = ElementTree.parse(Path(os.environ["RUNNER_TEMP"]) / "native-numerics.xml") + cases = report.findall(".//testcase") + assert cases and not any(case.find("skipped") is not None for case in cases), "native numerical contracts must execute" + PY + io-tools: name: I/O / real optional formats and benchmark smoke runs-on: blacksmith-2vcpu-ubuntu-2404 @@ -156,6 +186,37 @@ jobs: if-no-files-found: ignore retention-days: 7 + browser: + name: Browser / streamer and web clients + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - uses: actions/setup-node@v4 + with: + node-version: "24" + cache: npm + cache-dependency-path: | + open4d/webclients/system/Server/package-lock.json + open4d/webclients/system/WebClient/package-lock.json + - name: Install browser test dependencies and standalone streamer + run: python -m pip install -e '.[dev]' -e open4d/streamer 'DracoPy>=2,<3' scikit-image + - name: Check installed streamer and run its headless tests + run: | + python -I -c "import streamer; print(streamer.__file__)" + python -m pytest open4d/streamer/streamer_tests + - name: Install and build browser clients + run: | + npm ci --prefix open4d/webclients/system/Server + npm ci --prefix open4d/webclients/system/WebClient + npm run build --prefix open4d/webclients/system/WebClient + - name: Run web client and server JavaScript tests + run: node --test open4d/webclients/tests/*.js open4d/webclients/system/Server/test_*.js + package: name: Install / ${{ matrix.os }} / Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} diff --git a/README.md b/README.md index d3cf8b19..10ab5a1c 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ texture coordinates and custom attributes. Choose a research codec explicitly. For example, after setting up V-DMC: ```python -encoded = open4d.encode(sequence, "wave.v4d", codec="vdmc") +encoded = open4d.encode(sequence, "wave.vmesh", codec="vdmc") decoded = open4d.decode(encoded) open4d.visualize(decoded) decoded.close() @@ -67,7 +67,7 @@ decoded.close() artifact. Close a decoded mesh sequence when finished, or use `with`: ```python -with open4d.decode("wave.v4d") as decoded: +with open4d.decode("wave.vmesh") as decoded: print(len(decoded), "frames") ``` @@ -76,31 +76,40 @@ reconstruction and streaming in separate short cells. Short `.vmesh` examples: - [Preserve compressed state through USDC](examples/vmesh/01_container_and_usdc.ipynb) -- [TVMC, TSMC, V-DMC, faster V-DMC and N4MC](examples/vmesh/02_mesh_codecs.ipynb) +- [The eight mesh codecs](examples/vmesh/02_mesh_codecs.ipynb) - [Vega, QUEEN and 3DGStream](examples/vmesh/03_gaussian_codecs.ipynb) - [ReRF](examples/vmesh/04_rerf.ipynb) | Codec | Input | Output | Backend setup | | --- | --- | --- | --- | -| `vdmc` | Mesh sequence | `.vmesh` or `.v4d` | Build the V-DMC submodule and configure its encoder and decoder | -| `faster_vdmc` | Mesh sequence | `.vmesh` or `.v4d` | Build the faster V-DMC submodule and configure its encoder and decoder | -| `tvmc` | Mesh sequence | `.vmesh` file or `.tvmc` directory | [TVMC setup](open4d/codecs/tvmc/README.md) | -| `tsmc` | Mesh sequence | `.vmesh` file or `.tsmc` directory | [TSMC setup](open4d/codecs/tsmc/README.md) | -| `klt` | Mesh sequence converted to TSDF volumes | `.k4d` | Research source and `.[klt]` | -| `n4mc` | Mesh sequence converted to TSDF volumes | `.vmesh` or `.n4d` | Research source and `.[n4mc]` | -| `qndf`, `qndf-int8` | Mesh frames | `.q4d`, `.qi4d` | Research source and `.[qndf]` | -| `vega` | Gaussian splat frames or native run | `.vmesh` or `.vega` directory | [Vega CUDA environment](open4d/reconstruction/vega/README.md) | +| `vdmc` | Mesh sequence | `.vmesh` | Build the V-DMC submodule and configure its encoder and decoder | +| `faster_vdmc` | Mesh sequence | `.vmesh` | Build the faster V-DMC submodule and configure its encoder and decoder | +| `tvmc` | Mesh sequence | `.vmesh` | [TVMC setup](open4d/codecs/tvmc/README.md) | +| `tsmc` | Mesh sequence | `.vmesh` | [TSMC setup](open4d/codecs/tsmc/README.md) | +| `klt` | Mesh sequence converted to TSDF volumes | `.vmesh` | Research source and `.[klt]` | +| `n4mc` | Mesh sequence converted to TSDF volumes | `.vmesh` | Research source and `.[n4mc]` | +| `qndf`, `qndf-int8` | Mesh frames | `.vmesh` | Research source and `.[qndf]` | +| `vega` | Gaussian splat frames or native run | `.vmesh` | [Vega CUDA environment](open4d/reconstruction/vega/README.md) | | `queen`, `3dgstream`, `rerf` | Native temporal research output | `.vmesh`, with native USDC interchange | Method-specific CUDA runtime for evaluation | -QNDF-int8 now writes version 2 artifacts. Version 1 artifacts must be encoded again. - -TVMC, TSMC, V-DMC, Vega, QUEEN, 3DGStream, ReRF and N4MC support -[native `.vmesh` carriage](examples/vmesh/01_container_and_usdc.ipynb). Mesh codecs support mesh USDC -import/export; neural methods preserve their compressed temporal state in an -O4D native USDC schema. Existing native outputs can be packed without recompression. -N4MC carries a shared model and independent frame latents as an explicit -exception to the temporal-dependency requirement. -This experimental O4D V3C application format requires O4D decoding support. +All twelve public codecs write standalone `.vmesh` files. The embedded profile +identifies the codec; there is no separate public compressed output type for each +method. Existing native outputs can be packed without recompression. Standard +PLY, OBJ and USD import/export remain available. + +Mesh codecs decode to mesh sequences for ordinary USDC geometry export. +To preserve a compressed artifact exactly, construct `open4d.NativeSequence` +and save it to USDC using the custom `VMESH` prim, as shown in the +[container notebook](examples/vmesh/01_container_and_usdc.ipynb). This works with +every VMESH profile. Gaussian and field methods load as `NativeSequence` and +evaluate their native models explicitly. N4MC shares a model across independent +frame latents; QNDF has independent frame models. +[VMESH](docs/api.md#vmesh-format) is a standalone custom format: its own versioned +header and metadata directly carry native codec payloads. It has no private +Open4D/V3C wrapper and does not claim MPEG V-DMC interoperability. A V-DMC +profile carries the actual encoder bitstream as a native payload. Older artifacts +require explicit migration with `open4d.migrate_legacy`; normal load/save paths +accept VMESH rather than silently interpreting an older compressed type. KLT, N4MC and QNDF run in Python. TVMC, TSMC, V-DMC and Gaussian methods use separate research runtimes. N4MC and QNDF currently process frames independently; @@ -211,7 +220,7 @@ Read splat frames, then encode them with the local Vega adaptation: from open4d import load_gaussians, encode, decode frames = [load_gaussians("frame_0000.ply"), load_gaussians("frame_0001.ply")] -encoded = encode(frames, "capture.vega", codec="vega") +encoded = encode(frames, "capture.vmesh", codec="vega") decoded = decode(encoded) ``` @@ -245,11 +254,10 @@ requires its native viewer. The Qt viewer and TCP stream currently take meshes. - `open4d demo`, `open4d inspect` and `open4d view` provide command-line access. - `open4d.io.write_sequence` exports mesh folders; `open4d.save` writes OpenUSD - or explicitly selected codec artifacts. There is no default `.o4d` encoder. + or `.vmesh` with an explicitly selected codec. - `open4d.compare_sequences` measures mesh error with the `.[metrics]` extra. - [Viewer examples](examples/visualization/README.md) include GIF export and comparisons. - [Contributor setup and tests](CONTRIBUTING.md) cover optional dependencies and packaging. -The general `.o4d` format is separate work. Existing codec-specific formats -remain in use. Publication is still blocked by the unresolved component rights +Publication is still blocked by the unresolved component rights in [THIRD_PARTY.md](THIRD_PARTY.md); preparing the package does not resolve them. diff --git a/SECURITY.md b/SECURITY.md index 5630a353..5de40b9e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -28,4 +28,39 @@ private data, and identifying capture content. references. - Revoke exposed credentials before removing them from Git history. +## Known limitations + +VMESH framing and hashes validate container integrity; they do not authenticate +the publisher or make native payloads safe to execute. Inspection, packing, +and compressed USD round trips do not invoke native models. Decoding can: + +- [QUEEN's compressed loader](open4d/reconstruction/queen/scene/gaussian_model.py) + uses `pickle.load`, which can execute code from a malicious residual. +- [ReRF's player](open4d/reconstruction/rerf/rerf_stream/bitstream.py) loads + checkpoints with `torch.load` without explicitly selecting `weights_only`; + older Torch runtimes can execute serialized code. Other research entry points + also load executable checkpoints. A worker subprocess is not a security sandbox. + +The [Unity TVMC backend](integrations/unity/TVMCUnity/CPP_Backend/src) has +unchecked OBJ face indices, a decoder access to row 100 without a minimum-row +check, and a `FetchFrame` interface without an output-capacity argument. +Its matrix loader also permits very large allocations before checking that all +declared bytes exist. These are source-level memory safety and resource risks; +runtime exploitation and sanitizer validation have not been performed here. + +The Node control API, decoded-frame TCP transport, and research WebSocket +bridges have no authentication. Origin and Host checks do not authenticate +native clients. Some bridges bind to all interfaces, and static file routes +follow symlinks; their network exposure and asset roots must be controlled. + +Vendored QUEEN and 3DGStream trainer/renderer argument loaders evaluate +`cfg_args`. Some 3DGStream conversion scripts and the ReRF LLFF loader construct +shell commands from paths or options. Use trusted inputs for those entry points; +the VMESH import path uses JSON instead of these evaluated configurations. + +Legacy ZIP/NPZ paths do not consistently bound aggregate expanded bytes. +Repacking a legacy N4MC archive can exhaust disk before VMESH validation, and +neural architecture or volume dimensions can trigger large allocations even +after safe deserialization. + The absence of a warning is not a claim that a component is production-ready. diff --git a/docs/api.md b/docs/api.md index b5ba8084..18dde07a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -15,8 +15,9 @@ open4d.visualize("capture-copy.usdc") ``` `.usd`, `.usda`, `.usdc`, and `.usdz` are OpenUSD interchange containers. -The registered codec suffixes identify codec artifacts. Both carry whole -sequences and use the same `Sequence` interface. `open4d.unload(sequence)` is an +All registered compression methods use `.vmesh`; its descriptor identifies the +codec. Mesh profiles expose `Sequence`, while native Gaussian and field profiles +expose `NativeSequence`. `open4d.unload(sequence)` is an explicit, idempotent alternative to the context manager. Frame folders and individual meshes remain supported as import paths; frames @@ -40,9 +41,9 @@ out of a V-DMC bitstream. `open4d.Representation` is the axis to gate on, and taxonomy but have no concrete type yet; they land with the camera model they need in order to be comparable at a known pose. -The containers and codecs described below are the mesh path, and the one most -completely covered. Check a specific codec before assuming it round-trips -points or Gaussians. +The mesh codecs accept triangle meshes. Native Gaussian and field methods require +their own trained states or calibrated reconstruction inputs; a mesh file alone +does not provide those inputs. ## Writing sequences @@ -58,8 +59,8 @@ color and GLB/glTF quantize canonical float colors to eight bits. STL also requires `allow_lossy=True`: it discards unused vertices and vertex identity, so exported manifests clear correspondence and topology guarantees. STL/GLB/glTF reject geometry without triangles. Gaussian clouds cannot be -exported through these mesh writers, and USD writing currently accepts only -triangle meshes. +exported through these mesh writers. Ordinary USD geometry writing accepts +triangle meshes; compressed VMESH USD interchange is described below. OpenUSD is the public interchange container. `--pack-usd out.usdc` packs any source into one compressed `.usdc` file carrying the frame rate, the key-frame @@ -110,7 +111,7 @@ other. It is not this API and shares no code with it. ## Codecs -TVMC and TSMC can encode directly to a single `.vmesh` file: +Every public codec encodes to a single `.vmesh` file: ```python open4d.encode("capture.usdc", "capture.vmesh", codec="tvmc") @@ -118,41 +119,39 @@ with open4d.load("capture.vmesh") as sequence: open4d.save(sequence, "decoded.usdc") ``` -The embedded codec is detected on load. The experimental O4D V3C application -format preserves native compressed payloads and timing; existing `.tvmc` and -`.tsmc` directories remain supported. See the [native `.vmesh` notebook](../examples/vmesh/01_container_and_usdc.ipynb) +The embedded codec is detected on load. The standalone [VMESH format](#vmesh-format) +preserves native compressed payloads and timing. See the [native `.vmesh` notebook](../examples/vmesh/01_container_and_usdc.ipynb) for inspection, packing, extraction, and an exact compressed-state USDC round trip. The [mesh example](../examples/vmesh/02_mesh_codecs.ipynb) demonstrates encoding and saving decoded geometry as USDC. -Vega, QUEEN, 3DGStream and ReRF also support native `.vmesh` output. Import a +Vega, QUEEN, 3DGStream and ReRF carry native compressed models. Import a research run with `open4d.import_native(run, codec="queen", config=...)`, or use `open4d.encode(run, "capture.vmesh", codec="queen", config=...)`. Loading returns a `NativeSequence`; `native.decode(runtime=..., python=...)` evaluates the actual temporal models. Saving it to USDC preserves the exact -compressed representation using O4D's native USD schema. This route does not +compressed representation using the custom `VMESH` USD prim. This route does not train a neural scene from an arbitrary mesh USD. See the linked guide for required model/configuration files and representation-specific outputs. -N4MC also accepts `.vmesh` output from mesh sequences or mesh USDC using +KLT, N4MC, QNDF and QNDF-int8 accept mesh sequences or mesh USDC, for example `open4d.encode(..., codec="n4mc")`. Loading reconstructs a mesh `Sequence`. -`open4d.codec.pack_vmesh("old.n4d", "new.vmesh")` migrates existing O4D N4MC -artifacts without retraining. Its profile explicitly identifies independent -quantized TSDF latents sharing one model; it does not claim temporal prediction. +N4MC's profile identifies independent quantized TSDF latents sharing one model; +it does not claim temporal prediction. KLT carries a shared transform basis and +compressed frame coefficients. QNDF carries coarse meshes and independent +displacement models; QNDF-int8 uses quantized model weights. The guide also documents exact compressed-state preservation in native USDC. `available_codecs()` lists the public research adapters: `klt`, `n4mc`, `qndf`, `qndf-int8`, `vdmc`, `faster_vdmc`, `tvmc`, `tsmc`, `vega`, `queen`, -`3dgstream`, and `rerf`. The NumPy -reference codecs and temporal experiments are internal benchmark baselines; -`npz` and `.o4d` are not supported public save defaults. Use USD for interchange -or select a registered codec and its suffix. +`3dgstream`, and `rerf`. Use `.vmesh` for every registered compression method +and USD for interchange. NumPy archives, model checkpoints and Draco files are +native payload encodings within a profile, not alternate public sequence types. The lightweight wheel includes the adapters but excludes their research implementations. KLT, N4MC and QNDF need a source checkout and their optional dependencies; set `OPEN4D_RESEARCH_ROOT` to that checkout when using an installed -wheel. Open4D's separate `temporal-delta` and `temporal-pca` experiments are -not the repository's TVMC or TSMC pipelines. The V-DMC adapters do not execute +wheel. Internal benchmark experiments are not public codec choices. The V-DMC adapters do not execute shell scripts, but they do invoke configured native encoder and decoder processes once per sequence. Callers can also register another `open4d.codec.Codec`. @@ -176,6 +175,113 @@ KLT decoding also accepts `device="auto"`. ### Test coverage Normal CI runs dependency-complete CPU encode/fresh-decode contracts for KLT, -N4MC, QNDF, and QNDF-int8. The larger two-format Rafa quality/export matrix is +N4MC, QNDF, and QNDF-int8. The larger Rafa quality/export matrix is an additional CUDA acceptance test gated by `OPEN4D_TEST_RESEARCH_CODECS=1` and `OPEN4D_RAFA_DATASET`; it is not presented as part of ordinary CI coverage. + +## VMESH format + +VMESH is a standalone custom data type for a compressed sequence. Its header, +metadata, codec profiles and records belong to VMESH. It does not manufacture a +V3C stream, insert private SEI messages, or serialize Open4D geometry objects. +It is not an interoperable MPEG V-DMC interchange format. A V-DMC profile can +carry a real encoder-produced V3C bitstream as its native payload. + +The Python API uses `Sequence` or `NativeSequence` to expose a file. Those are +in-memory interfaces, not data types embedded in the file. + +### Binary layout + +All integers in the container are unsigned and big endian. The file starts with +eight bytes: `56 4d 45 53 48 00 01 00` (`VMESH`, zero, version 1, reserved zero). +Each following record is: + +| Field | Bytes | Meaning | +| --- | --- | --- | +| Record length | 4 | Length of the record header plus its data | +| Record version | 1 | Must be 1 | +| Kind | 1 | 0 = manifest, 1 = native payload chunk, 2 = end | +| File ID | 4 | Zero-based native file index; zero for manifest/end | +| Offset | 8 | Offset within the native payload; zero for manifest/end | +| Data | variable | JSON manifest, native bytes, or end digest | + +There is exactly one manifest, followed by all payload chunks in manifest file +order, followed by exactly one end record. The end data is the 32-byte SHA-256 +of the exact manifest bytes. No trailing bytes are permitted. + +The UTF-8 JSON manifest has `schema: "vmesh/1"`, `codec`, `native_version: 1`, +`representation`, `dependency_mode`, `frame_count`, `sequence`, `files`, and +codec-specific `native` settings when required. `sequence` contains version 1, +the codec ID, frame indices, timestamps in seconds, JSON metadata, and optional +normalization/topology fields. It cannot carry an application `schema` field. +Each file record contains `id`, `name`, `role`, byte `size`, and hexadecimal +`sha256`. Filenames and roles must exactly match the selected codec profile; +input cannot supply arbitrary extraction paths. + +The manifest is at most 16 MiB; payload chunks are at most 1 MiB. Frame and +profile file counts are bounded at 65,536 (including the generated adapter +metadata in the profile file limit). Some profiles have lower frame limits +because they require several files per frame. Duplicate JSON keys, non-finite +numbers, reordered chunks, offsets, unknown versions, hash failures and extra +records are rejected. These hashes detect corruption; they do not authenticate +a publisher or make an executable research payload safe to decode. + +### Native profiles + +| Codec | Native payloads | Dependencies | +| --- | --- | --- | +| KLT | `decoder_context.pt`, `_quantized_indices.zst`, `_quantized_metadata.npz` | Shared basis, independent coefficients | +| QNDF / QNDF-int8 | `frame_.pt`: subdivided base mesh, displacement context and model | Independent frames; int8 decode on CPU | +| N4MC | `checkpoint.pt`, `frame_.npz` | Shared model, independent TSDF latents | +| TVMC | `reference.drc`, `displacement_.drc` and `.npy` | Shared mesh reference; displacement and vertex order | +| TSMC | `reference.drc`, `B_matrix.txt`, `T_matrix.txt`, `delta_trajectories_encoded.npy`, `entropy_model.npz` | Whole temporal group | +| V-DMC / faster V-DMC | `sequence.vmesh`, optional `decoder.cfg` | Actual native encoder bitstream | +| Vega | `manifest.json`, `color_model.pt`, `frame_.pt` | Shared appearance and key/residual state | +| QUEEN | `initial.ply`, `frame_.pkl` for residual frames | Previous Gaussian frame | +| 3DGStream | `initial.ply`, `ntc_config.json`, `ntc_.pth`, optional `added_.ply` | Previous frame and transform architecture | +| ReRF | Field/model configuration, `rgb_net.tar`, headers, occupancy/channel entropy files, optional PCA/motion | Previous field and group keys | + +Frame ordinals are zero-padded to six digits except Vega's four-digit native +index. ReRF uses its declared native frame IDs and channel suffixes. The profile +defines exact names and order; this table is not a user-supplied extraction map. +The separate `frames` delivery profile carries standard per-frame PLY, Draco, +splat or image payloads and is not another research compression method. + +Native files are copied verbatim. A directory's `metadata.json` is parsed into +the VMESH manifest; it is **not a carried file**. Unpacking reconstructs a +`metadata.json` only as an input for the existing codec adapters. JSON whitespace +in that generated adapter file is not preserved. Native payload bytes are. + +Use `open4d.codec.inspect_vmesh`, `pack_vmesh`, and `unpack_vmesh` to inspect, +create and extract files without loading native models. Inspection verifies +all hashes. Extraction is staged and published only after complete validation. + +### USD interchange + +Compressed interchange uses a custom prim of type `VMESH` at `/VMESH`, with +`vmesh:schema = "vmesh.usd/1"`, timeline samples in `vmesh:frameIndex`, and +static `vmesh:payload:chunkNNNNNN` byte arrays. `vmesh:payloadManifest` describes +the exact carried VMESH file and its hash. This prim preserves compressed state; +it does not claim to be a renderable `UsdGeomMesh` or Gaussian schema. + +### Compatibility + +Normal load, save, encode and decode accept standalone VMESH for compressed +sequences. Older artifacts require explicit `open4d.migrate_legacy` conversion; +renaming their extension is insufficient. Use `open4d.import_native` for a +supported trained native run and `pack_vmesh` for a validated profile directory. +Neither operation trains a model or turns arbitrary native files into a sequence. +Native research encodings remain inside the VMESH profile. Previously exported +USD compressed state must use the custom `VMESH` prim described above. +Raw native MPEG V-DMC bitstreams remain supported as read/import inputs using +external frame timing; new compressed output is always standalone VMESH. + +```python +open4d.migrate_legacy("older_artifact", "converted.vmesh", fps=30.0) +``` + +Migration covers earlier research-codec exports and packed browser clips. Older +private benchmark artifacts must be re-encoded from their original geometry. +Migration is an explicit compatibility operation. `overwrite=False` protects an +existing destination, and `fps` supplies timing only when the old input did not +carry it. The result is validated VMESH; normal loading does not silently migrate. diff --git a/examples/open4d_sequence_codec.ipynb b/examples/open4d_sequence_codec.ipynb index 82ce2ebe..93a710a4 100644 --- a/examples/open4d_sequence_codec.ipynb +++ b/examples/open4d_sequence_codec.ipynb @@ -54,9 +54,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Encode\n", - "\n", - "After building V-DMC, set `OPEN4D_VDMC_ENCODER` and `OPEN4D_VDMC_DECODER` to its programs. The codec is always chosen explicitly. This writes the existing V-DMC artifact; it does not define the future general `.o4d` format.\n" + "After building V-DMC, set `OPEN4D_VDMC_ENCODER` and `OPEN4D_VDMC_DECODER` to its programs. Choose the codec explicitly. Every public codec writes a standalone `.vmesh` file containing its native payload and timeline." ], "id": "open4d-04" }, @@ -70,7 +68,7 @@ }, "outputs": [], "source": [ - "encoded = open4d.encode(sequence, \"wave.v4d\", codec=\"vdmc\")\n" + "encoded = open4d.encode(sequence, \"wave.vmesh\", codec=\"vdmc\")" ], "id": "open4d-05" }, @@ -209,14 +207,14 @@ "Then run the sender in the other process:\n", "\n", "```python\n", - "from open4d import stream\n", + "from open4d import send\n", "from open4d.demo import mesh_sequence\n", "\n", - "stream(mesh_sequence(frames=30))\n", + "send(mesh_sequence(frames=30))\n", "```\n", "\n", "This sends the mesh frames at their recorded timing over TCP on this computer.\n", - "It does not compress them. Both sides use port 7000; pass `host=` and `port=`\n", + "It does not compress them. Both sides use port 47004; pass `host=` and `port=`\n", "for another address. Use a trusted network or SSH tunnel for remote transport.\n" ], "id": "open4d-15" @@ -231,7 +229,7 @@ "\n", "```python\n", "frames = [open4d.load_gaussians(\"frame_0000.ply\"), open4d.load_gaussians(\"frame_0001.ply\")]\n", - "encoded = open4d.encode(frames, \"capture.vega\", codec=\"vega\")\n", + "encoded = open4d.encode(frames, \"capture.vmesh\", codec=\"vega\")\n", "decoded = open4d.decode(encoded)\n", "```\n", "\n", diff --git a/examples/visualization/README.md b/examples/visualization/README.md index 390d32aa..2f5f4b8b 100644 --- a/examples/visualization/README.md +++ b/examples/visualization/README.md @@ -110,7 +110,7 @@ one and say so. | Folder of `.obj` or `.ply` frames | nothing | | Folder of `.stl` `.off` `.glb` `.gltf` frames | `.[tools]` | | One USD file (`.usd` `.usda` `.usdc` `.usdz`) | `.[usd]` | -| Raw MPEG V-DMC bitstream (`.vmesh`) | a compatible native V-DMC decoder | +| Standalone `.vmesh` with a mesh profile | the codec's configured decoder/runtime | | One mesh file | as above | Frames are ordered by **the last number in the filename**, so `frame_2.obj` comes @@ -118,9 +118,11 @@ before `frame_10.obj` — but `frame_003_qp9.obj` sorts on 9, not 3. A codec tha puts a parameter last will silently misalign every frame and look far worse than it is; rename before comparing. A frame with no faces is drawn as a point cloud. -### Raw V-DMC bitstreams +### Compressed VMESH sequences -Raw `.vmesh` input needs a compatible native decoder: +The VMESH descriptor selects the codec and preserves frame indices, timestamps +and coordinate metadata. Configure that codec's runtime before loading. For +example, a V-DMC profile needs its native decoder: ```bash export OPEN4D_VDMC_DECODER=/path/to/vmesh-decoder @@ -128,10 +130,10 @@ export OPEN4D_VDMC_DECODER_CONFIG=/path/to/decoder.cfg # optional python examples/visualization/visualize_sequence.py capture.vmesh --info ``` -This path is read-only and geometry-only. It defaults to 30 fps, accepts -`--fps N`, and displays the coordinates emitted by the decoder. Open4D removes -the temporary decoded OBJ files when the sequence closes. Use `.v4d` when the -sequence also needs Open4D timestamps, metadata, and coordinate bounds. +The viewer displays decoded geometry. Open4D removes temporary decoded files +when the sequence closes. `--fps N` overrides playback speed without replacing +the stored VMESH timestamps. Native Gaussian and neural-field profiles use +their representation-specific decode/render paths rather than this mesh viewer. ## Flags @@ -169,7 +171,7 @@ with open4d.load("capture.usdc") as sequence: print(len(sequence), sequence.duration, sequence.fps) open4d.visualize(sequence) -open4d.visualize("capture.o4d") +open4d.visualize("capture.vmesh") ``` Loading and playback are lazy — frames decode on access. Frame directories and diff --git a/examples/visualization/compare_sequences.py b/examples/visualization/compare_sequences.py index 8dda7ea1..c2714e60 100644 --- a/examples/visualization/compare_sequences.py +++ b/examples/visualization/compare_sequences.py @@ -6,8 +6,8 @@ --metric plane --csv error.csv --save compare.gif This is the comparison viewer; `visualize_sequence.py` is the plain one. Both -read the same sources — `.o4d` codec artifacts, time-sampled USD files, frame -directories, raw V-DMC `.vmesh` bitstreams, or standalone mesh imports — +read the same sources — `.vmesh` mesh profiles, time-sampled USD files, frame +directories, or standalone mesh imports — through the same loader. Two synchronized panes: the reference as geometry, the decoded mesh coloured by diff --git a/examples/visualization/frame_sources.py b/examples/visualization/frame_sources.py index 7332a86b..3a5d07cf 100644 --- a/examples/visualization/frame_sources.py +++ b/examples/visualization/frame_sources.py @@ -6,11 +6,11 @@ from open4d import load as _open_sequence from open4d.codec import available_codecs +from open4d.codec._vmesh_format import probe_codec from open4d.io import available_formats, inspect_sequence DEFAULT_FPS = 30.0 _USD_SUFFIXES = {".usd", ".usda", ".usdc", ".usdz"} -_RAW_CODEC_SUFFIXES = {".vmesh"} def _codec_suffixes(): @@ -30,8 +30,8 @@ def supported_formats() -> str: ) target = sequence_lines if info.id == "usd" else frame_lines target.append(f" {'/'.join(info.suffixes):<24}{extra}".rstrip()) - sequence_lines.append(" research codec files/directories: " + ", ".join(sorted(_codec_suffixes()))) - sequence_lines.append(" .vmesh needs a native V-DMC decoder") + sequence_lines.append(" research codec files: " + ", ".join(sorted(_codec_suffixes())) + + " (needs the selected codec runtime)") return "\n".join((*sequence_lines, *frame_lines)) @@ -41,7 +41,7 @@ def source_kind(path: Path | str) -> str: if not path.exists(): raise SystemExit(f"{path} does not exist") if path.suffix.lower() in ( - _USD_SUFFIXES | _codec_suffixes() | _RAW_CODEC_SUFFIXES + _USD_SUFFIXES | _codec_suffixes() ): return "sequence-file" if path.is_dir(): @@ -56,14 +56,15 @@ def source_kind(path: Path | str) -> str: def open_sequence(path: Path | str, fps: float | None = None): """Load a source, using ``fps`` for manifest-free frame timing.""" path = Path(path) - uses_import_fps = (path.is_dir() and path.suffix.lower() not in _codec_suffixes()) or path.suffix.lower() in _RAW_CODEC_SUFFIXES - return _open_sequence(path, fps=fps if uses_import_fps else None) + uses_import_fps = path.is_dir() and path.suffix.lower() not in _codec_suffixes() + raw_vdmc = path.is_file() and path.suffix.lower() == ".vmesh" and probe_codec(path) is None + return _open_sequence(path, fps=fps if uses_import_fps or raw_vdmc else None) def describe_source(path: Path | str, frame_count: int | None = None) -> str: """Describe a source without eagerly parsing its frame geometry.""" path = Path(path) - if path.suffix.lower() in _codec_suffixes() | _RAW_CODEC_SUFFIXES: + if path.suffix.lower() in _codec_suffixes(): if frame_count is None: with _open_sequence(path) as opened: frame_count = len(opened) diff --git a/examples/visualization/tests/test_frame_sources.py b/examples/visualization/tests/test_frame_sources.py index 1457ccd1..f680c0ab 100644 --- a/examples/visualization/tests/test_frame_sources.py +++ b/examples/visualization/tests/test_frame_sources.py @@ -24,13 +24,13 @@ def test_example_helpers_treat_codec_artifacts_as_whole_sequences(tmp_path, monk from open4d.codec import _api from open4d.codec._npz import NumPyZipCodec - monkeypatch.setitem(_api._CODECS, "fixture", NumPyZipCodec()) + monkeypatch.setitem(_api._CODECS, "npz", NumPyZipCodec()) mesh = TriangleMesh( [[0.0, 0, 0], [1, 0, 0], [0, 1, 0]], [[0, 1, 2]] ) artifact = open4d.save( Sequence(MemoryFrameProvider([Frame(0, 0.0, mesh)])), - tmp_path / "capture.o4d", codec="fixture", + tmp_path / "capture.vmesh", codec="npz", ) assert frame_sources.source_kind(artifact) == "sequence-file" @@ -72,9 +72,11 @@ def test_raw_vmesh_fps_is_forwarded_to_the_public_loader(tmp_path, monkeypatch): assert received == {"path": bitstream, "fps": 24} -def test_native_directory_uses_artifact_timing(tmp_path, monkeypatch): - source = tmp_path / "capture.tvmc" - source.mkdir() +def test_standalone_vmesh_uses_artifact_timing(tmp_path, monkeypatch): + from open4d.codec._npz import NumPyZipCodec + mesh = TriangleMesh([[0., 0, 0], [1, 0, 0], [0, 1, 0]], [[0, 1, 2]]) + source = NumPyZipCodec().encode(Sequence(MemoryFrameProvider([Frame(0, 1.25, mesh)])), + tmp_path / "capture.vmesh") received = {} monkeypatch.setattr(frame_sources, "_open_sequence", lambda path, **options: received.update(path=path, **options)) diff --git a/examples/visualization/tests/test_visualize_sequence.py b/examples/visualization/tests/test_visualize_sequence.py index 93ed9517..77a5b9f4 100644 --- a/examples/visualization/tests/test_visualize_sequence.py +++ b/examples/visualization/tests/test_visualize_sequence.py @@ -15,8 +15,8 @@ def test_malformed_codec_source_is_reported_without_a_traceback( tmp_path, monkeypatch ): - path = tmp_path / "broken.o4d" - path.write_bytes(b"not an Open4D artifact") + path = tmp_path / "broken.vmesh" + path.write_bytes(b"not a VMESH artifact") monkeypatch.setattr(sys, "argv", ["visualize_sequence.py", str(path), "--info"]) with pytest.raises(SystemExit) as caught: @@ -62,7 +62,7 @@ def get_frame(self, index): return Frame(index, index / 30, TriangleMesh(positions, [[0, 1, 2]])) sequence = Sequence(Provider()) - path = tmp_path / "capture.o4d" + path = tmp_path / "capture.vmesh" path.touch() captured = {} monkeypatch.setattr(cli, "open_sequence", lambda *args, **kwargs: sequence) diff --git a/examples/visualization/visualize_sequence.py b/examples/visualization/visualize_sequence.py index b9cf29d5..4d6ec685 100644 --- a/examples/visualization/visualize_sequence.py +++ b/examples/visualization/visualize_sequence.py @@ -6,7 +6,7 @@ python examples/visualization/visualize_sequence.py my_capture.usdc Point it at your own data. A source is normally a whole-sequence file such as -`.o4d`, a time-sampled `.usdc`, or a raw V-DMC `.vmesh` bitstream. A folder +`.vmesh` with a mesh codec profile or a time-sampled `.usdc`. A folder holding one mesh per frame and a standalone mesh remain useful import paths. Meshes and point clouds are both handled: a frame with no faces is drawn as a point cloud. @@ -131,7 +131,7 @@ def main() -> None: "path", type=Path, nargs="?", - help="your sequence: an .o4d, USD, or raw .vmesh file; a frame " + help="your sequence: a .vmesh mesh profile or USD file; a frame " "directory; or a standalone mesh import", ) parser.add_argument( @@ -141,8 +141,8 @@ def main() -> None: "--fps", type=float, default=None, - help="override playback; for a manifest-free frame directory or raw " - ".vmesh this also defines imported timestamps (default: 30 fps)", + help="override playback; manifest-free frame directories or raw native " + "inputs also use this for imported timestamps (default: 30 fps)", ) parser.add_argument( "--up", diff --git a/examples/vmesh/01_container_and_usdc.ipynb b/examples/vmesh/01_container_and_usdc.ipynb index 90ff7b12..31da1cb0 100644 --- a/examples/vmesh/01_container_and_usdc.ipynb +++ b/examples/vmesh/01_container_and_usdc.ipynb @@ -8,7 +8,7 @@ "# Preserve a .vmesh through USDC\n", "\n", "Set the paths below. This works with every supported codec and needs OpenUSD, but no codec runtime.\n", - "The USDC stores compressed native state in O4D's custom schema; it is not a renderable mesh export." + "The USDC stores compressed native state in a custom VMESH prim; it is not a renderable mesh export." ] }, { @@ -64,7 +64,7 @@ " restored = open4d.save(native, output / \"restored.vmesh\", overwrite=True)\n", "\n", "with source.open(\"rb\") as a, restored.open(\"rb\") as b:\n", - " assert hashlib.file_digest(a, \"sha256\").digest() == hashlib.file_digest(b, \"sha256\").digest()\n", + " assert hashlib.sha256(a.read()).digest() == hashlib.sha256(b.read()).digest()\n", "print(\"Identical compressed bytes\")" ] }, @@ -73,7 +73,7 @@ "id": "01-05", "metadata": {}, "source": [ - "To inspect or migrate native files without decoding:" + "To extract and repack the same native payloads without decoding:" ] }, { diff --git a/examples/vmesh/02_mesh_codecs.ipynb b/examples/vmesh/02_mesh_codecs.ipynb index 79a7bbb0..da274eff 100644 --- a/examples/vmesh/02_mesh_codecs.ipynb +++ b/examples/vmesh/02_mesh_codecs.ipynb @@ -7,8 +7,8 @@ "source": [ "# Mesh USDC → .vmesh → mesh USDC\n", "\n", - "Set your input and choose `tvmc`, `tsmc`, `vdmc`, `faster_vdmc` or `n4mc`.\n", - "Configure its [native backend](../../README.md) first. These examples encode geometry only." + "Set your input and choose `klt`, `n4mc`, `qndf`, `qndf-int8`, `tvmc`, `tsmc`, `vdmc` or `faster_vdmc`.\n", + "Configure its [research backend](../../README.md) first. These examples encode geometry only." ] }, { @@ -71,8 +71,7 @@ "id": "02-05", "metadata": {}, "source": [ - "Existing O4D `.tvmc` / `.tsmc` directories or `.n4d` files can be migrated without re-encoding.\n", - "Run the next cell only when you have such an artifact." + "Older compressed artifacts require explicit migration. Run the next cell only when you have an older artifact; ordinary encode/decode use `.vmesh`." ] }, { @@ -86,9 +85,7 @@ }, "outputs": [], "source": [ - "from open4d.codec import pack_vmesh\n", - "\n", - "pack_vmesh(\"existing.tvmc\", output / \"migrated.vmesh\", overwrite=True)" + "open4d.migrate_legacy(\"older_artifact\", output / \"migrated.vmesh\", overwrite=True)" ] } ], diff --git a/examples/vmesh/03_gaussian_codecs.ipynb b/examples/vmesh/03_gaussian_codecs.ipynb index e6140df8..1a312dbb 100644 --- a/examples/vmesh/03_gaussian_codecs.ipynb +++ b/examples/vmesh/03_gaussian_codecs.ipynb @@ -47,7 +47,7 @@ }, "outputs": [], "source": [ - "with open4d.import_native(\"runs/capture.vega\", codec=\"vega\") as native:\n", + "with open4d.import_native(\"runs/capture\", codec=\"vega\") as native:\n", " open4d.save(native, output / \"vega.vmesh\", overwrite=True)\n", " print(len(native), \"Vega frames packed\")" ] diff --git a/open4d/__init__.py b/open4d/__init__.py index 4ff80fe3..f3742b26 100644 --- a/open4d/__init__.py +++ b/open4d/__init__.py @@ -25,7 +25,7 @@ ) from ._api import load, save, unload, reconstruct, stream from ._streamer import StreamerDependencyError -from .codec import available_codecs, decode_sequence as decode, encode_sequence as encode +from .codec import available_codecs, decode_sequence as decode, encode_sequence as encode, migrate_legacy from .streaming import receive, send from .gaussians import GaussianSplats, GaussianRun, NeuralGaussianFrame, load_gaussians from .metrics import compare_meshes, compare_sequences @@ -63,6 +63,7 @@ "available_codecs", "decode", "encode", + "migrate_legacy", "compare_meshes", "compare_sequences", "dtypes", diff --git a/open4d/_api.py b/open4d/_api.py index 42bbae44..2ca2fd9a 100644 --- a/open4d/_api.py +++ b/open4d/_api.py @@ -48,7 +48,7 @@ def load( fps: float | None = None, options: Mapping[str, object] | None = None, ) -> Sequence | NativeSequence | tuple[NeuralGaussianFrame, ...]: - """Open a sequence artifact, V3C .vmesh bitstream, or geometry source.""" + """Open VMESH, native V-DMC input, or a geometry source.""" if format is not None and codec is not None: raise TypeError("format and codec are mutually exclusive") values = _options(options) diff --git a/open4d/_cli.py b/open4d/_cli.py index d96a8600..8b8a9bb2 100644 --- a/open4d/_cli.py +++ b/open4d/_cli.py @@ -9,7 +9,8 @@ import sys from . import load -from .codec import CodecError, available_codecs +from .codec import CodecError +from .codec._vmesh_format import is_mesh_profile from .core import Sequence from .demo import write_demo from .io import Open4DError @@ -145,8 +146,7 @@ def main(argv: list[str] | None = None) -> int: source = args.source.expanduser() if not source.exists(): raise FileNotFoundError(f"sequence source does not exist: {source}") - if any(source.suffix.lower() in info.suffixes and info.representation != "triangle_mesh" - for info in available_codecs()): + if source.suffix.lower() == ".vmesh" and not is_mesh_profile(source): raise ValueError("this command takes mesh sequences; use open4d.decode for Gaussian artifacts") if args.command == "view": # Check optional dependencies before an expensive codec decode. @@ -154,6 +154,8 @@ def main(argv: list[str] | None = None) -> int: _qt.check_available() with load(source, format=args.format, fps=args.input_fps) as sequence: + if not isinstance(sequence, Sequence): + raise ValueError("this command takes mesh sequences; use the native renderer for Gaussian or field data") if args.command == "inspect": info = _inspect(sequence, source) if args.json: diff --git a/open4d/codec/__init__.py b/open4d/codec/__init__.py index 9c177cc8..2d1ed391 100644 --- a/open4d/codec/__init__.py +++ b/open4d/codec/__init__.py @@ -13,7 +13,8 @@ from ._qndf import QNDFCodec from ._vmesh import VMeshCodec from ._tracked import TrackedMeshCodec -from ._v3c import inspect_vmesh, pack_vmesh, unpack_vmesh +from ._migration import migrate_legacy +from ._vmesh_format import inspect_vmesh, pack_vmesh, unpack_vmesh __all__ = [ "Codec", @@ -30,5 +31,6 @@ "register_codec", "inspect_vmesh", "pack_vmesh", + "migrate_legacy", "unpack_vmesh", ] diff --git a/open4d/codec/_api.py b/open4d/codec/_api.py index 396053f6..997e1c27 100644 --- a/open4d/codec/_api.py +++ b/open4d/codec/_api.py @@ -14,10 +14,11 @@ from ._klt import KLT_CODEC from ._n4mc import N4MC_CODEC from ._protocol import Codec, CodecError +from ._metadata import require_vmesh_destination from ._qndf import QNDF_CODEC, QNDF_INT8_CODEC from ._tracked import TVMC_CODEC, TSMC_CODEC from ._vmesh import FASTER_VDMC_CODEC, VDMC_CODEC -from ._v3c import probe_codec +from ._vmesh_format import probe_codec from ._native_temporal import QUEEN_CODEC, GSTREAM_CODEC, RERF_CODEC if TYPE_CHECKING: @@ -105,7 +106,7 @@ def encode_sequence( """Encode mesh/Gaussian data, or carry native temporal neural outputs.""" if "overwrite" in options and not isinstance(options["overwrite"], bool): raise TypeError("overwrite must be bool") - path = Path(destination) + path = require_vmesh_destination(destination) implementation = _codec(codec, path) if getattr(implementation, "representation", "triangle_mesh") != "triangle_mesh": if input_format is not None: @@ -128,7 +129,7 @@ def encode_sequence( def decode_sequence( source: str | Path, *, codec: str | Codec | None = None, **options ) -> Sequence | NativeSequence | tuple[NeuralGaussianFrame, ...]: - """Open mesh reconstructions, native temporal state, or legacy Vega frames.""" + """Open mesh reconstructions, native temporal state.""" path = Path(source) if path.suffix.lower() == ".vmesh" and not path.is_dir(): detected = probe_codec(path) @@ -137,8 +138,22 @@ def decode_sequence( if selected is not None and selected != detected: raise CodecError(f".vmesh contains {detected}, but codec={selected!r} was requested") if "fps" in options: - raise TypeError("O4D .vmesh stores frame timestamps; fps cannot override them") - codec = detected if codec is None else codec + raise TypeError("VMESH stores frame timestamps; fps cannot override them") + if detected == "frames": + if options: + raise TypeError("frame delivery VMESH accepts no native decoder options") + from open4d._streamer import _require + _require() + from streamer.sequence import open_frames + return open_frames(path) + if detected not in _CODECS and (codec is None or codec == detected): + from ._npz import REFERENCE_CODECS + from ._draco import DRACO_CODEC + from ._temporal import TEMPORAL_DELTA_CODEC, TEMPORAL_PCA_CODEC + private = {item.id: item for item in (*REFERENCE_CODECS, DRACO_CODEC, TEMPORAL_DELTA_CODEC, TEMPORAL_PCA_CODEC)} + codec = private.get(detected, detected) + else: + codec = detected if codec is None else codec elif codec is None: codec = "vdmc" return _codec(codec, path).decode(path, **options) diff --git a/open4d/codec/_draco.py b/open4d/codec/_draco.py index 311ca4c5..8b3a3e0d 100644 --- a/open4d/codec/_draco.py +++ b/open4d/codec/_draco.py @@ -2,22 +2,20 @@ from __future__ import annotations -from collections.abc import Mapping from importlib import import_module import json from pathlib import Path import tempfile from types import MappingProxyType -from zipfile import BadZipFile, ZIP_STORED, ZipFile import numpy as np from open4d.core import Frame, Sequence, TopologyMode, TriangleMesh -from ._npz import _json_value +from ._metadata import _json_value +from ._vmesh_format import pack_vmesh, probe_codec, unpack_vmesh from ._protocol import CodecError -_SCHEMA = "open4d.draco-sequence/v1" def _backend(): @@ -52,8 +50,8 @@ def _encoder_arrays(mesh: TriangleMesh): class _DracoProvider: - def __init__(self, archive: ZipFile, manifest: dict) -> None: - self.archive = archive + def __init__(self, temporary, native: Path, manifest: dict) -> None: + self.temporary, self.native = temporary, native self.frames = manifest["frames"] self.metadata = MappingProxyType(manifest.get("metadata", {})) self.topology = TopologyMode(manifest.get("topology", "unknown")) @@ -76,7 +74,7 @@ def get_frame(self, index: int) -> Frame: raise IndexError("frame index out of range") record = self.frames[index] try: - decoded = _backend().decode(self.archive.read(record["member"])) + decoded = _backend().decode((self.native / f"frame_{index:06d}.drc").read_bytes()) except Exception as error: raise CodecError(f"could not decode Draco frame {index}: {error}") from error colors = decoded.colors @@ -95,25 +93,20 @@ def get_frame(self, index: int) -> Frame: ) def close(self) -> None: - self.archive.close() + self.temporary.cleanup() class DracoCodec: """Store each mesh frame as a real Google Draco bitstream.""" id = "draco" - suffixes = (".d4d",) + suffixes = (".vmesh",) backend = "python-binding" lossless = False preserves = ("positions", "triangles", "colors", "normals", "texture_coordinates") def can_decode(self, source: Path) -> bool: - try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - return isinstance(manifest, Mapping) and manifest.get("schema") == _SCHEMA - except (OSError, BadZipFile, KeyError, json.JSONDecodeError): - return False + return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id def encode( self, @@ -124,16 +117,17 @@ def encode( quantization_bits: int = 14, compression_level: int = 7, ) -> Path: - backend = _backend() if not isinstance(sequence, Sequence): raise TypeError("sequence must be an open4d.Sequence") destination = Path(destination).absolute() + if destination.suffix.lower() != ".vmesh": + raise ValueError("Draco destination must have a .vmesh extension") if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") + backend = _backend() destination.parent.mkdir(parents=True, exist_ok=True) manifest = { - "schema": _SCHEMA, - "codec": self.id, + "version": 1, "codec": self.id, "native": {"profile": "draco/1"}, "metadata": _json_value(sequence.metadata, "sequence"), "topology": sequence.topology.value, "has_constant_vertex_count": sequence.has_constant_vertex_count, @@ -143,80 +137,60 @@ def encode( } encoded_vertex_counts = [] split_uv_corners = False - with tempfile.NamedTemporaryFile( - prefix=f".{destination.name}.", suffix=".tmp", - dir=destination.parent, delete=False, - ) as stream: - temporary = Path(stream.name) - try: - with ZipFile(temporary, "w", compression=ZIP_STORED) as archive: - for ordinal, frame in enumerate(sequence): - mesh = frame.geometry - if mesh.attributes: - raise CodecError("Draco custom attributes are not yet supported") - positions, triangles, colors, normals, texture_coordinates = ( - _encoder_arrays(mesh) - ) - encoded_vertex_counts.append(len(positions)) - split_uv_corners |= ( - mesh.texture_coordinates is not None - and mesh.texture_coordinates.ndim == 3 - ) - member = f"frames/{ordinal:06d}.drc" - archive.writestr(member, backend.encode( - positions, - triangles, - colors=colors, - normals=normals, - tex_coord=texture_coordinates, - quantization_bits=quantization_bits, - compression_level=compression_level, - preserve_order=True, - )) - manifest["frames"].append({ - "frame_index": frame.frame_index, - "timestamp": frame.timestamp, - "metadata": _json_value(frame.metadata, f"frame {ordinal}"), - "member": member, - }) - if split_uv_corners: - manifest["topology"] = TopologyMode.UNKNOWN.value - manifest["has_constant_vertex_count"] = ( - len(set(encoded_vertex_counts)) == 1 - ) - manifest["has_vertex_correspondence"] = None - archive.writestr("manifest.json", json.dumps( - manifest, separators=(",", ":"), sort_keys=True, + with tempfile.TemporaryDirectory(prefix="open4d-draco-") as directory: + native = Path(directory) + for ordinal, frame in enumerate(sequence): + mesh = frame.geometry + if mesh.attributes: + raise CodecError("Draco custom attributes are not yet supported") + positions, triangles, colors, normals, texture_coordinates = ( + _encoder_arrays(mesh) + ) + encoded_vertex_counts.append(len(positions)) + split_uv_corners |= ( + mesh.texture_coordinates is not None + and mesh.texture_coordinates.ndim == 3 + ) + (native / f"frame_{ordinal:06d}.drc").write_bytes(backend.encode( + positions, + triangles, + colors=colors, + normals=normals, + tex_coord=texture_coordinates, + quantization_bits=quantization_bits, + compression_level=compression_level, + preserve_order=True, )) - temporary.replace(destination) - except Exception: - temporary.unlink(missing_ok=True) - raise + manifest["frames"].append({ + "frame_index": frame.frame_index, + "timestamp": frame.timestamp, + "metadata": _json_value(frame.metadata, f"frame {ordinal}"), + }) + if split_uv_corners: + manifest["topology"] = TopologyMode.UNKNOWN.value + manifest["has_constant_vertex_count"] = ( + len(set(encoded_vertex_counts)) == 1 + ) + manifest["has_vertex_correspondence"] = None + (native / "metadata.json").write_text(json.dumps(manifest, allow_nan=False), encoding="utf-8") + pack_vmesh(native, destination, overwrite=overwrite) return destination def decode(self, source: Path) -> Sequence: source = Path(source).absolute() - if not source.is_file(): - raise FileNotFoundError(f"codec artifact does not exist: {source}") - archive = None + if source.suffix.lower() != ".vmesh": + raise CodecError("Draco decoding requires .vmesh; re-encode older private artifacts") + if probe_codec(source) != self.id: + raise CodecError("VMESH does not contain Draco") + temporary = tempfile.TemporaryDirectory(prefix="open4d-draco-decode-") try: - archive = ZipFile(source) - manifest = json.loads(archive.read("manifest.json")) - if not isinstance(manifest, Mapping): - raise CodecError("Draco artifact manifest root must be an object") - if manifest.get("schema") != _SCHEMA: - raise CodecError("unsupported Draco artifact schema") - if not isinstance(manifest.get("frames"), list): - raise CodecError("Draco artifact manifest has no frame list") - return Sequence(_DracoProvider(archive, manifest)) - except Exception as error: - if archive is not None: - archive.close() - if isinstance(error, CodecError): - raise - if not isinstance(error, (BadZipFile, KeyError, json.JSONDecodeError)): - raise - raise CodecError(f"invalid Draco artifact {source}: {error}") from error + native = Path(temporary.name) / "native" + unpack_vmesh(source, native) + manifest = json.loads((native / "metadata.json").read_text(encoding="utf-8")) + return Sequence(_DracoProvider(temporary, native, manifest)) + except BaseException: + temporary.cleanup() + raise DRACO_CODEC = DracoCodec() diff --git a/open4d/codec/_klt.py b/open4d/codec/_klt.py index 84c75e8b..adc373bc 100644 --- a/open4d/codec/_klt.py +++ b/open4d/codec/_klt.py @@ -1,4 +1,4 @@ -"""In-process adapter for Open4D's KLT TSDF codec.""" +"""In-process KLT TSDF codec using standalone VMESH carriage.""" from __future__ import annotations @@ -7,25 +7,68 @@ import json import os from pathlib import Path +import pickle import shutil from types import MappingProxyType, SimpleNamespace import tempfile -from zipfile import BadZipFile, ZIP_DEFLATED, ZipFile +from zipfile import BadZipFile, ZipFile import numpy as np from open4d.core import Frame, Sequence, TopologyMode, TriangleMesh from open4d.io import open_sequence -from ._npz import _json_value, _publish_file, _validate_manifest +from ._metadata import _json_value, _validate_manifest from ._protocol import CodecError from ._research import research_module from ._tsdf import write_tsdf_sequence from ._torch import torch_device +from ._vmesh_format import pack_vmesh, probe_codec, unpack_vmesh _SCHEMA = "open4d.klt-sequence/v1" +def _extract_legacy_klt(source: Path, destination: Path) -> dict: + """Convert the retired archive explicitly; never used by normal decoding.""" + try: + with ZipFile(source) as archive: + members = archive.infolist() + if len({member.filename for member in members}) != len(members): + raise CodecError("duplicate KLT archive member") + if archive.getinfo("manifest.json").file_size > 16 * 1024 * 1024: + raise CodecError("oversized KLT manifest") + manifest = json.loads(archive.read("manifest.json")) + _validate_manifest(manifest, schema=_SCHEMA, codec="klt") + _normalization(manifest) + metadata = {key: value for key, value in manifest.items() if key != "schema"} + metadata.update(version=1, native={"profile": "klt/1"}) + from ._native_profiles import layout + required = [name for name, _ in layout("klt", len(metadata["frames"]), metadata["native"])[1:]] + if set(archive.namelist()) != {"manifest.json", *required}: + raise CodecError("unexpected KLT archive payloads") + for name in required: + member = archive.getinfo(name) + limit = 16 * 1024 * 1024 if name.endswith((".pt", ".npz")) else 128 * 1024 * 1024 + if not 0 < member.file_size <= limit: + raise CodecError(f"KLT legacy payload outside limits: {name}") + with archive.open(member) as payload, (destination / name).open("xb") as output: + shutil.copyfileobj(payload, output) + torch = import_module("torch") + checkpoint = destination / "decoder_context.pt" + context = torch.load(checkpoint, map_location="cpu", weights_only=True) + if not isinstance(context, dict) or context.pop("schema", None) != "open4d.klt/v1": + raise CodecError("invalid legacy KLT context") + context["version"] = 1 + # Old encoders could advertise more components than their training rank. + context["num_components"] = len(context["basis"]) + _backend().validate_decoder_context(context) + torch.save(context, checkpoint) + (destination / "metadata.json").write_text(json.dumps(metadata, allow_nan=False), encoding="utf-8") + return metadata + except (BadZipFile, KeyError, ValueError, TypeError, RuntimeError, pickle.UnpicklingError) as error: + raise CodecError(f"invalid legacy KLT artifact: {error}") from error + + def _normalization(manifest): try: normalization = manifest["normalization"] @@ -87,18 +130,13 @@ def close(self): class KLTCodec: id = "klt" - suffixes = (".k4d",) + suffixes = (".vmesh",) backend = "python-in-process" lossless = False preserves = ("positions", "triangles") def can_decode(self, source: Path) -> bool: - try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - return isinstance(manifest, dict) and manifest.get("schema") == _SCHEMA - except (OSError, BadZipFile, KeyError, ValueError, TypeError): - return False + return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id def encode( self, sequence: Sequence, destination: Path, *, overwrite: bool = False, @@ -107,8 +145,24 @@ def encode( verbose: bool = False, ) -> Path: destination = Path(destination).absolute() + if destination.suffix.lower() != ".vmesh": + raise ValueError("KLT destination must have a .vmesh extension") if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") + if type(resolution) is not int or not 7 <= resolution <= 255: + raise ValueError("KLT resolution must be an integer from 7 to 255") + if type(block_size) is not int or not 1 <= block_size <= min(16, resolution + 1): + raise ValueError("KLT block_size must be an integer from 1 to 16 within the grid") + if type(num_components) is not int or not 1 <= num_components <= min(block_size ** 3, 512): + raise ValueError("KLT num_components exceeds the block dimensions") + if type(k_total) is not int or not 1 <= k_total <= 65536: + raise ValueError("KLT k_total must be an integer from 1 to 65536") + training_frames = tuple(training_frames) + if not training_frames or any(type(index) is not int or not 0 <= index < len(sequence) + for index in training_frames): + raise ValueError("KLT training_frames must select existing frames") + if type(frame_rate) not in (int, float) or not np.isfinite(frame_rate) or frame_rate <= 0: + raise ValueError("KLT frame_rate must be finite and positive") for frame in sequence: mesh = frame.geometry if any((mesh.colors is not None, mesh.normals is not None, @@ -116,7 +170,7 @@ def encode( raise CodecError("KLT's TSDF profile cannot preserve mesh attributes") destination.parent.mkdir(parents=True, exist_ok=True) manifest = { - "schema": _SCHEMA, "codec": self.id, + "version": 1, "codec": self.id, "native": {"profile": "klt/1"}, "metadata": _json_value(sequence.metadata, "sequence"), "allow_nonmonotonic_timestamps": sequence.allow_nonmonotonic_timestamps, "frames": [{ @@ -141,39 +195,30 @@ def encode( else: with open(os.devnull, "w") as sink, redirect_stdout(sink), redirect_stderr(sink): backend.run_compression(arguments, verify_decode=False) - with tempfile.NamedTemporaryFile( - prefix=f".{destination.name}.", suffix=".tmp", - dir=destination.parent, delete=False, - ) as stream: - temporary = Path(stream.name) - try: - shutil.copyfile(work / "encoded/compressed_archive.zip", temporary) - with ZipFile(temporary, "a", compression=ZIP_DEFLATED) as archive: - archive.writestr("manifest.json", json.dumps(manifest)) - _publish_file(temporary, destination, overwrite=overwrite) - except Exception: - temporary.unlink(missing_ok=True) - raise + native = work / "encoded/compressed" + (native / "metadata.json").write_text(json.dumps(manifest, allow_nan=False), encoding="utf-8") + pack_vmesh(native, destination, overwrite=overwrite) return destination def decode(self, source: Path, *, device=None) -> Sequence: if device == "auto": device = str(torch_device(import_module("torch"), device)) source = Path(source).absolute() + if source.suffix.lower() != ".vmesh": + raise CodecError("KLT decoding requires .vmesh; migrate the legacy artifact explicitly") + if probe_codec(source) != self.id: + raise CodecError("VMESH does not contain KLT") temporary = tempfile.TemporaryDirectory(prefix="open4d-klt-decode-") work = Path(temporary.name) decoded = None try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - _validate_manifest(manifest, schema=_SCHEMA, codec=self.id) - _normalization(manifest) - members = [item for item in archive.infolist() if item.filename != "manifest.json"] - if any(Path(item.filename).is_absolute() or ".." in Path(item.filename).parts - for item in members): - raise CodecError("unsafe path in KLT artifact") - archive.extractall(work / "compressed", members) - _backend().decode_compressed(work / "compressed", work / "decoded", device) + unpack_vmesh(source, work / "compressed") + manifest = json.loads((work / "compressed/metadata.json").read_text(encoding="utf-8")) + _normalization(manifest) + try: + _backend().decode_compressed(work / "compressed", work / "decoded", device) + except (ValueError, KeyError, TypeError, RuntimeError, BadZipFile, pickle.UnpicklingError) as error: + raise CodecError(f"invalid KLT native payload: {error}") from error decoded = open_sequence(work / "decoded") if len(decoded) != len(manifest["frames"]): raise CodecError( diff --git a/open4d/codec/_metadata.py b/open4d/codec/_metadata.py new file mode 100644 index 00000000..de43369d --- /dev/null +++ b/open4d/codec/_metadata.py @@ -0,0 +1,70 @@ +"""Finite sequence metadata shared by VMESH adapters.""" + +from collections.abc import Mapping +import math +from pathlib import Path +import numpy as np +from open4d._files import publish_file as _publish_file +from ._protocol import CodecError + + +def _json_value(value, name: str): + if value is None or isinstance(value, (str, int, bool)): + return value + if isinstance(value, float): + if not math.isfinite(value): + raise CodecError(f"{name} metadata numbers must be finite") + return value + if isinstance(value, np.generic): + return _json_value(value.item(), name) + if isinstance(value, Path): + return str(value) + if isinstance(value, Mapping): + if any(not isinstance(key, str) for key in value): + raise CodecError(f"{name} metadata keys must be strings") + return {key: _json_value(item, f"{name}.{key}") for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_value(item, name) for item in value] + raise CodecError(f"{name} metadata value {type(value).__name__} is not serializable") + + +def _validate_manifest(manifest, *, schema: str | None, codec: str) -> dict: + if not isinstance(manifest, dict): + raise CodecError("artifact manifest root must be an object") + if manifest.get("schema") != schema or manifest.get("codec") != codec: + raise CodecError(f"unsupported {codec} artifact schema or codec") + frames = manifest.get("frames") + if not isinstance(frames, list): + raise CodecError("artifact manifest must contain a frame list") + nonmonotonic = manifest.get("allow_nonmonotonic_timestamps", False) + if not isinstance(nonmonotonic, bool): + raise CodecError("allow_nonmonotonic_timestamps must be boolean") + for name in ("has_constant_vertex_count", "has_vertex_correspondence"): + if manifest.get(name) is not None and not isinstance(manifest[name], bool): + raise CodecError(f"{name} must be boolean or null") + if not isinstance(manifest.get("metadata", {}), dict): + raise CodecError("sequence metadata must be an object") + _json_value(manifest.get("metadata", {}), "sequence") + previous = None + for ordinal, record in enumerate(frames): + if not isinstance(record, dict): + raise CodecError(f"invalid frame record {ordinal}") + index, timestamp = record.get("frame_index"), record.get("timestamp") + if type(index) is not int or index < 0: + raise CodecError(f"invalid frame index at {ordinal}") + if type(timestamp) not in (int, float) or not math.isfinite(timestamp): + raise CodecError(f"invalid frame timestamp at {ordinal}") + if previous is not None and not nonmonotonic and timestamp < previous: + raise CodecError("frame timestamps must be nondecreasing") + previous = timestamp + if not isinstance(record.get("metadata", {}), dict): + raise CodecError(f"frame {ordinal} metadata must be an object") + _json_value(record.get("metadata", {}), f"frame {ordinal}") + return manifest + + +def require_vmesh_destination(destination: str | Path) -> Path: + path = Path(destination).absolute() + if path.suffix.lower() != ".vmesh": + raise ValueError("compressed output requires a .vmesh extension") + return path diff --git a/open4d/codec/_migration.py b/open4d/codec/_migration.py new file mode 100644 index 00000000..e817a2b1 --- /dev/null +++ b/open4d/codec/_migration.py @@ -0,0 +1,108 @@ +"""Explicit, one-way conversion of retired sequence artifacts to VMESH.""" + +from __future__ import annotations + +import json +import pickle +from pathlib import Path +import shutil +import tempfile +from zipfile import BadZipFile, ZipFile + +from ._metadata import _validate_manifest, require_vmesh_destination +from ._protocol import CodecError +from ._vmesh_format import _json, pack_vmesh + +_MAX_PAYLOAD = 8 * 1024**3 + + +def _extract_vdmc(source: Path, destination: Path) -> None: + from ._vmesh import _position_normalization + + try: + with ZipFile(source) as archive: + names = archive.namelist() + if len(names) != len(set(names)) or archive.getinfo("manifest.json").file_size > 16 * 1024**2: + raise CodecError("duplicate members or oversized migration manifest") + if set(names) - {"manifest.json", "sequence.vmesh", "decoder.cfg"}: + raise CodecError("unexpected V-DMC migration member") + manifest = _json(archive.read("manifest.json")) + codec = manifest.get("codec") + if codec not in ("vdmc", "faster_vdmc"): + raise CodecError("unsupported native V-DMC codec") + _validate_manifest(manifest, schema="open4d.vmesh-sequence/v1", codec=codec) + if not 1 <= len(manifest["frames"]) <= 65536: + raise CodecError("migration frame count outside limits") + _position_normalization(manifest) + required = ["sequence.vmesh"] + if "decoder.cfg" in names: + required.append("decoder.cfg") + for name in required: + size = archive.getinfo(name).file_size + maximum = 16 * 1024**2 if name == "decoder.cfg" else _MAX_PAYLOAD + if not 0 < size <= maximum: + raise CodecError("migration payload size outside limits") + with archive.open(name) as payload, (destination / name).open("xb") as output: + shutil.copyfileobj(payload, output) + manifest.pop("schema") + manifest.update(version=1, native={"profile": f"{codec}/1", "decoder_config": len(required) == 2}) + (destination / "metadata.json").write_text(json.dumps(manifest, allow_nan=False), encoding="utf-8") + except (BadZipFile, KeyError, ValueError, TypeError, AttributeError) as error: + raise CodecError(f"invalid migration input: {error}") from error + + +def migrate_legacy(source: str | Path, destination: str | Path, *, overwrite: bool = False, + fps: float = 30.0) -> Path: + """Convert an older artifact explicitly; normal readers accept VMESH only. + + Native payloads are preserved except redundant checkpoint schema fields. + Tensor checkpoints use restricted weights-only loading. Executable early + QNDF-int8 contexts are deliberately unsupported. + """ + source = Path(source).absolute() + destination = require_vmesh_destination(destination) + if not isinstance(overwrite, bool): + raise TypeError("overwrite must be bool") + if destination.exists() and not overwrite: + raise FileExistsError(destination) + if source.is_symlink(): + raise CodecError("linked migration source") + if source.is_dir(): + if (source / "metadata.json").is_file(): + return pack_vmesh(source, destination, overwrite=overwrite) + from open4d.native import import_native, save_native + with import_native(source, codec="vega", fps=fps) as native: + return save_native(native, destination, overwrite=overwrite) + if source.suffix.lower() == ".seq": + from open4d._streamer import _require + _require() + from streamer.sequence import convert_legacy + return convert_legacy(source, destination, fps=fps, overwrite=overwrite) + with tempfile.TemporaryDirectory(prefix="vmesh-migration-") as folder: + native = Path(folder) + suffix = source.suffix.lower() + if suffix == ".k4d": + from ._klt import _extract_legacy_klt + _extract_legacy_klt(source, native) + elif suffix in (".q4d", ".qi4d"): + from ._qndf import _extract_legacy_qndf + _extract_legacy_qndf(source, native, int8=suffix == ".qi4d") + elif suffix == ".n4d": + from ._n4mc import _extract_n4d, _write_native_metadata + manifest = _extract_n4d(source, native) + from importlib import import_module + torch = import_module("torch") + try: + checkpoint = torch.load(native / "checkpoint.pt", map_location="cpu", weights_only=True) + except (OSError, RuntimeError, ValueError, TypeError, pickle.UnpicklingError) as error: + raise CodecError("invalid N4MC migration checkpoint") from error + if not isinstance(checkpoint, dict): + raise CodecError("invalid N4MC checkpoint") + checkpoint.pop("schema", None) + torch.save(checkpoint, native / "checkpoint.pt") + _write_native_metadata(manifest, native) + elif suffix == ".v4d": + _extract_vdmc(source, native) + else: + raise ValueError("unsupported migration input; import native payload directories explicitly") + return pack_vmesh(native, destination, overwrite=overwrite) diff --git a/open4d/codec/_n4mc.py b/open4d/codec/_n4mc.py index b4f84a5e..721c7e2c 100644 --- a/open4d/codec/_n4mc.py +++ b/open4d/codec/_n4mc.py @@ -9,7 +9,7 @@ import shutil import tempfile import warnings -from zipfile import BadZipFile, ZIP_DEFLATED, ZipFile +from zipfile import BadZipFile, ZipFile import numpy as np @@ -17,12 +17,12 @@ from open4d.io import open_sequence from ._klt import _KLTProvider, _normalization -from ._npz import _json_value, _publish_file, _validate_manifest +from ._metadata import _json_value, _validate_manifest, require_vmesh_destination from ._protocol import CodecError from ._research import research_module from ._torch import torch_device from ._tsdf import write_tsdf_sequence -from ._v3c import pack_vmesh, probe_codec, unpack_vmesh +from ._vmesh_format import _json, pack_vmesh, probe_codec, unpack_vmesh _SCHEMA = "open4d.n4mc-sequence/v1" @@ -36,10 +36,14 @@ def _extract_n4d(source: Path, destination: Path) -> dict: raise CodecError("duplicate N4MC archive member") if archive.getinfo("manifest.json").file_size > 16 * 1024 * 1024: raise CodecError("oversized N4MC manifest") - manifest = json.loads(archive.read("manifest.json")) + manifest = _json(archive.read("manifest.json")) _validate_manifest(manifest, schema=_SCHEMA, codec="n4mc") _normalization(manifest) + if archive.getinfo("checkpoint.pt").file_size > 64 * 1024**2: + raise CodecError("N4MC migration checkpoint exceeds 64 MiB") required = ["checkpoint.pt", *(f"frame_{i:06d}.npz" for i in range(len(manifest["frames"])))] + if len(required) > 65535 or sum(archive.getinfo(name).file_size for name in required) > 8 * 1024**3: + raise CodecError("N4MC migration payload budget exceeded") for name in required: if archive.getinfo(name).file_size == 0: raise CodecError(f"empty N4MC payload: {name}") @@ -130,20 +134,13 @@ def _reconstruct_mesh(volume, metrics): class N4MCCodec: id = "n4mc" - suffixes = (".n4d", ".vmesh") + suffixes = (".vmesh",) backend = "python-in-process" lossless = False preserves = ("positions", "triangles") def can_decode(self, source: Path) -> bool: - if Path(source).suffix.lower() == ".vmesh": - return probe_codec(source) == self.id - try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - return isinstance(manifest, dict) and manifest.get("schema") == _SCHEMA - except (OSError, BadZipFile, KeyError, ValueError, TypeError): - return False + return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id def encode( self, sequence: Sequence, destination: Path, *, overwrite: bool = False, @@ -151,10 +148,10 @@ def encode( latent_channels: int = 64, learning_rate: float = 1e-4, device: str | None = None, seed: int = 7, ) -> Path: - destination = Path(destination).absolute() + destination = require_vmesh_destination(destination) if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") - if destination.suffix.lower() == ".vmesh" and len(sequence) < 2: + if len(sequence) < 2: raise CodecError("N4MC .vmesh carriage requires at least two frames") for frame in sequence: mesh = frame.geometry @@ -180,7 +177,7 @@ def encode( "lambda_band": 2., "lambda_sign": .2, } manifest = { - "schema": _SCHEMA, "codec": self.id, + "version": 1, "codec": self.id, "metadata": _json_value(sequence.metadata, "sequence"), "allow_nonmonotonic_timestamps": sequence.allow_nonmonotonic_timestamps, "frames": [{ @@ -207,10 +204,9 @@ def encode( model.eval() checkpoint = work / "checkpoint.pt" torch.save({ - "schema": "open4d.n4mc/v1", "model": model.state_dict(), + "model": model.state_dict(), "model_config": model_config, }, checkpoint) - packs = [] with torch.inference_mode(): for ordinal, path in enumerate(volumes): encoded = model.encode(_volume(torch, path, target_device).unsqueeze(0)) @@ -221,26 +217,8 @@ def encode( original_shape=encoded["original_shape"].cpu().numpy(), bottleneck_shape=encoded["bottleneck_shape"].cpu().numpy(), ) - packs.append(pack) - if destination.suffix.lower() == ".vmesh": - _write_native_metadata(manifest, work) - return pack_vmesh(work, destination, overwrite=overwrite) - with tempfile.NamedTemporaryFile( - prefix=f".{destination.name}.", suffix=".tmp", - dir=destination.parent, delete=False, - ) as stream: - temporary = Path(stream.name) - try: - with ZipFile(temporary, "w", compression=ZIP_DEFLATED) as archive: - archive.write(checkpoint, checkpoint.name) - for pack in packs: - archive.write(pack, pack.name) - archive.writestr("manifest.json", json.dumps(manifest)) - _publish_file(temporary, destination, overwrite=overwrite) - except Exception: - temporary.unlink(missing_ok=True) - raise - return destination + _write_native_metadata(manifest, work) + return pack_vmesh(work, destination, overwrite=overwrite) def decode( self, source: Path, *, device: str | None = None, @@ -251,17 +229,12 @@ def decode( work = Path(temporary.name) decoded = None try: - if source.suffix.lower() == ".vmesh": - detected = probe_codec(source) - if detected != self.id: - raise CodecError(f".vmesh contains {detected!r}, not {self.id}") - native = unpack_vmesh(source, work / "native") - manifest = json.loads((native / "metadata.json").read_text()) - _validate_manifest(manifest, schema=None, codec=self.id) - _normalization(manifest) - else: - native = work - manifest = _extract_n4d(source, native) + if source.suffix.lower() != ".vmesh" or probe_codec(source) != self.id: + raise CodecError("N4MC decode requires its VMESH profile; use migrate_legacy for older artifacts") + native = unpack_vmesh(source, work / "native") + manifest = json.loads((native / "metadata.json").read_text()) + _validate_manifest(manifest, schema=None, codec=self.id) + _normalization(manifest) torch, models, _, metrics = _backend() target_device = _device(torch, device) checkpoint = torch.load( @@ -274,7 +247,10 @@ def decode( model.eval() output = work / "decoded" output.mkdir() - with torch.inference_mode(): + with torch.inference_mode(), torch.backends.cudnn.flags( + enabled=torch.backends.cudnn.enabled, benchmark=False, + deterministic=True, allow_tf32=torch.backends.cudnn.allow_tf32, + ): for ordinal in range(len(manifest["frames"])): with np.load(native / f"frame_{ordinal:06d}.npz", allow_pickle=False) as pack: latent = torch.from_numpy(pack["quantized_latent"]).unsqueeze(0).to(target_device) diff --git a/open4d/codec/_native_profiles.py b/open4d/codec/_native_profiles.py index e5ff1785..6b29f5b9 100644 --- a/open4d/codec/_native_profiles.py +++ b/open4d/codec/_native_profiles.py @@ -8,6 +8,19 @@ from ._protocol import CodecError PROFILES = { + "klt": ("tsdf", "shared-basis-independent-frames"), + "qndf": ("triangle_mesh", "independent"), + "qndf-int8": ("triangle_mesh", "independent"), + "frames": ("frame_payloads", "independent"), + "npz": ("triangle_mesh", "independent"), + "raw": ("triangle_mesh", "independent"), + "deflate": ("triangle_mesh", "independent"), + "bzip2": ("triangle_mesh", "independent"), + "lzma": ("triangle_mesh", "independent"), + "rle": ("triangle_mesh", "independent"), + "draco": ("triangle_mesh", "independent"), + "temporal-delta": ("triangle_mesh", "shared-reference"), + "temporal-pca": ("triangle_mesh", "shared-trajectory-basis"), "tvmc": ("triangle_mesh", "shared-reference"), "tsmc": ("triangle_mesh", "whole-group"), "vega": ("neural_gaussians", "shared-reference"), @@ -20,6 +33,9 @@ # prediction or residual dependency between the quantized frame latents. "n4mc": ("neural_tsdf", "shared-model-independent-frames"), } +MAX_FILES = 65536 +MAX_FRAMES = 65536 + NEURAL_CODECS = frozenset(("vega", "queen", "3dgstream", "rerf")) @@ -31,7 +47,19 @@ def _require(condition, message): def layout(codec, count, native=None): """Validate a profile and return its exact ordered filename/role pairs.""" _require(codec in PROFILES, f"unsupported codec {codec!r}") - _require(type(count) is int and count > 0, "invalid frame count") + _require(type(count) is int and 0 < count <= MAX_FRAMES, "frame count outside limits") + if codec == "tvmc": + _require(2 + 2 * count <= MAX_FILES, "payload file count outside limits") + elif codec == "klt": + _require(2 + 2 * count <= MAX_FILES, "payload file count outside limits") + elif codec in ("qndf", "qndf-int8", "frames", "npz", "raw", "deflate", "bzip2", "lzma", "rle", "draco"): + _require(1 + count <= MAX_FILES, "payload file count outside limits") + elif codec == "vega": + _require(3 + count <= MAX_FILES, "payload file count outside limits") + elif codec == "n4mc": + _require(2 + count <= MAX_FILES, "payload file count outside limits") + elif codec == "queen": + _require(1 + count <= MAX_FILES, "payload file count outside limits") files = [("metadata.json", "sequence-metadata")] if codec in ("tvmc", "tsmc"): _require(native is None, "unexpected tracked-mesh profile") @@ -51,6 +79,35 @@ def layout(codec, count, native=None): _require(set(native) == {"profile"}, "unexpected N4MC configuration") files.append(("checkpoint.pt", "shared-neural-model")) files += [(f"frame_{i:06d}.npz", "quantized-tsdf-latent") for i in range(count)] + elif codec == "klt": + _require(set(native) == {"profile"}, "unexpected KLT configuration") + files.append(("decoder_context.pt", "shared-klt-basis")) + files += [(f"{i:06d}_quantized_{suffix}", role) for i in range(count) + for suffix, role in (("indices.zst", "quantized-tsdf-coefficients"), + ("metadata.npz", "tsdf-quantizer"))] + elif codec in ("qndf", "qndf-int8"): + _require(set(native) == {"profile"}, "unexpected QNDF configuration") + files += [(f"frame_{i:06d}.pt", "neural-displacement-context") for i in range(count)] + elif codec in ("npz", "raw", "deflate", "bzip2", "lzma", "rle"): + _require(set(native) == {"profile", "rle"} and type(native["rle"]) is bool, + "unexpected array configuration") + files += [(f"frame_{i:06d}.npz", "mesh-arrays") for i in range(count)] + elif codec == "draco": + _require(set(native) == {"profile"}, "unexpected Draco configuration") + files += [(f"frame_{i:06d}.drc", "frame-mesh") for i in range(count)] + elif codec in ("temporal-delta", "temporal-pca"): + _require(set(native) == {"profile"}, "unexpected temporal configuration") + files.append(("sequence.npz", "quantized-trajectories")) + elif codec == "frames": + _require(set(native) == {"profile", "suffix", "representation"}, "unexpected frame configuration") + suffix = native["suffix"] + _require(suffix in ("ply", "drc", "splat", "jpg", "jpeg", "png"), "unsupported frame payload suffix") + representation = native["representation"] + compatible = {"pixels": ("jpg", "jpeg", "png"), "gaussians": ("ply", "splat"), + "mesh": ("ply", "drc"), "points": ("ply", "drc")} + _require(isinstance(representation, str) and representation in compatible + and suffix in compatible[representation], "incompatible frame representation/suffix") + files += [(f"frame_{i:06d}.{suffix}", "frame-payload") for i in range(count)] elif codec == "vega": _require(set(native) == {"profile"}, "unexpected Vega configuration") files += [("manifest.json", "native-frame-index"), ("color_model.pt", "shared-neural-appearance")] @@ -72,6 +129,7 @@ def layout(codec, count, native=None): _require(isinstance(added, list) and len(added) == count and all(type(v) is bool for v in added) and not added[0], "invalid added-Gaussian index") files += [("initial.ply", "initial-gaussians"), ("ntc_config.json", "neural-transform-architecture")] for i in range(1, count): + _require(len(files) + 1 + int(added[i]) <= MAX_FILES, "payload file count outside limits") files.append((f"ntc_{i:06d}.pth", "neural-transform")) if added[i]: files.append((f"added_{i:06d}.ply", "added-gaussians")) @@ -95,6 +153,8 @@ def layout(codec, count, native=None): halves = frame["channels"] _require(isinstance(halves, list) and len(halves) == (2 if not key and pca else 1) and all(type(n) is int and 0 < n <= 4096 for n in halves), "invalid ReRF entropy channels") + _require(len(files) + 2 + sum(halves) + int(not key and pca) + 2 * int(motion) + <= MAX_FILES, "payload file count outside limits") files += [(f"header_{index}.json", "field-frame-header"), (f"mask_{index}.rerf", "occupancy-mask")] # The native ac_dc_encode2 writes one file per channel; .rerf is # a prefix, not an aggregate entropy file. diff --git a/open4d/codec/_native_temporal.py b/open4d/codec/_native_temporal.py index 1e79ca17..a52bc14d 100644 --- a/open4d/codec/_native_temporal.py +++ b/open4d/codec/_native_temporal.py @@ -4,10 +4,12 @@ from ._native_profiles import PROFILES from ._protocol import CodecError -from ._v3c import probe_codec +from ._metadata import require_vmesh_destination +from ._vmesh_format import probe_codec def encode_native(source, destination, *, codec, overwrite=False, **options): + destination = require_vmesh_destination(destination) from open4d.native import NativeSequence, import_native, save_native if isinstance(source, NativeSequence): if options: @@ -38,12 +40,14 @@ def __init__(self, identifier): self.representation = PROFILES[identifier][0] def can_decode(self, path): - return probe_codec(path) == self.id + return Path(path).suffix.lower() == ".vmesh" and probe_codec(path) == self.id def encode(self, sequence, destination, **options): return encode_native(sequence, destination, codec=self.id, **options) def decode(self, source, *, runtime=None, python=None): + if Path(source).suffix.lower() != ".vmesh": + raise CodecError(f"{self.id} decode requires a .vmesh extension") from open4d.native import NativeSequence result = NativeSequence(source, runtime=runtime, python=python) if result.codec != self.id: diff --git a/open4d/codec/_npz.py b/open4d/codec/_npz.py index cb046ed0..cbe95ef9 100644 --- a/open4d/codec/_npz.py +++ b/open4d/codec/_npz.py @@ -1,8 +1,7 @@ -"""Lossless reference sequence codec using NumPy arrays inside ZIP.""" +"""Lossless array profiles using standard NPZ payloads inside VMESH.""" from __future__ import annotations -from collections.abc import Mapping from io import BytesIO import json import math @@ -21,89 +20,55 @@ import numpy as np from open4d.core import Frame, Sequence, TopologyMode, TriangleMesh -from open4d._files import publish_file as _publish_file from ._protocol import CodecError +from ._metadata import _json_value +from ._vmesh_format import pack_vmesh, probe_codec, unpack_vmesh -_SCHEMA = "open4d.numpy-zip/v1" +_MAX_ARRAY_BYTES = 256 * 1024 * 1024 _FIELDS = ("positions", "triangles", "colors", "normals", "texture_coordinates") -def _json_value(value, name: str): - if value is None or isinstance(value, (str, int, bool)): - return value - if isinstance(value, float): - if not math.isfinite(value): - raise CodecError(f"{name} metadata numbers must be finite") - return value - if isinstance(value, np.generic): - return _json_value(value.item(), name) - if isinstance(value, Path): - return str(value) - if isinstance(value, Mapping): - if any(not isinstance(key, str) for key in value): - raise CodecError(f"{name} metadata keys must be strings") - return {key: _json_value(item, f"{name}.{key}") for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [_json_value(item, name) for item in value] - raise CodecError(f"{name} metadata value {type(value).__name__} is not serializable") - - -def _validate_manifest(manifest, *, schema: str | None, codec: str) -> dict: - if not isinstance(manifest, dict): - raise CodecError("artifact manifest root must be an object") - if manifest.get("schema") != schema or manifest.get("codec") != codec: - raise CodecError(f"unsupported {codec} artifact schema or codec") - frames = manifest.get("frames") - if not isinstance(frames, list): - raise CodecError("artifact manifest must contain a frame list") - nonmonotonic = manifest.get("allow_nonmonotonic_timestamps", False) - if not isinstance(nonmonotonic, bool): - raise CodecError("allow_nonmonotonic_timestamps must be boolean") - for name in ("has_constant_vertex_count", "has_vertex_correspondence"): - if manifest.get(name) is not None and not isinstance(manifest[name], bool): - raise CodecError(f"{name} must be boolean or null") - if not isinstance(manifest.get("metadata", {}), dict): - raise CodecError("sequence metadata must be an object") - _json_value(manifest.get("metadata", {}), "sequence") - previous = None - for ordinal, record in enumerate(frames): - if not isinstance(record, dict): - raise CodecError(f"invalid frame record {ordinal}") - index, timestamp = record.get("frame_index"), record.get("timestamp") - if type(index) is not int or index < 0: - raise CodecError(f"invalid frame index at {ordinal}") - if type(timestamp) not in (int, float) or not math.isfinite(timestamp): - raise CodecError(f"invalid frame timestamp at {ordinal}") - if previous is not None and not nonmonotonic and timestamp < previous: - raise CodecError("frame timestamps must be nondecreasing") - previous = timestamp - if not isinstance(record.get("metadata", {}), dict): - raise CodecError(f"frame {ordinal} metadata must be an object") - _json_value(record.get("metadata", {}), f"frame {ordinal}") - return manifest - - def _array_bytes(array: np.ndarray) -> bytes: stream = BytesIO() np.save(stream, array, allow_pickle=False) return stream.getvalue() -def _read_array(archive: ZipFile, name: str, codec: "NumPyZipCodec") -> np.ndarray: +def _array_from_bytes(payload: bytes) -> np.ndarray: + stream = BytesIO(payload) + version = np.lib.format.read_magic(stream) + readers = {(1, 0): np.lib.format.read_array_header_1_0, + (2, 0): np.lib.format.read_array_header_2_0} + if version not in readers: + raise CodecError("unsupported array NPY version") + shape, _, dtype = readers[version](stream, max_header_size=4096) + if dtype.hasobject or math.prod(shape) * dtype.itemsize != len(payload) - stream.tell(): + raise CodecError("array NPY size/type disagrees with payload") + return np.load(BytesIO(payload), allow_pickle=False, max_header_size=4096) + + +def _read_array(path: Path, name: str, codec: "NumPyZipCodec") -> np.ndarray: try: - payload = archive.read(name) - except KeyError as error: - raise CodecError(f"artifact is missing {name}") from error - return np.load(BytesIO(codec.unpack(payload)), allow_pickle=False) + with ZipFile(path) as archive: + member = archive.getinfo(f"{name}.npy") + if not 0 < member.file_size <= _MAX_ARRAY_BYTES: + raise CodecError("array payload outside limits") + value = _array_from_bytes(archive.read(member)) + if codec.rle: + if value.dtype != np.uint8 or value.ndim != 1: + raise CodecError("RLE array payload must contain encoded bytes") + value = _array_from_bytes(codec.unpack(value.tobytes())) + return value + except (BadZipFile, KeyError, ValueError, TypeError) as error: + raise CodecError(f"invalid array payload {name}: {error}") from error class _ZipProvider: def __init__( - self, source: Path, archive: ZipFile, manifest: dict, codec: "NumPyZipCodec" + self, temporary, native: Path, manifest: dict, codec: "NumPyZipCodec" ) -> None: - self.source = source - self.archive = archive + self.temporary, self.native = temporary, native self.codec = codec self.frames = manifest["frames"] self.metadata = MappingProxyType(manifest.get("metadata", {})) @@ -134,12 +99,12 @@ def get_frame(self, index: int) -> Frame: raise IndexError("frame index out of range") from error arrays = record["arrays"] values = { - name: _read_array(self.archive, path, self.codec) + name: _read_array(self.native / f"frame_{index:06d}.npz", path, self.codec) for name, path in arrays.items() if name != "attributes" } attributes = { - name: _read_array(self.archive, path, self.codec) + name: _read_array(self.native / f"frame_{index:06d}.npz", path, self.codec) for name, path in arrays.get("attributes", {}).items() } return Frame( @@ -150,13 +115,13 @@ def get_frame(self, index: int) -> Frame: ) def close(self) -> None: - self.archive.close() + self.temporary.cleanup() class NumPyZipCodec: - """Losslessly store canonical Open4D arrays in a portable `.o4d` ZIP.""" + """Losslessly carry standard NumPy frame arrays in standalone VMESH.""" - suffixes = (".o4d",) + suffixes = (".vmesh",) backend = "python" lossless = True preserves = (*_FIELDS, "attributes") @@ -196,18 +161,13 @@ def unpack(self, payload: bytes) -> bytes: raise CodecError("invalid RLE payload") expected = int.from_bytes(payload[:8], "little") pairs = np.frombuffer(payload[8:], dtype=np.uint8).reshape(-1, 2) - decoded = np.repeat(pairs[:, 1], pairs[:, 0]).tobytes() - if len(decoded) != expected: - raise CodecError("RLE payload length does not match its header") - return decoded + if (expected > _MAX_ARRAY_BYTES or np.any(pairs[:, 0] == 0) + or sum(int(count) for count in pairs[:, 0]) != expected): + raise CodecError("RLE payload length does not match its bounded header") + return np.repeat(pairs[:, 1], pairs[:, 0]).tobytes() def can_decode(self, source: Path) -> bool: - try: - with ZipFile(source, "r") as archive: - manifest = json.loads(archive.read("manifest.json")) - return isinstance(manifest, Mapping) and manifest.get("codec") == self.id - except (OSError, BadZipFile, KeyError, ValueError, TypeError): - return False + return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id def encode( self, @@ -223,12 +183,14 @@ def encode( if level is not None and not 0 <= level <= 9: raise ValueError("compression_level must be in [0, 9]") destination = Path(destination).absolute() + if destination.suffix.lower() != ".vmesh": + raise ValueError("array destination must have a .vmesh extension") if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") destination.parent.mkdir(parents=True, exist_ok=True) manifest = { - "schema": _SCHEMA, - "codec": self.id, + "version": 1, "codec": self.id, + "native": {"profile": f"{self.id}/1", "rle": self.rle}, "metadata": _json_value(sequence.metadata, "sequence"), "topology": sequence.topology.value, "has_constant_vertex_count": sequence.has_constant_vertex_count, @@ -236,67 +198,69 @@ def encode( "allow_nonmonotonic_timestamps": sequence.allow_nonmonotonic_timestamps, "frames": [], } - with tempfile.NamedTemporaryFile( - prefix=f".{destination.name}.", suffix=".tmp", - dir=destination.parent, delete=False, - ) as stream: - temporary = Path(stream.name) - try: - with ZipFile( - temporary, "w", compression=self.compression, - compresslevel=level, - ) as archive: - for ordinal, frame in enumerate(sequence): - prefix = f"frames/{ordinal:06d}" - arrays = {} + with tempfile.TemporaryDirectory(prefix="open4d-arrays-") as directory: + native = Path(directory) + for ordinal, frame in enumerate(sequence): + arrays = {} + with ZipFile(native / f"frame_{ordinal:06d}.npz", "w", compression=self.compression, + compresslevel=level) as archive: for name in _FIELDS: value = getattr(frame.geometry, name) if value is not None: - member = f"{prefix}/{name}.npy" - archive.writestr(member, self.pack(_array_bytes(value))) - arrays[name] = member + payload = self.pack(_array_bytes(value)) + if self.rle: + payload = _array_bytes(np.frombuffer(payload, dtype=np.uint8)) + archive.writestr(f"{name}.npy", payload) + arrays[name] = name attributes = {} for number, (name, value) in enumerate(frame.geometry.attributes.items()): - member = f"{prefix}/attribute_{number:04d}.npy" - archive.writestr(member, self.pack(_array_bytes(value))) - attributes[name] = member + key = f"attribute_{number:04d}" + payload = self.pack(_array_bytes(value)) + if self.rle: + payload = _array_bytes(np.frombuffer(payload, dtype=np.uint8)) + archive.writestr(f"{key}.npy", payload) + attributes[name] = key arrays["attributes"] = attributes - manifest["frames"].append( - { - "frame_index": frame.frame_index, - "timestamp": frame.timestamp, - "metadata": _json_value(frame.metadata, f"frame {ordinal}"), - "arrays": arrays, - } - ) - archive.writestr( - "manifest.json", - json.dumps(manifest, separators=(",", ":"), sort_keys=True), - ) - _publish_file(temporary, destination, overwrite=overwrite) - except Exception: - temporary.unlink(missing_ok=True) - raise + manifest["frames"].append({ + "frame_index": frame.frame_index, "timestamp": frame.timestamp, + "metadata": _json_value(frame.metadata, f"frame {ordinal}"), "arrays": arrays, + }) + (native / "metadata.json").write_text(json.dumps(manifest, allow_nan=False), encoding="utf-8") + pack_vmesh(native, destination, overwrite=overwrite) return destination def decode(self, source: Path) -> Sequence: source = Path(source).absolute() - if not source.is_file(): - raise FileNotFoundError(f"codec artifact does not exist: {source}") - archive = None + if source.suffix.lower() != ".vmesh": + raise CodecError("array decoding requires .vmesh; re-encode older private artifacts") + if probe_codec(source) != self.id: + raise CodecError(f"VMESH does not contain {self.id}") + temporary = tempfile.TemporaryDirectory(prefix="open4d-arrays-decode-") try: - archive = ZipFile(source, "r") - manifest = json.loads(archive.read("manifest.json")) - _validate_manifest(manifest, schema=_SCHEMA, codec=self.id) - return Sequence(_ZipProvider(source, archive, manifest, self)) - except Exception as error: - if archive is not None: - archive.close() - if isinstance(error, CodecError): - raise - if not isinstance(error, (BadZipFile, KeyError, ValueError, TypeError)): - raise - raise CodecError(f"invalid Open4D artifact {source}: {error}") from error + native = Path(temporary.name) / "native" + unpack_vmesh(source, native) + manifest = json.loads((native / "metadata.json").read_text(encoding="utf-8")) + for ordinal, frame in enumerate(manifest["frames"]): + arrays = frame.get("arrays") + if (not isinstance(arrays, dict) or not {"positions", "triangles"} <= set(arrays) + or not set(arrays) <= {*_FIELDS, "attributes"} + or any(value != name for name, value in arrays.items() if name != "attributes")): + raise CodecError("invalid mesh array field index") + attributes = arrays.get("attributes", {}) + if (not isinstance(attributes, dict) or not all(isinstance(name, str) for name in attributes) + or list(attributes.values()) != [f"attribute_{i:04d}" for i in range(len(attributes))]): + raise CodecError("invalid mesh attribute field index") + expected = {f"{key}.npy" for key in arrays if key != "attributes"} + expected.update(f"{key}.npy" for key in attributes.values()) + with ZipFile(native / f"frame_{ordinal:06d}.npz") as archive: + members = archive.namelist() + if len(members) != len(set(members)) or set(members) != expected: + raise CodecError("unexpected NPZ frame arrays") + decoder = NumPyZipCodec(self.id, rle=manifest["native"]["rle"]) + return Sequence(_ZipProvider(temporary, native, manifest, decoder)) + except BaseException: + temporary.cleanup() + raise REFERENCE_CODECS = ( diff --git a/open4d/codec/_qndf.py b/open4d/codec/_qndf.py index bc8e0efb..66d6288c 100644 --- a/open4d/codec/_qndf.py +++ b/open4d/codec/_qndf.py @@ -1,4 +1,4 @@ -"""In-process adapters for Open4D's neural displacement field codecs.""" +"""Neural displacement field codecs with direct standalone VMESH carriage.""" from __future__ import annotations @@ -9,30 +9,41 @@ import math from numbers import Real from pathlib import Path +import pickle import tempfile -from zipfile import BadZipFile, ZIP_DEFLATED, ZipFile +from zipfile import BadZipFile, ZipFile import numpy as np from open4d.core import Frame, MemoryFrameProvider, Sequence, TopologyMode, TriangleMesh -from ._npz import _json_value, _publish_file, _validate_manifest +from ._metadata import _json_value, _validate_manifest from ._protocol import CodecError from ._research import research_module from ._torch import torch_device +from ._vmesh_format import pack_vmesh, probe_codec, unpack_vmesh + +_MAX_CONTEXT_BYTES = 64 * 1024 * 1024 +_MAX_GRAPH_ELEMENTS = 64 * 1024 * 1024 def _backend(): try: return ( - import_module("torch"), - research_module("qndf.compress"), + *_model_backend(), research_module("qndf.build_dataset_open3d"), ) except ImportError as error: raise CodecError("QNDF dependencies are missing; install open4d[qndf]") from error +def _model_backend(): + try: + return import_module("torch"), research_module("qndf.compress") + except ImportError as error: + raise CodecError("QNDF decoding needs PyTorch; install open4d[qndf]") from error + + def _torch_bytes(torch, value) -> bytes: stream = BytesIO() torch.save(value, stream) @@ -57,7 +68,7 @@ def _quantized_engine(torch, required=None): def _manifest(sequence, codec): return { - "schema": f"open4d.{codec}-sequence/v1", "codec": codec, + "version": 1, "codec": codec, "native": {"profile": f"{codec}/1"}, "metadata": _json_value(sequence.metadata, "sequence"), "allow_nonmonotonic_timestamps": sequence.allow_nonmonotonic_timestamps, "frames": [{ @@ -82,6 +93,12 @@ def _validate_model_options(pe_dim, hidden_dim, num_layers, input_scale, output_ raise ValueError(f"{name} must be a positive integer") if pe_dim % 2: raise ValueError("pe_dim must be even") + if pe_dim > 128 or hidden_dim > 4096 or num_layers > 128: + raise ValueError("QNDF model dimensions outside limits") + parameters = (2 * 3 * pe_dim * hidden_dim + (num_layers + 1) * hidden_dim ** 2 + + 3 * (num_layers + 3) * hidden_dim + 3 * hidden_dim + 3) + if parameters > 16 * 1024 * 1024: + raise ValueError("QNDF model parameter count outside limits") for name, value in (("input_scale", input_scale), ("output_scale", output_scale)): if isinstance(value, bool) or not isinstance(value, Real) or not math.isfinite(value): raise ValueError(f"{name} must be a finite number") @@ -89,10 +106,16 @@ def _validate_model_options(pe_dim, hidden_dim, num_layers, input_scale, output_ raise ValueError("output_scale must be nonzero") -def _validate_context(torch, context, schema): +def _validate_context(torch, context, *, int8=False): try: - if not isinstance(context, Mapping) or context.get("schema") != schema: - raise ValueError("unsupported schema or context is not an object") + required = {"version", "coarse_vertices", "coarse_faces", "input_mean", "input_std", + "pe_dim", "hidden_dim", "num_layers", "input_scale", "output_scale", + "normalization", "model_state_dict"} + if int8: + required.add("quantized_engine") + if (not isinstance(context, Mapping) or set(context) != required + or type(context.get("version")) is not int or context["version"] != 1): + raise ValueError("unsupported context/version or fields") _validate_model_options(*(context[name] for name in ( "pe_dim", "hidden_dim", "num_layers", "input_scale", "output_scale", ))) @@ -105,6 +128,8 @@ def _validate_context(torch, context, schema): if ((name == "coarse_vertices" and (len(shape) != 2 or shape[1] != 3 or shape[0] == 0)) or (name != "coarse_vertices" and shape != (1, 3))): raise ValueError(f"invalid {name} dimensions") + if len(context["coarse_vertices"]) * 3 * context["pe_dim"] * 14 > _MAX_GRAPH_ELEMENTS: + raise ValueError("QNDF neighbor graph outside limits") if torch.any(context["input_std"] <= 0): raise ValueError("input_std must be positive") faces = context["coarse_faces"] @@ -122,10 +147,51 @@ def _validate_context(torch, context, schema): raise ValueError("normalization requires finite XYZ bounds and a positive scale") if not isinstance(context["model_state_dict"], Mapping): raise ValueError("model_state_dict must be an object") + if int8 and not isinstance(context["quantized_engine"], str): + raise ValueError("quantized_engine must be a string") except (KeyError, TypeError, ValueError) as error: raise CodecError(f"invalid QNDF frame context: {error}") from error +def _extract_legacy_qndf(source: Path, destination: Path, *, int8: bool) -> dict: + """Convert safe legacy weights explicitly, without carrying their archive.""" + codec = "qndf-int8" if int8 else "qndf" + version = 2 if int8 else 1 + try: + with ZipFile(source) as archive: + members = archive.infolist() + if len({member.filename for member in members}) != len(members): + raise CodecError("duplicate QNDF archive member") + if archive.getinfo("manifest.json").file_size > 16 * 1024 * 1024: + raise CodecError("oversized QNDF manifest") + manifest = json.loads(archive.read("manifest.json")) + if int8 and isinstance(manifest, dict) and manifest.get("schema") == "open4d.qndf-int8-sequence/v1": + raise CodecError("QNDF-int8 v1 contains executable models; re-encode it") + _validate_manifest(manifest, schema=f"open4d.{codec}-sequence/v{version}", codec=codec) + metadata = {key: value for key, value in manifest.items() if key != "schema"} + metadata.update(version=1, native={"profile": f"{codec}/1"}) + from ._native_profiles import layout + layout(codec, len(metadata["frames"]), metadata["native"]) + expected = {"manifest.json", *(f"frames/{i:06d}/context.pt" for i in range(len(metadata["frames"])))} + if set(archive.namelist()) != expected: + raise CodecError("unexpected QNDF archive payloads") + torch = import_module("torch") + for ordinal in range(len(metadata["frames"])): + member = archive.getinfo(f"frames/{ordinal:06d}/context.pt") + if not 0 < member.file_size <= _MAX_CONTEXT_BYTES: + raise CodecError("QNDF legacy context outside limits") + context = torch.load(BytesIO(archive.read(member)), map_location="cpu", weights_only=True) + if not isinstance(context, Mapping) or context.pop("schema", None) != f"open4d.{codec}/v{version}": + raise CodecError("invalid legacy QNDF frame context") + context["version"] = 1 + _validate_context(torch, context, int8=int8) + torch.save(context, destination / f"frame_{ordinal:06d}.pt") + (destination / "metadata.json").write_text(json.dumps(metadata, allow_nan=False), encoding="utf-8") + return metadata + except (BadZipFile, KeyError, ValueError, TypeError, RuntimeError, pickle.UnpicklingError) as error: + raise CodecError(f"invalid legacy {codec} artifact: {error}") from error + + class QNDFCodec: backend = "python-in-process" lossless = False @@ -134,17 +200,10 @@ class QNDFCodec: def __init__(self, *, int8: bool = False) -> None: self.int8 = int8 self.id = "qndf-int8" if int8 else "qndf" - self.suffixes = (".qi4d",) if int8 else (".q4d",) - self.version = 2 if int8 else 1 - self.schema = f"open4d.{self.id}-sequence/v{self.version}" + self.suffixes = (".vmesh",) def can_decode(self, source: Path) -> bool: - try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - return isinstance(manifest, dict) and manifest.get("schema") == self.schema - except (OSError, BadZipFile, KeyError, ValueError, TypeError): - return False + return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id def encode( self, sequence: Sequence, destination: Path, *, overwrite: bool = False, @@ -163,6 +222,8 @@ def encode( if type(num_subdiv) is not int or num_subdiv < 0: raise ValueError("num_subdiv must be a nonnegative integer") destination = Path(destination).absolute() + if destination.suffix.lower() != ".vmesh": + raise ValueError("QNDF destination must have a .vmesh extension") if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") for frame in sequence: @@ -177,88 +238,84 @@ def encode( torch.cuda.manual_seed_all(seed) destination.parent.mkdir(parents=True, exist_ok=True) manifest = _manifest(sequence, self.id) - manifest["schema"] = self.schema - with tempfile.NamedTemporaryFile( - prefix=f".{destination.name}.", suffix=".tmp", - dir=destination.parent, delete=False, - ) as stream: - temporary = Path(stream.name) - try: - with ZipFile(temporary, "w", compression=ZIP_DEFLATED) as archive: - for ordinal, frame in enumerate(sequence): - low, faces, projected, normalization = preprocessing.build_pair( - frame.geometry.positions, frame.geometry.triangles, - coarse_size, num_subdiv, - ) - coarse = torch.from_numpy(np.asarray(low, dtype=np.float32)).to(target) - triangles = torch.from_numpy(np.asarray(faces, dtype=np.int64)).to(target) - goal = torch.from_numpy(np.asarray(projected, dtype=np.float32)).to(target) - encoded, inputs, mean, std = _inputs( - torch, models, coarse, pe_dim, input_scale - ) - model = models.MLP(3 * pe_dim, hidden_dim, 3, num_layers).to(target) - optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate) - graph = models.MeshDataset( - encoded, inputs, triangles, (goal - coarse) * output_scale, - progress=verbose, - ) - loader = torch.utils.data.DataLoader( - graph, batch_size=batch_size, shuffle=False - ) - model.train() - for _ in range(epochs): - for inputs, neighbors, weights, expected in loader: - optimizer.zero_grad(set_to_none=True) - loss = torch.nn.functional.mse_loss( - model(inputs, neighbors, weights), expected - ) - loss.backward() - optimizer.step() - context = { - "schema": f"open4d.{self.id}/v{self.version}", - "coarse_vertices": coarse.cpu(), "coarse_faces": triangles.cpu(), - "input_mean": mean.cpu(), "input_std": std.cpu(), - "pe_dim": pe_dim, "hidden_dim": hidden_dim, - "num_layers": num_layers, "input_scale": input_scale, - "output_scale": output_scale, "normalization": normalization, - } - if self.int8: - context["quantized_engine"] = _quantized_engine(torch) - cpu_model = torch.ao.quantization.quantize_dynamic( - model.cpu().eval(), {torch.nn.Linear}, dtype=torch.qint8 + with tempfile.TemporaryDirectory(prefix="open4d-qndf-") as directory: + native = Path(directory) + for ordinal, frame in enumerate(sequence): + low, faces, projected, normalization = preprocessing.build_pair( + frame.geometry.positions, frame.geometry.triangles, + coarse_size, num_subdiv, + ) + coarse = torch.from_numpy(np.asarray(low, dtype=np.float32)).to(target) + triangles = torch.from_numpy(np.asarray(faces, dtype=np.int64)).to(target) + goal = torch.from_numpy(np.asarray(projected, dtype=np.float32)).to(target) + encoded, inputs, mean, std = _inputs( + torch, models, coarse, pe_dim, input_scale + ) + model = models.MLP(3 * pe_dim, hidden_dim, 3, num_layers).to(target) + optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate) + graph = models.MeshDataset( + encoded, inputs, triangles, (goal - coarse) * output_scale, + progress=verbose, + ) + loader = torch.utils.data.DataLoader( + graph, batch_size=batch_size, shuffle=False + ) + model.train() + for _ in range(epochs): + for inputs, neighbors, weights, expected in loader: + optimizer.zero_grad(set_to_none=True) + loss = torch.nn.functional.mse_loss( + model(inputs, neighbors, weights), expected ) - context["model_state_dict"] = cpu_model.state_dict() - else: - context["model_state_dict"] = model.state_dict() - _validate_context(torch, context, f"open4d.{self.id}/v{self.version}") - archive.writestr( - f"frames/{ordinal:06d}/context.pt", _torch_bytes(torch, context) + loss.backward() + optimizer.step() + context = { + "version": 1, + "coarse_vertices": coarse.cpu(), "coarse_faces": triangles.cpu(), + "input_mean": mean.cpu(), "input_std": std.cpu(), + "pe_dim": pe_dim, "hidden_dim": hidden_dim, + "num_layers": num_layers, "input_scale": input_scale, + "output_scale": output_scale, "normalization": normalization, + } + if self.int8: + context["quantized_engine"] = _quantized_engine(torch) + cpu_model = torch.ao.quantization.quantize_dynamic( + model.cpu().eval(), {torch.nn.Linear}, dtype=torch.qint8 ) - archive.writestr("manifest.json", json.dumps(manifest)) - _publish_file(temporary, destination, overwrite=overwrite) - except Exception: - temporary.unlink(missing_ok=True) - raise + context["model_state_dict"] = cpu_model.state_dict() + else: + context["model_state_dict"] = model.state_dict() + _validate_context(torch, context, int8=self.int8) + torch.save(context, native / f"frame_{ordinal:06d}.pt") + (native / "metadata.json").write_text(json.dumps(manifest, allow_nan=False), encoding="utf-8") + pack_vmesh(native, destination, overwrite=overwrite) return destination def decode( self, source: Path, *, device: str | None = None, verbose: bool = False ) -> Sequence: - try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - if self.int8 and isinstance(manifest, dict) and manifest.get("schema") == "open4d.qndf-int8-sequence/v1": - raise CodecError("QNDF-int8 v1 contains executable models; re-encode with the current version") - _validate_manifest(manifest, schema=self.schema, codec=self.id) - torch, models, _ = _backend() + source = Path(source).absolute() + if source.suffix.lower() != ".vmesh": + raise CodecError("QNDF decoding requires .vmesh; migrate the legacy artifact explicitly") + if probe_codec(source) != self.id: + raise CodecError(f"VMESH does not contain {self.id}") + with tempfile.TemporaryDirectory(prefix="open4d-qndf-decode-") as directory: + native = Path(directory) / "native" + unpack_vmesh(source, native) + manifest = json.loads((native / "metadata.json").read_text(encoding="utf-8")) + try: + torch, models = _model_backend() target = torch.device("cpu") if self.int8 else torch_device(torch, device) frames = [] for ordinal, record in enumerate(manifest["frames"]): + context_path = native / f"frame_{ordinal:06d}.pt" + if context_path.stat().st_size > _MAX_CONTEXT_BYTES: + raise CodecError("QNDF context outside limits") context = torch.load( - BytesIO(archive.read(f"frames/{ordinal:06d}/context.pt")), + context_path, map_location=target, weights_only=True, ) - _validate_context(torch, context, f"open4d.{self.id}/v{self.version}") + _validate_context(torch, context, int8=self.int8) coarse, faces = context["coarse_vertices"].to(target), context["coarse_faces"].to(target) inputs = (coarse * context["input_scale"] - context["input_mean"].to(target)) inputs /= context["input_std"].to(target) @@ -293,8 +350,8 @@ def decode( positions.cpu().numpy(), faces.cpu().numpy() ), record.get("metadata", {}), )) - except (BadZipFile, KeyError, json.JSONDecodeError) as error: - raise CodecError(f"invalid {self.id} artifact {source}: {error}") from error + except (KeyError, ValueError, TypeError, RuntimeError, pickle.UnpicklingError) as error: + raise CodecError(f"invalid {self.id} native payload: {error}") from error return Sequence(MemoryFrameProvider( frames, metadata=manifest.get("metadata", {}), topology=TopologyMode.CHANGING, has_constant_vertex_count=None, has_vertex_correspondence=False, diff --git a/open4d/codec/_temporal.py b/open4d/codec/_temporal.py index c566e572..c1498b2b 100644 --- a/open4d/codec/_temporal.py +++ b/open4d/codec/_temporal.py @@ -6,18 +6,21 @@ import json from pathlib import Path import tempfile +from zipfile import ZipFile import numpy as np from open4d.core import Frame, MemoryFrameProvider, Sequence, TopologyMode, TriangleMesh -from ._npz import _json_value, _publish_file, _validate_manifest +from ._metadata import _json_value +from ._npz import NumPyZipCodec, _read_array +from ._vmesh_format import pack_vmesh, probe_codec, unpack_vmesh from ._protocol import CodecError def _manifest(sequence, codec): return { - "schema": f"open4d.{codec}-sequence/v1", "codec": codec, + "version": 1, "codec": codec, "native": {"profile": f"{codec}/1"}, "metadata": _json_value(sequence.metadata, "sequence"), "allow_nonmonotonic_timestamps": sequence.allow_nonmonotonic_timestamps, "frames": [{ @@ -94,25 +97,21 @@ def __init__(self, identifier: str) -> None: if identifier not in {"temporal-delta", "temporal-pca"}: raise ValueError(f"unknown experimental temporal profile: {identifier}") self.id = identifier - self.suffixes = ((".td4d",) if identifier == "temporal-delta" else (".tp4d",)) - self.schema = f"open4d.{identifier}-sequence/v1" + self.suffixes = (".vmesh",) def can_decode(self, source: Path) -> bool: - try: - with np.load(source, allow_pickle=False) as artifact: - manifest = json.loads(artifact["manifest"].tobytes()) - return isinstance(manifest, dict) and manifest.get("schema") == self.schema - except (OSError, ValueError, TypeError, KeyError): - return False + return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id def encode( self, sequence: Sequence, destination: Path, *, overwrite: bool = False, face_budget: int = 3000, quantization_bits: int = 16, components: int = 5, ) -> Path: - if face_budget < 1: + if type(face_budget) is not int or face_budget < 1: raise ValueError("face_budget must be positive") destination = Path(destination).absolute() + if destination.suffix.lower() != ".vmesh": + raise ValueError("temporal destination must have a .vmesh extension") if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") reference, faces, fitted = _fit(sequence, face_budget) @@ -140,31 +139,33 @@ def encode( manifest = _manifest(sequence, self.id) manifest["quantization_bits"] = quantization_bits destination.parent.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile( - prefix=f".{destination.name}.", suffix=".tmp", - dir=destination.parent, delete=False, - ) as stream: - temporary = Path(stream.name) - try: - with temporary.open("wb") as stream: - np.savez_compressed( - stream, - manifest=np.frombuffer(json.dumps(manifest).encode(), dtype=np.uint8), - **payload, - ) - _publish_file(temporary, destination, overwrite=overwrite) - except Exception: - temporary.unlink(missing_ok=True) - raise + with tempfile.TemporaryDirectory(prefix="open4d-temporal-") as directory: + native = Path(directory) + np.savez_compressed(native / "sequence.npz", **payload) + (native / "metadata.json").write_text(json.dumps(manifest, allow_nan=False), encoding="utf-8") + pack_vmesh(native, destination, overwrite=overwrite) return destination def decode(self, source: Path, *, device: str | None = None) -> Sequence: if device not in (None, "cpu"): raise ValueError(f"{self.id} decoding is NumPy-based; device must be 'cpu'") + source = Path(source).absolute() + if source.suffix.lower() != ".vmesh": + raise CodecError("temporal decoding requires .vmesh; re-encode older private artifacts") + if probe_codec(source) != self.id: + raise CodecError(f"VMESH does not contain {self.id}") try: - with np.load(source, allow_pickle=False) as artifact: - manifest = json.loads(artifact["manifest"].tobytes()) - _validate_manifest(manifest, schema=self.schema, codec=self.id) + with tempfile.TemporaryDirectory(prefix="open4d-temporal-decode-") as directory: + native = Path(directory) / "native" + unpack_vmesh(source, native) + manifest = json.loads((native / "metadata.json").read_text(encoding="utf-8")) + with ZipFile(native / "sequence.npz") as archive: + members = archive.namelist() + if (len(members) != len(set(members)) + or set(members) != {f"{name}.npy" for name in _temporal_fields(self.id)}): + raise CodecError("unexpected temporal native arrays") + artifact = {name: _read_array(native / "sequence.npz", name, _PLAIN_ARRAYS) + for name in _temporal_fields(self.id)} reference, faces = artifact["reference"], artifact["triangles"] reference_mesh = TriangleMesh(reference, faces) count, vertices = len(manifest["frames"]), len(reference_mesh.positions) @@ -214,5 +215,12 @@ def decode(self, source: Path, *, device: str | None = None) -> Sequence: )) +def _temporal_fields(codec): + return ({"reference", "triangles", "displacement", "scale"} if codec == "temporal-delta" + else {"reference", "triangles", "coefficients", "basis", "mean", "scale"}) + + +_PLAIN_ARRAYS = NumPyZipCodec("raw", rle=False) + TEMPORAL_DELTA_CODEC = TemporalMeshCodec("temporal-delta") TEMPORAL_PCA_CODEC = TemporalMeshCodec("temporal-pca") diff --git a/open4d/codec/_tracked.py b/open4d/codec/_tracked.py index 84f586be..b3ae1bbf 100644 --- a/open4d/codec/_tracked.py +++ b/open4d/codec/_tracked.py @@ -10,15 +10,14 @@ import tempfile from types import MappingProxyType -from open4d._files import publish_directory from open4d.core import Frame, Sequence, TopologyMode, TriangleMesh from open4d.io import open_sequence from open4d.io._mesh import write_obj -from ._npz import _json_value, _validate_manifest +from ._metadata import _json_value, _validate_manifest, require_vmesh_destination from ._protocol import CodecError from ._native import run as _run_native -from ._v3c import pack_vmesh, probe_codec, unpack_vmesh +from ._vmesh_format import pack_vmesh, probe_codec, unpack_vmesh def _executable(value, label: str) -> str: @@ -95,17 +94,10 @@ class TrackedMeshCodec: def __init__(self, identifier): self.id = identifier - self.suffixes = (f".{identifier}", ".vmesh") + self.suffixes = (".vmesh",) def can_decode(self, source: Path) -> bool: - source = Path(source) - if source.is_file() and source.suffix.lower() == ".vmesh": - return probe_codec(source) == self.id - try: - _manifest(source, self.id) - except CodecError: - return False - return True + return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id def _settings(self, backend, python, encoder, decoder, *, encoding): variable = f"OPEN4D_{self.id.upper()}_ROOT" @@ -143,15 +135,12 @@ def encode( grid_resolution=512, key_frame=None, components=None, quantization=None, overwrite=False, ) -> Path: - """Encode native payloads to a .vmesh file or legacy codec directory.""" - destination = Path(destination).absolute() - container = destination.suffix.lower() == ".vmesh" + """Encode direct native payloads to VMESH.""" + destination = require_vmesh_destination(destination) if destination.exists() and not overwrite: raise FileExistsError(f"destination already exists: {destination}") - if container and destination.is_dir(): + if destination.is_dir(): raise IsADirectoryError(destination) - if not container and destination.exists() and not destination.is_dir(): - raise NotADirectoryError(destination) if len(sequence) < 2: raise CodecError(f"{self.id} requires at least two frames") _positive_integer(num_centers, "num_centers") @@ -226,49 +215,18 @@ def encode( if not (result / name).is_file() or not (result / name).stat().st_size: raise CodecError(f"{self.id} encoder produced no {name}") (result / "metadata.json").write_text(json.dumps(manifest), encoding="utf-8") - if container: - return pack_vmesh(result, destination, overwrite=overwrite) - previous = None - if destination.exists(): - if not overwrite: - raise FileExistsError(f"destination already exists: {destination}") - previous = Path(tempfile.mkdtemp( - prefix=f".{destination.name}.backup.", dir=destination.parent, - )) - previous.rmdir() - destination.rename(previous) - try: - if overwrite: - result.rename(destination) - else: - publish_directory(result, destination) - except BaseException: - if previous is not None: - try: - previous.rename(destination) - except OSError as error: - raise OSError( - f"Could not restore {destination}; original output remains at {previous}" - ) from error - raise - if previous is not None: - if previous.is_symlink(): - previous.unlink() - else: - shutil.rmtree(previous) - return destination + return pack_vmesh(result, destination, overwrite=overwrite) def decode(self, source: Path, *, backend=None, python=None, decoder=None) -> Sequence: - """Reconstruct frames from a .vmesh file or native codec directory.""" + """Reconstruct frames from VMESH.""" source = Path(source).absolute() temporary = tempfile.TemporaryDirectory(prefix=f"open4d-{self.id}-decode-") decoded = None try: work = Path(temporary.name) - if source.is_file() and source.suffix.lower() == ".vmesh": - if probe_codec(source) != self.id: - raise CodecError(f".vmesh does not contain {self.id} payloads") - source = unpack_vmesh(source, work / "native") + if source.suffix.lower() != ".vmesh" or probe_codec(source) != self.id: + raise CodecError(f"{self.id} decode requires VMESH; import native directories explicitly") + source = unpack_vmesh(source, work / "native") manifest = _manifest(source, self.id) settings = self._settings(backend, python, None, decoder, encoding=False) output = work / "decoded" diff --git a/open4d/codec/_vmesh.py b/open4d/codec/_vmesh.py index a1c627e9..a8cfe6bb 100644 --- a/open4d/codec/_vmesh.py +++ b/open4d/codec/_vmesh.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Mapping import json import math from numbers import Real @@ -10,7 +9,6 @@ from pathlib import Path import tempfile from types import MappingProxyType -from zipfile import BadZipFile, ZIP_STORED, ZipFile import numpy as np @@ -18,12 +16,11 @@ from open4d.io import Open4DError, open_sequence from open4d.io._mesh import write_obj -from ._npz import _json_value, _publish_file, _validate_manifest +from ._metadata import _json_value, _validate_manifest, require_vmesh_destination from ._protocol import CodecError from ._native import run as _run -from ._v3c import pack_vmesh, probe_codec, unpack_vmesh +from ._vmesh_format import pack_vmesh, probe_codec, unpack_vmesh -_SCHEMA = "open4d.vmesh-sequence/v1" _POSITION_BIT_DEPTH = 12 _DEFAULT_RAW_FPS = 30.0 @@ -164,7 +161,7 @@ def close(self) -> None: class VMeshCodec: """Invoke one external V-Mesh process per sequence direction.""" - suffixes = (".v4d", ".vmesh") + suffixes = (".vmesh",) backend = "native-sequence" lossless = False preserves = ("positions", "triangles") @@ -174,18 +171,7 @@ def __init__(self, identifier: str) -> None: self._environment = identifier.upper() def can_decode(self, source: Path) -> bool: - if Path(source).suffix.lower() == ".vmesh": - return probe_codec(source) == self.id - try: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - return ( - isinstance(manifest, Mapping) - and manifest.get("schema") == _SCHEMA - and manifest.get("codec") == self.id - ) - except (OSError, BadZipFile, KeyError, ValueError, TypeError): - return False + return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id def encode( self, @@ -197,12 +183,12 @@ def encode( decoder_config: str | os.PathLike[str] | None = None, overwrite: bool = False, ) -> Path: + destination = require_vmesh_destination(destination) executable = _executable(encoder, f"OPEN4D_{self._environment}_ENCODER") configs = [Path(value).absolute() for value in (encoder_config, decoder_config) if value] for config in configs: if not config.is_file(): raise CodecError(f"native codec configuration is missing: {config}") - destination = Path(destination).absolute() if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") if not len(sequence): @@ -215,7 +201,7 @@ def encode( lower = np.minimum(lower, frame.geometry.positions.min(0)) upper = np.maximum(upper, frame.geometry.positions.max(0)) manifest = { - "schema": _SCHEMA, "codec": self.id, + "version": 1, "codec": self.id, "metadata": _json_value(sequence.metadata, "sequence"), "topology": sequence.topology.value, "has_constant_vertex_count": sequence.has_constant_vertex_count, @@ -261,32 +247,12 @@ def encode( _run(command, f"{self.id} encoder") if not stream.is_file() or not stream.stat().st_size: raise CodecError(f"{self.id} encoder produced no bitstream") - if destination.suffix.lower() == ".vmesh": - # Preserve native V3C bytes plus the timing and normalization - # previously available only in the .v4d ZIP wrapper. - native_manifest = {k: v for k, v in manifest.items() if k != "schema"} - native_manifest.update(version=1, native={"profile": f"{self.id}/1", "decoder_config": bool(decoder_config)}) - (work / "metadata.json").write_text(json.dumps(native_manifest), encoding="utf-8") - if decoder_config: - import shutil - shutil.copyfile(Path(decoder_config).absolute(), work / "decoder.cfg") - return pack_vmesh(work, destination, overwrite=overwrite) - with tempfile.NamedTemporaryFile( - prefix=f".{destination.name}.", suffix=".tmp", - dir=destination.parent, delete=False, - ) as temporary_stream: - temporary = Path(temporary_stream.name) - try: - with ZipFile(temporary, "w", compression=ZIP_STORED) as archive: - archive.write(stream, "sequence.vmesh") - if decoder_config: - archive.write(Path(decoder_config).absolute(), "decoder.cfg") - archive.writestr("manifest.json", json.dumps(manifest)) - _publish_file(temporary, destination, overwrite=overwrite) - except Exception: - temporary.unlink(missing_ok=True) - raise - return destination + manifest["native"] = {"profile": f"{self.id}/1", "decoder_config": bool(decoder_config)} + (work / "metadata.json").write_text(json.dumps(manifest), encoding="utf-8") + if decoder_config: + import shutil + shutil.copyfile(Path(decoder_config).absolute(), work / "decoder.cfg") + return pack_vmesh(work, destination, overwrite=overwrite) def decode( self, @@ -297,7 +263,9 @@ def decode( fps: float | None = None, ) -> Sequence: source = Path(source).absolute() - carried = probe_codec(source) if source.suffix.lower() == ".vmesh" else None + if source.suffix.lower() != ".vmesh": + raise CodecError("V-DMC decode requires VMESH; use migrate_legacy for older artifacts") + carried = probe_codec(source) if carried is not None and carried != self.id: raise CodecError(f".vmesh contains {carried}, not {self.id}") raw = source.suffix.lower() == ".vmesh" and carried is None @@ -324,15 +292,6 @@ def decode( elif raw: stream = source manifest = None - else: - with ZipFile(source) as archive: - manifest = json.loads(archive.read("manifest.json")) - _validate_manifest(manifest, schema=_SCHEMA, codec=self.id) - _position_normalization(manifest) - archive.extract("sequence.vmesh", work) - if "decoder.cfg" in archive.namelist(): - archive.extract("decoder.cfg", work) - stream = work / "sequence.vmesh" output = work / "decoded" output.mkdir() # The pinned V-DMC decoder parses decTex even with zero attributes. diff --git a/open4d/codec/_v3c.py b/open4d/codec/_vmesh_format.py similarity index 61% rename from open4d/codec/_v3c.py rename to open4d/codec/_vmesh_format.py index 1eb779b1..3f89ab96 100644 --- a/open4d/codec/_v3c.py +++ b/open4d/codec/_vmesh_format.py @@ -1,81 +1,36 @@ -"""Experimental O4D native-codec carriage in V3C sample streams. +"""Standalone VMESH container for native codec payloads. -This deliberately implements one bounded application format, not a general -V3C parser or an ISO-conforming V-DMC geometry decoder. Usage examples are -in examples/vmesh/. -Native temporal payloads are carried; independent decoded frame sequences are -not an admitted profile. +VMESH owns its framing, sequence metadata and codec profiles. No V3C units, +private SEI messages or serialized Open4D objects are used. Native payloads +are copied verbatim and inspection never deserializes them. """ from __future__ import annotations import hashlib import json +import math from pathlib import Path import struct import tempfile -import uuid from open4d._files import publish_directory, publish_file -from ._npz import _validate_manifest +from ._metadata import _validate_manifest from ._protocol import CodecError -from ._native_profiles import PROFILES, layout as _layout +from ._native_profiles import MAX_FRAMES, PROFILES, layout as _layout -_UUID = uuid.UUID("e23b8c47-9135-4e34-90ad-421c6c57b63a").bytes -_SCHEMA = "open4d.vmesh.native/1" +_SCHEMA = "vmesh/1" _CHUNK = 1024 * 1024 _MAX_JSON = 16 * 1024 * 1024 -_MAX_UNIT = _MAX_JSON + _MAX_JSON // 255 + 128 _RECORD = struct.Struct(">BBIQ") # version, kind, file ID, byte offset _U32 = struct.Struct(">I") -_AD = b"\x08\x00\x00\x00" -_NAL = b"\x56\x01" # atlas PREFIX_NSEI (43), layer 0, temporal_id_plus1 1 - - -def _bootstrap() -> bytes: - """VPS syntax from MPEG V-DMC v14, with an unspecified (type 0) extension.""" - bits = [] - - def put(value, width): - bits.append(f"{value:0{width}b}") - - def ue(value): - encoded = f"{value + 1:b}" - bits.append("0" * (len(encoded) - 1) + encoded) - - # PTL: codec group 1, toolset 0, unconstrained reconstruction. These are - # syntax fields, not a claim of conformance to a standardized toolset. - for value, width in ((0, 1), (1, 7), (0, 8), (255, 8), (0, 1), (0, 8), - (0, 7), (15, 4), (0xfff, 12), (0, 8), (0, 6), - (0, 1), (0, 1)): - put(value, width) - for value, width in ((0, 4), (0, 8), (0, 6), (0, 6)): - put(value, width) # VPS 0, reserved, one atlas, atlas ID 0 - ue(1) # placeholder width/height; no atlas pictures or video components - ue(1) - put(0, 4) # one map - put(0, 4) # auxiliary, occupancy, geometry, attribute video absent - put(1, 1) - put(1, 8) # one extension - extension = _UUID + b"O4D\x01" - ue(3 + len(extension) - 1) - put(0, 8) # VPS_EXT_UNSPECIFIED - put(len(extension), 16) - for byte in extension: - put(byte, 8) - bits.append("1") - binary = "".join(bits) - binary += "0" * (-len(binary) % 8) - vps = b"\x00" * 4 + int(binary, 2).to_bytes(len(binary) // 8, "big") - return b"\x60" + _U32.pack(len(vps)) + vps - - -_BOOTSTRAP = _bootstrap() +_MAGIC = b"VMESH\x00\x01\x00" +_MAX_UNIT = _MAX_JSON + _RECORD.size def _error(message): - return CodecError(f"invalid O4D .vmesh: {message}") + return CodecError(f"invalid VMESH: {message}") def _read(stream, size): @@ -97,8 +52,14 @@ def pairs(items): def constant(value): raise ValueError(f"non-finite JSON value {value}") + def real(value): + number = float(value) + if not math.isfinite(number): + raise ValueError("non-finite JSON number") + return number + try: - return json.loads(data, object_pairs_hook=pairs, parse_constant=constant) + return json.loads(data, object_pairs_hook=pairs, parse_constant=constant, parse_float=real) except (ValueError, UnicodeError, RecursionError) as error: raise _error(f"invalid JSON: {error}") from error @@ -109,8 +70,10 @@ def _sequence_metadata(data, codec): if not isinstance(value, dict) or type(value.get("version")) is not int or value["version"] != 1: raise ValueError("unsupported native metadata version") _validate_manifest(value, schema=None, codec=codec) - if not value["frames"]: - raise ValueError("empty frame list") + if "schema" in value: + raise ValueError("VMESH metadata cannot contain an application schema") + if not 0 < len(value["frames"]) <= MAX_FRAMES: + raise ValueError("frame count outside limits") _layout(codec, len(value["frames"]), value.get("native")) except (ValueError, TypeError, KeyError) as error: raise _error(f"invalid sequence metadata: {error}") from error @@ -128,9 +91,9 @@ def _descriptor(data): or value.get("representation") != PROFILES[codec][0] or value.get("dependency_mode") != PROFILES[codec][1]): raise _error("unsupported native representation/dependencies") - if type(count) is not int or not 1 <= count <= _MAX_JSON // 16: + if type(count) is not int or not 1 <= count <= MAX_FRAMES: raise _error("invalid frame count") - expected = _layout(codec, count, value.get("native")) + expected = _layout(codec, count, value.get("native"))[1:] if not isinstance(files, list) or len(files) != len(expected): raise _error("invalid native file list") for index, (record, (name, role)) in enumerate(zip(files, expected)): @@ -142,51 +105,38 @@ def _descriptor(data): raise _error("invalid native payload size") if not isinstance(digest, str) or len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest): raise _error("invalid SHA-256 digest") - if files[0]["size"] > _MAX_JSON: - raise _error("sequence metadata exceeds size limit") + sequence = value.get("sequence") + if not isinstance(sequence, dict) or "schema" in sequence: + raise _error("invalid VMESH sequence metadata") + parsed = _sequence_metadata(json.dumps(sequence, allow_nan=False).encode(), codec) + if len(parsed["frames"]) != count or parsed.get("native") != value.get("native"): + raise _error("sequence metadata disagrees with codec profile") return value def _write_record(stream, kind, file_id, offset, data): - payload = _UUID + _RECORD.pack(1, kind, file_id, offset) + data - size = len(payload) - # Atlas SEI user_data_unregistered is payload type 4 (not video SEI 5). - sei = b"\x04" + b"\xff" * (size // 255) + bytes([size % 255]) + payload + b"\x80" - nal = _NAL + sei - unit = _AD + b"\x60" + _U32.pack(len(nal)) + nal + unit = _RECORD.pack(1, kind, file_id, offset) + data stream.write(_U32.pack(len(unit))) stream.write(unit) def _read_record(stream): size = _U32.unpack(_read(stream, 4))[0] - if not 45 <= size <= _MAX_UNIT: - raise _error("V3C unit size outside limits") - unit = _read(stream, size) - if unit[:5] != _AD + b"\x60" or _U32.unpack(unit[5:9])[0] != size - 9: - raise _error("invalid atlas unit or NAL sample length") - if unit[9:12] != _NAL + b"\x04": - raise _error("expected atlas prefix NSEI user_data_unregistered") - pos, payload_size = 12, 0 - while pos < size: - byte = unit[pos] - payload_size += byte - pos += 1 - if byte != 255: - break - if payload_size < 16 + _RECORD.size or pos + payload_size + 1 != size or unit[-1] != 0x80: - raise _error("invalid SEI payload length or trailing bits") - if unit[pos:pos + 16] != _UUID: - raise _error("unexpected application UUID") - version, kind, file_id, offset = _RECORD.unpack_from(unit, pos + 16) + if not _RECORD.size <= size <= _MAX_UNIT: + raise _error("record size outside limits") + version, kind, file_id, offset = _RECORD.unpack(_read(stream, _RECORD.size)) if version != 1 or kind not in (0, 1, 2): raise _error("unsupported record version/type") - return kind, file_id, offset, unit[pos + 16 + _RECORD.size:-1] + length = size - _RECORD.size + maximum = (_MAX_JSON, _CHUNK, 32)[kind] + if not 0 < length <= maximum: + raise _error("record payload size outside limits") + return kind, file_id, offset, _read(stream, length) def _start(stream): - if _read(stream, len(_BOOTSTRAP)) != _BOOTSTRAP: - raise _error("unsupported VPS/bootstrap") + if _read(stream, len(_MAGIC)) != _MAGIC: + raise _error("unsupported magic/version") kind, file_id, offset, data = _read_record(stream) if (kind, file_id, offset) != (0, 0, 0) or len(data) > _MAX_JSON: raise _error("missing or oversized application manifest") @@ -194,26 +144,41 @@ def _start(stream): def probe_codec(source: str | Path) -> str | None: - """Identify this application format without loading native libraries. + """Identify standalone VMESH without loading native libraries. - None means an ordinary/other V3C stream; recognized malformed O4D streams - raise CodecError. This probe does not validate the native payload hashes. + None means another format, including raw MPEG V-DMC. Recognizable corrupt + VMESH and the retired private O4D/V3C wrapper fail before native decoding. """ with Path(source).open("rb") as stream: - prefix = stream.read(len(_BOOTSTRAP)) - if prefix != _BOOTSTRAP: - if _UUID in prefix or b"O4D\x01" in prefix: - raise _error("damaged VPS/bootstrap") + prefix = stream.read(128) + if not prefix.startswith(_MAGIC): + if prefix.startswith(b"PK\x03\x04"): + raise _error("archive input requires explicit migration") + if prefix.startswith(b"VMESH") or prefix[1:8] == _MAGIC[1:]: + raise _error("damaged magic/version") + if bytes.fromhex("e23b8c4791354e3490ad421c6c57b63a") in prefix or b"O4D\x01" in prefix: + raise _error("retired O4D/V3C wrapper; repack the original native directory") return None stream.seek(0) descriptor, _ = _start(stream) return descriptor["codec"] +def is_mesh_profile(source: str | Path) -> bool: + """Classify VMESH for mesh-only tools without loading a decoder.""" + codec = probe_codec(source) + if codec in ("vega", "queen", "3dgstream", "rerf"): + return False + if codec == "frames": + with Path(source).open("rb") as stream: + descriptor, _ = _start(stream) + return descriptor["native"]["representation"] == "mesh" + return True + + def _consume(source, destination=None): with Path(source).open("rb") as stream: descriptor, manifest_hash = _start(stream) - sequence_data = bytearray() for record in descriptor["files"]: digest, offset = hashlib.sha256(), 0 target = (destination / record["name"]).open("xb") if destination is not None else None @@ -223,11 +188,11 @@ def _consume(source, destination=None): if ((kind, file_id, position) != (1, record["id"], offset) or not 0 < len(data) <= min(_CHUNK, record["size"] - offset)): raise _error("missing, reordered or oversized native payload chunk") + if descriptor["codec"] == "frames" and len(data) != min(_CHUNK, record["size"] - offset): + raise _error("frames profile requires fixed-size payload chunks") digest.update(data) if target is not None: target.write(data) - if record["id"] == 0: - sequence_data.extend(data) offset += len(data) finally: if target is not None: @@ -237,16 +202,16 @@ def _consume(source, destination=None): kind, file_id, offset, data = _read_record(stream) if (kind, file_id, offset, data) != (2, 0, 0, manifest_hash) or stream.read(1): raise _error("invalid end record or trailing data") - sequence = _sequence_metadata(sequence_data, descriptor["codec"]) - if len(sequence["frames"]) != descriptor["frame_count"]: - raise _error("frame count disagrees with sequence metadata") - if sequence.get("native") != descriptor.get("native"): - raise _error("native profile disagrees with sequence metadata") - return {**descriptor, "sequence": sequence} + if destination is not None: + # Adapter input is reconstructed from VMESH metadata. It is not a + # carried Open4D file or object in the container. + (destination / "metadata.json").write_text( + json.dumps(descriptor["sequence"], allow_nan=False), encoding="utf-8") + return descriptor def inspect_vmesh(source: str | Path) -> dict: - """Validate an O4D .vmesh and return its manifest and sequence metadata. + """Validate a standalone VMESH and return its manifest and sequence metadata. Streams and verifies all payload hashes without extracting/deserializing native data or requiring the research codecs' optional dependencies. @@ -255,7 +220,7 @@ def inspect_vmesh(source: str | Path) -> dict: def pack_vmesh(source: str | Path, destination: str | Path, *, overwrite: bool = False) -> Path: - """Pack an O4D native directory or legacy N4MC .n4d into one .vmesh. + """Pack a native codec directory into one .vmesh. Required native files are preserved byte for byte. Scratch files and decoded geometry are excluded. This does not recompress the native data. @@ -269,15 +234,6 @@ def pack_vmesh(source: str | Path, destination: str | Path, *, overwrite: bool = raise FileExistsError(destination) if destination.is_dir(): raise IsADirectoryError(destination) - if source.suffix.lower() == ".n4d" and source.is_file(): - if source.is_symlink(): - raise _error("linked N4MC source archive") - from ._n4mc import _extract_n4d, _write_native_metadata - with tempfile.TemporaryDirectory(prefix="open4d-n4mc-repack-") as directory: - native = Path(directory) - metadata = _extract_n4d(source, native) - _write_native_metadata(metadata, native) - return pack_vmesh(native, destination, overwrite=overwrite) metadata = source / "metadata.json" if metadata.is_symlink() or not metadata.is_file() or metadata.stat().st_size > _MAX_JSON: raise _error("missing, linked or oversized metadata.json") @@ -289,10 +245,10 @@ def pack_vmesh(source: str | Path, destination: str | Path, *, overwrite: bool = sequence = _sequence_metadata(data, codec) descriptor = dict(schema=_SCHEMA, codec=codec, native_version=1, representation=PROFILES[codec][0], frame_count=len(sequence["frames"]), - dependency_mode=PROFILES[codec][1], files=[]) + dependency_mode=PROFILES[codec][1], sequence=sequence, files=[]) if "native" in sequence: descriptor["native"] = sequence["native"] - for index, (name, role) in enumerate(_layout(codec, len(sequence["frames"]), sequence.get("native"))): + for index, (name, role) in enumerate(_layout(codec, len(sequence["frames"]), sequence.get("native"))[1:]): path = source / name if path.is_symlink() or not path.is_file(): raise _error(f"missing or linked native payload {name}") @@ -301,9 +257,6 @@ def pack_vmesh(source: str | Path, destination: str | Path, *, overwrite: bool = while chunk := stream.read(_CHUNK): digest.update(chunk) size += len(chunk) - # Preserve the exact metadata we validated even if the source changed. - if index == 0 and digest.digest() != hashlib.sha256(data).digest(): - raise _error("metadata changed during packing") descriptor["files"].append(dict(id=index, name=name, role=role, size=size, sha256=digest.hexdigest())) manifest = json.dumps(descriptor, separators=(",", ":"), allow_nan=False).encode("utf-8") if len(manifest) > _MAX_JSON: @@ -313,7 +266,7 @@ def pack_vmesh(source: str | Path, destination: str | Path, *, overwrite: bool = with tempfile.TemporaryDirectory(prefix=f".{destination.name}-", dir=destination.parent) as directory: temporary = Path(directory) / "stream" with temporary.open("xb") as output: - output.write(_BOOTSTRAP) + output.write(_MAGIC) _write_record(output, 0, 0, 0, manifest) for record in descriptor["files"]: digest, offset = hashlib.sha256(), 0 diff --git a/open4d/codec/tests/editor_numerics.cs b/open4d/codec/tests/editor_numerics.cs new file mode 100644 index 00000000..47a6a85f --- /dev/null +++ b/open4d/codec/tests/editor_numerics.cs @@ -0,0 +1,140 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Numerics; +using TVMEditor.Editing.AffinityCalculation; +using TVMEditor.Editing.CenterDeformation; +using TVMEditor.Editing.SurfaceDeformation; +using TVMEditor.Structures; + +sealed class TestAffinity : IAffinityCalculation +{ + float[,] affinity; + public TestAffinity(float[,] value) { affinity = value; } + public float[,] CalculateCentersAffinity(Vector3[][] centers) { return affinity; } + public float[,] GetCentersAffinity() { return affinity; } + public void SetCentersAffinity(float[,] value) { affinity = value; } +} + +static class EditorNumerics +{ + static readonly Vector3[] Vertices = { + new Vector3(1000f, 1200f, 900f), new Vector3(1010f, 1201f, 901f), + new Vector3(1005f, 1210f, 905f) + }; + static readonly Face[] Faces = {new Face {V1 = 0, V2 = 1, V3 = 2}}; + + static void Require(bool condition, string message) + { + if (!condition) throw new Exception(message); + } + + static Vector3[] ManyCenters() + { + return Enumerable.Range(0, 40).Select(i => new Vector3( + 1000f + 100f * (i % 5), 1200f + 100f * ((i / 5) % 4), 900f + 100f * (i / 20) + )).ToArray(); + } + + static void Deform(Vector3[] centers, bool denseAffinity, Vector3 translation) + { + var affinity = new float[centers.Length, centers.Length]; + if (denseAffinity) + for (int i = 0; i < centers.Length; i++) + for (int j = 0; j < centers.Length; j++) affinity[i, j] = 1f; + var deformation = new CustomSurfaceDeformation(new TestAffinity(affinity)); + var transforms = centers.Select(c => DualQuaternion.Translation(translation)).ToArray(); + var moved = centers.Select(c => c + translation).ToArray(); + var result = deformation.DeformSurface(Vertices, Faces, centers, moved, 0, transforms); + Require(result.Vertices.Length == Vertices.Length && result.Faces.Length == Faces.Length, + "Deformation changed the mesh's element counts"); + for (int i = 0; i < Vertices.Length; i++) + { + var actual = result.Vertices[i]; + Require(float.IsFinite(actual.X) && float.IsFinite(actual.Y) && float.IsFinite(actual.Z), + "Surface deformation produced nonfinite coordinates"); + Require(Vector3.Distance(actual, Vertices[i] + translation) < 0.01f, + "Blended identical transformations failed to preserve the expected translation"); + } + Require(result.Faces[0].V1 == 0 && result.Faces[0].V2 == 1 && result.Faces[0].V3 == 2, + "Surface deformation changed connectivity"); + } + + static void EmptyCenters() + { + try { Deform(Array.Empty(), false, Vector3.Zero); } + catch (ArgumentException) { return; } + catch (AggregateException error) + { + if (error.Flatten().InnerExceptions.All(e => e is ArgumentException)) return; + throw; + } + throw new Exception("Empty deformation centers were accepted"); + } + + static void FewCenters() + { + Deform(new[] {Vertices[0]}, false, new Vector3(1f, 2f, 3f)); + Deform(new[] {Vertices[0], Vertices[1]}, false, new Vector3(1f, 2f, 3f)); + } + + static void Effectors(bool partial) + { + var centers = ManyCenters(); + var affinity = new float[centers.Length, centers.Length]; + for (int i = 0; i < centers.Length; i++) affinity[i, i] = 1f; + var deformation = new AffinityCenterDeformation(new TestAffinity(affinity)); + var indices = partial ? new[] {0, 2} : Array.Empty(); + var transforms = new DualQuaternion[centers.Length]; + var first = new Vector3(1f, 2f, 3f); + var second = new Vector3(-2f, 1f, -4f); + if (partial) + { + transforms[0] = DualQuaternion.Translation(first); + transforms[2] = DualQuaternion.Translation(second); + } + var moved = indices.Select(i => transforms[i].Transform(centers[i])).ToArray(); + var result = deformation.DeformCenters(centers, indices, moved, ref transforms); + for (int i = 0; i < centers.Length; i++) + { + var offset = partial && i == 0 ? first : partial && i == 2 ? second : Vector3.Zero; + Require(Vector3.Distance(result[i], centers[i] + offset) < 0.01f, + "Centers without effector influence must stay unchanged"); + Require(float.IsFinite(transforms[i].Norm()) && Math.Abs(transforms[i].Norm() - 1f) < 0.00001f, + "Center deformation produced a nonfinite or invalid unit transformation"); + } + } + + static void InvalidAffinity() + { + foreach (var invalid in new[] {-1f, float.NaN, float.PositiveInfinity}) + { + var affinity = new float[,] {{invalid}}; + var deformation = new AffinityCenterDeformation(new TestAffinity(affinity)); + var transformations = new[] {DualQuaternion.Identity()}; + try + { + deformation.DeformCenters(new[] {Vertices[0]}, new[] {0}, + new[] {Vertices[0]}, ref transformations); + } + catch (ArgumentException) { continue; } + throw new Exception("Invalid total affinity was accepted"); + } + } + + public static int Main(string[] arguments) + { + var cases = new Dictionary { + {"zero_affinity", () => Deform(ManyCenters(), false, Vector3.Zero)}, + {"few_centers", FewCenters}, + {"duplicate_centers", () => Deform(new[] {Vertices[0], Vertices[0], Vertices[1], Vertices[1]}, false, Vector3.Zero)}, + {"empty_centers", EmptyCenters}, + {"finite_deformation", () => Deform(ManyCenters(), true, new Vector3(3f, -4f, 2f))}, + {"partial_effectors", () => Effectors(true)}, {"empty_effectors", () => Effectors(false)}, + {"invalid_affinity", InvalidAffinity} + }; + cases[arguments[0]](); + Console.WriteLine(arguments[0] + ": passed"); + return 0; + } +} diff --git a/open4d/codec/tests/test_artifact_validation.py b/open4d/codec/tests/test_artifact_validation.py index 6efb0f7f..bc461188 100644 --- a/open4d/codec/tests/test_artifact_validation.py +++ b/open4d/codec/tests/test_artifact_validation.py @@ -14,6 +14,7 @@ from open4d.codec import CodecError from open4d.codec._npz import NumPyZipCodec from open4d.codec import _klt, _n4mc, _qndf, _temporal, _vmesh +from open4d.codec._vmesh_format import pack_vmesh, unpack_vmesh def sequence(): @@ -23,9 +24,25 @@ def sequence(): ])) +def research_artifact(tmp_path, codec, payload, *, normalization=None): + native = tmp_path / "native" + native.mkdir() + metadata = {"version": 1, "codec": codec, "native": {"profile": f"{codec}/1"}, + "frames": [{"frame_index": 0, "timestamp": 0}]} + if codec == "klt": + metadata["normalization"] = normalization or {"center": [0, 0, 0], "scale": 1} + (native / "decoder_context.pt").write_bytes(payload) + (native / "000000_quantized_indices.zst").write_bytes(b"indices") + (native / "000000_quantized_metadata.npz").write_bytes(b"quantizer") + else: + (native / "frame_000000.pt").write_bytes(payload) + (native / "metadata.json").write_text(json.dumps(metadata)) + return pack_vmesh(native, tmp_path / "take.vmesh") + + def test_reference_encode_does_not_replace_a_file_created_during_encoding(tmp_path, monkeypatch): codec = NumPyZipCodec() - destination = tmp_path / "take.o4d" + destination = tmp_path / "take.vmesh" original = codec.pack def pack(payload): @@ -39,8 +56,8 @@ def pack(payload): def test_vmesh_preserves_neighbor_temporary_file(tmp_path, monkeypatch): - destination = tmp_path / "take.v4d" - neighbor = tmp_path / ".take.v4d.tmp" + destination = tmp_path / "take.vmesh" + neighbor = tmp_path / ".take.vmesh.tmp" neighbor.write_bytes(b"unrelated user file") def run(command, label): @@ -53,13 +70,7 @@ def run(command, label): def test_klt_closes_decoded_provider_if_output_count_is_wrong(tmp_path, monkeypatch): - artifact = tmp_path / "take.k4d" - with ZipFile(artifact, "w") as archive: - archive.writestr("manifest.json", json.dumps({ - "schema": "open4d.klt-sequence/v1", "codec": "klt", - "normalization": {"center": [0, 0, 0], "scale": 1}, - "frames": [{"frame_index": 0, "timestamp": 0}], - })) + artifact = research_artifact(tmp_path, "klt", b"context") decoded = sequence() monkeypatch.setattr(_klt, "_backend", lambda: SimpleNamespace(decode_compressed=lambda *args: None)) monkeypatch.setattr(_klt, "open_sequence", lambda *args: decoded) @@ -71,14 +82,16 @@ def test_klt_closes_decoded_provider_if_output_count_is_wrong(tmp_path, monkeypa def test_temporal_delta_rejects_broadcasted_displacements(tmp_path): from open4d.codec._temporal import TEMPORAL_DELTA_CODEC - artifact = tmp_path / "broken.td4d" - manifest = {"schema": "open4d.temporal-delta-sequence/v1", "codec": "temporal-delta", + native = tmp_path / "native" + native.mkdir() + manifest = {"version": 1, "codec": "temporal-delta", "native": {"profile": "temporal-delta/1"}, "quantization_bits": 16, "frames": [{"frame_index": i, "timestamp": i / 30} for i in range(2)]} - with artifact.open("wb") as stream: - np.savez(stream, manifest=np.frombuffer(json.dumps(manifest).encode(), dtype=np.uint8), - reference=sequence()[0].geometry.positions, triangles=np.array([[0, 1, 2]], dtype=np.uint32), - displacement=np.zeros((2, 1, 3), dtype=np.int16), scale=1.0) + np.savez(native / "sequence.npz", reference=sequence()[0].geometry.positions, + triangles=np.array([[0, 1, 2]], dtype=np.uint32), + displacement=np.zeros((2, 1, 3), dtype=np.int16), scale=1.0) + (native / "metadata.json").write_text(json.dumps(manifest)) + artifact = pack_vmesh(native, tmp_path / "broken.vmesh") with pytest.raises(CodecError, match="shape|displacement"): TEMPORAL_DELTA_CODEC.decode(artifact) @@ -96,24 +109,23 @@ def test_codec_detection_returns_false_for_malformed_manifest(tmp_path, codec, m @pytest.mark.parametrize("frame", [{"frame_index": -1, "timestamp": 0}, {"frame_index": 0, "timestamp": float("nan")}, {"frame_index": 0, "timestamp": 0, "metadata": []}]) -def test_reference_decode_rejects_invalid_frame_metadata_without_reading_arrays(tmp_path, frame): - artifact = tmp_path / "invalid.o4d" - with ZipFile(artifact, "w") as archive: - archive.writestr("manifest.json", json.dumps({ - "schema": "open4d.numpy-zip/v1", "codec": "npz", "frames": [frame], - })) +def test_reference_decode_rejects_invalid_frame_metadata_without_reading_arrays(tmp_path, frame, monkeypatch): + from open4d.codec import _npz + from open4d.codec.tests.test_vmesh_format import records, rewrite + artifact = NumPyZipCodec().encode(sequence(), tmp_path / "invalid.vmesh") + items = records(artifact) + manifest = json.loads(items[0][3]) + manifest["sequence"]["frames"][0] = frame + items[0] = 0, 0, 0, json.dumps(manifest).encode() + rewrite(artifact, items) + monkeypatch.setattr(_npz, "_read_array", lambda *args: pytest.fail("invalid metadata read arrays")) with pytest.raises(CodecError): NumPyZipCodec().decode(artifact) def test_klt_rejects_broadcast_normalization_before_decoding(tmp_path, monkeypatch): - artifact = tmp_path / "invalid.k4d" - with ZipFile(artifact, "w") as archive: - archive.writestr("manifest.json", json.dumps({ - "schema": "open4d.klt-sequence/v1", "codec": "klt", - "frames": [{"frame_index": 0, "timestamp": 0}], - "normalization": {"center": 0, "scale": 1}, - })) + artifact = research_artifact(tmp_path, "klt", b"context", + normalization={"center": 0, "scale": 1}) monkeypatch.setattr(_klt, "open_sequence", lambda *args: sequence()[:1]) monkeypatch.setattr(_klt, "_backend", lambda: SimpleNamespace(decode_compressed=lambda *args: None)) with pytest.raises(CodecError, match="normalization"): @@ -135,7 +147,8 @@ def test_vmesh_rejects_invalid_position_normalization(tmp_path, monkeypatch, bou monkeypatch.setattr(_vmesh, "_run", lambda *args: None) monkeypatch.setattr(_vmesh, "open_sequence", lambda *args, **kwargs: sequence()[:1]) with pytest.raises(CodecError, match="position"): - _vmesh.VDMC_CODEC.decode(artifact) + from open4d.codec import migrate_legacy + migrate_legacy(artifact, tmp_path / "converted.vmesh") @pytest.mark.torch @@ -161,13 +174,15 @@ def capture(*args, **kwargs): monkeypatch.setattr(torch.ao.quantization, "quantize_dynamic", capture) codec = _qndf.QNDF_INT8_CODEC - path = codec.encode(sequence(), tmp_path / "take.qi4d", coarse_size=3, + path = codec.encode(sequence(), tmp_path / "take.vmesh", coarse_size=3, num_subdiv=0, pe_dim=2, hidden_dim=4, num_layers=3, epochs=1, batch_size=16, input_scale=input_scale, output_scale=output_scale, device="cpu") - with ZipFile(path) as archive: - assert not any(name.endswith("/model.pt") for name in archive.namelist()) - context = torch.load(BytesIO(archive.read("frames/000000/context.pt")), weights_only=True) + native = tmp_path / "unpacked" + unpack_vmesh(path, native) + assert not any(name.name == "model.pt" for name in native.iterdir()) + context = torch.load(native / "frame_000000.pt", weights_only=True) + assert "schema" not in context coarse = context["coarse_vertices"] encoded, normalized, _, _ = _qndf._inputs(torch, models, coarse, 2, input_scale) graph = models.MeshDataset(encoded, normalized, context["coarse_faces"], torch.zeros_like(coarse), progress=False) @@ -187,7 +202,9 @@ def test_qndf_int8_rejects_executable_artifact_version(tmp_path, monkeypatch): "frames": [{"frame_index": 0, "timestamp": 0}], })) with pytest.raises(CodecError, match="executable|re-encode"): - _qndf.QNDF_INT8_CODEC.decode(path) + destination = tmp_path / "legacy" + destination.mkdir() + _qndf._extract_legacy_qndf(path, destination, int8=True) @pytest.mark.torch @@ -205,7 +222,7 @@ def test_qndf_rejects_malformed_frame_context(tmp_path, monkeypatch, field): monkeypatch.setattr(_qndf, "_backend", lambda: (torch, models, None)) coarse = torch.tensor([[0., 0, 0], [1., 0, 0], [0., 1, 0]]) context = { - "schema": "open4d.qndf/v1", "coarse_vertices": coarse, + "version": 1, "coarse_vertices": coarse, "coarse_faces": torch.tensor([[0, 1, 2]]), "input_mean": torch.zeros((1, 3)), "input_std": torch.ones((1, 3)), "input_scale": 1., "output_scale": 1., "pe_dim": 2, "hidden_dim": 4, "num_layers": 1, @@ -227,13 +244,7 @@ def test_qndf_rejects_malformed_frame_context(tmp_path, monkeypatch, field): context["normalization"][field.split(".")[1]] = bad_values[field] else: context[field.split(".")[0]] = bad_values[field] - path = tmp_path / "invalid.q4d" - with ZipFile(path, "w") as archive: - archive.writestr("manifest.json", json.dumps({ - "schema": "open4d.qndf-sequence/v1", "codec": "qndf", - "frames": [{"frame_index": 0, "timestamp": 0}], - })) - archive.writestr("frames/000000/context.pt", _qndf._torch_bytes(torch, context)) + path = research_artifact(tmp_path, "qndf", _qndf._torch_bytes(torch, context)) with pytest.raises(CodecError, match="context"): _qndf.QNDF_CODEC.decode(path, device="cpu") @@ -248,4 +259,4 @@ def unexpected_backend(): monkeypatch.setattr(_qndf, "_backend", unexpected_backend) with pytest.raises(ValueError): - _qndf.QNDF_CODEC.encode(sequence(), tmp_path / "invalid.q4d", **options) + _qndf.QNDF_CODEC.encode(sequence(), tmp_path / "invalid.vmesh", **options) diff --git a/open4d/codec/tests/test_codec.py b/open4d/codec/tests/test_codec.py index 01ebb9f7..357120af 100644 --- a/open4d/codec/tests/test_codec.py +++ b/open4d/codec/tests/test_codec.py @@ -73,7 +73,7 @@ def sequence() -> Sequence: def test_numpy_zip_round_trip_is_lazy_and_preserves_geometry(tmp_path, monkeypatch): source = sequence() - artifact = encode_sequence(source, tmp_path / "take.o4d", codec=REFERENCE) + artifact = encode_sequence(source, tmp_path / "take.vmesh", codec=REFERENCE) calls = [] import open4d.codec._npz as implementation @@ -107,7 +107,7 @@ def recording_read(*args): def test_numpy_zip_preserves_reversed_view_timing_policy(tmp_path): source = sequence()[::-1] - artifact = encode_sequence(source, tmp_path / "reversed.o4d", codec=REFERENCE) + artifact = encode_sequence(source, tmp_path / "reversed.vmesh", codec=REFERENCE) decoded = decode_sequence(artifact, codec=REFERENCE) @@ -132,25 +132,24 @@ def test_n4mc_component_filter_is_disabled_by_default(): def test_encode_refuses_to_overwrite_and_decode_rejects_corruption(tmp_path): - artifact = encode_sequence(sequence(), tmp_path / "take.o4d", codec=REFERENCE) + artifact = encode_sequence(sequence(), tmp_path / "take.vmesh", codec=REFERENCE) with pytest.raises(FileExistsError): encode_sequence(sequence(), artifact, codec=REFERENCE) - broken = tmp_path / "broken.o4d" + broken = tmp_path / "broken.vmesh" broken.write_bytes(b"not a zip") - with pytest.raises(CodecError, match="invalid Open4D artifact"): + with pytest.raises(CodecError, match="VMESH"): decode_sequence(broken, codec=REFERENCE) -@pytest.mark.parametrize(("suffix", "codec"), ( - (".o4d", REFERENCE), (".d4d", DRACO_CODEC), (".v4d", None), -)) -def test_non_object_codec_manifests_are_codec_errors(tmp_path, suffix, codec): - artifact = tmp_path / f"invalid{suffix}" - with ZipFile(artifact, "w") as archive: - archive.writestr("manifest.json", "[]") - +@pytest.mark.parametrize("codec", (REFERENCE, DRACO_CODEC, None)) +def test_non_object_codec_manifests_are_codec_errors(tmp_path, codec): + from open4d.codec import _vmesh_format + artifact = tmp_path / "invalid.vmesh" + with artifact.open("wb") as stream: + stream.write(_vmesh_format._MAGIC) + _vmesh_format._write_record(stream, 0, 0, 0, b"[]") options = {} if codec is None else {"codec": codec} - with pytest.raises(CodecError, match="manifest|no known codec"): + with pytest.raises(CodecError, match="schema"): decode_sequence(artifact, **options) @@ -158,7 +157,7 @@ def test_encode_failure_removes_partial_artifact(tmp_path): bad = Sequence( MemoryFrameProvider(tuple(sequence()), metadata={"bad": object()}) ) - destination = tmp_path / "bad.o4d" + destination = tmp_path / "bad.vmesh" with pytest.raises(CodecError, match="not serializable"): encode_sequence(bad, destination, codec=REFERENCE) @@ -179,7 +178,7 @@ def test_codec_import_does_not_load_optional_dependencies(): @pytest.mark.parametrize("codec", REFERENCE_CODECS) def test_reference_codecs_remain_usable_privately(tmp_path, codec): source = sequence() - artifact = encode_sequence(source, tmp_path / f"{codec}.o4d", codec=codec) + artifact = encode_sequence(source, tmp_path / f"{codec}.vmesh", codec=codec) decoded = decode_sequence(artifact, codec=codec) assert len(decoded) == len(source) @@ -209,7 +208,7 @@ def test_public_registry_contains_research_codecs_only(): @pytest.mark.parametrize("codec,suffix", ( - (TEMPORAL_DELTA_CODEC, ".td4d"), (TEMPORAL_PCA_CODEC, ".tp4d"), + (TEMPORAL_DELTA_CODEC, ".vmesh"), (TEMPORAL_PCA_CODEC, ".vmesh"), )) def test_temporal_codecs_fresh_decode_without_processes(tmp_path, codec, suffix): frames = [Frame( @@ -246,7 +245,7 @@ def test_encode_accepts_a_supported_path_without_codec_specific_io(tmp_path): "v 0 0 0\nv 1 0 0\nv 0 1 0\nf 1 2 3\n", encoding="ascii" ) - artifact = encode_sequence(source, tmp_path / "frame.o4d", fps=24, codec=REFERENCE) + artifact = encode_sequence(source, tmp_path / "frame.vmesh", fps=24, codec=REFERENCE) decoded = decode_sequence(artifact, codec=REFERENCE) assert len(decoded) == 1 @@ -256,13 +255,14 @@ def test_encode_accepts_a_supported_path_without_codec_specific_io(tmp_path): def test_path_reader_options_are_rejected_for_an_open_sequence(tmp_path): with pytest.raises(TypeError, match="apply only to path inputs"): - encode_sequence(sequence(), tmp_path / "frame.o4d", fps=24, codec=REFERENCE) + encode_sequence(sequence(), tmp_path / "frame.vmesh", fps=24, codec=REFERENCE) -def test_caller_supplied_codec_is_used_without_a_registry(tmp_path): +def test_caller_supplied_codec_is_used_without_a_registry(tmp_path, monkeypatch): + import open4d.codec._api as implementation class Codec: id = "memory-test" - suffixes = (".test",) + suffixes = (".vmesh",) def encode(self, value, destination, **options): self.value = value @@ -275,7 +275,8 @@ def decode(self, source, **options): return sequence() codec = Codec() - destination = tmp_path / "sequence.test" + destination = tmp_path / "sequence.vmesh" + monkeypatch.setattr(implementation, "probe_codec", lambda path: codec.id) assert encode_sequence(sequence(), destination, codec=codec, level=2) == destination assert codec.options == {"level": 2} assert len(decode_sequence(destination, codec=codec, verify=True)) == 2 @@ -289,7 +290,7 @@ def test_registered_codec_can_be_selected_by_name(tmp_path, monkeypatch): class RegisteredCodec: id = "registered-test" - suffixes = (".registered",) + suffixes = (".vmesh",) def encode(self, value, destination, **options): return destination @@ -300,7 +301,8 @@ def decode(self, source, **options): codec = RegisteredCodec() register_codec(codec) - destination = tmp_path / "sequence.registered" + destination = tmp_path / "sequence.vmesh" + monkeypatch.setattr(implementation, "probe_codec", lambda path: codec.id) assert encode_sequence(sequence(), destination, codec=codec.id) == destination assert len(decode_sequence(destination)) == 2 with pytest.raises(ValueError, match="already registered"): @@ -315,7 +317,7 @@ def test_missing_research_source_has_an_actionable_error(tmp_path, monkeypatch): research_module("klt.klt") -@pytest.mark.parametrize("identifier,suffix", [("native-test", ".v4d"), ("vdmc", ".vmesh"), ("faster_vdmc", ".vmesh")]) +@pytest.mark.parametrize("identifier,suffix", [("vdmc", ".vmesh"), ("faster_vdmc", ".vmesh")]) def test_vmesh_uses_one_native_call_per_sequence_direction(tmp_path, monkeypatch, identifier, suffix): import open4d.codec._vmesh as implementation @@ -439,9 +441,11 @@ def prepare(sequence, destination, *, resolution): def compress(args, *, verify_decode): calls.append(("encode", args.num_frames, verify_decode)) - Path(args.output_path).mkdir() - with ZipFile(Path(args.output_path) / "compressed_archive.zip", "w") as archive: - archive.writestr("decoder_context.pt", b"saved-klt-context") + native = Path(args.output_path) / "compressed" + native.mkdir(parents=True) + (native / "decoder_context.pt").write_bytes(b"saved-klt-context") + (native / "000000_quantized_indices.zst").write_bytes(b"indices") + (native / "000000_quantized_metadata.npz").write_bytes(b"quantizer") def decompress(source, destination, device): calls.append(("decode", (source / "decoder_context.pt").read_bytes(), device)) @@ -458,7 +462,7 @@ def decompress(source, destination, device): monkeypatch.setattr(implementation, "_backend", lambda: SimpleNamespace( run_compression=compress, decode_compressed=decompress, )) - artifact = encode_sequence(source, tmp_path / "take.k4d", codec="klt") + artifact = encode_sequence(source, tmp_path / "take.vmesh", codec="klt") decoded = decode_sequence(artifact, device=requested) assert calls == [ @@ -473,7 +477,7 @@ def decompress(source, destination, device): def test_encode_rejects_text_overwrite_flag_without_changing_output(tmp_path): from open4d.codec._npz import NumPyZipCodec - output = tmp_path / "existing.o4d" + output = tmp_path / "existing.vmesh" output.write_bytes(b"original") with pytest.raises(TypeError, match="overwrite"): encode_sequence(sequence(), output, codec=NumPyZipCodec(), overwrite="false") diff --git a/open4d/codec/tests/test_draco.py b/open4d/codec/tests/test_draco.py index 69b4fc28..01a55fce 100644 --- a/open4d/codec/tests/test_draco.py +++ b/open4d/codec/tests/test_draco.py @@ -16,7 +16,7 @@ def _round_trip(mesh: TriangleMesh, tmp_path) -> TriangleMesh: source = Sequence(MemoryFrameProvider([Frame(7, 1.25, mesh)])) decoded = DRACO_CODEC.decode( - DRACO_CODEC.encode(source, tmp_path / "attributes.d4d") + DRACO_CODEC.encode(source, tmp_path / "attributes.vmesh") ) assert decoded[0].frame_index == 7 assert decoded[0].timestamp == 1.25 @@ -93,7 +93,7 @@ def test_draco_recomputes_declarations_after_uv_corner_splitting(tmp_path): )) decoded = DRACO_CODEC.decode(DRACO_CODEC.encode( - source, tmp_path / "mixed-layout.d4d" + source, tmp_path / "mixed-layout.vmesh" )) assert [len(frame.geometry.positions) for frame in decoded] == [4, 6] diff --git a/open4d/codec/tests/test_editor_numerics.py b/open4d/codec/tests/test_editor_numerics.py new file mode 100644 index 00000000..d94d5971 --- /dev/null +++ b/open4d/codec/tests/test_editor_numerics.py @@ -0,0 +1,54 @@ +import os +from pathlib import Path +import re +import shutil +import subprocess + +import pytest + +pytestmark = pytest.mark.cpu + + +@pytest.fixture(scope="module", params=("tvmc", "tsmc")) +def editor_program(request, tmp_path_factory): + dotnet = os.environ.get("OPEN4D_TEST_DOTNET") or shutil.which("dotnet") + if not dotnet: + pytest.skip("a .NET SDK is required for native editor regressions") + root = Path(os.environ.get( + "OPEN4D_EDITOR_TEST_SOURCE_ROOT", Path(__file__).resolve().parents[2] / "codecs" + )) + source = root / request.param / "tvm-editing/TVMEditor" + if not (source / "TVMEditor.csproj").is_file(): + pytest.skip(f"{request.param} editor sources are unavailable") + sdk = subprocess.check_output([dotnet, "--list-sdks"], text=True, timeout=15) + installed = [int(value) for value in re.findall(r"^(\d+)\.", sdk, re.MULTILINE)] + if not installed or max(installed) < 3: + pytest.skip("a .NET SDK supporting netstandard2.1 is required") + work = tmp_path_factory.mktemp(f"editor-{request.param}") + shutil.copytree(source, work / "TVMEditor", ignore=shutil.ignore_patterns("bin", "obj")) + shutil.copyfile(Path(__file__).with_name("editor_numerics.cs"), work / "Program.cs") + (work / "EditorRegression.csproj").write_text(f""" +Exenet{max(installed)}.0 +false + + +""") + built = subprocess.run( + [dotnet, "build", str(work / "EditorRegression.csproj"), "--disable-build-servers", + "--nologo", "--verbosity", "quiet", "--output", str(work / "bin")], + text=True, capture_output=True, timeout=180, + ) + assert built.returncode == 0, built.stdout + built.stderr + return dotnet, work / "bin/EditorRegression.dll" + + +@pytest.mark.parametrize("case", ( + "zero_affinity", "few_centers", "duplicate_centers", "empty_centers", "finite_deformation", + "partial_effectors", "empty_effectors", "invalid_affinity", +)) +def test_native_editor_numeric_contracts(editor_program, case): + dotnet, program = editor_program + result = subprocess.run([dotnet, str(program), case], text=True, + capture_output=True, timeout=10) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == f"{case}: passed" diff --git a/open4d/codec/tests/test_klt_quantization.py b/open4d/codec/tests/test_klt_quantization.py new file mode 100644 index 00000000..9596cbee --- /dev/null +++ b/open4d/codec/tests/test_klt_quantization.py @@ -0,0 +1,88 @@ +from types import SimpleNamespace + +import numpy as np +import pytest + +torch = pytest.importorskip("torch") +pytest.importorskip("zstd") +pytest.importorskip("trimesh") + +from open4d.codec._research import research_module + + +def test_compression_quantizes_the_variance_of_each_coefficient_column(tmp_path, monkeypatch): + backend = research_module("klt.klt") + source = tmp_path / "tsdf" + source.mkdir() + grid = np.indices((4, 4, 4), dtype=np.float32) + volume = grid[0] * .2 + grid[1] ** 2 * .03 + grid[2] ** 3 * .01 - .5 + np.savez_compressed(source / "000000.npz", sdf=volume[..., None]) + quantize = backend.quantize_coeffs + calls = [] + + def check_variances(coefficients, variances, **options): + torch.testing.assert_close(variances, coefficients.var(dim=0, unbiased=True)) + calls.append(variances) + return quantize(coefficients, variances, **options) + + monkeypatch.setattr(backend, "quantize_coeffs", check_variances) + backend.run_compression(SimpleNamespace( + input_path=str(source), output_path=str(tmp_path / "encoded"), + num_frames=1, training_frames=[0], block_size=2, num_components=2, + k_total=16, fps=30, + ), verify_decode=False) + assert len(calls) == 1 + + +def test_zero_variance_column_does_not_erase_a_changing_coefficient(): + backend = research_module("klt.klt") + coefficients = torch.tensor([[-2., 0.], [-1., 0.], [1., 0.], [2., 0.]]) + indices, centers, fixed = backend.quantize_coeffs( + coefficients, coefficients.var(dim=0, unbiased=True), K_total=16 + ) + reconstructed = backend.decompress_coeffs(indices, centers, fixed, H=2) + assert fixed == {1: 0.} + torch.testing.assert_close(reconstructed, coefficients) + + +def test_constant_coefficients_round_trip_without_quantization_bins(): + backend = research_module("klt.klt") + coefficients = torch.tensor([[2., -3.], [2., -3.]]) + indices, centers, fixed = backend.quantize_coeffs( + coefficients, coefficients.var(dim=0, unbiased=True), K_total=16 + ) + reconstructed = backend.decompress_coeffs(indices, centers, fixed, H=2) + assert fixed == {0: 2., 1: -3.} + torch.testing.assert_close(reconstructed, coefficients) + + +def test_single_block_compression_round_trip(tmp_path, monkeypatch): + backend = research_module("klt.klt") + source = tmp_path / "tsdf" + source.mkdir() + volume = np.linspace(-1., 1., 64, dtype=np.float32).reshape(4, 4, 4) + np.savez_compressed(source / "000000.npz", sdf=volume[..., None]) + quantize = backend.quantize_coeffs + + def check_single_block(coefficients, variances, **options): + assert torch.isfinite(variances).all() + torch.testing.assert_close(variances, torch.zeros_like(variances)) + return quantize(coefficients, variances, **options) + + monkeypatch.setattr(backend, "quantize_coeffs", check_single_block) + backend.run_compression(SimpleNamespace( + input_path=str(source), output_path=str(tmp_path / "encoded"), + num_frames=1, training_frames=[0], block_size=4, num_components=1, + k_total=16, fps=30, + ), verify_decode=False) + folder = tmp_path / "encoded/compressed" + decoder = torch.load(folder / "decoder_context.pt", weights_only=True) + indices, centers, fixed, shape = backend.load_quantized_coeffs( + str(folder / "000000_quantized"), "cpu" + ) + coefficients = backend.decompress_coeffs(indices, centers, fixed, H=1) + blocks = backend.reconstruct_blocks_torch( + coefficients, decoder["basis"], decoder["mean"], 1 + ) + reconstructed = backend.reconstruct_volume_from_blocks_torch(blocks, shape, 4) + torch.testing.assert_close(reconstructed, torch.from_numpy(volume)) diff --git a/open4d/codec/tests/test_n4mc_determinism.py b/open4d/codec/tests/test_n4mc_determinism.py new file mode 100644 index 00000000..1396ee7a --- /dev/null +++ b/open4d/codec/tests/test_n4mc_determinism.py @@ -0,0 +1,62 @@ +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from open4d.codec import encode_sequence +from open4d.codec.tests.test_research_cpu import moving_cube + + +@pytest.mark.gpu +def test_n4mc_cuda_decode_is_stable_in_fresh_processes(tmp_path): + torch = pytest.importorskip("torch") + pytest.importorskip("point_cloud_utils") + pytest.importorskip("skimage") + if not torch.cuda.is_available(): + pytest.skip("CUDA is unavailable") + artifact = encode_sequence( + moving_cube(), tmp_path / "cube.vmesh", codec="n4mc", device="cuda", + resolution=15, epochs=30, hidden_channels=(4, 8), latent_channels=4, + learning_rate=3e-3, + ) + program = """ +import hashlib, json, sys, torch +from open4d.codec import decode_sequence +from open4d.codec._research import research_module +torch.backends.cudnn.benchmark = True +torch.backends.cudnn.deterministic = False +before = (torch.backends.cudnn.enabled, torch.backends.cudnn.benchmark, + torch.backends.cudnn.deterministic, torch.backends.cudnn.allow_tf32) +model = research_module('n4mc.models').TSDFCompressionAutoencoder +original = model.decode_quantized_latent +def checked(self, *arguments): + assert torch.backends.cudnn.deterministic + assert not torch.backends.cudnn.benchmark + assert torch.backends.cudnn.enabled == before[0] + assert torch.backends.cudnn.allow_tf32 == before[3] + return original(self, *arguments) +model.decode_quantized_latent = checked +with decode_sequence(sys.argv[1], device='cuda') as decoded: + frames = [{'index': f.frame_index, 'timestamp': f.timestamp, + 'positions': hashlib.sha256(f.geometry.positions.tobytes()).hexdigest(), + 'triangles': hashlib.sha256(f.geometry.triangles.tobytes()).hexdigest()} + for f in decoded] +after = (torch.backends.cudnn.enabled, torch.backends.cudnn.benchmark, + torch.backends.cudnn.deterministic, torch.backends.cudnn.allow_tf32) +assert after == before +print(json.dumps(frames)) +""" + root = Path(__file__).resolve().parents[3] + environment = dict(os.environ) + environment["PYTHONPATH"] = os.pathsep.join(filter(None, ( + str(root), environment.get("PYTHONPATH"), + ))) + results = [json.loads(subprocess.check_output( + [sys.executable, "-c", program, str(artifact)], env=environment, text=True, + )) for _ in range(2)] + assert results[0] == results[1] + assert [frame["index"] for frame in results[0]] == [41, 73] + assert [frame["timestamp"] for frame in results[0]] == [1.25, 2.75] diff --git a/open4d/codec/tests/test_n4mc_vmesh.py b/open4d/codec/tests/test_n4mc_vmesh.py index a37f123f..768001af 100644 --- a/open4d/codec/tests/test_n4mc_vmesh.py +++ b/open4d/codec/tests/test_n4mc_vmesh.py @@ -1,12 +1,13 @@ """N4MC carries a shared model and independent latents, not temporal residuals.""" import json +from io import BytesIO from zipfile import ZipFile import numpy as np import pytest import open4d -from open4d.codec import CodecError, inspect_vmesh, pack_vmesh, unpack_vmesh +from open4d.codec import CodecError, inspect_vmesh, migrate_legacy, pack_vmesh, unpack_vmesh from open4d.codec import _n4mc pytestmark = pytest.mark.cpu @@ -18,8 +19,10 @@ def legacy_archive(path, *, omit=None, duplicate=False, normalization=None): metadata={"scene": "fixture"}, allow_nonmonotonic_timestamps=False, frames=[dict(frame_index=7, timestamp=.125, metadata={"key": "first"}), dict(frame_index=19, timestamp=.875, metadata={"key": "last"})]) - # These bytes only test carriage, never neural reconstruction. - payloads = {"checkpoint.pt": b"opaque shared model", "frame_000000.npz": b"opaque latent 0", + torch = pytest.importorskip("torch") + checkpoint = BytesIO() + torch.save({"model": {}, "model_config": {}, "schema": "open4d.n4mc/v1"}, checkpoint) + payloads = {"checkpoint.pt": checkpoint.getvalue(), "frame_000000.npz": b"opaque latent 0", "frame_000001.npz": b"opaque latent 1"} with ZipFile(path, "w") as archive: archive.writestr("manifest.json", json.dumps(manifest)) @@ -37,9 +40,9 @@ def test_n4d_migration_preserves_native_payloads_without_loading_models(tmp_path source = tmp_path / "legacy.n4d" manifest, payloads = legacy_archive(source) monkeypatch.setattr(_n4mc, "_backend", lambda: pytest.fail("repacking must not load a model")) - artifact = pack_vmesh(source, tmp_path / "motion.vmesh") + artifact = migrate_legacy(source, tmp_path / "motion.vmesh") source.unlink() - assert artifact.read_bytes()[0] == 0x60 + assert artifact.read_bytes()[:8] == b"VMESH\x00\x01\x00" info = inspect_vmesh(artifact) assert info["codec"] == "n4mc" assert info["representation"] == "neural_tsdf" @@ -50,7 +53,12 @@ def test_n4d_migration_preserves_native_payloads_without_loading_models(tmp_path recovered = unpack_vmesh(artifact, tmp_path / "recovered") assert sorted(p.name for p in recovered.iterdir()) == sorted(["metadata.json", *payloads]) for name, data in payloads.items(): - assert (recovered / name).read_bytes() == data + if name == "checkpoint.pt": + import torch + checkpoint = torch.load(recovered / name, weights_only=True) + assert checkpoint == {"model": {}, "model_config": {}} + else: + assert (recovered / name).read_bytes() == data repacked = pack_vmesh(recovered, tmp_path / "again.vmesh") assert artifact.read_bytes() == repacked.read_bytes() with pytest.raises(CodecError, match="contains n4mc"): @@ -64,7 +72,7 @@ def test_invalid_n4d_never_publishes_vmesh(tmp_path, options): legacy_archive(source, **options) destination = tmp_path / "result.vmesh" with pytest.raises(CodecError): - pack_vmesh(source, destination) + migrate_legacy(source, destination) assert not destination.exists() @@ -72,7 +80,7 @@ def test_n4mc_native_usdc_preserves_compressed_state_without_decoding(tmp_path, pytest.importorskip("pxr.Usd") source = tmp_path / "legacy.n4d" legacy_archive(source) - artifact = pack_vmesh(source, tmp_path / "motion.vmesh") + artifact = migrate_legacy(source, tmp_path / "motion.vmesh") monkeypatch.setattr(_n4mc, "_backend", lambda: pytest.fail("native USD must not decode N4MC")) with open4d.NativeSequence(artifact) as native: usd = open4d.save(native, tmp_path / "native.usdc") @@ -91,10 +99,10 @@ def test_n4mc_native_usdc_preserves_compressed_state_without_decoding(tmp_path, def test_corrupt_n4mc_vmesh_fails_before_loading_model(tmp_path, monkeypatch): source = tmp_path / "original.n4d" legacy_archive(source) - artifact = pack_vmesh(source, tmp_path / "original.vmesh") + artifact = migrate_legacy(source, tmp_path / "original.vmesh") data = artifact.read_bytes() - assert b"opaque shared model" in data - artifact.write_bytes(data.replace(b"opaque shared model", b"broken shared model")) + assert b"opaque latent 0" in data + artifact.write_bytes(data.replace(b"opaque latent 0", b"broken latent 0")) monkeypatch.setattr(_n4mc, "_backend", lambda: pytest.fail("corrupt carriage must not load a model")) with pytest.raises(CodecError, match="SHA-256"): open4d.load(artifact) @@ -117,6 +125,9 @@ def test_real_n4mc_mesh_usdc_vmesh_decode_matches_legacy_latents(tmp_path): latent_channels=4, learning_rate=3e-3) native = unpack_vmesh(artifact, tmp_path / "native") info = inspect_vmesh(artifact) + import torch + checkpoint = torch.load(native / "checkpoint.pt", weights_only=True) + assert "schema" not in checkpoint # Reconstruct the legacy packaging from exactly the same native model and # latents, proving the new carrier does not change neural reconstruction. legacy = tmp_path / "same-latents.n4d" @@ -125,9 +136,14 @@ def test_real_n4mc_mesh_usdc_vmesh_decode_matches_legacy_latents(tmp_path): manifest["schema"] = "open4d.n4mc-sequence/v1" with ZipFile(legacy, "w") as archive: archive.writestr("manifest.json", json.dumps(manifest)) - for record in info["files"][1:]: - archive.write(native / record["name"], record["name"]) - with open4d.load(artifact, options={"device": "cpu"}) as decoded, open4d.load(legacy, options={"device": "cpu"}) as original: + for record in info["files"]: + if record["name"] == "checkpoint.pt": + stream = BytesIO() + torch.save({**checkpoint, "schema": "open4d.n4mc/v1"}, stream) + archive.writestr("checkpoint.pt", stream.getvalue()) + else: + archive.write(native / record["name"], record["name"]) + with open4d.load(artifact, options={"device": "cpu"}) as decoded, open4d.load(migrate_legacy(legacy, tmp_path / "converted.vmesh"), options={"device": "cpu"}) as original: assert len(decoded) == len(original) == 2 assert decoded.timestamps == original.timestamps == (1.25, 2.75) assert decoded.metadata == original.metadata diff --git a/open4d/codec/tests/test_native_temporal.py b/open4d/codec/tests/test_native_temporal.py index 3d7076da..b8b5982d 100644 --- a/open4d/codec/tests/test_native_temporal.py +++ b/open4d/codec/tests/test_native_temporal.py @@ -143,6 +143,13 @@ def test_self_contained_usdc_restores_identical_native_bytes_without_runtime(tmp with open4d.import_native(source, codec=codec, timestamps=[1.125, 3.75], frame_indices=[5, 11], **options) as original: expected = original.path.read_bytes() usd = open4d.save(original, tmp_path / "take.usdc") + from pxr import Usd + stage = Usd.Stage.Open(str(usd)) + prim = stage.GetPrimAtPath("/VMESH") + assert prim.GetTypeName() == "VMESH" + assert all(not attr.GetName().startswith("open4d:") for attr in prim.GetAttributes()) + assert not stage.GetPrimAtPath("/Open4DNative") + del stage shutil.rmtree(source) with open4d.load(usd) as restored: assert restored.timestamps == (1.125, 3.75) @@ -185,12 +192,12 @@ def test_usd_rejects_tampered_payload_and_timeline(tmp_path): with open4d.import_native(source, codec="queen", **options) as native: usd = open4d.save(native, tmp_path / "take.usdc") stage = Usd.Stage.Open(str(usd)) - prim = stage.GetPrimAtPath("/Open4DNative") - prim.GetAttribute("open4d:frameIndex").Set(999, 0) + prim = stage.GetPrimAtPath("/VMESH") + prim.GetAttribute("vmesh:frameIndex").Set(999, 0) stage.GetRootLayer().Save() with pytest.raises(CodecError, match="timeline"): open4d.load(usd) - payload = prim.GetAttribute("open4d:payload:chunk000000") + payload = prim.GetAttribute("vmesh:payload:chunk000000") values = payload.Get() values[100] ^= 1 payload.Set(values) diff --git a/open4d/codec/tests/test_qndf_preprocess.py b/open4d/codec/tests/test_qndf_preprocess.py index 2b46858c..d6d326bb 100644 --- a/open4d/codec/tests/test_qndf_preprocess.py +++ b/open4d/codec/tests/test_qndf_preprocess.py @@ -72,4 +72,39 @@ def test_simplification_retains_every_disconnected_component(): assert len(component_counts) == 2 assert sum(item["allocated_coarse_faces"] for item in metadata["components"]) == 8 assert all(item["actual_coarse_faces"] > 0 for item in metadata["components"]) + assert all(item["simplification_target_faces"] == item["allocated_coarse_faces"] + for item in metadata["components"]) + assert metadata["actual_coarse_faces"] == 8 + assert metadata["coarse_face_budget_overshoot"] == 0 assert target.shape == low.shape + + +@pytest.mark.open3d +def test_simplification_retries_collapsing_real_component_and_reports_overshoot(): + o3d = pytest.importorskip("open3d") + from open4d.codecs.qndf.build_dataset_open3d import build_pair + + # C5/mitch frame 100, component 62 after the builder's normalization. + vertices = np.array([ + [0.3591478930135873, 0.08521303404473901, 0.04937343443180466], + [0.3676691964180612, 0.060651630114196595, 0.04761904843676592], + [0.36140354932215, 0.0571428581241191, 0.04987468757324431], + [0.35639101790775357, 0.059899750402037134, 0.04135338416877041], + [0, 0, 0], [1, 0, 0], [0, 1, 0], + ], dtype=np.float64) + faces = np.array([[0, 1, 2], [0, 2, 3], [0, 3, 1], [4, 5, 6]], dtype=np.int32) + + low, low_faces, target, metadata = build_pair(vertices, faces, coarse_size=2, num_subdiv=0) + mesh = o3d.geometry.TriangleMesh( + o3d.utility.Vector3dVector(low), o3d.utility.Vector3iVector(low_faces) + ) + assert len(mesh.cluster_connected_triangles()[1]) == 2 + component = metadata["components"][0] + assert component["original_faces"] == 3 + assert component["allocated_coarse_faces"] == 1 + assert component["simplification_target_faces"] == 2 + assert component["actual_coarse_faces"] == 2 + assert metadata["actual_coarse_faces"] == 3 + assert metadata["coarse_face_budget_overshoot"] == 1 + assert np.isfinite(low).all() and np.isfinite(target).all() + assert np.min(target[:component["training_vertices"], 2]) > 0.04 diff --git a/open4d/codec/tests/test_queen_training_exports.py b/open4d/codec/tests/test_queen_training_exports.py new file mode 100644 index 00000000..e91a8adb --- /dev/null +++ b/open4d/codec/tests/test_queen_training_exports.py @@ -0,0 +1,68 @@ +import ast +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +def export_function(): + tree = ast.parse(Path(__file__).parents[2].joinpath("reconstruction/queen/train.py").read_text()) + function = next(node for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "_save_frame_outputs") + namespace = {} + exec(compile(ast.Module(body=[function], type_ignores=[]), "train.py", "exec"), namespace) + return namespace[function.name] + + +class Gate: + training = True + + def eval(self): + self.training = False + + def train(self, value): + self.training = value + + +@pytest.mark.parametrize("frame_idx", [1, 2]) +def test_dense_and_compressed_exports_use_final_state(frame_idx): + gate = Gate() + state = SimpleNamespace(points=4681, gate_atts=gate) + saved = [] + + def save(iteration, **kwargs): + assert not gate.training + assert kwargs == {"save_point_cloud": True} + saved.append(("ply", state.points)) + + def save_compressed(iteration, parameters): + assert not gate.training + saved.append(("compressed", state.points)) + + scene = SimpleNamespace(save=save, save_compressed=save_compressed) + export_function()(scene, state, SimpleNamespace(log_ply=True, log_compressed=True), + frame_idx, 70, None) + expected = [("ply", 4681)] + ([("compressed", 4681)] if frame_idx > 1 else []) + assert saved == expected + assert gate.training + + +def test_failed_export_restores_gate_mode(): + gate = Gate() + + def fail(*args, **kwargs): + raise OSError("disk full") + + with pytest.raises(OSError, match="disk full"): + export_function()(SimpleNamespace(save=fail), SimpleNamespace(gate_atts=gate), + SimpleNamespace(log_ply=True, log_compressed=True), 1, 70, None) + assert gate.training + + +def test_camera_iteration_does_not_publish_canonical_state(): + tree = ast.parse(Path(__file__).parents[2].joinpath("reconstruction/queen/train.py").read_text()) + training = next(node for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "training") + canonical = [node for node in ast.walk(training) if isinstance(node, ast.Call) + and any(key.arg == "save_point_cloud" for key in node.keywords)] + assert canonical == [] diff --git a/open4d/codec/tests/test_real_dataset.py b/open4d/codec/tests/test_real_dataset.py index 260c24b4..3babe8bc 100644 --- a/open4d/codec/tests/test_real_dataset.py +++ b/open4d/codec/tests/test_real_dataset.py @@ -29,7 +29,7 @@ def test_rafa_obj_frames_encode_and_decode(tmp_path): source = open_sequence(dataset, fps=30) sample = source[:2] - artifact = NumPyZipCodec().encode(sample, tmp_path / "rafa.o4d") + artifact = NumPyZipCodec().encode(sample, tmp_path / "rafa.vmesh") decoded = NumPyZipCodec().decode(artifact) assert len(source) >= 2 @@ -52,7 +52,7 @@ def test_real_draco_round_trip_on_rafa(tmp_path): pytest.skip("Rafa_Approves_hd_4k is not available") source = open_sequence(dataset, fps=30)[:2] - artifact = encode_sequence(source, tmp_path / "rafa.d4d", codec=DRACO_CODEC) + artifact = encode_sequence(source, tmp_path / "rafa.vmesh", codec=DRACO_CODEC) decoded = DRACO_CODEC.decode(artifact) assert len(decoded) == len(source) diff --git a/open4d/codec/tests/test_reference_vmesh.py b/open4d/codec/tests/test_reference_vmesh.py new file mode 100644 index 00000000..2301a43a --- /dev/null +++ b/open4d/codec/tests/test_reference_vmesh.py @@ -0,0 +1,60 @@ +from io import BytesIO +import json + +import numpy as np +import pytest + +from open4d.codec import CodecError, decode_sequence, inspect_vmesh, unpack_vmesh +from open4d.codec._npz import NumPyZipCodec, REFERENCE_CODECS, _array_from_bytes +from open4d.codec._temporal import TEMPORAL_DELTA_CODEC, TEMPORAL_PCA_CODEC +from open4d.codec.tests.test_codec import sequence + + +@pytest.mark.parametrize("codec", REFERENCE_CODECS) +def test_reference_vmesh_carries_standard_npz_arrays_without_application_manifest(tmp_path, codec): + source = sequence() + artifact = codec.encode(source, tmp_path / "take.vmesh") + manifest = inspect_vmesh(artifact) + assert artifact.read_bytes().startswith(b"VMESH\x00\x01\x00") + assert manifest["codec"] == codec.id and "open4d." not in json.dumps(manifest) + assert [record["name"] for record in manifest["files"]] == ["frame_000000.npz", "frame_000001.npz"] + native = tmp_path / "native" + unpack_vmesh(artifact, native) + with np.load(native / "frame_000000.npz", allow_pickle=False) as arrays: + assert "manifest" not in arrays.files + if codec.rle: + assert arrays["positions"].dtype == np.uint8 + else: + np.testing.assert_array_equal(arrays["positions"], source[0].geometry.positions) + with decode_sequence(artifact) as decoded: + np.testing.assert_array_equal(decoded[0].geometry.positions, source[0].geometry.positions) + np.testing.assert_array_equal(decoded[0].geometry.attributes["labels"], source[0].geometry.attributes["labels"]) + + +@pytest.mark.parametrize("payload", ( + (2 ** 40).to_bytes(8, "little") + bytes([255, 1]), + (1).to_bytes(8, "little") + bytes([255, 1]), + (1).to_bytes(8, "little") + bytes([0, 1]), +)) +def test_rle_rejects_invalid_length_before_repeat_allocates(payload, monkeypatch): + monkeypatch.setattr(np, "repeat", lambda *args: pytest.fail("invalid RLE allocated decoded values")) + with pytest.raises(CodecError, match="RLE"): + NumPyZipCodec("rle", rle=True).unpack(payload) + + +def test_npy_shape_bomb_fails_before_numpy_load_allocates(monkeypatch): + stream = BytesIO() + np.lib.format.write_array_header_1_0(stream, { + "descr": " +Exe{target} +false + + +""") + built = subprocess.run( + [dotnet, "build", str(work / "TrackerRegression.csproj"), "--disable-build-servers", + "--nologo", "--verbosity", "quiet", "--output", str(work / "bin")], + text=True, capture_output=True, timeout=180, + ) + assert built.returncode == 0, built.stdout + built.stderr + return dotnet, work / "bin/TrackerRegression.dll" + + +@pytest.mark.parametrize("case", ( + "self_distance", "empty_weighted", "zero_weights", "empty_unweighted", + "nonfinite_kd", "finite_kd", +)) +def test_native_tracker_numeric_contracts(tracker_program, case): + dotnet, program = tracker_program + result = subprocess.run([dotnet, str(program), case], text=True, + capture_output=True, timeout=10) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == f"{case}: passed" diff --git a/open4d/codec/tests/test_v3c.py b/open4d/codec/tests/test_vmesh_format.py similarity index 70% rename from open4d/codec/tests/test_v3c.py rename to open4d/codec/tests/test_vmesh_format.py index 599a8013..24832326 100644 --- a/open4d/codec/tests/test_v3c.py +++ b/open4d/codec/tests/test_vmesh_format.py @@ -11,7 +11,7 @@ import open4d from open4d.codec import CodecError, decode_sequence, inspect_vmesh, pack_vmesh, unpack_vmesh -from open4d.codec import _api, _tracked, _v3c +from open4d.codec import _api, _tracked, _vmesh_format from open4d.io._mesh import write_obj pytestmark = pytest.mark.cpu @@ -42,18 +42,18 @@ def native_directory(root, codec, *, large=False): def records(path): with path.open("rb") as stream: - stream.seek(len(_v3c._BOOTSTRAP)) + stream.seek(len(_vmesh_format._MAGIC)) result = [] while stream.peek(1): - result.append(_v3c._read_record(stream)) + result.append(_vmesh_format._read_record(stream)) return result def rewrite(path, items): with path.open("wb") as stream: - stream.write(_v3c._BOOTSTRAP) + stream.write(_vmesh_format._MAGIC) for item in items: - _v3c._write_record(stream, *item) + _vmesh_format._write_record(stream, *item) @pytest.mark.parametrize("codec", ["tvmc", "tsmc"]) @@ -64,7 +64,7 @@ def test_native_bytes_and_irregular_timing_survive_single_file_carriage(tmp_path path = pack_vmesh(source, tmp_path / "take.vmesh") shutil.rmtree(source) assert path.is_file() - assert _v3c.probe_codec(path) == codec + assert _vmesh_format.probe_codec(path) == codec manifest = inspect_vmesh(path) assert manifest["sequence"] == metadata assert manifest["frame_count"] == 2 @@ -72,55 +72,43 @@ def test_native_bytes_and_irregular_timing_survive_single_file_carriage(tmp_path for record in manifest["files"]: assert hashlib.sha256(expected[record["name"]]).hexdigest() == record["sha256"] recovered = unpack_vmesh(path, tmp_path / "recovered") - assert {p.name: p.read_bytes() for p in recovered.iterdir()} == expected + assert json.loads((recovered / "metadata.json").read_text()) == metadata + assert {p.name: p.read_bytes() for p in recovered.iterdir() if p.name != "metadata.json"} == { + k: v for k, v in expected.items() if k != "metadata.json"} with pytest.raises(FileExistsError): unpack_vmesh(path, recovered) -def test_wire_is_v3c_atlas_sei_with_no_private_top_level_units(tmp_path): - source, _ = native_directory(tmp_path, "tvmc") +def test_wire_is_standalone_vmesh_with_direct_native_payloads(tmp_path): + source, metadata = native_directory(tmp_path, "tvmc") data = pack_vmesh(source, tmp_path / "take.vmesh").read_bytes() - # Independent framing walk: V3C four-byte lengths; VPS then AD; each AD - # has an atlas NAL sample stream containing user_data_unregistered SEI. - assert data[0] == 0x60 - pos, types, messages = 1, [], [] + assert data[:8] == b"VMESH\x00\x01\x00" + pos, messages = 8, [] while pos < len(data): size = int.from_bytes(data[pos:pos + 4], "big") unit = data[pos + 4:pos + 4 + size] assert len(unit) == size + assert unit[0] == 1 # record version + messages.append(unit[1]) pos += 4 + size - kind = int.from_bytes(unit[:4], "big") >> 27 - types.append(kind) - if kind == 0: - assert len(types) == 1 - assert unit[:4] == bytes(4) - assert b"O4D\x01" in unit - continue - assert unit[:5] == b"\x08\x00\x00\x00\x60" - assert int.from_bytes(unit[5:9], "big") == len(unit) - 9 - assert unit[9:12] == b"\x56\x01\x04" - index, length = 12, 0 - while unit[index] == 255: - length += 255 - index += 1 - length += unit[index] - payload = unit[index + 1:-1] - assert len(payload) == length and unit[-1] == 128 - assert payload[:16].hex() == "e23b8c4791354e3490ad421c6c57b63a" - assert payload[16] == 1 - messages.append(payload[17]) - assert pos == len(data) and types == [0] + [1] * (len(types) - 1) - assert messages == [0] + [1] * 6 + [2] - assert b"PK\x03\x04" not in data # TVMC fixture contains no ZIP data + assert pos == len(data) and messages == [0] + [1] * 5 + [2] + manifest = json.loads(records(tmp_path / "take.vmesh")[0][3]) + assert manifest["schema"] == "vmesh/1" + assert manifest["sequence"]["frames"] == metadata["frames"] + assert "schema" not in manifest["sequence"] + assert "metadata.json" not in [f["name"] for f in manifest["files"]] + assert b"open4d." not in data + assert b"O4D\x01" not in data + assert bytes.fromhex("e23b8c4791354e3490ad421c6c57b63a") not in data @pytest.mark.parametrize("length", [224, 225, 226, 480, 481, 1024 * 1024, 1024 * 1024 + 1]) -def test_sei_extended_lengths_and_chunk_boundaries(tmp_path, length): +def test_record_lengths_and_chunk_boundaries(tmp_path, length): source, _ = native_directory(tmp_path, "tvmc") payload = bytes(range(256)) * (length // 256) + bytes(range(length % 256)) (source / "reference.drc").write_bytes(payload) path = pack_vmesh(source, tmp_path / "take.vmesh") - chunks = [item for item in records(path) if item[0] == 1 and item[1] == 1] + chunks = [item for item in records(path) if item[0] == 1 and item[1] == 0] assert b"".join(item[3] for item in chunks) == payload assert all(len(item[3]) <= 1024 * 1024 for item in chunks) recovered = unpack_vmesh(path, tmp_path / "native") @@ -128,7 +116,7 @@ def test_sei_extended_lengths_and_chunk_boundaries(tmp_path, length): @pytest.mark.parametrize("damage", ["truncated", "no-end", "trailing", "hash", "order", "offset", - "bootstrap", "huge-unit", "wrong-nal", "wrong-end"]) + "bootstrap", "huge-unit", "wrong-record", "wrong-end"]) def test_malformed_stream_never_publishes_or_launches_native_decode(tmp_path, monkeypatch, damage): source, _ = native_directory(tmp_path, "tvmc") path = pack_vmesh(source, tmp_path / "take.vmesh") @@ -156,9 +144,9 @@ def test_malformed_stream_never_publishes_or_launches_native_decode(tmp_path, mo elif damage == "bootstrap": data[0] ^= 1 elif damage == "huge-unit": - data[len(_v3c._BOOTSTRAP):len(_v3c._BOOTSTRAP) + 4] = b"\xff" * 4 + data[len(_vmesh_format._MAGIC):len(_vmesh_format._MAGIC) + 4] = b"\xff" * 4 else: - data[len(_v3c._BOOTSTRAP) + 4 + 9] ^= 2 + data[len(_vmesh_format._MAGIC) + 4] ^= 2 path.write_bytes(data) monkeypatch.setattr(_tracked, "_run", lambda *args: pytest.fail("native worker called on corrupt input")) with pytest.raises(CodecError): @@ -192,11 +180,7 @@ def test_manifest_rejects_unsafe_or_inconsistent_native_descriptions(tmp_path, c elif change == "mode": manifest["dependency_mode"] = "independent-frames" else: - sequence = json.loads(items[1][3]) - sequence["frames"][1]["timestamp"] = float("nan") - payload = json.dumps(sequence).encode() - items[1] = (1, 0, 0, payload) - manifest["files"][0].update(size=len(payload), sha256=hashlib.sha256(payload).hexdigest()) + manifest["sequence"]["frames"][1]["timestamp"] = float("nan") blob = json.dumps(manifest).encode() items[0], items[-1] = (0, 0, 0, blob), (2, 0, 0, hashlib.sha256(blob).digest()) rewrite(path, items) @@ -211,19 +195,19 @@ def test_pack_preserves_existing_file_on_failure_and_requires_explicit_overwrite path.write_bytes(b"original") with pytest.raises(FileExistsError): pack_vmesh(source, path) - original_writer = _v3c._write_record + original_writer = _vmesh_format._write_record def failing_writer(stream, kind, file_id, offset, data): if kind == 1: raise OSError("disk full") original_writer(stream, kind, file_id, offset, data) - monkeypatch.setattr(_v3c, "_write_record", failing_writer) + monkeypatch.setattr(_vmesh_format, "_write_record", failing_writer) with pytest.raises(OSError, match="disk full"): pack_vmesh(source, path, overwrite=True) assert path.read_bytes() == b"original" assert not list(tmp_path.glob(".take.vmesh-*")) - monkeypatch.setattr(_v3c, "_write_record", original_writer) + monkeypatch.setattr(_vmesh_format, "_write_record", original_writer) pack_vmesh(source, path, overwrite=True) assert inspect_vmesh(path)["codec"] == "tsmc" @@ -340,3 +324,76 @@ def run(python, action, request): with open4d.load(reconstructed) as result: assert result.timestamps == timestamps np.testing.assert_allclose(result[1].geometry.positions, sequence[1].geometry.positions) + + +def test_retired_o4d_wrapper_is_rejected_before_native_decode(tmp_path): + path = tmp_path / 'old.vmesh' + path.write_bytes(b'\x60' + bytes(20) + bytes.fromhex('e23b8c4791354e3490ad421c6c57b63a') + b'O4D\x01') + with pytest.raises(CodecError, match='retired O4D/V3C'): + decode_sequence(path) + + +def test_nested_open4d_schema_is_not_a_vmesh_metadata_type(tmp_path): + source, _ = native_directory(tmp_path, 'tvmc') + metadata = json.loads((source / 'metadata.json').read_text()) + metadata['schema'] = 'open4d.some-custom-type/1' + (source / 'metadata.json').write_text(json.dumps(metadata)) + with pytest.raises(CodecError): + pack_vmesh(source, tmp_path / 'bad.vmesh') + + +def test_profile_expansion_is_bounded_before_allocating_payload_names(): + from open4d.codec._native_profiles import layout + with pytest.raises(CodecError, match='outside limits'): + layout('tvmc', 65536) + # Sixteen small JSON frame records could otherwise expand to 131k names. + profile = dict(profile='rerf/1', group_size=1, pca=False, + pca_channels=[7, 13], frames=[dict(id=i, quality=90, motion=False, + channels=[4096]) for i in range(16)]) + with pytest.raises(CodecError, match='file count outside limits'): + layout('rerf', 16, profile) + + +@pytest.mark.parametrize('data', [b'{"x":1e999}', b'{"x":NaN}', b'{"x":1,"x":2}']) +def test_json_rejects_overflow_and_ambiguous_values(data): + with pytest.raises(CodecError): + _vmesh_format._json(data) + + +@pytest.mark.parametrize('codec', ['vdmc', 'faster_vdmc']) +def test_vdmc_adapter_carries_actual_backend_bytes_and_restores_timing(tmp_path, monkeypatch, codec): + from open4d.codec import _vmesh + frames = [open4d.Frame(i + 7, t, open4d.TriangleMesh( + [[float(i), 0, 0], [i + 1., 0, 0], [float(i), 1, 0]], [[0, 1, 2]])) + for i, t in enumerate((.125, .875))] + sequence = open4d.Sequence(open4d.MemoryFrameProvider(frames)) + native_bytes = b'\x60native encoder-produced bitstream fixture' + config = tmp_path / 'decoder.cfg' + config.write_text('native decoder configuration') + normalized = [] + + def run(command, label): + options = dict(item[2:].split('=', 1) for item in command[1:] if item.startswith('--')) + stream = Path(options['compressed']) + if label.endswith('encoder'): + with open4d.load(Path(options['srcMesh']).parent) as input_sequence: + normalized.extend(f.geometry for f in input_sequence) + stream.write_bytes(native_bytes) + else: + assert stream.read_bytes() == native_bytes + assert Path(options['config']).read_text() == config.read_text() + for i, mesh in enumerate(normalized): + write_obj(Path(options['decMesh'] % i), mesh.positions, mesh.triangles) + + monkeypatch.setattr(_vmesh, '_run', run) + artifact = open4d.encode(sequence, tmp_path / 'take.vmesh', codec=codec, + encoder=sys.executable, decoder_config=config) + info = inspect_vmesh(artifact) + assert info['codec'] == codec + assert [record['name'] for record in info['files']] == ['sequence.vmesh', 'decoder.cfg'] + extracted = unpack_vmesh(artifact, tmp_path / 'native') + assert (extracted / 'sequence.vmesh').read_bytes() == native_bytes + with open4d.decode(artifact, decoder=sys.executable) as decoded: + assert decoded.timestamps == sequence.timestamps + for expected, actual in zip(sequence, decoded): + np.testing.assert_allclose(actual.geometry.positions, expected.geometry.positions, atol=1 / 4095) diff --git a/open4d/codec/tests/test_vmesh_only.py b/open4d/codec/tests/test_vmesh_only.py new file mode 100644 index 00000000..9ef42d94 --- /dev/null +++ b/open4d/codec/tests/test_vmesh_only.py @@ -0,0 +1,129 @@ +import pytest + +from open4d.codec import available_codecs, encode_sequence +from open4d.codec._api import _CODECS + + +def test_public_codecs_share_one_custom_output_extension(): + assert len(available_codecs()) == 12 + assert all(info.suffixes == (".vmesh",) for info in available_codecs()) + + +@pytest.mark.parametrize("identifier", sorted(_CODECS)) +def test_invalid_destination_fails_before_codec_backend(identifier, tmp_path): + from open4d.codec.tests.test_research_cpu import moving_cube + + destination = tmp_path / "retired.custom" + with moving_cube() as sequence: + with pytest.raises(ValueError, match=".vmesh"): + _CODECS[identifier].encode(sequence, destination) + with pytest.raises(ValueError, match=".vmesh"): + encode_sequence(sequence, destination, codec=identifier) + assert not destination.exists() + + +def test_pack_is_not_an_implicit_archive_converter(tmp_path): + from open4d.codec import CodecError, pack_vmesh + + source = tmp_path / "archive.n4d" + source.write_bytes(b"not a native directory") + destination = tmp_path / "output.vmesh" + with pytest.raises(CodecError): + pack_vmesh(source, destination) + assert not destination.exists() + + +@pytest.mark.parametrize("identifier", ("vega", "queen", "3dgstream", "rerf")) +def test_native_codec_rejects_a_retired_extension_before_loading(identifier, tmp_path): + from open4d.codec import CodecError + + source = tmp_path / "renamed.retired" + source.write_bytes(b"VMESH\x00\x01\x00") + assert not _CODECS[identifier].can_decode(source) + with pytest.raises(CodecError, match=".vmesh"): + _CODECS[identifier].decode(source) + + +@pytest.mark.parametrize("identifier", ("npz", "raw", "deflate", "bzip2", "lzma", "rle")) +def test_private_profile_can_be_selected_explicitly_without_registration(identifier, tmp_path): + from open4d.codec import decode_sequence + from open4d.codec._npz import REFERENCE_CODECS + from open4d.codec.tests.test_codec import sequence + + codec = next(item for item in REFERENCE_CODECS if item.id == identifier) + with sequence() as source: + artifact = codec.encode(source, tmp_path / "arrays.vmesh") + with decode_sequence(artifact, codec=identifier) as decoded: + assert len(decoded) == len(source) + assert decoded.timestamps == source.timestamps + + +@pytest.fixture +def installed_streamer_namespace(monkeypatch): + import sys + from pathlib import Path + + monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / "streamer")) + for name in tuple(sys.modules): + if name.startswith("open4d.streamer."): + monkeypatch.delitem(sys.modules, name) + monkeypatch.setitem(sys.modules, "open4d.streamer", None) + + +def browser_ply(): + return (b"ply\nformat ascii 1.0\nelement vertex 3\nproperty float x\n" + b"property float y\nproperty float z\nelement face 1\n" + b"property list uchar int vertex_indices\nend_header\n" + b"0 0 0\n1 0 0\n0 1 0\n3 0 1 2\n") + + +def legacy_browser_clip(path): + import struct + + payload = browser_ply() + path.write_bytes(struct.pack("<8sIII", b"O4DSEQ\x00\x00", 1, 1, 3) + + b"ply" + struct.pack("; + +static class TrackerNumerics +{ + static void Require(bool condition, string message) + { + if (!condition) throw new Exception(message); + } + + static void Reject(Action action) + { + try { action(); } + catch (ArgumentException) { return; } + throw new Exception("Invalid input was accepted"); + } + + static void SelfDistance() + { + var points = new List(); + for (int i = 0; i < 22; i++) + points.Add(new Vector4(1000f + i * 0.125f, 1200f + i * 0.25f, 900f + i * 0.375f, 0f)); + var distance = new TransformDistance(points); + var rotation = Matrix.Build.DenseOfArray(new float[,] { + {0.9553365f, -0.2955202f, 0f}, {0.2955202f, 0.9553365f, 0f}, {0f, 0f, 1f} + }); + var translation = FloatVector.Build.DenseOfArray(new float[] {33f, -52f, 7f}); + Require(distance.GetTransformDistance(rotation, rotation, translation, translation) == 0f, + "A transform's distance to itself must be exactly zero at large coordinates"); + Require(distance.GetTransformDistance(rotation, rotation.Clone(), translation, translation.Clone()) == 0f, + "Value-equal cloned transforms must also have zero distance"); + var origin = new TransformDistance(new[] {Vector4.Zero}); + var identity = Matrix.Build.DenseIdentity(3); + var zero = FloatVector.Build.Dense(3); + var offset = FloatVector.Build.DenseOfArray(new float[] {3f, -4f, 0f}); + Require(origin.GetTransformDistance(identity, identity.Clone(), zero, offset) == 25f, + "Different translations must retain their squared transform distance"); + } + + static void EmptyWeighted() + { + Reject(() => ARAP.GetTransform(Array.Empty(), Array.Empty(), Array.Empty())); + } + + static void ZeroWeights() + { + var source = new[] {Vector4.Zero}; + var target = new[] {new Vector4(1f, 2f, 3f, 0f)}; + foreach (var value in new[] {0f, -1f, float.NaN, float.PositiveInfinity}) + Reject(() => ARAP.GetTransform(source, target, new[] {value})); + var valid = ARAP.GetTransform(source, target, new[] {1f}); + Require(Vector4.Distance(valid.Apply(source[0]), target[0]) == 0f, + "A valid isolated point must retain its translation"); + } + + static void EmptyUnweighted() + { + Reject(() => ARAP.GetTransformUnweighted(new[] {Vector4.Zero}, new[] {Vector4.Zero}, new List())); + } + + static void NonfiniteKD() + { + foreach (var point in new[] { + new Vector4(float.NaN, 0f, 0f, 0f), + new Vector4(0f, float.PositiveInfinity, 0f, 0f), + new Vector4(0f, 0f, float.NegativeInfinity, 0f) + }) Reject(() => new KDTree(new[] {point})); + } + + static void FiniteKD() + { + var points = new[] { + new Vector4(-7f, 1f, 0f, 0f), new Vector4(2f, 8f, 4f, 0f), + new Vector4(3f, 2f, -1f, 0f), new Vector4(1f, -9f, 1f, 0f), + new Vector4(5f, 0f, 7f, 0f) + }; + var tree = new KDTree(points); + Require(tree.FindNearest(points[2], out float exact) == 2 && exact == 0f, + "Finite exact nearest-neighbor query failed"); + Require(tree.FindNearest(points[2] + new Vector4(0.01f, 0f, 0f, 0f), out float near) == 2 + && Math.Abs(near - 0.01f) < 0.00001f, "Finite nearby query failed"); + var duplicates = new KDTree(new[] {Vector4.One, Vector4.One, Vector4.One}); + Require(duplicates.FindNearest(Vector4.One, out float same) >= 0 && same == 0f, + "Finite duplicate coordinates must remain supported"); + } + + public static int Main(string[] arguments) + { + var cases = new Dictionary { + {"self_distance", SelfDistance}, {"empty_weighted", EmptyWeighted}, + {"zero_weights", ZeroWeights}, {"empty_unweighted", EmptyUnweighted}, + {"nonfinite_kd", NonfiniteKD}, {"finite_kd", FiniteKD} + }; + cases[arguments[0]](); + Console.WriteLine(arguments[0] + ": passed"); + return 0; + } +} diff --git a/open4d/codecs/klt/klt.py b/open4d/codecs/klt/klt.py index 2477504d..c515f166 100644 --- a/open4d/codecs/klt/klt.py +++ b/open4d/codecs/klt/klt.py @@ -19,6 +19,9 @@ """ import argparse +from collections.abc import Mapping +from io import BytesIO +import math import os import time import zipfile @@ -28,6 +31,7 @@ import torch import trimesh import zstd +import zstandard from tqdm import tqdm try: @@ -37,6 +41,56 @@ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +_MAX_VOXELS = 256 ** 3 +_MAX_INDEX_BYTES = 128 * 1024 * 1024 +_MAX_CONTEXT_BYTES = 16 * 1024 * 1024 +_MAX_METADATA_BYTES = 16 * 1024 * 1024 + + +def validate_decoder_context(context): + if (not isinstance(context, Mapping) or set(context) != { + "version", "basis", "mean", "block_size", "num_components"} + or type(context["version"]) is not int or context["version"] != 1): + raise ValueError("invalid KLT decoder context/version") + block, count = context["block_size"], context["num_components"] + if (type(block) is not int or not 1 <= block <= 16 + or type(count) is not int or not 1 <= count <= min(block ** 3, 512)): + raise ValueError("KLT decoder dimensions outside limits") + for name, shape in (("basis", (count, block ** 3)), ("mean", (1, block ** 3))): + value = context[name] + if (not isinstance(value, torch.Tensor) or value.layout != torch.strided + or value.dtype != torch.float32 or tuple(value.shape) != shape + or not torch.isfinite(value).all()): + raise ValueError(f"invalid KLT decoder {name}") + + +def _quantizer_metadata(path): + """Read bounded, non-object NPY members before NumPy allocates their arrays.""" + if os.path.getsize(path) > _MAX_METADATA_BYTES: + raise ValueError("KLT quantizer file outside limits") + result = {} + with zipfile.ZipFile(path) as archive: + members = archive.infolist() + if (len(members) > 517 or sum(member.file_size for member in members) > _MAX_METADATA_BYTES + or len({member.filename for member in members}) != len(members)): + raise ValueError("KLT quantizer metadata outside limits") + for member in members: + name = member.filename + if not name.endswith(".npy") or "/" in name or "\\" in name: + raise ValueError("invalid KLT quantizer member") + data = archive.read(member) + stream = BytesIO(data) + version = np.lib.format.read_magic(stream) + readers = {(1, 0): np.lib.format.read_array_header_1_0, + (2, 0): np.lib.format.read_array_header_2_0} + if version not in readers: + raise ValueError("unsupported KLT NPY version") + shape, _, dtype = readers[version](stream, max_header_size=4096) + if (dtype.hasobject or dtype.kind not in "iufUS" + or math.prod(shape) * dtype.itemsize != len(data) - stream.tell()): + raise ValueError("invalid KLT NPY payload size/type") + result[name[:-4]] = np.load(BytesIO(data), allow_pickle=False, max_header_size=4096) + return result def extract_training_blocks_torch(tsdf_volumes, block_size=4): @@ -138,11 +192,11 @@ def quantize_coeffs(coeffs, eigenvalues, K_total=256, max_iterations=100, tol=1e eigenvalues = eigenvalues.clamp(min=0) V_total = eigenvalues.sum() if V_total == 0: - raise ValueError("All eigenvalues are zero; cannot assign bins.") - stds = torch.sqrt(eigenvalues) - std_total = torch.sqrt(V_total) - K_h = torch.floor(K_total * stds / std_total).long() - K_h = torch.clamp(K_h, min=1) + K_h = torch.ones_like(eigenvalues, dtype=torch.long) + else: + stds = torch.sqrt(eigenvalues) + std_total = torch.sqrt(V_total) + K_h = torch.floor(K_total * stds / std_total).long().clamp(min=1) # Identify fixed dimensions (zero variance) fixed_dims = {} @@ -244,20 +298,66 @@ def save_quantized_coeffs( np.savez_compressed(f"{output_path}_metadata.npz", **metadata) -def load_quantized_coeffs(output_path, target_device=device): - metadata = np.load(f"{output_path}_metadata.npz", allow_pickle=False) +def load_quantized_coeffs(output_path, target_device=device, *, decoder=None): + metadata = _quantizer_metadata(f"{output_path}_metadata.npz") + for name, dimensions in (("indices_shape", 2), ("volume_shape", 3)): + value = metadata.get(name) + if (not isinstance(value, np.ndarray) or value.dtype != np.int64 + or value.shape != (dimensions,) or np.any(value <= 0)): + raise ValueError(f"invalid KLT {name}") shape = tuple(int(value) for value in metadata["indices_shape"]) - dtype = np.dtype(str(metadata["indices_dtype"])) - with open(f"{output_path}_indices.zst", "rb") as stream: - values = np.frombuffer(zstd.decompress(stream.read()), dtype=dtype).reshape(shape) - centers = [ - torch.from_numpy(metadata[f"bin_centers_{h}"].copy()).to(target_device) - for h in range(shape[1]) - ] - fixed_dims = dict(zip( - metadata["fixed_indices"].tolist(), metadata["fixed_values"].tolist() - )) volume_shape = tuple(int(value) for value in metadata["volume_shape"]) + if (not 1 <= shape[1] <= 512 or len(set(volume_shape)) != 1 + or not 2 <= volume_shape[0] <= 256 or math.prod(volume_shape) > _MAX_VOXELS): + raise ValueError("KLT frame dimensions outside limits") + value = metadata.get("indices_dtype") + if not isinstance(value, np.ndarray) or value.shape != () or value.dtype.kind not in "US": + raise ValueError("invalid KLT index dtype") + dtype = np.dtype(str(value)) + if dtype not in (np.dtype("uint8"), np.dtype("uint16"), np.dtype("uint32")): + raise ValueError("unsupported KLT index dtype") + expected = math.prod(shape) * dtype.itemsize + if expected > _MAX_INDEX_BYTES: + raise ValueError("KLT coefficient storage outside limits") + if decoder is not None: + block = decoder["block_size"] + blocks = math.prod((axis + block - 1) // block for axis in volume_shape) + if shape != (blocks, decoder["num_components"]): + raise ValueError("KLT coefficient dimensions disagree with decoder/grid") + required = {"indices_shape", "indices_dtype", "volume_shape", "fixed_indices", "fixed_values", + *(f"bin_centers_{h}" for h in range(shape[1]))} + if set(metadata) != required: + raise ValueError("unexpected KLT quantizer metadata") + arrays = [metadata[f"bin_centers_{h}"] for h in range(shape[1])] + if any(value.dtype != np.float32 or value.ndim != 1 or not 0 < len(value) <= 65536 + or not np.isfinite(value).all() for value in arrays): + raise ValueError("invalid KLT quantizer centers") + fixed, fixed_values = metadata["fixed_indices"], metadata["fixed_values"] + if (fixed.dtype != np.int64 or fixed.ndim != 1 or len(fixed) > shape[1] + or np.any(fixed < 0) or np.any(fixed >= shape[1]) or len(set(fixed.tolist())) != len(fixed) + or fixed_values.dtype != np.float32 or fixed_values.shape != fixed.shape + or not np.isfinite(fixed_values).all()): + raise ValueError("invalid KLT fixed coefficients") + with open(f"{output_path}_indices.zst", "rb") as stream: + data = stream.read(_MAX_INDEX_BYTES + 1) + if len(data) > _MAX_INDEX_BYTES: + raise ValueError("KLT compressed coefficients outside limits") + try: + declared = zstandard.frame_content_size(data) + if declared not in (expected, zstandard.CONTENTSIZE_UNKNOWN): + raise ValueError("KLT coefficient frame size disagrees with quantizer") + decoded = zstandard.ZstdDecompressor(max_window_size=_MAX_INDEX_BYTES // 1024).decompress( + data, max_output_size=expected, allow_extra_data=False + ) + except zstandard.ZstdError as error: + raise ValueError(f"invalid KLT coefficient stream: {error}") from error + if len(decoded) != expected: + raise ValueError("KLT coefficient output size disagrees with quantizer") + values = np.frombuffer(decoded, dtype=dtype).reshape(shape) + if any(np.any(values[:, h] >= len(centers)) for h, centers in enumerate(arrays)): + raise ValueError("KLT coefficient index outside quantizer") + centers = [torch.from_numpy(value.copy()).to(target_device) for value in arrays] + fixed_dims = dict(zip(fixed.tolist(), fixed_values.tolist())) return torch.from_numpy(values.astype(np.int64)).to(target_device), centers, fixed_dims, volume_shape @@ -327,13 +427,15 @@ def decode_compressed(input_path, output_path, target_device=None): """Decode saved KLT coefficients without access to encoder-side tensors.""" target_device = torch.device(target_device or device) input_path, output_path = os.fspath(input_path), os.fspath(output_path) + context_path = os.path.join(input_path, "decoder_context.pt") + if os.path.getsize(context_path) > _MAX_CONTEXT_BYTES: + raise ValueError("KLT decoder context outside limits") decoder = torch.load( - os.path.join(input_path, "decoder_context.pt"), + context_path, map_location=target_device, weights_only=True, ) - if decoder.get("schema") != "open4d.klt/v1": - raise ValueError(f"Unsupported KLT decoder context: {decoder.get('schema')!r}") + validate_decoder_context(decoder) os.makedirs(output_path, exist_ok=True) metadata_paths = sorted(glob(os.path.join(input_path, "*_quantized_metadata.npz"))) if not metadata_paths: @@ -342,7 +444,7 @@ def decode_compressed(input_path, output_path, target_device=None): for metadata_path in metadata_paths: prefix = metadata_path.removesuffix("_metadata.npz") indices, centers, fixed, volume_shape = load_quantized_coeffs( - prefix, target_device + prefix, target_device, decoder=decoder ) coefficients = decompress_coeffs( indices, centers, fixed, H=indices.shape[1] @@ -385,13 +487,14 @@ def run_compression(args, *, verify_decode=True): training_tsdfs = [load_tsdf_tensor(tsdf_paths[i]) for i in args.training_frames] training_blocks = extract_training_blocks_torch(training_tsdfs, block_size=args.block_size) klt_basis, mean_vec = compute_klt_basis_torch(training_blocks) + components = min(args.num_components, len(klt_basis)) context_path = os.path.join(compressed_folder, "decoder_context.pt") torch.save({ - "schema": "open4d.klt/v1", - "basis": klt_basis[:args.num_components].cpu(), + "version": 1, + "basis": klt_basis[:components].cpu(), "mean": mean_vec.cpu(), "block_size": args.block_size, - "num_components": args.num_components, + "num_components": components, }, context_path) decoder = ( torch.load(context_path, map_location=device, weights_only=True) @@ -413,7 +516,7 @@ def run_compression(args, *, verify_decode=True): num_components=args.num_components) # 2) Quantize + save - eigenvalues = torch.linalg.eigvalsh(coeffs.T @ coeffs / (coeffs.shape[0] - 1)) + eigenvalues = coeffs.var(dim=0, unbiased=coeffs.shape[0] > 1) quantized_indices, bin_centers, fixed_dims = quantize_coeffs( coeffs, eigenvalues, K_total=args.k_total) diff --git a/open4d/codecs/qndf/README.md b/open4d/codecs/qndf/README.md index 7933320a..49214e74 100644 --- a/open4d/codecs/qndf/README.md +++ b/open4d/codecs/qndf/README.md @@ -45,6 +45,12 @@ all connected components, and simplifies, subdivides, and projects each componen independently. A JSON transform is saved beside the training pair so decoded vertices can be restored to the input coordinate system. +The coarse-face budget specifies a target. If decimation removes every triangle +of a component, the builder increases that component's target until it retains +valid faces, keeping the original component as the final fallback. This can +exceed the requested budget. Transform metadata records each allocated and used +target, `actual_coarse_faces`, and `coarse_face_budget_overshoot`. + Run the complete basketball sequence with: ```bash diff --git a/open4d/codecs/qndf/build_dataset_open3d.py b/open4d/codecs/qndf/build_dataset_open3d.py index a931d96e..7ec9d156 100644 --- a/open4d/codecs/qndf/build_dataset_open3d.py +++ b/open4d/codecs/qndf/build_dataset_open3d.py @@ -107,6 +107,18 @@ def combine(meshes: list[o3d.geometry.TriangleMesh]) -> o3d.geometry.TriangleMes return result +def simplify_component(original: o3d.geometry.TriangleMesh, budget: int): + """Increase a collapsing target until the component retains valid faces.""" + target = int(budget) + original_faces = len(original.triangles) + while target < original_faces: + coarse = clean_mesh(original.simplify_quadric_decimation(target)) + if len(coarse.triangles): + return coarse, target + target = min(original_faces, target * 2) + return o3d.geometry.TriangleMesh(original), original_faces + + def build_pair(vertices, triangles, coarse_size: int, num_subdiv: int): """Create QNDF's coarse/subdivided input and projected training target.""" source = o3d.geometry.TriangleMesh() @@ -127,13 +139,7 @@ def build_pair(vertices, triangles, coarse_size: int, num_subdiv: int): budgets = allocate_face_budget(face_counts, coarse_size) inputs, targets, details = [], [], [] for index, (original, budget) in enumerate(zip(components, budgets)): - coarse = ( - original.simplify_quadric_decimation(int(budget)) - if budget < len(original.triangles) else o3d.geometry.TriangleMesh(original) - ) - coarse = clean_mesh(coarse) - if not len(coarse.triangles): - raise RuntimeError(f"component {index} vanished during simplification") + coarse, simplification_target = simplify_component(original, int(budget)) subdivided = o3d.geometry.TriangleMesh(coarse) for _ in range(num_subdiv): subdivided = subdivided.subdivide_midpoint(number_of_iterations=1) @@ -147,15 +153,19 @@ def build_pair(vertices, triangles, coarse_size: int, num_subdiv: int): "index": index, "original_vertices": len(original.vertices), "original_faces": len(original.triangles), "allocated_coarse_faces": int(budget), + "simplification_target_faces": simplification_target, "actual_coarse_vertices": len(coarse.vertices), "actual_coarse_faces": len(coarse.triangles), "training_vertices": len(subdivided.vertices), "training_faces": len(subdivided.triangles), }) low, target = combine(inputs), combine(targets) + actual_coarse_faces = sum(component["actual_coarse_faces"] for component in details) metadata = { "bbox_min": bbox_min.tolist(), "scale": scale, "component_count": len(components), "coarse_face_budget": coarse_size, + "actual_coarse_faces": actual_coarse_faces, + "coarse_face_budget_overshoot": max(0, actual_coarse_faces - coarse_size), "num_subdiv": num_subdiv, "components": details, } return ( diff --git a/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/ARAP.cs b/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/ARAP.cs index 25ec2356..bee2f99f 100644 --- a/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/ARAP.cs +++ b/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/ARAP.cs @@ -21,6 +21,8 @@ public static class ARAP public static Transform GetTransformUnweighted(Vector4[] pts1, Vector4[] pts2, List ind) { int n = ind.Count; + if (n == 0) + throw new ArgumentException("ARAP requires at least one point", nameof(ind)); Vector4[] a = new Vector4[n]; Vector4[] b = new Vector4[n]; @@ -94,6 +96,8 @@ public static Transform GetTransform(Vector4[] a, Vector4[] b, float[] weights) Vector4 cB = new Vector4(); float wSum = 0; int n = a.Length; + if (n == 0) + throw new ArgumentException("ARAP requires at least one point", nameof(a)); //Compute weighted centroids for (int i = 0; i < n; i++) @@ -103,6 +107,9 @@ public static Transform GetTransform(Vector4[] a, Vector4[] b, float[] weights) cB += weights[i] * b[i]; } + if (!float.IsFinite(wSum) || wSum <= 0) + throw new ArgumentException("ARAP requires a finite positive total weight", nameof(weights)); + cA /= wSum; cB /= wSum; diff --git a/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/KDTree.cs b/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/KDTree.cs index 784485ae..f5de3cb4 100644 --- a/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/KDTree.cs +++ b/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/KDTree.cs @@ -70,6 +70,10 @@ public KDTree(KDTree source) public KDTree(Vector4[] points) { + for (int i = 0; i < points.Length; i++) + if (!float.IsFinite(points[i].X) || !float.IsFinite(points[i].Y) || !float.IsFinite(points[i].Z)) + throw new ArgumentException("KD-tree points must have finite XYZ coordinates", nameof(points)); + this.pnts = points; List list = new List(); for (int i = 0; i < points.Length; i++) diff --git a/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/TransformDistance.cs b/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/TransformDistance.cs index 63358405..56fc0061 100644 --- a/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/TransformDistance.cs +++ b/open4d/codecs/tsmc/arap-volume-tracking/Framework/Util/TransformDistance.cs @@ -95,6 +95,9 @@ public static void TestTransformDistance() public float GetTransformDistance(Matrix R1, Matrix R2, Vector t1, Vector t2) { + if (R1.Equals(R2) && t1.Equals(t2)) + return 0; + var dif = t1 - t2; var greenTerm = 2 * (dif).DotProduct(R1 * qSum); var blueTerm = -2 * (dif).DotProduct(R2 * qSum); diff --git a/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs b/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs index 60c069f6..943c262a 100644 --- a/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs +++ b/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs @@ -47,10 +47,12 @@ public Vector3[] DeformCenters(Vector3[] centers, int[] centerIndices, Vector3[] } // Divide by sum of weights - weightedDifference /= weightSum; - // weightedDifference /= newPositions.Length; - // Normalize DQ - weightedDifference = weightedDifference.Normalize(); + if (!float.IsFinite(weightSum) || weightSum < 0) + throw new ArgumentException("Center deformation requires finite nonnegative affinity weights"); + if (weightSum == 0) + weightedDifference = DualQuaternion.Identity(); + else + weightedDifference = (weightedDifference / weightSum).Normalize(); // Apply transformation to original center newTransformations[i] = weightedDifference; diff --git a/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs b/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs index 8094623b..b89fa8bd 100644 --- a/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs +++ b/open4d/codecs/tsmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs @@ -284,14 +284,16 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI { //return ComputeEmbeddedDeformationsWeightsForVertex(centersKdTree, vertices, vertex, affinity, affinityThreshold, oldCenters); // Find k-nearest center - var searchCount = Neighbors; + if (Neighbors <= 0 || centersKdTree.Count == 0) + throw new ArgumentException("Surface deformation requires centers and a positive neighbor count"); + var neighborCount = System.Math.Min(Neighbors, centersKdTree.Count); + var searchCount = neighborCount; var found = 0; KdTreeNode[] nearestCenters = null; int nearestCenter; do { nearestCenters = centersKdTree.GetNearestNeighbours(new float[] { vertex.X, vertex.Y, vertex.Z }, searchCount); - searchCount *= 2; nearestCenter = nearestCenters[0].Value; found = 0; for (var i = 0; i < nearestCenters.Length; i++) @@ -306,12 +308,16 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } } + if (found >= neighborCount || searchCount == centersKdTree.Count) + break; + searchCount = searchCount > centersKdTree.Count / 2 + ? centersKdTree.Count : searchCount * 2; if (searchCount > 200) affinityThreshold = 0; - } while (found < Neighbors); + } while (true); - var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(Neighbors).ToArray(); + var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(neighborCount).ToArray(); // var nearestCenter = kdTree.Nearest(new double[] { vertices[v].X, vertices[v].Y, vertices[v].Z }).Value; @@ -324,7 +330,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI var mostAffineCenters = centerAffinities.Select((a, i) => (a, i)).OrderByDescending(x => x.a).Take(Neighbors.Value).Select(x => x.i).ToArray();*/ // Compute distances - var distances = new float[Neighbors]; + var distances = new float[mostAffineCenters.Length]; var distMin = float.PositiveInfinity; for (var i = 0; i < distances.Length; i++) { @@ -334,7 +340,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } // Compute soft minimums vector - var softmin = new float[Neighbors]; + var softmin = new float[mostAffineCenters.Length]; var softminSum = 0f; var softminMin = float.PositiveInfinity; for (var i = 0; i < softmin.Length; i++) @@ -351,7 +357,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } } - var weights1 = new float[Neighbors]; + var weights1 = new float[mostAffineCenters.Length]; var weightsSum = 0f; for (var i = 0; i < weights1.Length; i++) { @@ -372,14 +378,16 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI private (int[], float[]) ComputeEmbeddedDeformationsWeightsForVertex(KdTree centersKdTree, Vector3[] vertices, Vector3 vertex, float[,] affinity, double affinityThreshold, Vector3[] oldCenters) { // Find k-nearest center - var searchCount = Neighbors; + if (Neighbors <= 0 || centersKdTree.Count == 0) + throw new ArgumentException("Surface deformation requires centers and a positive neighbor count"); + var neighborCount = System.Math.Min(Neighbors, centersKdTree.Count); + var searchCount = neighborCount; var found = 0; KdTreeNode[] nearestCenters = null; int nearestCenter; do { nearestCenters = centersKdTree.GetNearestNeighbours(new float[] { vertex.X, vertex.Y, vertex.Z }, searchCount); - searchCount *= 2; nearestCenter = nearestCenters[0].Value; found = 0; for (var i = 0; i < nearestCenters.Length; i++) @@ -394,15 +402,19 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } } + if (found >= neighborCount || searchCount == centersKdTree.Count) + break; + searchCount = searchCount > centersKdTree.Count / 2 + ? centersKdTree.Count : searchCount * 2; if (searchCount > 200) affinityThreshold = 0; - } while (found < Neighbors); + } while (true); - var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(Neighbors).ToArray(); + var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(neighborCount).ToArray(); // Compute distances - var distances = new float[Neighbors]; + var distances = new float[mostAffineCenters.Length]; var distMax = float.PositiveInfinity; for (var i = 0; i < distances.Length; i++) { @@ -411,7 +423,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI distMax = distances[i]; } - var weights1 = new float[Neighbors]; + var weights1 = new float[mostAffineCenters.Length]; var weightsSum = 0f; for (var i = 0; i < weights1.Length; i++) { diff --git a/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/ARAP.cs b/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/ARAP.cs index 25ec2356..bee2f99f 100644 --- a/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/ARAP.cs +++ b/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/ARAP.cs @@ -21,6 +21,8 @@ public static class ARAP public static Transform GetTransformUnweighted(Vector4[] pts1, Vector4[] pts2, List ind) { int n = ind.Count; + if (n == 0) + throw new ArgumentException("ARAP requires at least one point", nameof(ind)); Vector4[] a = new Vector4[n]; Vector4[] b = new Vector4[n]; @@ -94,6 +96,8 @@ public static Transform GetTransform(Vector4[] a, Vector4[] b, float[] weights) Vector4 cB = new Vector4(); float wSum = 0; int n = a.Length; + if (n == 0) + throw new ArgumentException("ARAP requires at least one point", nameof(a)); //Compute weighted centroids for (int i = 0; i < n; i++) @@ -103,6 +107,9 @@ public static Transform GetTransform(Vector4[] a, Vector4[] b, float[] weights) cB += weights[i] * b[i]; } + if (!float.IsFinite(wSum) || wSum <= 0) + throw new ArgumentException("ARAP requires a finite positive total weight", nameof(weights)); + cA /= wSum; cB /= wSum; diff --git a/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/KDTree.cs b/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/KDTree.cs index 784485ae..f5de3cb4 100644 --- a/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/KDTree.cs +++ b/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/KDTree.cs @@ -70,6 +70,10 @@ public KDTree(KDTree source) public KDTree(Vector4[] points) { + for (int i = 0; i < points.Length; i++) + if (!float.IsFinite(points[i].X) || !float.IsFinite(points[i].Y) || !float.IsFinite(points[i].Z)) + throw new ArgumentException("KD-tree points must have finite XYZ coordinates", nameof(points)); + this.pnts = points; List list = new List(); for (int i = 0; i < points.Length; i++) diff --git a/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/TransformDistance.cs b/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/TransformDistance.cs index 63358405..56fc0061 100644 --- a/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/TransformDistance.cs +++ b/open4d/codecs/tvmc/arap-volume-tracking/Framework/Util/TransformDistance.cs @@ -95,6 +95,9 @@ public static void TestTransformDistance() public float GetTransformDistance(Matrix R1, Matrix R2, Vector t1, Vector t2) { + if (R1.Equals(R2) && t1.Equals(t2)) + return 0; + var dif = t1 - t2; var greenTerm = 2 * (dif).DotProduct(R1 * qSum); var blueTerm = -2 * (dif).DotProduct(R2 * qSum); diff --git a/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs b/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs index 60c069f6..943c262a 100644 --- a/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs +++ b/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/CenterDeformation/AffinityCenterDeformation.cs @@ -47,10 +47,12 @@ public Vector3[] DeformCenters(Vector3[] centers, int[] centerIndices, Vector3[] } // Divide by sum of weights - weightedDifference /= weightSum; - // weightedDifference /= newPositions.Length; - // Normalize DQ - weightedDifference = weightedDifference.Normalize(); + if (!float.IsFinite(weightSum) || weightSum < 0) + throw new ArgumentException("Center deformation requires finite nonnegative affinity weights"); + if (weightSum == 0) + weightedDifference = DualQuaternion.Identity(); + else + weightedDifference = (weightedDifference / weightSum).Normalize(); // Apply transformation to original center newTransformations[i] = weightedDifference; diff --git a/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs b/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs index 8094623b..b89fa8bd 100644 --- a/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs +++ b/open4d/codecs/tvmc/tvm-editing/TVMEditor/Editing/SurfaceDeformation/CustomSurfaceDeformation.cs @@ -284,14 +284,16 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI { //return ComputeEmbeddedDeformationsWeightsForVertex(centersKdTree, vertices, vertex, affinity, affinityThreshold, oldCenters); // Find k-nearest center - var searchCount = Neighbors; + if (Neighbors <= 0 || centersKdTree.Count == 0) + throw new ArgumentException("Surface deformation requires centers and a positive neighbor count"); + var neighborCount = System.Math.Min(Neighbors, centersKdTree.Count); + var searchCount = neighborCount; var found = 0; KdTreeNode[] nearestCenters = null; int nearestCenter; do { nearestCenters = centersKdTree.GetNearestNeighbours(new float[] { vertex.X, vertex.Y, vertex.Z }, searchCount); - searchCount *= 2; nearestCenter = nearestCenters[0].Value; found = 0; for (var i = 0; i < nearestCenters.Length; i++) @@ -306,12 +308,16 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } } + if (found >= neighborCount || searchCount == centersKdTree.Count) + break; + searchCount = searchCount > centersKdTree.Count / 2 + ? centersKdTree.Count : searchCount * 2; if (searchCount > 200) affinityThreshold = 0; - } while (found < Neighbors); + } while (true); - var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(Neighbors).ToArray(); + var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(neighborCount).ToArray(); // var nearestCenter = kdTree.Nearest(new double[] { vertices[v].X, vertices[v].Y, vertices[v].Z }).Value; @@ -324,7 +330,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI var mostAffineCenters = centerAffinities.Select((a, i) => (a, i)).OrderByDescending(x => x.a).Take(Neighbors.Value).Select(x => x.i).ToArray();*/ // Compute distances - var distances = new float[Neighbors]; + var distances = new float[mostAffineCenters.Length]; var distMin = float.PositiveInfinity; for (var i = 0; i < distances.Length; i++) { @@ -334,7 +340,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } // Compute soft minimums vector - var softmin = new float[Neighbors]; + var softmin = new float[mostAffineCenters.Length]; var softminSum = 0f; var softminMin = float.PositiveInfinity; for (var i = 0; i < softmin.Length; i++) @@ -351,7 +357,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } } - var weights1 = new float[Neighbors]; + var weights1 = new float[mostAffineCenters.Length]; var weightsSum = 0f; for (var i = 0; i < weights1.Length; i++) { @@ -372,14 +378,16 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI private (int[], float[]) ComputeEmbeddedDeformationsWeightsForVertex(KdTree centersKdTree, Vector3[] vertices, Vector3 vertex, float[,] affinity, double affinityThreshold, Vector3[] oldCenters) { // Find k-nearest center - var searchCount = Neighbors; + if (Neighbors <= 0 || centersKdTree.Count == 0) + throw new ArgumentException("Surface deformation requires centers and a positive neighbor count"); + var neighborCount = System.Math.Min(Neighbors, centersKdTree.Count); + var searchCount = neighborCount; var found = 0; KdTreeNode[] nearestCenters = null; int nearestCenter; do { nearestCenters = centersKdTree.GetNearestNeighbours(new float[] { vertex.X, vertex.Y, vertex.Z }, searchCount); - searchCount *= 2; nearestCenter = nearestCenters[0].Value; found = 0; for (var i = 0; i < nearestCenters.Length; i++) @@ -394,15 +402,19 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI } } + if (found >= neighborCount || searchCount == centersKdTree.Count) + break; + searchCount = searchCount > centersKdTree.Count / 2 + ? centersKdTree.Count : searchCount * 2; if (searchCount > 200) affinityThreshold = 0; - } while (found < Neighbors); + } while (true); - var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(Neighbors).ToArray(); + var mostAffineCenters = nearestCenters.Select(n => n.Value).Take(neighborCount).ToArray(); // Compute distances - var distances = new float[Neighbors]; + var distances = new float[mostAffineCenters.Length]; var distMax = float.PositiveInfinity; for (var i = 0; i < distances.Length; i++) { @@ -411,7 +423,7 @@ private void ComputeWeights(Vector3[] vertices, Vector3[] oldCenters, int frameI distMax = distances[i]; } - var weights1 = new float[Neighbors]; + var weights1 = new float[mostAffineCenters.Length]; var weightsSum = 0f; for (var i = 0; i < weights1.Length; i++) { diff --git a/open4d/core/tests/test_gaussians.py b/open4d/core/tests/test_gaussians.py index ae1fd985..12a65332 100644 --- a/open4d/core/tests/test_gaussians.py +++ b/open4d/core/tests/test_gaussians.py @@ -326,19 +326,40 @@ def test_real_vega_round_trip(tmp_path): def test_public_encode_decode_dispatch_to_vega(tmp_path, monkeypatch): import open4d - from types import SimpleNamespace + from open4d.codec import inspect_vmesh + from open4d.native import NativeSequence frames = [splats(), splats()] - output = tmp_path / "capture.vega" + output = tmp_path / "capture.vmesh" calls = [] + def encode(values, path, **options): calls.append((values, path, options)) - return SimpleNamespace(path=path) + native_bitstream(path) + return gaussians.VegaRun(path, tmp_path, sys.executable) + monkeypatch.setattr(gaussians, "encode_gaussians", encode) - monkeypatch.setattr(gaussians, "decode_gaussians", lambda path, **options: tuple(frames)) assert open4d.encode(frames, output, codec="vega", key_iterations=10) == output - assert calls == [(frames, output, {"key_iterations": 10})] - assert open4d.decode(output) == tuple(frames) + assert calls[0][0] is frames + assert calls[0][1].name == "native" + assert calls[0][2] == {"key_iterations": 10} + assert not calls[0][1].exists() + assert inspect_vmesh(output)["codec"] == "vega" + with open4d.decode(output) as restored: + assert isinstance(restored, NativeSequence) + assert restored.codec == "vega" + assert restored.frame_indices == (0, 1) + assert restored.timestamps == (0, 1 / 30) + + +@pytest.mark.parametrize("extension", ["vega", "o4d", ""]) +def test_public_vega_output_requires_vmesh(tmp_path, monkeypatch, extension): + import open4d + monkeypatch.setattr(gaussians, "encode_gaussians", lambda *args, **kwargs: pytest.fail("native encoder launched")) + output = tmp_path / (f"capture.{extension}" if extension else "capture") + with pytest.raises(ValueError, match=".vmesh"): + open4d.encode([splats(), splats()], output, codec="vega") + assert not output.exists() def test_public_reconstruction_dispatch(tmp_path, monkeypatch): diff --git a/open4d/core/tests/test_metrics.py b/open4d/core/tests/test_metrics.py index 47f15ef1..b1db17a5 100644 --- a/open4d/core/tests/test_metrics.py +++ b/open4d/core/tests/test_metrics.py @@ -89,7 +89,7 @@ def test_sequence_weights_frames_equally_and_uses_one_peak(): def test_comparison_after_codec_round_trip(tmp_path): with mesh_sequence(side=4, frames=3) as reference: - artifact = save(reference, tmp_path / "wave.o4d", codec=NumPyZipCodec()) + artifact = save(reference, tmp_path / "wave.vmesh", codec=NumPyZipCodec()) with load(artifact, codec=NumPyZipCodec()) as decoded: result = compare_sequences(reference, decoded, peak=10) assert result.symmetric_rms == 0 diff --git a/open4d/gaussians.py b/open4d/gaussians.py index 1d757484..e7a89b46 100644 --- a/open4d/gaussians.py +++ b/open4d/gaussians.py @@ -467,42 +467,35 @@ def decode_gaussians( class _VegaCodec: id = "vega" - suffixes = (".vega", ".vmesh") + suffixes = (".vmesh",) representation = "gaussian_splats" backend = "research-subprocess" lossless = False preserves = ("positions", "scales", "rotations", "opacities", "neural_appearance") def can_decode(self, source): - if Path(source).suffix.lower() == ".vmesh": - from .codec._v3c import probe_codec - return probe_codec(source) == self.id - return Path(source).is_dir() and (Path(source) / "manifest.json").is_file() + from .codec._vmesh_format import probe_codec + return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id def encode(self, sequence, destination: Path, **options) -> Path: - if destination.suffix.lower() == ".vmesh": - from .native import NativeSequence, import_native, save_native - from .codec._native_temporal import encode_native - if isinstance(sequence, (NativeSequence, VegaRun, str, os.PathLike)): - return encode_native(sequence, destination, codec="vega", **options) - overwrite = options.pop("overwrite", False) - if destination.exists() and not overwrite: - raise FileExistsError(destination) - timeline = {key: options.pop(key) for key in ("fps", "timestamps", "frame_indices", "metadata", "frame_metadata") if key in options} - with tempfile.TemporaryDirectory(prefix="open4d-vega-encode-") as folder: - encoded = encode_gaussians(sequence, Path(folder) / "native", **options) - with import_native(encoded, **timeline) as native: - return save_native(native, destination, overwrite=overwrite) - if "overwrite" in options: - if options.pop("overwrite"): - raise ValueError("Vega directory encoding does not support overwrite; use a new directory or .vmesh") - return encode_gaussians(sequence, destination, **options).path - - def decode(self, source: Path, **options) -> tuple[NeuralGaussianFrame, ...]: - if source.suffix.lower() == ".vmesh": - from .codec._native_temporal import NativeTemporalCodec - return NativeTemporalCodec("vega").decode(source, **options) - return decode_gaussians(source, **options) + from .codec._metadata import require_vmesh_destination + from .native import NativeSequence, import_native, save_native + from .codec._native_temporal import encode_native + destination = require_vmesh_destination(destination) + if isinstance(sequence, (NativeSequence, VegaRun, str, os.PathLike)): + return encode_native(sequence, destination, codec="vega", **options) + overwrite = options.pop("overwrite", False) + if destination.exists() and not overwrite: + raise FileExistsError(destination) + timeline = {key: options.pop(key) for key in ("fps", "timestamps", "frame_indices", "metadata", "frame_metadata") if key in options} + with tempfile.TemporaryDirectory(prefix="open4d-vega-encode-") as folder: + encoded = encode_gaussians(sequence, Path(folder) / "native", **options) + with import_native(encoded, **timeline) as native: + return save_native(native, destination, overwrite=overwrite) + + def decode(self, source: Path, **options): + from .codec._native_temporal import NativeTemporalCodec + return NativeTemporalCodec("vega").decode(source, **options) VEGA_CODEC = _VegaCodec() diff --git a/open4d/io/_native_usd.py b/open4d/io/_native_usd.py index 80f1face..d283ecca 100644 --- a/open4d/io/_native_usd.py +++ b/open4d/io/_native_usd.py @@ -1,6 +1,6 @@ -"""Self-contained O4D native temporal data in USD, without executable loading. +"""Self-contained VMESH native temporal data in USD, without executable loading. -The authored Xform is an O4D application schema, not a standard renderable +The authored VMESH prim is a custom data type, not a standard renderable Gaussian or neural-field prim. Compressed bytes survive USDC round trips. """ from __future__ import annotations @@ -14,11 +14,11 @@ from open4d._files import publish_file from open4d.codec._protocol import CodecError -from open4d.codec._v3c import _json, inspect_vmesh +from open4d.codec._vmesh_format import _json, inspect_vmesh from ._usd import _pxr -_PRIM = "/Open4DNative" -_SCHEMA = "open4d.usd-native-temporal/1" +_PRIM = "/VMESH" +_SCHEMA = "vmesh.usd/1" _CHUNK = 1024 * 1024 @@ -38,7 +38,7 @@ def _open(path): def is_native_usd(path): _, prim = _open(path) - return bool(prim and prim.HasAttribute("open4d:nativeSchema")) + return bool(prim and prim.HasAttribute("vmesh:schema")) def write_native_usd(sequence, destination, *, overwrite=False): @@ -48,7 +48,7 @@ def write_native_usd(sequence, destination, *, overwrite=False): raise ValueError("native USD interchange supports .usd, .usda and .usdc") if destination.exists() and not overwrite: raise FileExistsError(destination) - Sdf, Usd, UsdGeom, Vt = _pxr() + Sdf, Usd, _, Vt = _pxr() descriptor = inspect_vmesh(sequence.path) if any(right <= left for left, right in zip(sequence.timestamps, sequence.timestamps[1:])): raise CodecError("native USD export requires strictly increasing timestamps") @@ -58,13 +58,13 @@ def write_native_usd(sequence, destination, *, overwrite=False): with tempfile.TemporaryDirectory(prefix=f".{destination.name}-", dir=destination.parent) as directory: temporary = Path(directory) / destination.name stage = Usd.Stage.CreateNew(str(temporary)) - prim = UsdGeom.Xform.Define(stage, _PRIM).GetPrim() + prim = stage.DefinePrim(_PRIM, "VMESH") stage.SetDefaultPrim(prim) stage.SetTimeCodesPerSecond(1.0) stage.SetStartTimeCode(sequence.timestamps[0]) stage.SetEndTimeCode(sequence.timestamps[-1]) - prim.CreateAttribute("open4d:nativeSchema", Sdf.ValueTypeNames.String).Set(_SCHEMA) - frame = prim.CreateAttribute("open4d:frameIndex", Sdf.ValueTypeNames.Int64) + prim.CreateAttribute("vmesh:schema", Sdf.ValueTypeNames.String).Set(_SCHEMA) + frame = prim.CreateAttribute("vmesh:frameIndex", Sdf.ValueTypeNames.Int64) for index, timestamp in zip(sequence.frame_indices, sequence.timestamps): frame.Set(index, timestamp) digest = hashlib.sha256() @@ -72,13 +72,13 @@ def write_native_usd(sequence, destination, *, overwrite=False): for index in range(count): data = source.read(_CHUNK) digest.update(data) - prim.CreateAttribute(f"open4d:payload:chunk{index:06d}", Sdf.ValueTypeNames.UCharArray).Set( + prim.CreateAttribute(f"vmesh:payload:chunk{index:06d}", Sdf.ValueTypeNames.UCharArray).Set( Vt.UCharArray.FromNumpy(np.frombuffer(data, dtype=np.uint8))) if source.read(1): raise CodecError("native stream changed while exporting USD") header = dict(codec=sequence.codec, bytes=size, chunks=count, sha256=digest.hexdigest(), representation=descriptor["representation"]) - prim.CreateAttribute("open4d:payloadManifest", Sdf.ValueTypeNames.String).Set(json.dumps(header, separators=(",", ":"))) + prim.CreateAttribute("vmesh:payloadManifest", Sdf.ValueTypeNames.String).Set(json.dumps(header, separators=(",", ":"))) stage.GetRootLayer().Save() del stage # Verify the USD actually contains the original stream before publishing. @@ -93,9 +93,9 @@ def read_native_usd(source, *, runtime=None, python=None): from open4d.native import NativeSequence stage, prim = _open(source) - if not prim or prim.GetAttribute("open4d:nativeSchema").Get() != _SCHEMA: - raise CodecError("unsupported O4D native USD schema") - raw = prim.GetAttribute("open4d:payloadManifest").Get() + if not prim or prim.GetTypeName() != "VMESH" or prim.GetAttribute("vmesh:schema").Get() != _SCHEMA: + raise CodecError("unsupported VMESH USD schema") + raw = prim.GetAttribute("vmesh:payloadManifest").Get() if not isinstance(raw, str) or len(raw) > 1024 * 1024: raise CodecError("invalid native USD payload manifest") header = _json(raw) @@ -105,8 +105,8 @@ def read_native_usd(source, *, runtime=None, python=None): if (type(count) is not int or not 1 <= count <= 65536 or type(size) is not int or not (count - 1) * _CHUNK < size <= count * _CHUNK): raise CodecError("invalid native USD payload bounds") - chunks = sorted(a.GetName() for a in prim.GetAttributes() if a.GetName().startswith("open4d:payload:chunk")) - if chunks != [f"open4d:payload:chunk{i:06d}" for i in range(count)]: + chunks = sorted(a.GetName() for a in prim.GetAttributes() if a.GetName().startswith("vmesh:payload:chunk")) + if chunks != [f"vmesh:payload:chunk{i:06d}" for i in range(count)]: raise CodecError("missing or extra native USD chunks") temporary = tempfile.TemporaryDirectory(prefix="open4d-native-usd-") try: @@ -127,7 +127,7 @@ def read_native_usd(source, *, runtime=None, python=None): result = NativeSequence(path, temporary=temporary, runtime=runtime, python=python) if result.codec != header.get("codec") or result.representation != header.get("representation"): raise CodecError("native USD profile disagrees with carried stream") - frame = prim.GetAttribute("open4d:frameIndex") + frame = prim.GetAttribute("vmesh:frameIndex") if (stage.GetTimeCodesPerSecond() != 1.0 or tuple(frame.GetTimeSamples()) != result.timestamps or tuple(frame.Get(t) for t in result.timestamps) != result.frame_indices): raise CodecError("USD timeline disagrees with native temporal payload") diff --git a/open4d/io/tests/test_unified_api.py b/open4d/io/tests/test_unified_api.py index 424ccdb1..c39ce085 100644 --- a/open4d/io/tests/test_unified_api.py +++ b/open4d/io/tests/test_unified_api.py @@ -21,7 +21,7 @@ def sequence() -> Sequence: def test_top_level_codec_round_trip_and_unload(tmp_path): - artifact = open4d.save(sequence(), tmp_path / "capture.o4d", codec=NumPyZipCodec()) + artifact = open4d.save(sequence(), tmp_path / "capture.vmesh", codec=NumPyZipCodec()) loaded = open4d.load(artifact, codec=NumPyZipCodec()) np.testing.assert_array_equal(loaded[0].geometry.triangles, [[0, 1, 2]]) @@ -92,7 +92,7 @@ def test_top_level_save_requires_a_sequence_file_extension(tmp_path, name): open4d.save(sequence(), tmp_path / name) -@pytest.mark.parametrize("suffix", [".v4d", ".vmesh"]) +@pytest.mark.parametrize("suffix", [".vmesh"]) def test_top_level_save_requires_a_codec_for_ambiguous_codec_suffix(tmp_path, suffix): with pytest.raises(ValueError, match="ambiguous.*codec"): open4d.save(sequence(), tmp_path / f"capture{suffix}") @@ -105,11 +105,11 @@ def test_top_level_api_exports_visualize(): @pytest.mark.parametrize("codec", ["draco", "raw", "npz", "temporal-delta"]) def test_utility_codecs_are_not_public_choices(tmp_path, codec): with pytest.raises(ValueError, match="unknown codec"): - open4d.encode(sequence(), tmp_path / "take.o4d", codec=codec) + open4d.encode(sequence(), tmp_path / "take.vmesh", codec=codec) def test_encoding_requires_an_explicit_codec(tmp_path): with pytest.raises(TypeError, match="codec"): - open4d.encode(sequence(), tmp_path / "take.o4d") - with pytest.raises(ValueError, match="sequence-file extension"): - open4d.save(sequence(), tmp_path / "take.o4d") + open4d.encode(sequence(), tmp_path / "take.vmesh") + with pytest.raises(ValueError, match="ambiguous.*codec"): + open4d.save(sequence(), tmp_path / "take.vmesh") diff --git a/open4d/native.py b/open4d/native.py index d1ac6707..a6fc3baf 100644 --- a/open4d/native.py +++ b/open4d/native.py @@ -1,4 +1,4 @@ -"""Portable native temporal runs and lossless USDC / V3C interchange. +"""Portable native temporal runs and lossless USDC / VMESH interchange. Opening a run never executes its models. ``decode`` and ``render`` explicitly invoke the selected research runtime. USD interchange retains the compressed @@ -21,9 +21,9 @@ from .codec._native import run from .codec._native_profiles import NEURAL_CODECS, layout -from .codec._npz import _json_value +from .codec._metadata import _json_value from .codec._protocol import CodecError -from .codec._v3c import _json, inspect_vmesh, pack_vmesh, unpack_vmesh +from .codec._vmesh_format import _json, inspect_vmesh, pack_vmesh, unpack_vmesh from ._files import publish_file @@ -243,6 +243,8 @@ def decode(self, *, runtime=None, python=None): formats. Call this only for trusted native runs, in their configured runtime. Generic inspection/load/save never deserializes these models. """ + if self.codec == "frames": + raise NotImplementedError("Use unpack() or browser playback for point, Gaussian and image frame payloads") source = self._native_directory() root, executable = self._settings(runtime, python) if self.codec == "vega": @@ -342,7 +344,7 @@ def import_native(source, *, codec=None, config=None, initial_model=None, ntc_co def save_native(sequence, destination, *, overwrite=False): - """Save native temporal state as V3C .vmesh or a self-contained O4D USD.""" + """Save native temporal state as standalone .vmesh or a self-contained VMESH USD.""" if not isinstance(sequence, NativeSequence): raise TypeError("sequence must be a NativeSequence") if not isinstance(overwrite, bool): diff --git a/open4d/reconstruction/queen/train.py b/open4d/reconstruction/queen/train.py index 52e2fd0f..ea05cd2b 100644 --- a/open4d/reconstruction/queen/train.py +++ b/open4d/reconstruction/queen/train.py @@ -74,6 +74,23 @@ except ImportError: WANDB_FOUND = False +def _save_frame_outputs(scene, gaussians, dataset, frame_idx, iteration, qp): + if not (dataset.log_ply or dataset.log_compressed): + return + gate = gaussians.gate_atts + was_training = gate.training if gate is not None else False + if gate is not None: + gate.eval() + try: + if dataset.log_ply or frame_idx == 1: + scene.save(iteration, save_point_cloud=True) + if dataset.log_compressed and frame_idx > 1: + scene.save_compressed(-1, qp) + finally: + if gate is not None: + gate.train(was_training) + + def training(dataset: ModelParams, opt: OptimizationParams, pipe: PipelineParams, qp:QuantizeParams, testing_iterations: list, saving_iterations: list, checkpoint_iterations, checkpoint: str, debug_from, args): """Main training function for QUEEN compressed Gaussian splatting.""" @@ -800,13 +817,6 @@ def training(dataset: ModelParams, opt: OptimizationParams, pipe: PipelineParams if dataset.log_images: save_image(gt_image,os.path.join(scene.model_path, "gt.png")) - if dataset.log_ply: - scene.save(iteration, save_point_cloud=True) - - if dataset.log_compressed: - if frame_idx == 1: - scene.save(frame_idx, save_point_cloud=True) - if frame_idx>1 and (dataset.adaptive_render and dataset.adaptive_update_period>0.0) and dataset.update_mask!="none": torchvision.utils.save_image(train_cameras[cam_idx].mask.unsqueeze(0)*gt_image, os.path.join(scene.model_path, "mask.png")) @@ -876,19 +886,7 @@ def training(dataset: ModelParams, opt: OptimizationParams, pipe: PipelineParams else: raise ValueError(f"Invalid save format {args.save_format}") - if dataset.log_compressed: - if frame_idx == 1: - scene.save(frame_idx, save_point_cloud=True) - else: - gate = gaussians.gate_atts - was_training = gate.training if gate is not None else False - if gate is not None: - gate.eval() - try: - scene.save_compressed(-1, qp) - finally: - if gate is not None: - gate.train(was_training) + _save_frame_outputs(scene, gaussians, dataset, frame_idx, opt.iterations, qp) # Update previous frame's attributes and latents for next frame's residual encoding diff --git a/open4d/streamer/streamer/client/viewer.html b/open4d/streamer/streamer/client/viewer.html index 5c9bee3f..1b98e14f 100644 --- a/open4d/streamer/streamer/client/viewer.html +++ b/open4d/streamer/streamer/client/viewer.html @@ -937,48 +937,173 @@

session

return dot < 0 ? "" : path.slice(dot).toLowerCase(); } -const SEQ_MAGIC = "O4DSEQ\0\0"; -//: magic(8) + version(4) + count(4) + suffix length(4) -const SEQ_PREAMBLE = 20; +const SEQ_MAGIC = "VMESH\0\x01\0"; +const SEQ_PREAMBLE = 26; +const VMESH_CHUNK = 1048576; + +/* Pure SHA-256 also works on LAN HTTP pages, where Web Crypto is unavailable. */ +function vmeshSHA256(data) { + const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); + const k = [0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2]; + const h = [0x6a09e667,0xbb67ae85,0x3c6ef372,0xa54ff53a,0x510e527f,0x9b05688c,0x1f83d9ab,0x5be0cd19]; + const rotate = (x, n) => (x >>> n) | (x << (32 - n)); + const blocks = Math.ceil((bytes.length + 9) / 64), words = new Uint32Array(64); + const bitLength = bytes.length * 8; + for (let block = 0; block < blocks; block++) { + words.fill(0); + for (let i = 0; i < 64; i++) { + const at = block * 64 + i; + let value = at < bytes.length ? bytes[at] : at === bytes.length ? 128 : 0; + if (at >= blocks * 64 - 8) { + const shift = (blocks * 64 - 1 - at) * 8; + value = shift >= 32 ? (Math.floor(bitLength / 4294967296) >>> (shift - 32)) & 255 + : (bitLength >>> shift) & 255; + } + words[i >>> 2] |= value << (24 - (i % 4) * 8); + } + for (let i = 16; i < 64; i++) { + const x = words[i - 15], y = words[i - 2]; + words[i] = words[i - 16] + (rotate(x,7)^rotate(x,18)^(x>>>3)) + + words[i - 7] + (rotate(y,17)^rotate(y,19)^(y>>>10)); + } + let [a,b,c,d,e,f,g,j] = h; + for (let i = 0; i < 64; i++) { + const t = (j + (rotate(e,6)^rotate(e,11)^rotate(e,25)) + ((e&f)^(~e&g)) + k[i] + words[i]) | 0; + const u = ((rotate(a,2)^rotate(a,13)^rotate(a,22)) + ((a&b)^(a&c)^(b&c))) | 0; + j=g; g=f; f=e; e=(d+t)|0; d=c; c=b; b=a; a=(t+u)|0; + } + [a,b,c,d,e,f,g,j].forEach((value, i) => { h[i] = (h[i] + value) >>> 0; }); + } + return h.map(value => value.toString(16).padStart(8, "0")).join(""); +} -/* The header of a container, from its first bytes. Parsed separately from the - * body so a download that has only started can already say how many frames are - * coming, which is what a progress count needs. */ -function readSequenceHeader(buffer) { - const bytes = new Uint8Array(buffer); - if (bytes.length < SEQ_PREAMBLE) throw new Error("sequence header truncated"); - let magic = ""; - for (let i = 0; i < 8; i++) magic += String.fromCharCode(bytes[i]); - if (magic !== SEQ_MAGIC) throw new Error("not a sequence container"); +function vmeshJSON(text) { + const value = JSON.parse(text), stack = []; + for (const token of text.match(/"(?:[^"\\]|\\.)*"|[{}\[\],:]/g) || []) { + if (token === "{") stack.push({keys:new Set(), key:true}); + else if (token === "[") stack.push(null); + else if (token === "}" || token === "]") stack.pop(); + else if (token === ":") stack[stack.length - 1].key = false; + else if (token === ",") { + const current = stack[stack.length - 1]; + if (current) current.key = true; + } else { + const current = stack[stack.length - 1]; + if (current && current.key) { + const key = JSON.parse(token); + if (current.keys.has(key)) throw new Error("duplicate VMESH JSON key"); + current.keys.add(key); + } + } + } + return value; +} + +function vmeshRecord(buffer, at, kind, fileId, offset, length) { + if (at < 0 || at + 18 + length > buffer.byteLength) throw new Error("VMESH record truncated"); const view = new DataView(buffer); - const version = view.getUint32(8, true); - if (version !== 1) throw new Error(`sequence version ${version} is not 1`); - const count = view.getUint32(12, true); - const suffixLength = view.getUint32(16, true); - let suffix = ""; - for (let i = 0; i < suffixLength; i++) { - suffix += String.fromCharCode(bytes[SEQ_PREAMBLE + i]); - } - const table = SEQ_PREAMBLE + suffixLength; - if (bytes.length < table + 8 * count) throw new Error("sequence table truncated"); + const position = Number(view.getBigUint64(at + 10)); + if (!Number.isSafeInteger(position) || view.getUint32(at) !== 14 + length + || view.getUint8(at + 4) !== 1 || view.getUint8(at + 5) !== kind + || view.getUint32(at + 6) !== fileId || position !== offset) { + throw new Error("invalid VMESH record version, kind, file ID, offset or length"); + } +} + +/* The manifest is enough to calculate frame record ranges before body arrival. */ +function readSequenceHeader(buffer) { + const bytes = new Uint8Array(buffer), view = new DataView(buffer); + if (bytes.length < SEQ_PREAMBLE) throw new Error("VMESH manifest truncated"); + if (String.fromCharCode(...bytes.subarray(0, 8)) !== SEQ_MAGIC) throw new Error("not a VMESH container"); + const length = view.getUint32(8) - 14; + if (length <= 0 || length > 16777216) throw new Error("VMESH manifest exceeds limits"); + vmeshRecord(buffer, 8, 0, 0, 0, length); + const manifestBytes = bytes.subarray(SEQ_PREAMBLE, SEQ_PREAMBLE + length); + let manifest; + try { manifest = vmeshJSON(new TextDecoder("utf-8", {fatal:true}).decode(manifestBytes)); } + catch (_) { throw new Error("invalid VMESH manifest JSON"); } + const object = value => value !== null && typeof value === "object" && !Array.isArray(value); + const finiteJSON = value => { + if (typeof value === "number" && !Number.isFinite(value)) throw new Error("non-finite VMESH metadata"); + if (value !== null && typeof value === "object") Object.values(value).forEach(finiteJSON); + }; + finiteJSON(manifest); + if (!object(manifest)) throw new Error("VMESH manifest must be an object"); + const native = manifest.native, count = manifest.frame_count, sequence = manifest.sequence; + const suffixes = ["ply", "drc", "splat", "jpg", "jpeg", "png"]; + if (!object(manifest) || manifest.schema !== "vmesh/1" || manifest.codec !== "frames" + || manifest.native_version !== 1 || manifest.representation !== "frame_payloads" + || manifest.dependency_mode !== "independent" || !Number.isInteger(count) || count < 1 || count > 65536 + || !object(native) || native.profile !== "frames/1" || !suffixes.includes(native.suffix) + || Object.keys(native).sort().join(",") !== "profile,representation,suffix" + || !({mesh:["ply","drc"], points:["ply","drc"], gaussians:["ply","splat"], pixels:["jpg","jpeg","png"]}[native.representation] || []).includes(native.suffix) + || !Array.isArray(manifest.files) || manifest.files.length !== count + || !object(sequence) || Object.hasOwn(sequence, "schema") || sequence.version !== 1 || sequence.codec !== "frames" + || !object(sequence.native) || Object.keys(sequence.native).sort().join(",") !== "profile,representation,suffix" + || sequence.native.profile !== native.profile || sequence.native.suffix !== native.suffix + || sequence.native.representation !== native.representation + || !Array.isArray(sequence.frames) || sequence.frames.length !== count + || (sequence.allow_nonmonotonic_timestamps !== undefined && typeof sequence.allow_nonmonotonic_timestamps !== "boolean") + || (sequence.metadata !== undefined && !object(sequence.metadata)) + || ["has_constant_vertex_count", "has_vertex_correspondence"].some(name => + sequence[name] !== undefined && sequence[name] !== null && typeof sequence[name] !== "boolean")) { + throw new Error("unsupported VMESH browser frame profile"); + } const entries = []; + let offset = SEQ_PREAMBLE + length, previous = null; for (let i = 0; i < count; i++) { - entries.push({ - offset: view.getUint32(table + 8 * i, true), - length: view.getUint32(table + 8 * i + 4, true), - }); + const file = manifest.files[i], frame = sequence.frames[i]; + if (!object(file) || file.id !== i || file.name !== `frame_${String(i).padStart(6,"0")}.${native.suffix}` + || file.role !== "frame-payload" || !Number.isSafeInteger(file.size) || file.size <= 0 + || typeof file.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(file.sha256) + || !object(frame) || !Number.isSafeInteger(frame.frame_index) || frame.frame_index < 0 + || typeof frame.timestamp !== "number" || !Number.isFinite(frame.timestamp) + || (previous !== null && !sequence.allow_nonmonotonic_timestamps && frame.timestamp < previous) + || (frame.metadata !== undefined && !object(frame.metadata))) { + throw new Error("invalid VMESH frame metadata or payload descriptor"); + } + entries.push({offset, length:file.size, sha256:file.sha256}); + offset += file.size + 18 * Math.ceil(file.size / VMESH_CHUNK); + if (!Number.isSafeInteger(offset + 50)) throw new Error("VMESH offsets exceed browser integer limit"); + previous = frame.timestamp; } - return { suffix, entries }; + return {suffix:native.suffix, representation:native.representation, entries, endOffset:offset, + manifestHash:vmeshSHA256(manifestBytes)}; } -/* One frame's bytes, as its own ArrayBuffer. - * - * Sliced rather than viewed: the decode worker takes ownership of what it is - * given, and handing it a view into the container would transfer the whole - * container on the first frame and leave the rest unreachable. */ +function validateSequenceEnd(buffer, header) { + if (buffer.byteLength !== header.endOffset + 50) throw new Error("VMESH is truncated or has trailing data"); + vmeshRecord(buffer, header.endOffset, 2, 0, 0, 32); + const digest = new Uint8Array(buffer, header.endOffset + 18, 32); + const hex = Array.from(digest, byte => byte.toString(16).padStart(2,"0")).join(""); + if (hex !== header.manifestHash) throw new Error("VMESH end manifest SHA-256 mismatch"); +} + +/* Assemble chunk records into an owned buffer, then verify before decoding. */ function sequenceFrame(buffer, header, index) { + if (!Number.isInteger(index) || index < 0 || index >= header.entries.length) throw new Error("VMESH frame index out of range"); const entry = header.entries[index]; - return buffer.slice(entry.offset, entry.offset + entry.length); + if (entry.offset + entry.length + 18 * Math.ceil(entry.length / VMESH_CHUNK) > buffer.byteLength) { + throw new Error("VMESH frame payload truncated"); + } + const output = new Uint8Array(entry.length); + let at = entry.offset, copied = 0; + while (copied < entry.length) { + const length = Math.min(VMESH_CHUNK, entry.length - copied); + vmeshRecord(buffer, at, 1, index, copied, length); + output.set(new Uint8Array(buffer, at + 18, length), copied); + at += 18 + length; + copied += length; + } + if (vmeshSHA256(output) !== entry.sha256) throw new Error("VMESH frame SHA-256 mismatch"); + return output.buffer; } /* ------------------------------------------------------------- adaptation --- @@ -1445,18 +1570,22 @@

session

const fetched = performance.now(); this.stats.bytes += buffer.byteLength; const header = readSequenceHeader(buffer); + validateSequenceEnd(buffer, header); if (header.entries.length !== total) { throw new Error(`${url}: holds ${header.entries.length} frames,` + ` clip lists ${total}`); } // The frame list survives packing, and this is one reason it has to: a // frame is decoded by the codec its *own* name selects. Handing the - // container's name to the decode instead is a `.seq` suffix in a table of + // container's name to the decode instead is a `.vmesh` suffix in a table of // `.ply` and `.splat`, which is how every Gaussian pane went blank -- and // silently, because pixels are decoded from the media type on the main // thread and were unaffected, so it looked like one method being broken // rather than one code path. const names = this.frames; + if (this.clip.representation && this.clip.representation !== header.representation) { + throw new Error(`${url}: VMESH representation disagrees with clip`); + } if (suffixOf(names[0]) !== `.${header.suffix}`) { throw new Error(`${url}: holds .${header.suffix} frames, clip lists` + ` ${suffixOf(names[0])} -- one of them would be decoded wrongly`); @@ -1489,7 +1618,9 @@

session

*/ async _readPacked(response, expected, total, onProgress) { if (!response.body || !response.body.getReader) { - return response.arrayBuffer(); + const buffer = await response.arrayBuffer(); + if (buffer.byteLength > this.byteBudget) throw new Error("VMESH download exceeds byte budget"); + return buffer; } const reader = response.body.getReader(); const chunks = []; @@ -1498,8 +1629,12 @@

session

const { done, value } = await reader.read(); if (done) break; if (this.abandoned) { reader.cancel().catch(() => {}); return null; } - chunks.push(value); got += value.byteLength; + if (got > this.byteBudget) { + reader.cancel().catch(() => {}); + throw new Error("VMESH download exceeds byte budget"); + } + chunks.push(value); if (onProgress && expected) { // Bytes shown as frames, because frames are the unit the caller counts // and the two phases have to share one scale to make a bar that only @@ -2775,7 +2910,7 @@

session

* rejection: the pane stayed blank, nothing was logged where anyone * would look, and the page went on insisting it was fine. That is how a * decoder error reads as "this method is broken" -- which is what - * happened when every Gaussian frame was handed the container's `.seq` + * happened when every Gaussian frame was handed the container's `.vmesh` * name instead of its own. A pane that cannot show its content has to * say why. */ if (token !== app.downloadToken) return; diff --git a/open4d/streamer/streamer/codecs.py b/open4d/streamer/streamer/codecs.py index 4c6fc5b6..858a5ec4 100644 --- a/open4d/streamer/streamer/codecs.py +++ b/open4d/streamer/streamer/codecs.py @@ -5,12 +5,11 @@ at all: in what format do its frames travel, and can the client turn them back into that representation? -Until this existed the two questions were conflated, and the cost was concrete. -Open4D's codecs produce ``.d4d``, ``.v4d``, ``.q4d`` and six more; the client -could decode ``.ply``, ``.splat``, ``.jpg`` and ``.png``. The two sets did not -intersect at all, and nothing in the code said so — a producer found out by -watching a pane stay blank. A registry that names the wire format and where it -decodes makes that a lookup instead of a discovery. +Research sequences use ``.vmesh``; the manifest identifies their codec and +native payload dependencies. The browser decodes whole-frame ``.ply``, +``.drc``, ``.splat``, ``.jpg`` and ``.png`` payloads, including those carried +by VMESH's ``frames/1`` profile. Other research profiles need their native +server runtime. The registry makes that distinction explicit. **The key is (representation, suffix), not suffix.** ``.ply`` is claimed by three representations here and needs two different parsers: a 3DGS PLY and a mesh PLY @@ -29,8 +28,8 @@ ReRF's entropy coder exists only as a CPython 3.8 binary. Those are decoded and rendered where the GPU is, and what reaches the client is ``pixels``. That is the codec's output representation as the platform sees it, which is - why ``.rerf`` is registered as producing ``pixels`` rather than a volume it - never delivers. + why ReRF VMESH is registered for server-side rendering into ``pixels``. + Its field payload is selected by the VMESH manifest. """ from __future__ import annotations @@ -257,7 +256,7 @@ def media_types() -> dict[str, str]: )) register(CodecSpec( name="rerf", - suffix=".rerf", + suffix=".vmesh", representation=Representation.PIXELS, decodes="server", lossy=True, diff --git a/open4d/streamer/streamer/sequence.py b/open4d/streamer/streamer/sequence.py index 53c4c19f..e6b5f759 100644 --- a/open4d/streamer/streamer/sequence.py +++ b/open4d/streamer/streamer/sequence.py @@ -1,61 +1,51 @@ -"""A whole clip as one file. - -A clip is a directory of frames, and fetching it a frame at a time is how this -started: thirty requests for thirty frames. On loopback that is free, which is -why it survived. Over a link it is thirty round trips before anything plays -- -600 ms of pure latency on a 20 ms connection -- and it is thirty chances for -one frame to arrive late and stall a clip that was otherwise complete. - -For **on demand** none of that buys anything. A viewer with a free camera has -to hold the whole sequence in memory anyway, because a frame it has thrown away -cannot be redrawn from a new angle. So the unit of transfer should be the -sequence, and there is no reason to have asked for it in pieces. - -Hence a container: a header naming the frames, then their bytes end to end. One -request, one response, one progress bar. Deliberately not a zip or a tar -- -both would need a decoder in the client and neither buys anything here, since -the frames are already compressed and an archive's own compression would only -spend CPU to save nothing. - - header magic "O4DSEQ\\0\\0" | version | frame count | suffix - then per frame: offset, length -- both uint32 little-endian - body each frame's bytes, in playback order, unpadded - -The offsets are absolute within the file, so a client that has the header can -slice any frame out without walking the ones before it -- which is what lets a -partially arrived download start playing from the beginning while the rest -lands. +"""Whole browser clips in standalone VMESH with native frame payloads. + +The ``frames/1`` profile carries PLY, Draco, splat or image bytes verbatim. +VMESH owns the manifest, timestamps, record framing and SHA-256 integrity. +A manifest prefix is enough to locate any frame's chunk records, so downloads +and HTTP byte ranges retain stable offsets without a second private format. """ from __future__ import annotations import dataclasses +import hashlib +from io import BytesIO +import json +import math import struct +import tempfile from dataclasses import dataclass from pathlib import Path -from typing import Sequence +from typing import Sequence as TypingSequence +from open4d._files import publish_directory, publish_file +from open4d.codec import _vmesh_format as vmesh +from open4d.codec._protocol import CodecError from . import representations -MAGIC = b"O4DSEQ\x00\x00" +MAGIC = vmesh._MAGIC VERSION = 1 -#: ``magic + version + count + suffix length``, before the suffix itself. -_PREAMBLE = struct.calcsize("<8sIII") +_SUFFIXES = frozenset(("ply", "drc", "splat", "jpg", "jpeg", "png")) +_MAX_SAFE_INTEGER = 2**53 - 1 @dataclass(frozen=True) class Entry: - """Where one frame sits inside the container.""" + """First VMESH payload record, native byte count and native SHA-256.""" offset: int length: int + sha256: str @dataclass(frozen=True) class Sequence: - """A container's header: what is in it and where.""" + """A validated VMESH frame manifest, usable before its payload arrives.""" suffix: str entries: tuple[Entry, ...] + end_offset: int + manifest_hash: bytes @property def frames(self) -> int: @@ -66,99 +56,221 @@ def bytes(self) -> int: return sum(entry.length for entry in self.entries) -def pack(frames: Sequence[Path | str], destination: Path | str) -> Sequence: - """Write ``frames`` into one container at ``destination``. +def _fps(value): + if type(value) not in (int, float) or not math.isfinite(value) or value <= 0: + raise ValueError("fps must be positive and finite") + return value - Every frame must share a suffix. A container holding two formats would need - the client to switch decoders mid-sequence, and a clip whose frames are not - all one codec is a clip that should have been two. - """ - paths = [Path(f) for f in frames] + +def _header(descriptor, manifest_hash, start): + if descriptor["codec"] != "frames": + raise ValueError("VMESH is not a browser frame-payload profile") + entries, offset = [], start + for record in descriptor["files"]: + length = record["size"] + entries.append(Entry(offset, length, record["sha256"])) + offset += length + 18 * ((length + vmesh._CHUNK - 1) // vmesh._CHUNK) + if offset + 50 > _MAX_SAFE_INTEGER: + raise ValueError("VMESH frame offsets exceed the browser integer limit") + return Sequence(descriptor["native"]["suffix"], tuple(entries), offset, manifest_hash) + + +def pack(frames: TypingSequence[Path | str], destination: Path | str, *, + fps: float = 30, representation: str | None = None, + overwrite: bool = False) -> Sequence: + """Write native whole frames into one .vmesh without recompression.""" + destination = Path(destination) + if destination.suffix.lower() != ".vmesh": + raise ValueError("destination must have a .vmesh extension") + if destination.exists() and not overwrite: + raise FileExistsError(destination) + if not isinstance(overwrite, bool): + raise TypeError("overwrite must be bool") + fps = _fps(fps) + paths = [Path(path) for path in frames] if not paths: raise ValueError("a sequence needs at least one frame") + if len(paths) > 65536: + raise ValueError("VMESH frame count exceeds limits") suffixes = {path.suffix.lower().lstrip(".") for path in paths} if len(suffixes) != 1: - raise ValueError( - f"frames must share one suffix; got {', '.join(sorted(suffixes))}" - ) + raise ValueError(f"frames must share one suffix; got {', '.join(sorted(suffixes))}") suffix = suffixes.pop() - missing = [str(path) for path in paths if not path.is_file()] - if missing: - raise FileNotFoundError(f"{len(missing)} frame(s) missing, e.g. {missing[0]}") - - encoded = suffix.encode() - header = _PREAMBLE + len(encoded) + 8 * len(paths) - entries, offset = [], header - for path in paths: - length = path.stat().st_size - entries.append(Entry(offset=offset, length=length)) - offset += length - - destination = Path(destination) + if suffix not in _SUFFIXES: + raise ValueError(f"unsupported browser frame suffix {suffix!r}") + if representation is None: + representation = "pixels" if suffix in ("jpg", "jpeg", "png") else "gaussians" if suffix == "splat" else "mesh" + native = dict(profile="frames/1", suffix=suffix, representation=representation) + metadata = dict(version=1, codec="frames", native=native, metadata={}, + frames=[dict(frame_index=i, timestamp=i / fps, metadata={}) + for i in range(len(paths))]) + descriptor = dict(schema="vmesh/1", codec="frames", native_version=1, + representation="frame_payloads", dependency_mode="independent", + frame_count=len(paths), native=native, sequence=metadata, files=[]) + for index, path in enumerate(paths): + if path.is_symlink() or not path.is_file(): + raise FileNotFoundError(f"missing or linked frame: {path}") + digest, size = hashlib.sha256(), 0 + with path.open("rb") as stream: + while chunk := stream.read(vmesh._CHUNK): + digest.update(chunk) + size += len(chunk) + descriptor["files"].append(dict(id=index, name=f"frame_{index:06d}.{suffix}", + role="frame-payload", size=size, + sha256=digest.hexdigest())) + manifest = json.dumps(descriptor, separators=(",", ":"), allow_nan=False).encode() + try: + vmesh._descriptor(manifest) + except CodecError as error: + raise ValueError(str(error)) from error + if len(manifest) > vmesh._MAX_JSON: + raise ValueError("VMESH manifest exceeds size limit") + manifest_hash = hashlib.sha256(manifest).digest() + header = _header(descriptor, manifest_hash, 26 + len(manifest)) destination.parent.mkdir(parents=True, exist_ok=True) - with open(destination, "wb") as out: - out.write(struct.pack("<8sIII", MAGIC, VERSION, len(paths), len(encoded))) - out.write(encoded) - for entry in entries: - out.write(struct.pack(" Sequence: - """The header of a container, from its first bytes. - - Takes bytes rather than a path so a client that has only the start of a - download can already know what is coming -- which is what a progress bar - counting frames needs. - """ - if len(data) < _PREAMBLE: - raise ValueError("not enough bytes for a sequence header") - magic, version, count, suffix_length = struct.unpack_from("<8sIII", data) - if magic != MAGIC: - raise ValueError(f"not a sequence container: magic is {magic!r}") - if version != VERSION: - raise ValueError(f"sequence version {version} is not {VERSION}") - at = _PREAMBLE - suffix = data[at:at + suffix_length].decode() - at += suffix_length - needed = at + 8 * count - if len(data) < needed: - raise ValueError(f"header needs {needed} bytes, got {len(data)}") - entries = tuple( - Entry(*struct.unpack_from(" bytes: - """One frame's bytes out of a container.""" + """Recover and hash-check one frame's VMESH chunk records.""" header = header or read_header(data) + if type(index) is not int or not 0 <= index < header.frames: + raise IndexError("frame index out of range") entry = header.entries[index] - return data[entry.offset:entry.offset + entry.length] + stream, output, digest = BytesIO(data), bytearray(), hashlib.sha256() + stream.seek(entry.offset) + try: + while len(output) < entry.length: + kind, file_id, offset, chunk = vmesh._read_record(stream) + length = min(vmesh._CHUNK, entry.length - len(output)) + if (kind, file_id, offset, len(chunk)) != (1, index, len(output), length): + raise ValueError("invalid VMESH frame chunk order or length") + digest.update(chunk) + output.extend(chunk) + except CodecError as error: + raise ValueError(str(error)) from error + if digest.hexdigest() != entry.sha256: + raise ValueError("VMESH frame SHA-256 mismatch") + return bytes(output) def unpack(path: Path | str, destination: Path | str) -> list[Path]: - """Write a container back out as numbered frame files. - - The inverse, for checking a container round-trips and for anyone who wants - the frames on disk again. - """ - data = Path(path).read_bytes() - header = read_header(data) + """Validate the complete VMESH before publishing recovered native frames.""" destination = Path(destination) - destination.mkdir(parents=True, exist_ok=True) - written = [] - for index in range(header.frames): - target = destination / f"frame_{index:04d}.{header.suffix}" - target.write_bytes(frame(data, index, header)) - written.append(target) - return written - - -def pack_clip(bundle_dir, clip, *, keep_frames: bool = False) -> dict: + if destination.exists(): + raise FileExistsError(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + try: + descriptor = vmesh.inspect_vmesh(path) + if descriptor["codec"] != "frames": + raise ValueError("VMESH is not a browser frame-payload profile") + with tempfile.TemporaryDirectory(prefix=f".{destination.name}-", dir=destination.parent) as directory: + native, output = Path(directory) / "native", Path(directory) / "frames" + vmesh.unpack_vmesh(path, native) + output.mkdir() + names = [record["name"] for record in descriptor["files"]] + for name in names: + (native / name).replace(output / name) + publish_directory(output, destination) + return [destination / name for name in names] + except CodecError as error: + raise ValueError(str(error)) from error + + +def convert_legacy(source: Path | str, destination: Path | str, *, + fps: float = 30, representation: str | None = None, + overwrite: bool = False) -> Path: + """Explicitly migrate a bounded retired O4DSEQ clip to real VMESH records.""" + source, destination = Path(source), Path(destination) + if destination.suffix.lower() != ".vmesh": + raise ValueError("destination must have a .vmesh extension") + if not isinstance(overwrite, bool): + raise TypeError("overwrite must be bool") + if destination.exists() and not overwrite: + raise FileExistsError(destination) + fps = _fps(fps) + with source.open("rb") as stream: + preamble = stream.read(20) + if len(preamble) != 20: + raise ValueError("truncated legacy sequence header") + magic, version, count, suffix_length = struct.unpack("<8sIII", preamble) + if magic != b"O4DSEQ\x00\x00" or version != 1: + raise ValueError("unsupported legacy sequence magic/version") + if not 0 < count <= 65536 or not 0 < suffix_length <= 8: + raise ValueError("legacy sequence header exceeds limits") + try: + suffix = stream.read(suffix_length).decode("ascii") + except UnicodeError as error: + raise ValueError("invalid legacy sequence suffix") from error + if suffix not in _SUFFIXES: + raise ValueError("unsupported legacy frame suffix") + table = stream.read(8 * count) + if len(table) != 8 * count: + raise ValueError("truncated legacy sequence table") + offset = 20 + suffix_length + 8 * count + records = [] + for index in range(count): + position, length = struct.unpack_from(" dict: """Pack one clip's frames into a container beside the bundle root. Returns the mapping a `bundle.Clip` records as its ``sequence``. The frame @@ -167,8 +279,12 @@ def pack_clip(bundle_dir, clip, *, keep_frames: bool = False) -> dict: pieces. """ root = Path(bundle_dir) - relative = f"{clip.name}.seq" - header = pack([root / frame for frame in clip.frames], root / relative) + if not clip.name or clip.name in (".", "..") or any(c in clip.name for c in ("/", "\\", "\x00")): + raise ValueError("clip name must be a safe filename") + paths = _clip_paths(root, clip.frames) + relative = f"{clip.name}.vmesh" + header = pack(paths, root / relative, fps=fps, representation=clip.representation, + overwrite=True) if not keep_frames: directories = {Path(frame).parts[0] for frame in clip.frames} for frame in clip.frames: @@ -196,7 +312,7 @@ def pack_clip(bundle_dir, clip, *, keep_frames: bool = False) -> dict: def pack_bundle( bundle_dir: Path | str, *, - names: Sequence[str] | None = None, + names: TypingSequence[str] | None = None, keep_frames: bool = False, ) -> list[tuple[str, dict]]: """Pack a bundle's clips into containers and rewrite its manifest. @@ -204,7 +320,9 @@ def pack_bundle( Live clips are skipped rather than refused: a bundle is usually a mix, and a whole-bundle command that failed on the first stream would be unusable on exactly the bundles that have both. Clips already packed are skipped too, - so running this twice is not an error. + so running this twice is not an error. Retired packed clips are explicitly + converted and their URLs updated; originals are removed after the manifest + commits unless ``keep_frames`` is set. """ from . import bundle as _bundle @@ -220,12 +338,12 @@ def pack_bundle( if missing: raise KeyError(f"no clip named {', '.join(sorted(missing))}") - packed, changed = [], [] + packed, changed, retired = [], [], [] for clip in clips: skip = ( (wanted is not None and clip.name not in wanted) or clip.stream is not None - or clip.sequence is not None + or (clip.sequence is not None and not str(clip.sequence.get("url", "")).endswith(".seq")) or not clip.frames ) if skip: @@ -233,7 +351,31 @@ def pack_bundle( continue # Do not remove inputs until every clip is packed and the manifest # has committed. A later codec or filesystem failure must be retryable. - entry = pack_clip(root, clip, keep_frames=True) + if clip.sequence is not None: + _clip_paths(root, clip.frames) + old = Path(clip.sequence["url"]) + if old.is_absolute() or ".." in old.parts or "\\" in str(old): + raise ValueError("legacy sequence must use a safe relative path") + source = root / old + if source.is_symlink() or not source.resolve().is_relative_to(root.resolve()): + raise ValueError("legacy sequence must stay inside the bundle") + relative = str(old.with_suffix(".vmesh")) + target = root / relative + convert_legacy(source, target, fps=index.get("fps", 30), + representation=clip.representation, overwrite=True) + with target.open("rb") as stream: + descriptor, digest = vmesh._start(stream) + header = _header(descriptor, digest, stream.tell()) + if header.frames != len(clip.frames): + raise ValueError("legacy sequence disagrees with its logical frame list") + entry = dict(clip.sequence, url=relative, frames=header.frames, + bytes=target.stat().st_size, suffix=header.suffix) + media_type = representations.media_types().get(f".{header.suffix}") + if media_type: + entry["media_type"] = media_type + retired.append(source) + else: + entry = pack_clip(root, clip, keep_frames=True, fps=index.get("fps", 30)) changed.append(dataclasses.replace(clip, sequence=entry)) packed.append((clip.name, entry)) @@ -256,6 +398,9 @@ def pack_bundle( for path in clip.frames} - retained for path in removable: (root / path).unlink(missing_ok=True) + for path in set(retired): + if path.relative_to(root).as_posix() not in retained: + path.unlink() for directory in {root / Path(path).parts[0] for path in removable}: if directory.is_dir() and not any(directory.iterdir()): directory.rmdir() @@ -293,5 +438,68 @@ def main(argv=None) -> int: return 0 +class _MeshFrames: + """Keep validated native payloads alive for lazy canonical mesh decoding.""" + + def __init__(self, native, manifest, temporary): + from types import MappingProxyType + from open4d.core import TopologyMode + self.native, self.manifest, self.temporary = native, manifest, temporary + self.records = manifest["sequence"]["frames"] + self.metadata = MappingProxyType(manifest["sequence"].get("metadata", {})) + self.topology = TopologyMode(manifest["sequence"].get("topology", "unknown")) + self.has_constant_vertex_count = manifest["sequence"].get("has_constant_vertex_count") + self.has_vertex_correspondence = manifest["sequence"].get("has_vertex_correspondence") + self.allow_nonmonotonic_timestamps = manifest["sequence"].get("allow_nonmonotonic_timestamps", False) + + @property + def frame_count(self): + return len(self.records) + + @property + def timestamps(self): + return tuple(record["timestamp"] for record in self.records) + + def get_frame(self, index): + from open4d.core import Frame, TriangleMesh + if not 0 <= index < self.frame_count: + raise IndexError("frame index out of range") + record = self.records[index] + path = self.native / self.manifest["files"][index]["name"] + if self.manifest["native"]["suffix"] == "ply": + from open4d.io._mesh import read_ply + positions, triangles, colors = read_ply(path) + geometry = TriangleMesh(positions, triangles, colors=colors) + else: + from open4d.codec._draco import _backend + decoded = _backend().decode(path.read_bytes()) + colors = decoded.colors.astype("float32") / 255 if decoded.colors is not None else None + geometry = TriangleMesh(decoded.points, decoded.faces, colors=colors, + normals=decoded.normals, texture_coordinates=decoded.tex_coord) + return Frame(record["frame_index"], record["timestamp"], geometry, record.get("metadata", {})) + + def close(self): + self.temporary.cleanup() + + +def open_frames(path: Path | str): + """Open mesh payloads lazily, or retain other browser payloads natively.""" + from open4d.core import Sequence as MeshSequence + from open4d.native import NativeSequence + manifest = vmesh.inspect_vmesh(path) + if manifest["codec"] != "frames": + raise CodecError("VMESH is not a browser frame-payload profile") + if manifest["native"]["representation"] != "mesh": + return NativeSequence(path) + temporary = tempfile.TemporaryDirectory(prefix="vmesh-browser-mesh-") + try: + native = Path(temporary.name) / "native" + vmesh.unpack_vmesh(path, native) + return MeshSequence(_MeshFrames(native, manifest, temporary)) + except BaseException: + temporary.cleanup() + raise + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/open4d/streamer/streamer/server/__init__.py b/open4d/streamer/streamer/server/__init__.py index 19949e78..66c2b7f8 100644 --- a/open4d/streamer/streamer/server/__init__.py +++ b/open4d/streamer/streamer/server/__init__.py @@ -218,7 +218,7 @@ class _Handler(http.server.SimpleHTTPRequestHandler): # A packed clip is opaque: the frames inside keep their own type, which # the manifest names, so guessing one for the container would only be # wrong. See streamer/sequence.py. - ".seq": "application/octet-stream", + ".vmesh": "application/octet-stream", } def do_GET(self): # noqa: N802 - the base class names it diff --git a/open4d/streamer/streamer_tests/test_codecs.py b/open4d/streamer/streamer_tests/test_codecs.py index d931a7ee..08cce749 100644 --- a/open4d/streamer/streamer_tests/test_codecs.py +++ b/open4d/streamer/streamer_tests/test_codecs.py @@ -61,7 +61,7 @@ def test_by_name_finds_a_codec(): def test_rerf_is_registered_as_server_decoded(): """Not an omission: its entropy coder is a sourceless CPython 3.8 binary.""" - spec = codecs.for_frame("pixels", ".rerf") + spec = codecs.for_frame("pixels", ".vmesh") assert spec.decodes == "server" assert "browser" in spec.cost @@ -176,14 +176,14 @@ def test_adding_a_codec_is_one_registration(monkeypatch): codecs.register( CodecSpec( name="mesh-vdmc", - suffix=".v4d", + suffix=".vmesh", representation=Representation.MESH, lossy=True, cost="V-DMC is lossy at any rate worth using", ) ) - assert codecs.for_frame("mesh", ".v4d").name == "mesh-vdmc" - assert ".v4d" in codecs.media_types() + assert codecs.for_frame("mesh", ".vmesh").name == "mesh-vdmc" + assert ".vmesh" in codecs.media_types() assert "mesh-vdmc" in {spec.name for spec in codecs.client_decodable("mesh")} diff --git a/open4d/streamer/streamer_tests/test_representations.py b/open4d/streamer/streamer_tests/test_representations.py index 530eb129..db77d107 100644 --- a/open4d/streamer/streamer_tests/test_representations.py +++ b/open4d/streamer/streamer_tests/test_representations.py @@ -75,11 +75,11 @@ def test_registering_a_codec_changes_what_a_representation_serves(monkeypatch): monkeypatch.setattr(codecs, "_REGISTRY", dict(codecs._REGISTRY)) codecs.register( codecs.CodecSpec( - name="mesh-vdmc", suffix=".v4d", representation=Representation.MESH + name="mesh-vdmc", suffix=".vmesh", representation=Representation.MESH ) ) - assert ".v4d" in representations.spec("mesh").media_types - assert ".v4d" in representations.media_types() + assert ".vmesh" in representations.spec("mesh").media_types + assert ".vmesh" in representations.media_types() def test_a_representation_exposes_its_codecs(): diff --git a/open4d/streamer/streamer_tests/test_sequence.py b/open4d/streamer/streamer_tests/test_sequence.py index 512e5119..5e3fdacf 100644 --- a/open4d/streamer/streamer_tests/test_sequence.py +++ b/open4d/streamer/streamer_tests/test_sequence.py @@ -54,53 +54,51 @@ def _frames(directory: Path, sizes: list[int], suffix: str = "splat") -> list[Pa def test_a_packed_container_round_trips_every_frame(tmp_path): frames = _frames(tmp_path / "clip", [10, 300, 7, 4096]) - header = sequence.pack(frames, tmp_path / "clip.seq") + header = sequence.pack(frames, tmp_path / "clip.vmesh") assert header.frames == 4 assert header.suffix == "splat" assert header.bytes == 10 + 300 + 7 + 4096 - data = (tmp_path / "clip.seq").read_bytes() + data = (tmp_path / "clip.vmesh").read_bytes() for index, path in enumerate(frames): assert sequence.frame(data, index, header) == path.read_bytes() -def test_offsets_are_absolute_and_leave_no_gaps(tmp_path): +def test_offsets_name_real_vmesh_records(tmp_path): frames = _frames(tmp_path / "clip", [10, 300, 7]) - header = sequence.pack(frames, tmp_path / "clip.seq") - size = (tmp_path / "clip.seq").stat().st_size - - # Absolute, so a client with the header can slice any frame without having - # walked the ones before it -- which is what lets a partial download play - # from the beginning. + header = sequence.pack(frames, tmp_path / "clip.vmesh") + data = (tmp_path / "clip.vmesh").read_bytes() + assert data[:8] == b"VMESH\x00\x01\x00" first = header.entries[0] - assert first.offset == size - header.bytes + assert first.offset == 26 + struct.unpack_from(">I", data, 8)[0] - 14 for before, after in zip(header.entries, header.entries[1:]): - assert before.offset + before.length == after.offset + assert before.offset + before.length + 18 == after.offset last = header.entries[-1] - assert last.offset + last.length == size + assert last.offset + last.length + 18 == header.end_offset + assert header.end_offset + 50 == len(data) def test_a_header_read_back_matches_the_one_written(tmp_path): frames = _frames(tmp_path / "clip", [10, 300, 7]) - written = sequence.pack(frames, tmp_path / "clip.seq") - read = sequence.read_header((tmp_path / "clip.seq").read_bytes()) + written = sequence.pack(frames, tmp_path / "clip.vmesh") + read = sequence.read_header((tmp_path / "clip.vmesh").read_bytes()) assert read == written def test_the_header_can_be_read_before_the_body_arrives(tmp_path): frames = _frames(tmp_path / "clip", [4096, 4096, 4096]) - written = sequence.pack(frames, tmp_path / "clip.seq") + written = sequence.pack(frames, tmp_path / "clip.vmesh") # The point of a fixed header: a progress count knows how many frames are # coming from the first few hundred bytes, not the last. - prefix = (tmp_path / "clip.seq").read_bytes()[: written.entries[0].offset] + prefix = (tmp_path / "clip.vmesh").read_bytes()[: written.entries[0].offset] assert sequence.read_header(prefix) == written def test_unpacking_restores_the_frames(tmp_path): frames = _frames(tmp_path / "clip", [10, 300, 7]) - sequence.pack(frames, tmp_path / "clip.seq") - restored = sequence.unpack(tmp_path / "clip.seq", tmp_path / "out") + sequence.pack(frames, tmp_path / "clip.vmesh") + restored = sequence.unpack(tmp_path / "clip.vmesh", tmp_path / "out") assert [path.read_bytes() for path in restored] == [ path.read_bytes() for path in frames ] @@ -111,7 +109,7 @@ def test_unpacking_restores_the_frames(tmp_path): def test_a_container_needs_a_frame(tmp_path): with pytest.raises(ValueError, match="at least one frame"): - sequence.pack([], tmp_path / "clip.seq") + sequence.pack([], tmp_path / "clip.vmesh") def test_frames_must_share_a_suffix(tmp_path): @@ -121,30 +119,30 @@ def test_frames_must_share_a_suffix(tmp_path): # Two formats in one container would need the client to switch decoders # mid-clip. A clip whose frames are not one codec is two clips. with pytest.raises(ValueError, match="share one suffix"): - sequence.pack(frames + [odd], tmp_path / "clip.seq") + sequence.pack(frames + [odd], tmp_path / "clip.vmesh") def test_a_missing_frame_is_named(tmp_path): frames = _frames(tmp_path / "clip", [10, 10]) with pytest.raises(FileNotFoundError, match="frame_0009"): sequence.pack( - frames + [tmp_path / "clip" / "frame_0009.splat"], tmp_path / "clip.seq" + frames + [tmp_path / "clip" / "frame_0009.splat"], tmp_path / "clip.vmesh" ) @pytest.mark.parametrize( "mangle, complaint", [ - (lambda data: b"NOTASEQ\x00" + data[8:], "not a sequence container"), - (lambda data: data[:8] + struct.pack(" str: return "\n".join(chunks) -READER = ("SEQ_MAGIC", "SEQ_PREAMBLE", "readSequenceHeader", "sequenceFrame") +READER = ("SEQ_MAGIC", "SEQ_PREAMBLE", "VMESH_CHUNK", "vmeshSHA256", + "vmeshJSON", "vmeshRecord", "readSequenceHeader", "validateSequenceEnd", "sequenceFrame") def run_js(body: str, tmp_path: Path, names=READER, name: str = "q.mjs") -> object: @@ -278,19 +277,20 @@ def run_js(body: str, tmp_path: Path, names=READER, name: str = "q.mjs") -> obje @requires_node def test_the_client_reads_a_header_python_wrote(tmp_path): frames = _frames(tmp_path / "clip", [10, 300, 7, 4096]) - header = sequence.pack(frames, tmp_path / "clip.seq") + header = sequence.pack(frames, tmp_path / "clip.vmesh") result = run_js( LOAD + f""" - const header = readSequenceHeader(load({str(tmp_path / "clip.seq")!r})); + const header = readSequenceHeader(load({str(tmp_path / "clip.vmesh")!r})); process.stdout.write(JSON.stringify(header)); """, tmp_path, ) assert result["suffix"] == header.suffix assert result["entries"] == [ - {"offset": entry.offset, "length": entry.length} for entry in header.entries + {"offset": entry.offset, "length": entry.length, "sha256": entry.sha256} + for entry in header.entries ] @@ -299,12 +299,12 @@ def test_the_client_slices_the_same_bytes(tmp_path): # Sizes chosen unequal and unaligned: equal-length frames would let an # off-by-one in the offset table pass every frame but the last. frames = _frames(tmp_path / "clip", [10, 300, 7, 4096, 33]) - sequence.pack(frames, tmp_path / "clip.seq") + sequence.pack(frames, tmp_path / "clip.vmesh") result = run_js( LOAD + f""" - const buffer = load({str(tmp_path / "clip.seq")!r}); + const buffer = load({str(tmp_path / "clip.vmesh")!r}); const header = readSequenceHeader(buffer); const out = []; for (let i = 0; i < header.entries.length; i++) {{ @@ -325,11 +325,11 @@ def test_the_client_slices_the_same_bytes(tmp_path): @requires_node def test_a_sliced_frame_does_not_alias_the_container(tmp_path): - sequence.pack(_frames(tmp_path / "clip", [64, 64]), tmp_path / "clip.seq") + sequence.pack(_frames(tmp_path / "clip", [64, 64]), tmp_path / "clip.vmesh") result = run_js( LOAD + f""" - const buffer = load({str(tmp_path / "clip.seq")!r}); + const buffer = load({str(tmp_path / "clip.vmesh")!r}); const header = readSequenceHeader(buffer); const first = sequenceFrame(buffer, header, 0); new Uint8Array(first).fill(0xff); @@ -338,7 +338,7 @@ def test_a_sliced_frame_does_not_alias_the_container(tmp_path): // the first frame and leave every later one unreachable. const container = new Uint8Array(buffer); process.stdout.write(JSON.stringify({{ - inContainer: container[header.entries[0].offset], + inContainer: container[header.entries[0].offset + 18], inFrame: new Uint8Array(first)[0], }})); """, @@ -351,8 +351,7 @@ def test_a_sliced_frame_does_not_alias_the_container(tmp_path): # The container's whole point is the request count, so that is what these # assert on: one request for a clip, not one per frame. -SCHEDULER = ("INDEPENDENT", "dependencyOf", "chain", "suffixOf", "SEQ_MAGIC", - "SEQ_PREAMBLE", "readSequenceHeader", "sequenceFrame", "Scheduler") +SCHEDULER = ("INDEPENDENT", "dependencyOf", "chain", "suffixOf", *READER, "Scheduler") HARNESS = """ import {readFileSync} from "node:fs"; @@ -382,15 +381,15 @@ def test_a_sliced_frame_does_not_alias_the_container(tmp_path): @requires_node def test_a_packed_clip_is_fetched_in_one_request(tmp_path): frames = _frames(tmp_path / "clip", [10, 300, 7, 4096]) - sequence.pack(frames, tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 4, "bytes": - (tmp_path / "clip.seq").stat().st_size, + sequence.pack(frames, tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 4, "bytes": + (tmp_path / "clip.vmesh").stat().st_size, "media_type": "application/octet-stream"} result = run_js( HARNESS + f""" - globalThis.serve({str(tmp_path / "clip.seq")!r}); + globalThis.serve({str(tmp_path / "clip.vmesh")!r}); const s = new Scheduler(packedClip(4, {json.dumps(packed)}), "", decode, {{cacheSize: 1}}); (async () => {{ @@ -406,7 +405,7 @@ def test_a_packed_clip_is_fetched_in_one_request(tmp_path): names=SCHEDULER, ) - assert result["requests"] == ["clip.seq"] + assert result["requests"] == ["clip.vmesh"] assert result["held"] is True assert [frame["bytes"] for frame in result["seen"]] == [10, 300, 7, 4096] # Each frame's bytes are all its own index, so this catches a slice that @@ -426,14 +425,14 @@ def test_a_packed_clip_is_fetched_in_one_request(tmp_path): @requires_node def test_progress_climbs_to_the_frame_count(tmp_path): - sequence.pack(_frames(tmp_path / "clip", [4096] * 6), tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 6, - "bytes": (tmp_path / "clip.seq").stat().st_size} + sequence.pack(_frames(tmp_path / "clip", [4096] * 6), tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 6, + "bytes": (tmp_path / "clip.vmesh").stat().st_size} result = run_js( HARNESS + f""" - globalThis.serve({str(tmp_path / "clip.seq")!r}); + globalThis.serve({str(tmp_path / "clip.vmesh")!r}); const s = new Scheduler(packedClip(6, {json.dumps(packed)}), "", decode, {{}}); (async () => {{ const seen = []; @@ -456,13 +455,13 @@ def test_progress_climbs_to_the_frame_count(tmp_path): @requires_node def test_a_container_too_big_to_hold_is_refused_unfetched(tmp_path): - sequence.pack(_frames(tmp_path / "clip", [4096] * 4), tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 4, "bytes": 900e6} + sequence.pack(_frames(tmp_path / "clip", [4096] * 4), tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 4, "bytes": 900e6} result = run_js( HARNESS + f""" - globalThis.serve({str(tmp_path / "clip.seq")!r}); + globalThis.serve({str(tmp_path / "clip.vmesh")!r}); const s = new Scheduler(packedClip(4, {json.dumps(packed)}), "", decode, {{}}); (async () => {{ const held = await s.downloadAll(null); @@ -486,14 +485,14 @@ def test_a_container_too_big_to_hold_is_refused_unfetched(tmp_path): @requires_node def test_a_container_that_disagrees_with_the_clip_is_an_error(tmp_path): - sequence.pack(_frames(tmp_path / "clip", [4096] * 4), tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 5, - "bytes": (tmp_path / "clip.seq").stat().st_size} + sequence.pack(_frames(tmp_path / "clip", [4096] * 4), tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 5, + "bytes": (tmp_path / "clip.vmesh").stat().st_size} result = run_js( HARNESS + f""" - globalThis.serve({str(tmp_path / "clip.seq")!r}); + globalThis.serve({str(tmp_path / "clip.vmesh")!r}); const s = new Scheduler(packedClip(5, {json.dumps(packed)}), "", decode, {{}}); (async () => {{ let message = null; @@ -512,14 +511,14 @@ def test_a_container_that_disagrees_with_the_clip_is_an_error(tmp_path): @requires_node def test_seeking_a_packed_clip_pulls_the_container_once(tmp_path): - sequence.pack(_frames(tmp_path / "clip", [4096] * 4), tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 4, - "bytes": (tmp_path / "clip.seq").stat().st_size} + sequence.pack(_frames(tmp_path / "clip", [4096] * 4), tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 4, + "bytes": (tmp_path / "clip.vmesh").stat().st_size} result = run_js( HARNESS + f""" - globalThis.serve({str(tmp_path / "clip.seq")!r}); + globalThis.serve({str(tmp_path / "clip.vmesh")!r}); const s = new Scheduler(packedClip(4, {json.dumps(packed)}), "", decode, {{}}); (async () => {{ // Two seeks started together, before either could finish. Packing @@ -534,7 +533,7 @@ def test_seeking_a_packed_clip_pulls_the_container_once(tmp_path): tmp_path, names=SCHEDULER, ) - assert result["requests"] == ["clip.seq"] + assert result["requests"] == ["clip.vmesh"] assert result["firsts"] == [3, 4] @@ -565,9 +564,9 @@ def test_an_unpacked_clip_still_fetches_frame_by_frame(tmp_path): @requires_node def test_a_rung_without_a_container_is_not_served_the_base_one(tmp_path): - sequence.pack(_frames(tmp_path / "clip", [4096] * 3), tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 3, - "bytes": (tmp_path / "clip.seq").stat().st_size} + sequence.pack(_frames(tmp_path / "clip", [4096] * 3), tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 3, + "bytes": (tmp_path / "clip.vmesh").stat().st_size} result = run_js( HARNESS @@ -596,7 +595,7 @@ def test_a_rung_without_a_container_is_not_served_the_base_one(tmp_path): # ------------------------------------------------ decoded under its own name --- # Reported as "why is vega blank". A frame is decoded by the codec its *name* # selects (`worker.js` sniffs the suffix), so handing the decode the -# container's name means a `.seq` in a table of `.ply` and `.splat`. Every +# container's name means a `.vmesh` in a table of `.ply` and `.splat`. Every # Gaussian pane went blank; the pixel panes did not, because an image is # decoded from its media type on the main thread -- so it looked like one # method being broken rather than one code path. @@ -604,14 +603,14 @@ def test_a_rung_without_a_container_is_not_served_the_base_one(tmp_path): @requires_node def test_a_packed_frame_is_decoded_under_its_own_name(tmp_path): - sequence.pack(_frames(tmp_path / "clip", [64] * 3), tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 3, - "bytes": (tmp_path / "clip.seq").stat().st_size} + sequence.pack(_frames(tmp_path / "clip", [64] * 3), tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 3, + "bytes": (tmp_path / "clip.vmesh").stat().st_size} result = run_js( HARNESS + f""" - globalThis.serve({str(tmp_path / "clip.seq")!r}); + globalThis.serve({str(tmp_path / "clip.vmesh")!r}); // The real rule, from worker.js: the codec comes from the suffix. const CODECS = {{gaussians: {{".ply": 1, ".splat": 1}}}}; const pick = (url) => {{ @@ -648,14 +647,14 @@ def test_a_container_whose_frames_are_a_different_format_is_refused(tmp_path): # plausible files and the length check passes, so without this the frames # would be handed to the Gaussian parser and come out as noise. sequence.pack(_frames(tmp_path / "clip", [64] * 3, suffix="jpg"), - tmp_path / "clip.seq") - packed = {"url": "clip.seq", "frames": 3, - "bytes": (tmp_path / "clip.seq").stat().st_size} + tmp_path / "clip.vmesh") + packed = {"url": "clip.vmesh", "frames": 3, + "bytes": (tmp_path / "clip.vmesh").stat().st_size} result = run_js( HARNESS + f""" - globalThis.serve({str(tmp_path / "clip.seq")!r}); + globalThis.serve({str(tmp_path / "clip.vmesh")!r}); const s = new Scheduler(packedClip(3, {json.dumps(packed)}), "./", decode, {{}}); (async () => {{ let message = null; @@ -691,7 +690,7 @@ def cut(source: str) -> str: + cut(worker).replace("function suffixOf(", "function original(") + """ const cases = ["a/b.splat", "a/b.SPLAT", "a/b.ply?v=2", "b.tar.gz", - "nodot", "a.b/c", "", "a/b.seq"]; + "nodot", "a.b/c", "", "a/b.vmesh"]; const rows = cases.map((c) => [c, suffixOf(c), original(c)]); process.stdout.write(JSON.stringify(rows)); """ @@ -704,7 +703,7 @@ def cut(source: str) -> str: disagree = [row for row in rows if row[1] != row[2]] assert not disagree, f"suffixOf disagrees with the worker on {disagree}" # And the case that started this: a container is not a frame format. - assert dict((row[0], row[1]) for row in rows)["a/b.seq"] == ".seq" + assert dict((row[0], row[1]) for row in rows)["a/b.vmesh"] == ".vmesh" def test_a_failed_download_is_explained_in_the_pane(): @@ -755,3 +754,286 @@ def test_packing_keeps_frames_referenced_by_variants_or_unpacked_clips(tmp_path) bundle.write(tmp_path, title="test", source="test", clips=clips) sequence.pack_bundle(tmp_path, names=["first"]) assert all(p.is_file() for p in paths) + + +def test_browser_vmesh_is_validated_by_shared_codec_inspector(tmp_path): + from open4d.codec import inspect_vmesh + paths = _frames(tmp_path / "input", [13, 1_048_581], suffix="drc") + target = tmp_path / "clip.vmesh" + header = sequence.pack(paths, target, fps=24) + descriptor = inspect_vmesh(target) + assert descriptor["codec"] == "frames" + assert descriptor["native"] == {"profile": "frames/1", "suffix": "drc", "representation": "mesh"} + assert descriptor["sequence"]["frames"][1]["timestamp"] == 1 / 24 + assert all(sequence.frame(target.read_bytes(), i, header) == path.read_bytes() + for i, path in enumerate(paths)) + + +@pytest.mark.parametrize("suffix", ["seq", "o4d", "bin"]) +def test_only_vmesh_is_a_public_sequence_output(tmp_path, suffix): + with pytest.raises(ValueError, match=".vmesh extension"): + sequence.pack(_frames(tmp_path / "input", [12]), tmp_path / f"clip.{suffix}") + + +@pytest.mark.parametrize("fps", [0, -1, float("nan"), float("inf"), True]) +def test_sequence_timing_must_be_finite_positive(tmp_path, fps): + with pytest.raises(ValueError, match="fps"): + sequence.pack(_frames(tmp_path / "input", [12]), tmp_path / "clip.vmesh", fps=fps) + + +def _legacy_bytes(payloads=(b"abc", b"defg"), suffix="splat"): + header = struct.pack("<8sIII", b"O4DSEQ\x00\x00", 1, len(payloads), len(suffix)) + at = 20 + len(suffix) + 8 * len(payloads) + table = bytearray() + for payload in payloads: + table.extend(struct.pack(" {{ + const bytes = new Uint8Array(sequenceFrame(buffer, header, i)); + return [bytes.length, bytes[0], bytes[bytes.length - 1]]; + }}))); + """, tmp_path) + assert result == [[1_048_581, 1, 1], [17, 2, 2]] + + +@requires_node +@pytest.mark.parametrize("corruption", ["payload", "record", "end", "trailing", "legacy"]) +def test_browser_refuses_corrupt_vmesh_before_decoder(tmp_path, corruption): + target = tmp_path / "clip.vmesh" + header = sequence.pack(_frames(tmp_path / "input", [64, 64]), target) + data = bytearray(target.read_bytes()) + if corruption == "payload": + data[header.entries[0].offset + 18] ^= 1 + elif corruption == "record": + data[header.entries[0].offset + 5] = 2 + elif corruption == "end": + data[-1] ^= 1 + elif corruption == "trailing": + data += b"extra" + else: + data = _legacy_bytes() + target.write_bytes(data) + result = run_js(LOAD + f""" + let message = null; + try {{ + const buffer = load({str(target)!r}), header = readSequenceHeader(buffer); + validateSequenceEnd(buffer, header); + sequenceFrame(buffer, header, 0); + }} catch (error) {{ message = error.message; }} + process.stdout.write(JSON.stringify(message)); + """, tmp_path) + assert result + + +def test_public_frame_loader_preserves_mesh_geometry_and_timestamps(tmp_path): + import numpy as np + from open4d.io._mesh import write_ply + paths = [] + positions = np.array([[0, 0, 0], [1, 0, 0], [0, 1, 0]], dtype=np.float32) + triangles = np.array([[0, 1, 2]], dtype=np.uint32) + for index in range(2): + path = tmp_path / f"frame_{index}.ply" + write_ply(path, positions + index, triangles) + paths.append(path) + target = tmp_path / "mesh.vmesh" + sequence.pack(paths, target, fps=12, representation="mesh") + with sequence.open_frames(target) as restored: + assert restored.timestamps == (0, 1 / 12) + assert [frame.frame_index for frame in restored] == [0, 1] + np.testing.assert_array_equal(restored[1].geometry.positions, positions + 1) + np.testing.assert_array_equal(restored[0].geometry.triangles, triangles) + + +def test_public_frame_loader_preserves_images_without_deserialization(tmp_path): + from open4d.native import NativeSequence + target = tmp_path / "pixels.vmesh" + sequence.pack(_frames(tmp_path / "input", [23, 35], suffix="png"), target) + with sequence.open_frames(target) as restored: + assert isinstance(restored, NativeSequence) + assert restored.codec == "frames" + assert restored.manifest["native"]["representation"] == "pixels" + assert len(restored) == 2 + + +def test_packing_a_bundle_uses_its_real_fps_and_representation(tmp_path): + from open4d.codec import inspect_vmesh + clip = _clip(tmp_path, [23, 35]) + bundle.write(tmp_path, title="test", source="test", clips=[clip], fps=24) + sequence.pack_bundle(tmp_path) + manifest = inspect_vmesh(tmp_path / "clip.vmesh") + assert manifest["sequence"]["frames"][1]["timestamp"] == 1 / 24 + assert manifest["native"]["representation"] == "gaussians" + + +@pytest.mark.parametrize("representation", ["unknown", "pixels", "points"]) +def test_frame_suffix_must_match_its_representation(tmp_path, representation): + with pytest.raises(ValueError, match="representation"): + sequence.pack(_frames(tmp_path / "input", [12]), tmp_path / "bad.vmesh", representation=representation) + + +@requires_node +@pytest.mark.parametrize("text", [ + '{"codec":"frames","codec":"frames"}', + '{"nested":{"name":1,"na\\u006de":2}}', + '{"a":[{"x":1,"x":2}]}', +]) +def test_browser_refuses_duplicate_json_keys(tmp_path, text): + result = run_js(f""" + let message = null; + try {{ vmeshJSON({json.dumps(text)}); }} catch (error) {{ message = error.message; }} + process.stdout.write(JSON.stringify(message)); + """, tmp_path) + assert "duplicate" in result + + +@requires_node +def test_browser_partial_download_can_recover_first_complete_frame(tmp_path): + target = tmp_path / "clip.vmesh" + header = sequence.pack(_frames(tmp_path / "input", [71, 103]), target) + partial = tmp_path / "partial.bin" + partial.write_bytes(target.read_bytes()[:header.entries[1].offset]) + result = run_js(LOAD + f""" + const buffer = load({str(partial)!r}), header = readSequenceHeader(buffer); + const frame = new Uint8Array(sequenceFrame(buffer, header, 0)); + let unavailable = false; + try {{ sequenceFrame(buffer, header, 1); }} catch (_) {{ unavailable = true; }} + process.stdout.write(JSON.stringify([frame.length, frame[0], unavailable])); + """, tmp_path) + assert result == [71, 1, True] + + +def test_existing_packed_bundle_migrates_and_rewrites_its_url(tmp_path): + from open4d.codec import inspect_vmesh + old = tmp_path / "capture.seq" + old.write_bytes(_legacy_bytes()) + clip = bundle.Clip(name="capture", representation="gaussians", + frames=["capture/frame_0000.splat", "capture/frame_0001.splat"], + sequence={"url": "capture.seq", "frames": 2, "suffix": "splat", "bytes": old.stat().st_size}) + bundle.write(tmp_path, title="test", source="test", clips=[clip], fps=12) + converted = sequence.pack_bundle(tmp_path) + assert converted[0][1]["url"] == "capture.vmesh" + assert bundle.read(tmp_path)["clips"][0]["sequence"]["url"] == "capture.vmesh" + assert not old.exists() + assert inspect_vmesh(tmp_path / "capture.vmesh")["sequence"]["frames"][1]["timestamp"] == 1 / 12 + assert sequence.pack_bundle(tmp_path) == [] + + +def test_failed_bundle_migration_preserves_original_and_manifest(tmp_path, monkeypatch): + old = tmp_path / "capture.seq" + data = _legacy_bytes() + old.write_bytes(data) + clip = bundle.Clip(name="capture", representation="gaussians", + frames=["f0.splat", "f1.splat"], + sequence={"url": "capture.seq", "frames": 2}) + bundle.write(tmp_path, title="test", source="test", clips=[clip]) + original = (tmp_path / "view.json").read_bytes() + monkeypatch.setattr(bundle, "write", lambda *args, **kwargs: (_ for _ in ()).throw(OSError("manifest write failed"))) + with pytest.raises(OSError, match="manifest write failed"): + sequence.pack_bundle(tmp_path) + assert old.read_bytes() == data + assert (tmp_path / "view.json").read_bytes() == original + + +def test_public_open4d_load_supports_self_describing_browser_vmesh(tmp_path): + import open4d + import numpy as np + from open4d.io._mesh import write_ply + path = tmp_path / "frame.ply" + positions = np.array([[0, 0, 0], [1, 0, 0], [0, 1, 0]], dtype=np.float32) + triangles = np.array([[0, 1, 2]], dtype=np.uint32) + write_ply(path, positions, triangles) + target = tmp_path / "mesh.vmesh" + sequence.pack([path], target, representation="mesh") + with open4d.load(target) as restored: + assert restored.timestamps == (0,) + np.testing.assert_array_equal(restored[0].geometry.positions, positions) + + +def test_migration_keeps_a_legacy_container_referenced_by_an_unselected_clip(tmp_path): + old = tmp_path / "shared.seq" + old.write_bytes(_legacy_bytes()) + clips = [bundle.Clip(name=name, representation="gaussians", frames=["f0.splat", "f1.splat"], + sequence={"url": "shared.seq", "frames": 2}) for name in ("one", "two")] + bundle.write(tmp_path, title="test", source="test", clips=clips) + sequence.pack_bundle(tmp_path, names=["one"]) + assert old.is_file() + index = bundle.read(tmp_path) + assert index["clips"][0]["sequence"]["url"] == "shared.vmesh" + assert index["clips"][1]["sequence"]["url"] == "shared.seq" + + +def test_legacy_bundle_migration_never_removes_paths_outside_the_bundle(tmp_path): + root = tmp_path / "bundle" + root.mkdir() + sentinel = tmp_path / "keep.splat" + sentinel.write_bytes(b"keep") + old = root / "capture.seq" + old.write_bytes(_legacy_bytes()) + clip = bundle.Clip(name="capture", representation="gaussians", frames=["../keep.splat", "f1.splat"], + sequence={"url": "capture.seq", "frames": 2}) + bundle.write(root, title="test", source="test", clips=[clip]) + with pytest.raises(ValueError, match="safe relative paths"): + sequence.pack_bundle(root) + assert sentinel.read_bytes() == b"keep" + assert old.is_file() + assert not (root / "capture.vmesh").exists() diff --git a/open4d/streamer/streamer_tests/test_server_and_transfer.py b/open4d/streamer/streamer_tests/test_server_and_transfer.py index c9ae85ca..603d5c7a 100644 --- a/open4d/streamer/streamer_tests/test_server_and_transfer.py +++ b/open4d/streamer/streamer_tests/test_server_and_transfer.py @@ -39,9 +39,9 @@ def test_fetch_rejects_symlink_targets(tmp_path, monkeypatch, partial): def test_fetch_paths_include_variants_and_packed_sequences(): - index = {"clips": [{"frames": ["logical.ply"], "sequence": {"url": "clip.seq"}, + index = {"clips": [{"frames": ["logical.ply"], "sequence": {"url": "clip.vmesh"}, "variants": [{"frames": ["low.ply"]}, {"frames": ["low.ply"]}]}]} - assert transfer.frame_paths(index) == ("clip.seq", "low.ply") + assert transfer.frame_paths(index) == ("clip.vmesh", "low.ply") def make_bundle(root, *, frames: int = 3): @@ -673,10 +673,10 @@ def test_a_frame_container_is_never_compressed(tmp_path): something the client did not ask for, which is what the header prefetch relies on.""" root = _repetitive_bundle(tmp_path) - (root / "clip.seq").write_bytes(b"O4DSEQ\x00\x00" + bytes(4096)) + (root / "clip.vmesh").write_bytes(b"VMESH\x00\x01\x00" + bytes(4096)) with serving(root) as base: request = urllib.request.Request( - f"{base}/clip.seq", headers={"Accept-Encoding": "gzip"}) + f"{base}/clip.vmesh", headers={"Accept-Encoding": "gzip"}) with urllib.request.urlopen(request) as response: body = response.read() headers = dict(response.headers) diff --git a/open4d/visualization/_api.py b/open4d/visualization/_api.py index fc3e00cc..9a1aef59 100644 --- a/open4d/visualization/_api.py +++ b/open4d/visualization/_api.py @@ -76,14 +76,14 @@ def _open_source(source: Sequence | str | os.PathLike[str]) -> tuple[Sequence, b if isinstance(source, (str, os.PathLike)): from open4d import _api as public_api - from open4d.codec import available_codecs + from open4d.codec._vmesh_format import is_mesh_profile suffix = Path(source).suffix.lower() - if any(suffix in info.suffixes and info.representation != "triangle_mesh" - for info in available_codecs()): + if suffix == ".vmesh" and not is_mesh_profile(source): raise TypeError("the Qt viewer takes mesh sequences; Gaussian runs need their native renderer") sequence = public_api.load(source) if not isinstance(sequence, Sequence): + sequence.close() raise TypeError("the Qt viewer takes an open4d.Sequence") return sequence, True raise TypeError("source must be an open4d.Sequence or path-like sequence source") diff --git a/open4d/visualization/tests/test_visualization.py b/open4d/visualization/tests/test_visualization.py index d41a13b1..b2eef92a 100644 --- a/open4d/visualization/tests/test_visualization.py +++ b/open4d/visualization/tests/test_visualization.py @@ -136,13 +136,14 @@ def test_visualize_path_owns_and_closes_loaded_sequence(monkeypatch): monkeypatch.setattr( _api, "load", lambda source: value ) + monkeypatch.setattr("open4d.codec._vmesh_format.is_mesh_profile", lambda source: True) monkeypatch.setattr(_qt, "check_available", lambda **options: None) monkeypatch.setattr(_qt, "play", lambda frames, options: frames[0]) monkeypatch.setattr( value, "close", lambda: (close_calls.append(True), real_close())[1] ) - visualize("capture.o4d") + visualize("capture.vmesh") assert close_calls == [True] assert value.closed is True @@ -166,13 +167,14 @@ def test_visualize_path_closes_after_renderer_failure(monkeypatch): from open4d.visualization import _qt monkeypatch.setattr(_api, "load", lambda source: value) + monkeypatch.setattr("open4d.codec._vmesh_format.is_mesh_profile", lambda source: True) monkeypatch.setattr(_qt, "check_available", lambda **options: None) monkeypatch.setattr( _qt, "play", lambda frames, options: (_ for _ in ()).throw(RuntimeError("boom")) ) with pytest.raises(RuntimeError, match="boom"): - visualize("capture.o4d") + visualize("capture.vmesh") assert value.closed is True @@ -264,13 +266,24 @@ def test_gaussian_paths_are_rejected_before_native_decode(tmp_path, monkeypatch, from open4d.visualization import _api, _qt import open4d._api as public + import json + from open4d.codec import pack_vmesh + native = tmp_path / "native" + native.mkdir() + (native / "metadata.json").write_text(json.dumps({ + "version": 1, "codec": "vega", "native": {"profile": "vega/1"}, + "frames": [{"frame_index": i, "timestamp": i / 30} for i in range(2)], + })) + for name in ("manifest.json", "color_model.pt", "frame_0000.pt", "frame_0001.pt"): + (native / name).write_bytes(b"opaque native fixture") + source = pack_vmesh(native, tmp_path / "capture.vmesh") monkeypatch.setattr(_qt, "check_available", lambda **options: None) monkeypatch.setattr(public, "load", lambda *args: pytest.fail("must not decode Gaussian data")) with pytest.raises(TypeError, match="native renderer"): if render: - _api.render_gif(tmp_path / "capture.vega", tmp_path / "capture.gif") + _api.render_gif(source, tmp_path / "capture.gif") else: - _api.visualize(tmp_path / "capture.vega") + _api.visualize(source) @pytest.mark.parametrize("scale", [1e-10, 1e10, 1e20]) @@ -429,3 +442,26 @@ def fail_save(self, path, **options): assert output.read_bytes() == b"original" assert list(tmp_path.iterdir()) == [output] assert gif_scene == [True] + + +def test_mesh_vmesh_path_reaches_player_and_releases_owned_payloads(tmp_path, monkeypatch): + from open4d.io._mesh import write_ply + from open4d.streamer.streamer.sequence import pack + from open4d.visualization import _qt + source = tmp_path / "frame.ply" + mesh = sequence()[0].geometry + write_ply(source, mesh.positions, mesh.triangles) + target = tmp_path / "capture.vmesh" + pack([source], target, representation="mesh") + seen = {} + monkeypatch.setattr(_qt, "check_available", lambda **options: None) + + def play(frames, options): + seen["sequence"] = frames.sequence + seen["positions"] = frames[0].positions.copy() + assert not frames.sequence.closed + + monkeypatch.setattr(_qt, "play", play) + visualize(target) + np.testing.assert_array_equal(seen["positions"], mesh.positions) + assert seen["sequence"].closed diff --git a/open4d/webclients/README.md b/open4d/webclients/README.md index 61e30057..f78eef41 100644 --- a/open4d/webclients/README.md +++ b/open4d/webclients/README.md @@ -40,7 +40,12 @@ servers (`baselines/`) and the corpora live in `4DVideoStreaming`, so clients and run the tests. The Node control server binds to `127.0.0.1` by default. Set `HOST` explicitly -when remote clients need access; it provides no authentication. +when remote clients need access; it provides no authentication. Browser requests +must use this server's origin, or an origin explicitly listed in the comma-separated +`VS4D_ALLOWED_ORIGINS`. IP addresses and `localhost` are accepted as request hosts; +list additional DNS names in `VS4D_ALLOWED_HOSTS` (comma-separated), or bind using +that name with `HOST`. Host checks prevent an unconfigured DNS name from rebinding +to this local control server. ## Adding a method diff --git a/open4d/webclients/system/Server/path-safety.js b/open4d/webclients/system/Server/path-safety.js new file mode 100644 index 00000000..65417e23 --- /dev/null +++ b/open4d/webclients/system/Server/path-safety.js @@ -0,0 +1,26 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +function validDirectoryId(id, maxLength = 180) { + return typeof id === 'string' && id.length <= maxLength + && /^[A-Za-z0-9._-]+$/.test(id) && id !== '.' && id !== '..'; +} + +/** Resolve one directory component, rejecting traversal and existing symlinks. */ +function childDirectory(root, id, label = 'directory id', maxLength = 180) { + const invalid = () => Object.assign(new Error(`invalid ${label}`), { statusCode: 400 }); + if (!validDirectoryId(id, maxLength)) throw invalid(); + const base = path.resolve(root); + const target = path.resolve(base, id); + if (!target.startsWith(base + path.sep)) throw invalid(); + try { + if (fs.lstatSync(target).isSymbolicLink()) throw invalid(); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + } + return target; +} + +module.exports = { childDirectory, validDirectoryId }; diff --git a/open4d/webclients/system/Server/server.js b/open4d/webclients/system/Server/server.js index e635b390..e4855e01 100644 --- a/open4d/webclients/system/Server/server.js +++ b/open4d/webclients/system/Server/server.js @@ -6,6 +6,7 @@ const net = require("net"); const path = require("path"); const { spawn, execFile } = require("child_process"); const readline = require("readline"); // add here +const { childDirectory, validDirectoryId } = require("./path-safety"); // -------------------- Logging -------------------- // One line per event: [YYYY-MM-DD HH:MM:SS.mmm][LEVEL][COMPONENT] message @@ -22,7 +23,26 @@ const logWarn = (component, message) => log("WARN", component, message); const logError = (component, message) => log("ERROR", component, message); const app = express(); -app.use(cors({ origin: "*" })); +// Native/headset requests have no Origin. Browser requests must come from +// this server or an explicitly configured companion UI. +const allowedOrigins = new Set((process.env.VS4D_ALLOWED_ORIGINS || "") + .split(",").map(value => value.trim()).filter(Boolean)); +const allowedHostnames = new Set(["localhost", process.env.HOST, + ...(process.env.VS4D_ALLOWED_HOSTS || "").split(",")] + .filter(Boolean).map(value => value.trim().toLowerCase())); +app.use((req, res, next) => { + // A same-origin check alone trusts attacker-controlled DNS names that can + // rebind to loopback. Accept IP literals and explicitly configured names. + const hostname = req.hostname?.replace(/^\[|\]$/g, "").toLowerCase(); + if (!hostname || (!net.isIP(hostname) && !allowedHostnames.has(hostname))) + return res.status(403).json({ error: "request hostname is not allowed" }); + const origin = req.get("Origin"); + if (!origin) return next(); + const sameOrigin = origin === `${req.protocol}://${req.get("Host")}`; + if (!sameOrigin && !allowedOrigins.has(origin)) + return res.status(403).json({ error: "browser origin is not allowed" }); + return cors({ origin })(req, res, next); +}); app.use(express.json({ limit: "200mb" })); // -------------------- Paths -------------------- @@ -517,7 +537,7 @@ app.use("/api/study", createStudyRouter({ // live in server_results// so each run stays self-contained. function broadcastResultsDir(broadcastId) { const id = broadcastId || currentBroadcastId || "no-broadcast"; - const dir = path.join(SERVER_RESULTS_DIR, id); + const dir = childDirectory(SERVER_RESULTS_DIR, id, "broadcast id"); ensureDir(dir); return dir; } @@ -1008,7 +1028,7 @@ const SHAPED_INTERFACE = process.env.VS4D_SHAPED_INTERFACE || "eth0"; // so unlike every other system here, choosing objects means replacing the // process. That is why this exists rather than a query parameter. // -// This endpoint spawns processes and the server listens on 0.0.0.0, so the +// This endpoint spawns processes and can be bound to a network interface, so the // input is constrained hard rather than trusted: // * the baseline id must be one of the two configured ones, never a module // name from the request; @@ -1335,14 +1355,13 @@ app.get("/api/systems", async (req, res) => { app.get("/api/viewpoint-index", (req, res) => { const requested = String(req.query.dir || ""); - if (requested && !/^[A-Za-z0-9._-]+$/.test(requested)) { + if (requested && !validDirectoryId(requested)) { return res.status(400).json({ error: "invalid viewpoint directory" }); } const root = path.join(SYSTEM_ROOT_CLIENT, "viewpoints"); - const directory = requested ? path.join(root, requested) : root; - if (!path.resolve(directory).startsWith(path.resolve(root))) { - return res.status(400).json({ error: "viewpoint path escapes the client dir" }); - } + let directory; + try { directory = requested ? childDirectory(root, requested, "viewpoint directory") : root; } + catch (error) { return res.status(400).json({ error: error.message }); } if (!fs.existsSync(directory)) { return res.status(404).json({ error: `no viewpoint directory ${directory}` }); } @@ -2040,7 +2059,7 @@ function sceneRelativeTrajectoryPayload(value, preferredBroadcastId = "") { // older in-session runs fall back to playback-aligned Unity poses in the CSV. app.get("/api/offline-trajectory/:broadcastId", (req, res) => { const broadcastId = String(req.params.broadcastId || ""); - if (!/^[A-Za-z0-9._-]+$/.test(broadcastId)) + if (!validDirectoryId(broadcastId)) return res.status(400).json({ error: "invalid trajectory broadcast id" }); const filepath = path.join(SERVER_RESULTS_DIR, broadcastId, "viewport_trace.csv"); if (!fs.existsSync(filepath)) @@ -2157,6 +2176,9 @@ app.post("/api/client-log", (req, res) => { message: redactIpAddresses(entry.message), stack: entry.stack ? redactIpAddresses(entry.stack) : entry.stack, })); + if (broadcastId !== undefined && broadcastId !== null && broadcastId !== "" + && !validDirectoryId(broadcastId)) + return res.status(400).json({ error: "invalid broadcast id" }); const logFile = path.join(broadcastResultsDir(broadcastId), "client_log.jsonl"); fs.appendFileSync(logFile, safeEntries.map((entry) => JSON.stringify(entry)).join("\n") + "\n"); for (const entry of safeEntries) { @@ -2383,8 +2405,9 @@ app.get("/api/results", (req, res) => { }); function validViewportBroadcastId(value) { - return typeof value === "string" && /^[a-zA-Z0-9._-]+$/.test(value) - && fs.existsSync(path.join(SERVER_RESULTS_DIR, value)); + if (!validDirectoryId(value)) return false; + try { return fs.existsSync(childDirectory(SERVER_RESULTS_DIR, value, "broadcast id")); } + catch { return false; } } function latestViewportBroadcastId() { diff --git a/open4d/webclients/system/Server/study/routes.js b/open4d/webclients/system/Server/study/routes.js index 4fad5dbd..daf9818e 100644 --- a/open4d/webclients/system/Server/study/routes.js +++ b/open4d/webclients/system/Server/study/routes.js @@ -153,6 +153,9 @@ function createStudyRouter({ resultsRoot, shaper, tracesDir, layoutFile }) { })); router.post('/sessions/:id/trials/:position/finish', wrap(req => { + if (!live || live.session !== req.params.id || live.position !== Number(req.params.position)) { + throw Object.assign(new Error('this trial does not own the live link'), { statusCode: 409 }); + } const shaping = shaper.status(); const result = store.finishTrial(req.params.id, req.params.position, { ...(req.body || {}), shaping diff --git a/open4d/webclients/system/Server/study/store.js b/open4d/webclients/system/Server/study/store.js index 6cc04fe5..bbde920e 100644 --- a/open4d/webclients/system/Server/study/store.js +++ b/open4d/webclients/system/Server/study/store.js @@ -18,6 +18,7 @@ const crypto = require('crypto'); const fs = require('fs'); const path = require('path'); +const { childDirectory } = require('../path-safety'); const questionnaire = require('./questionnaire'); const { participantIndex, trialOrder } = require('./order'); @@ -25,7 +26,6 @@ const { parseTrace, windowTrace, summarize } = require('./trace'); const SCHEMA_VERSION = 1; const PARTICIPANT = /^[A-Za-z0-9._-]{1,40}$/; -const SESSION_ID = /^[A-Za-z0-9._-]{1,80}$/; /** * Every method the comparison page knows, and whether it can follow a camera @@ -116,10 +116,7 @@ class StudyStore { } _dir(sessionId) { - if (!SESSION_ID.test(String(sessionId || ''))) { - throw Object.assign(new Error('invalid session id'), { statusCode: 400 }); - } - return path.join(this.root, sessionId); + return childDirectory(this.root, sessionId, 'session id', 80); } _read(sessionId, name) { @@ -182,7 +179,7 @@ class StudyStore { // always gets the same row whatever order the methods were ticked in. const canonical = Object.keys(METHODS).filter(id => methods.includes(id)); const { row, order } = trialOrder(canonical, index); - const id = `${participant}-${timestamp()}`; + const id = `${participant}-${timestamp()}-${crypto.randomBytes(8).toString("hex")}`; const trajectory = spec.trajectory ? validateTrajectory(spec.trajectory) : null; const session = { @@ -209,6 +206,7 @@ class StudyStore { : null, trials: order.map(entry => ({ ...entry, status: 'pending' })) }; + fs.mkdirSync(this._dir(id)); this._write(id, 'session.json', session); fs.writeFileSync(path.join(this._dir(id), 'trace.csv'), spec.trace.text); if (trajectory) this._write(id, 'trajectory.json', trajectory); @@ -342,8 +340,11 @@ class StudyStore { ]; const cell = value => { if (value === null || value === undefined) return ''; - const text = String(value); - return /[",\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; + // Quoting CSV syntax alone does not prevent spreadsheet formulas. + const raw = String(value); + const text = typeof value === 'string' && /^[=+@\-\t\r\n]/.test(raw) + ? "'" + raw : raw; + return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; }; const rows = records.map(record => [ record.session, record.participant, record.position + 1, record.label, record.method, @@ -353,7 +354,7 @@ class StudyStore { record.questionnaire.timing.totalSeconds, ...metricKeys.map(key => record.metrics[key]) ].map(cell).join(',')); - return `${columns.join(',')}\n${rows.join('\n')}${rows.length ? '\n' : ''}`; + return `${columns.map(cell).join(',')}\n${rows.join('\n')}${rows.length ? '\n' : ''}`; } } diff --git a/open4d/webclients/tests/test_server_security.js b/open4d/webclients/tests/test_server_security.js new file mode 100644 index 00000000..978097b2 --- /dev/null +++ b/open4d/webclients/tests/test_server_security.js @@ -0,0 +1,63 @@ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const http = require('node:http'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { app } = require('../system/Server/server'); + +const results = path.resolve(__dirname, '../system/Server/server_results'); + +test('client log cannot write outside the results root', async t => { + const outside = fs.mkdtempSync(path.join(os.tmpdir(), 'open4d-log-security-')); + t.after(() => fs.rmSync(outside, { recursive: true, force: true })); + const server = await new Promise(resolve => { + const s = app.listen(0, '127.0.0.1', () => resolve(s)); + }); + t.after(() => new Promise(resolve => server.close(resolve))); + const response = await fetch(`http://127.0.0.1:${server.address().port}/api/client-log`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ broadcastId: path.relative(results, outside), + entries: [{ message: 'fixture' }] }) + }); + assert.equal(response.status, 400); + assert.equal(fs.existsSync(path.join(outside, 'client_log.jsonl')), false); +}); + +test('foreign websites cannot invoke the local control API', async t => { + const server = await new Promise(resolve => { + const s = app.listen(0, '127.0.0.1', () => resolve(s)); + }); + t.after(() => new Promise(resolve => server.close(resolve))); + const response = await fetch(`http://127.0.0.1:${server.address().port}/api/health`, { + headers: { Origin: 'https://untrusted.example' } + }); + assert.equal(response.status, 403); + assert.equal(response.headers.get('access-control-allow-origin'), null); +}); + +test('a forged hostname cannot bypass the browser origin check', async t => { + const server = await new Promise(resolve => { + const s = app.listen(0, '127.0.0.1', () => resolve(s)); + }); + t.after(() => new Promise(resolve => server.close(resolve))); + const status = await new Promise((resolve, reject) => { + const request = http.get({ host: '127.0.0.1', port: server.address().port, + path: '/api/health', headers: { Host: 'untrusted.example', Origin: 'http://untrusted.example' } + }, response => { response.resume(); resolve(response.statusCode); }); + request.on('error', reject); + }); + assert.equal(status, 403); +}); + +test('the same-origin UI can still read the API', async t => { + const server = await new Promise(resolve => { + const s = app.listen(0, '127.0.0.1', () => resolve(s)); + }); + t.after(() => new Promise(resolve => server.close(resolve))); + const base = `http://127.0.0.1:${server.address().port}`; + const response = await fetch(`${base}/api/health`, { headers: { Origin: base } }); + assert.equal(response.status, 200); + assert.equal(response.headers.get('access-control-allow-origin'), base); +}); diff --git a/open4d/webclients/tests/test_study_session.js b/open4d/webclients/tests/test_study_session.js index d31d7462..02018e14 100644 --- a/open4d/webclients/tests/test_study_session.js +++ b/open4d/webclients/tests/test_study_session.js @@ -291,3 +291,54 @@ test('abort frees the link without recording a result', { skip: NO_EXPRESS }, as server.close(); } }); + + +test('session ids cannot name the root or its parent', () => { + const store = new StudyStore(tempDir()); + for (const id of ['.', '..', '../outside', '/outside', ['p01']]) { + assert.throws(() => store._dir(id), /invalid session id/); + } +}); + +test('two sessions created in the same second never overwrite each other', () => { + const store = new StudyStore(tempDir()); + const spec = { participant: 'p03', methods: ['mesh'], durationSeconds: 20, + trace: { text: TRACE } }; + const sessions = Array.from({ length: 10 }, () => store.createSession(spec)); + assert.strictEqual(new Set(sessions.map(s => s.id)).size, sessions.length); + assert.strictEqual(store.listSessions().length, sessions.length); +}); + +test('finishing an abandoned trial cannot disarm another session', { skip: NO_EXPRESS }, async () => { + const { server, call, shaper } = await serve(); + try { + const make = async participant => (await call('POST', '/api/study/sessions', { + participant, methods: ['mesh'], durationSeconds: 20, + trace: { text: TRACE }, trajectory: trajectory() + })).json; + const first = await make('p10'); + const second = await make('p11'); + await call('POST', `/api/study/sessions/${first.id}/trials/0/start`); + await call('POST', '/api/study/abort'); + await call('POST', `/api/study/sessions/${second.id}/trials/0/start`); + const response = await call('POST', `/api/study/sessions/${first.id}/trials/0/finish`, + { metrics: {}, questionnaire: RESPONSE }); + assert.strictEqual(response.status, 409); + assert.strictEqual(shaper.armed, true); + assert.strictEqual((await call('GET', '/api/study/shaping')).json.live.session, second.id); + } finally { shaper.disarm(); server.close(); } +}); + +test('CSV exports keep uploaded formula-like strings as text', () => { + const store = new StudyStore(tempDir()); + const session = store.createSession({ participant: 'p12', methods: ['mesh'], + durationSeconds: 20, trace: { name: '=1+1', text: TRACE }, trajectory: trajectory() }); + store.startTrial(session.id, 0); + store.finishTrial(session.id, 0, { metrics: { note: '@SUM(1)', loss: -1, 'bad,key': 'safe' }, + questionnaire: RESPONSE }); + const csv = store.exportCsv(session.id); + assert.ok(csv.includes("'=1+1")); + assert.ok(csv.includes("'@SUM(1)")); + assert.ok(csv.includes(',-1,')); + assert.ok(csv.includes('"m_bad,key"')); +}); diff --git a/open4d/webclients/tests/test_v4ds_protocol.js b/open4d/webclients/tests/test_v4ds_protocol.js index 87e6beb4..7069c9ec 100644 --- a/open4d/webclients/tests/test_v4ds_protocol.js +++ b/open4d/webclients/tests/test_v4ds_protocol.js @@ -339,7 +339,7 @@ test('an empty-but-present selection list is distinguishable from absent', () => 'a u16 count of zero is still written'); }); -test('feedback round-trips through the Python decoder', () => { +test('feedback round-trips through the Python decoder', (t) => { // Encoding matching bytes proves the layout; this proves the Python side // actually accepts what we produce. const { execFileSync } = require('child_process'); @@ -353,8 +353,14 @@ test('feedback round-trips through the Python decoder', () => { }); const python = process.env.VS4D_TEST_PYTHON || '/home/ryan/miniconda3/envs/open4d/bin/python'; + const reference = path.join(__dirname, '..', + 'baselines/DeltaStream/orbitstream/protocol.py'); + if (!fs.existsSync(reference)) { + t.skip('upstream DeltaStream Python decoder is not vendored'); + return; + } if (!fs.existsSync(python)) { - console.log('skipping Python round-trip: no interpreter at', python); + t.skip(`Python reference interpreter is unavailable: ${python}`); return; } const script = ` diff --git a/pyproject.toml b/pyproject.toml index b16562db..d74cf24f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ player = ["PyQt6", "pyqtgraph", "PyOpenGL", "pillow>=9", "scipy"] open3d = ["open3d>=0.19,<0.20", "pillow>=9"] usd = ["usd-core>=24.3"] torch = ["torch"] -klt = ["torch", "trimesh", "zstd", "tqdm", "point-cloud-utils"] +klt = ["torch", "trimesh", "zstd", "zstandard", "tqdm", "point-cloud-utils"] n4mc = ["torch", "trimesh", "scikit-image", "point-cloud-utils"] qndf = ["torch", "open3d>=0.19,<0.20", "trimesh", "tqdm"] gaussians = ["plyfile"] diff --git a/scripts/smoke_installed_io.py b/scripts/smoke_installed_io.py index 50a5eb2b..6884c4a3 100644 --- a/scripts/smoke_installed_io.py +++ b/scripts/smoke_installed_io.py @@ -72,9 +72,11 @@ np.testing.assert_array_equal(frame.geometry.triangles, [[0, 1, 2]]) assert all(callable(getattr(open4d, name)) for name in ("encode", "decode", "visualize", "reconstruct", "stream", "receive")) - installed = {info.id for info in available_codecs()} - assert {"klt", "n4mc", "qndf", "qndf-int8", "vdmc", "faster_vdmc", "tvmc", "tsmc", "vega"} <= installed - assert not installed & {"npz", "raw", "draco", "temporal-delta", "temporal-pca"} + codecs = available_codecs() + installed = {info.id for info in codecs} + assert installed == {"klt", "n4mc", "qndf", "qndf-int8", "vdmc", "faster_vdmc", + "tvmc", "tsmc", "vega", "queen", "3dgstream", "rerf"} + assert all(info.suffixes == (".vmesh",) for info in codecs) from open4d.codec._research import research_module os.environ.pop("OPEN4D_RESEARCH_ROOT", None) try: @@ -85,9 +87,20 @@ raise AssertionError("wheel unexpectedly includes KLT research source") from concurrent.futures import ThreadPoolExecutor + from open4d.codec import inspect_vmesh + from open4d.codec._npz import REFERENCE_CODECS from open4d.demo import mesh_sequence with mesh_sequence(side=3, frames=2) as source: + reference = next(codec for codec in REFERENCE_CODECS if codec.id == "npz") + artifact = open4d.encode(source, Path(directory) / "reference.vmesh", codec=reference) + assert artifact.read_bytes().startswith(b"VMESH\x00\x01\x00") + assert inspect_vmesh(artifact)["codec"] == "npz" + with open4d.decode(artifact) as decoded: + assert decoded.timestamps == source.timestamps + for expected, actual in zip(source, decoded): + np.testing.assert_array_equal(actual.geometry.positions, expected.geometry.positions) + np.testing.assert_array_equal(actual.geometry.triangles, expected.geometry.triangles) with open4d.receive(port=0, timeout=5) as receiver, ThreadPoolExecutor(1) as pool: transfer = pool.submit(open4d.stream, source, *receiver.address, realtime=False) restored = list(receiver) @@ -95,4 +108,4 @@ for expected, actual in zip(source, restored): np.testing.assert_array_equal(actual.geometry.positions, expected.geometry.positions) assert actual.timestamp == expected.timestamp - print("Installed I/O, CLI, codec discovery and mesh streaming passed") + print("Installed I/O, CLI, VMESH round trip, codec discovery and mesh streaming passed") diff --git a/scripts/tests/test_benchmark_codec.py b/scripts/tests/test_benchmark_codec.py index 70a9312e..24878802 100644 --- a/scripts/tests/test_benchmark_codec.py +++ b/scripts/tests/test_benchmark_codec.py @@ -72,7 +72,7 @@ def fixed_timed(function): monkeypatch.setattr(benchmark_codec, "peak_bytes", lambda function, cleanup=None: 0) source = benchmark_codec.synthetic(3, 2) - result = benchmark_codec.run(source, Path(tmp_path) / "take.o4d") + result = benchmark_codec.run(source, Path(tmp_path) / "take.vmesh") assert result["decode_open_ms"] == 2000 assert result["decode_validate_s"] >= 0 @@ -100,7 +100,7 @@ def check(left, right): monkeypatch.setattr(benchmark_codec, "timed", marked_timed) monkeypatch.setattr(benchmark_codec, "_surface_errors", check) - benchmark_codec.run(benchmark_codec.synthetic(3, 2), tmp_path / "test.o4d") + benchmark_codec.run(benchmark_codec.synthetic(3, 2), tmp_path / "test.vmesh") def test_decode_peak_memory_excludes_surface_validation(tmp_path, monkeypatch): @@ -124,7 +124,7 @@ def checked_surface_errors(left, right): monkeypatch.setattr(benchmark_codec, "_surface_errors", checked_surface_errors) result = benchmark_codec.run( - benchmark_codec.synthetic(3, 2), Path(tmp_path) / "take.o4d" + benchmark_codec.synthetic(3, 2), Path(tmp_path) / "take.vmesh" ) assert result["decode_all_peak_bytes"] > 0 @@ -150,7 +150,7 @@ def decode(*args, **kwargs): has_vertex_correspondence=False)) monkeypatch.setattr(benchmark_codec, "decode_sequence", decode) - result = benchmark_codec.run(source, tmp_path / "test.o4d", codec="reordered") + result = benchmark_codec.run(source, tmp_path / "test.vmesh", codec="reordered") assert result["position_rms_error"] is None assert result["position_max_error"] is None assert result["surface_rms_error"] == 0 @@ -169,7 +169,7 @@ def decode(*args, **kwargs): monkeypatch.setattr(benchmark_codec, "decode_sequence", decode) with pytest.raises(AssertionError): - benchmark_codec.run(source, tmp_path / "test.o4d") + benchmark_codec.run(source, tmp_path / "test.vmesh") assert all(value.closed for value in opened) assert not source.closed diff --git a/scripts/tests/test_cli.py b/scripts/tests/test_cli.py index 57476655..0f061c6b 100644 --- a/scripts/tests/test_cli.py +++ b/scripts/tests/test_cli.py @@ -56,9 +56,9 @@ def test_inspect_uses_registered_codec_loader(tmp_path, capsys, monkeypatch): from open4d.codec import _api from open4d.codec._npz import NumPyZipCodec - monkeypatch.setitem(_api._CODECS, "test-fixture", NumPyZipCodec()) + monkeypatch.setitem(_api._CODECS, "npz", NumPyZipCodec()) with mesh_sequence(side=3, frames=2) as sequence: - path = save(sequence, tmp_path / "existing.o4d", codec="test-fixture") + path = save(sequence, tmp_path / "existing.vmesh", codec="npz") assert main(["inspect", str(path), "--json"]) == 0 info = json.loads(capsys.readouterr().out) assert info["frame_count"] == 2 @@ -100,7 +100,7 @@ def view(sequence, **options): def test_missing_player_is_reported_before_decoding(tmp_path, monkeypatch, capsys): from open4d.visualization import VisualizationDependencyError, _qt - path = tmp_path / "take.o4d" + path = tmp_path / "take.vmesh" path.touch() def unavailable(): From 1809cbf1289f2c67af3da49cbb09facc04233070 Mon Sep 17 00:00:00 2001 From: cicm Date: Wed, 30 Sep 2026 15:52:07 -0400 Subject: [PATCH 3/4] Make VMESH visualization test independent of import order --- open4d/visualization/tests/test_visualization.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/open4d/visualization/tests/test_visualization.py b/open4d/visualization/tests/test_visualization.py index b2eef92a..c846163f 100644 --- a/open4d/visualization/tests/test_visualization.py +++ b/open4d/visualization/tests/test_visualization.py @@ -445,9 +445,13 @@ def fail_save(self, path, **options): def test_mesh_vmesh_path_reaches_player_and_releases_owned_payloads(tmp_path, monkeypatch): + from pathlib import Path from open4d.io._mesh import write_ply - from open4d.streamer.streamer.sequence import pack from open4d.visualization import _qt + + monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / "streamer")) + from streamer.sequence import pack + source = tmp_path / "frame.ply" mesh = sequence()[0].geometry write_ply(source, mesh.positions, mesh.triangles) From ca5b31955beafecfa38477c6ef52abe1aee58ad4 Mon Sep 17 00:00:00 2001 From: cicm Date: Wed, 30 Sep 2026 17:01:44 -0400 Subject: [PATCH 4/4] Fix VMESH review findings: timeouts, probes and array bounds - Make OPEN4D_NATIVE_TIMEOUT opt-in so long native encodes are not killed, and report invalid values as CodecError. - Allow single-frame N4MC encodes and legacy .n4d migration again. - Enforce the NPZ/RLE decode size bound at encode time so the codec never writes artifacts it cannot read; size the RLE member bound for its worst case, vectorize the run-length check and open each frame archive once. - Keep can_decode a boolean probe via contains_codec, while codec inference still surfaces the specific damaged/archived/retired-wrapper diagnostic. - Align the Vega decode annotations and README with its NativeSequence result. - Detect native USD from layer metadata before a full open, so mesh USD is parsed once. - Use NEURAL_CODECS in is_mesh_profile, drop an unused import and document how stream() treats fps for timed sources. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 3 +- docs/api.md | 4 +- open4d/_streamer.py | 4 ++ open4d/codec/_api.py | 3 + open4d/codec/_draco.py | 4 +- open4d/codec/_klt.py | 4 +- open4d/codec/_metadata.py | 1 - open4d/codec/_n4mc.py | 6 +- open4d/codec/_native.py | 19 +++++- open4d/codec/_native_profiles.py | 2 +- open4d/codec/_native_temporal.py | 4 +- open4d/codec/_npz.py | 66 +++++++++++-------- open4d/codec/_protocol.py | 5 +- open4d/codec/_qndf.py | 4 +- open4d/codec/_temporal.py | 8 +-- open4d/codec/_tracked.py | 4 +- open4d/codec/_vmesh.py | 4 +- open4d/codec/_vmesh_format.py | 17 ++++- .../codec/tests/test_artifact_validation.py | 10 +++ open4d/codec/tests/test_codec_review.py | 36 ++++++++++ open4d/codec/tests/test_n4mc_vmesh.py | 16 +++-- open4d/gaussians.py | 12 ++-- open4d/io/_native_usd.py | 10 +++ open4d/io/tests/test_usd_sequence.py | 9 +++ 24 files changed, 190 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 10ab5a1c..762e7092 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,8 @@ from open4d import load_gaussians, encode, decode frames = [load_gaussians("frame_0000.ply"), load_gaussians("frame_0001.ply")] encoded = encode(frames, "capture.vmesh", codec="vega") -decoded = decode(encoded) +native = decode(encoded) # owned native state; inspecting it needs no CUDA +decoded = native.decode() # runs the Vega runtime, returning NeuralGaussianFrame values ``` A `GaussianSplats` frame contains `positions` `(N, 3)`, positive `scales` diff --git a/docs/api.md b/docs/api.md index 18dde07a..f8283429 100644 --- a/docs/api.md +++ b/docs/api.md @@ -155,8 +155,8 @@ wheel. Internal benchmark experiments are not public codec choices. The V-DMC ad shell scripts, but they do invoke configured native encoder and decoder processes once per sequence. Callers can also register another `open4d.codec.Codec`. -Native mesh processes default to a one-hour timeout; set -`OPEN4D_NATIVE_TIMEOUT` to a positive number of seconds for longer jobs. +Native mesh processes run without a time limit by default; set +`OPEN4D_NATIVE_TIMEOUT` to a positive number of seconds to bound them. For an all-registered-codec attempt using `4d_files/Rafa_Approves_hd_4k`, open [`examples/open4d_sequence_codec.ipynb`](../examples/open4d_sequence_codec.ipynb). diff --git a/open4d/_streamer.py b/open4d/_streamer.py index 601e32af..2df4595b 100644 --- a/open4d/_streamer.py +++ b/open4d/_streamer.py @@ -81,6 +81,10 @@ def stream( by default and the rest are what it can switch to, so a single-entry list is a fixed-quality stream and says so. + ``fps`` applies only to a path source carrying no timing of its own. A + loaded `Sequence` always has timestamps, so the bundle plays at their rate + and ``fps`` is ignored rather than allowed to contradict them. + With ``out_dir`` omitted the bundle goes to a temporary directory, which is **not** deleted afterwards. An export is minutes of encoding and the directory is the artifact; removing it when the server stops would throw diff --git a/open4d/codec/_api.py b/open4d/codec/_api.py index 997e1c27..f5fb13f0 100644 --- a/open4d/codec/_api.py +++ b/open4d/codec/_api.py @@ -87,6 +87,9 @@ def _codec(value: str | Codec | None, path: Path) -> Codec: if callable(getattr(codec, "can_decode", None)) and codec.can_decode(path) ] if not detected: + # can_decode is a boolean probe; surface why a damaged, archived or + # retired-wrapper file was rejected before the generic error. + probe_codec(path) raise CodecError(f"invalid Open4D artifact {path}: no known codec manifest") matches = detected if len(matches) != 1: diff --git a/open4d/codec/_draco.py b/open4d/codec/_draco.py index 8b3a3e0d..9d4e80cc 100644 --- a/open4d/codec/_draco.py +++ b/open4d/codec/_draco.py @@ -13,7 +13,7 @@ from open4d.core import Frame, Sequence, TopologyMode, TriangleMesh from ._metadata import _json_value -from ._vmesh_format import pack_vmesh, probe_codec, unpack_vmesh +from ._vmesh_format import contains_codec, pack_vmesh, probe_codec, unpack_vmesh from ._protocol import CodecError @@ -106,7 +106,7 @@ class DracoCodec: preserves = ("positions", "triangles", "colors", "normals", "texture_coordinates") def can_decode(self, source: Path) -> bool: - return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id + return contains_codec(source, self.id) def encode( self, diff --git a/open4d/codec/_klt.py b/open4d/codec/_klt.py index adc373bc..e8ebdfeb 100644 --- a/open4d/codec/_klt.py +++ b/open4d/codec/_klt.py @@ -23,7 +23,7 @@ from ._research import research_module from ._tsdf import write_tsdf_sequence from ._torch import torch_device -from ._vmesh_format import pack_vmesh, probe_codec, unpack_vmesh +from ._vmesh_format import contains_codec, pack_vmesh, probe_codec, unpack_vmesh _SCHEMA = "open4d.klt-sequence/v1" @@ -136,7 +136,7 @@ class KLTCodec: preserves = ("positions", "triangles") def can_decode(self, source: Path) -> bool: - return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id + return contains_codec(source, self.id) def encode( self, sequence: Sequence, destination: Path, *, overwrite: bool = False, diff --git a/open4d/codec/_metadata.py b/open4d/codec/_metadata.py index de43369d..d963c52a 100644 --- a/open4d/codec/_metadata.py +++ b/open4d/codec/_metadata.py @@ -4,7 +4,6 @@ import math from pathlib import Path import numpy as np -from open4d._files import publish_file as _publish_file from ._protocol import CodecError diff --git a/open4d/codec/_n4mc.py b/open4d/codec/_n4mc.py index 721c7e2c..f12bf6ca 100644 --- a/open4d/codec/_n4mc.py +++ b/open4d/codec/_n4mc.py @@ -22,7 +22,7 @@ from ._research import research_module from ._torch import torch_device from ._tsdf import write_tsdf_sequence -from ._vmesh_format import _json, pack_vmesh, probe_codec, unpack_vmesh +from ._vmesh_format import _json, contains_codec, pack_vmesh, probe_codec, unpack_vmesh _SCHEMA = "open4d.n4mc-sequence/v1" @@ -140,7 +140,7 @@ class N4MCCodec: preserves = ("positions", "triangles") def can_decode(self, source: Path) -> bool: - return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id + return contains_codec(source, self.id) def encode( self, sequence: Sequence, destination: Path, *, overwrite: bool = False, @@ -151,8 +151,6 @@ def encode( destination = require_vmesh_destination(destination) if destination.exists() and not overwrite: raise FileExistsError(f"artifact already exists: {destination}") - if len(sequence) < 2: - raise CodecError("N4MC .vmesh carriage requires at least two frames") for frame in sequence: mesh = frame.geometry if any((mesh.colors is not None, mesh.normals is not None, diff --git a/open4d/codec/_native.py b/open4d/codec/_native.py index 44c48b8b..ddf0f3e2 100644 --- a/open4d/codec/_native.py +++ b/open4d/codec/_native.py @@ -10,10 +10,23 @@ from ._protocol import CodecError -def run(command, label, *, cwd=None): - timeout = float(os.environ.get("OPEN4D_NATIVE_TIMEOUT", "3600")) +def _timeout(): + # Native encodes of long sequences can legitimately run for hours, so the + # bound is opt-in rather than a default that kills working jobs. + value = os.environ.get("OPEN4D_NATIVE_TIMEOUT", "").strip() + if not value: + return None + try: + timeout = float(value) + except ValueError: + timeout = math.nan if not math.isfinite(timeout) or timeout <= 0: - raise ValueError("OPEN4D_NATIVE_TIMEOUT must be finite and positive") + raise CodecError(f"OPEN4D_NATIVE_TIMEOUT must be a finite positive number of seconds, got {value!r}") + return timeout + + +def run(command, label, *, cwd=None): + timeout = _timeout() with tempfile.TemporaryFile(mode="w+", encoding="utf-8", errors="replace") as log: with subprocess.Popen( command, cwd=cwd, stdout=log, stderr=subprocess.STDOUT, diff --git a/open4d/codec/_native_profiles.py b/open4d/codec/_native_profiles.py index 6b29f5b9..3a2cfbe0 100644 --- a/open4d/codec/_native_profiles.py +++ b/open4d/codec/_native_profiles.py @@ -73,7 +73,7 @@ def layout(codec, count, native=None): ("entropy_model.npz", "entropy-model")] return files _require(isinstance(native, dict) and native.get("profile") == f"{codec}/1", "unsupported profile version") - if codec in NEURAL_CODECS or codec == "n4mc": + if codec in NEURAL_CODECS: _require(count >= 2, "native sequence profiles require at least two frames") if codec == "n4mc": _require(set(native) == {"profile"}, "unexpected N4MC configuration") diff --git a/open4d/codec/_native_temporal.py b/open4d/codec/_native_temporal.py index a52bc14d..0d8f8413 100644 --- a/open4d/codec/_native_temporal.py +++ b/open4d/codec/_native_temporal.py @@ -5,7 +5,7 @@ from ._native_profiles import PROFILES from ._protocol import CodecError from ._metadata import require_vmesh_destination -from ._vmesh_format import probe_codec +from ._vmesh_format import contains_codec def encode_native(source, destination, *, codec, overwrite=False, **options): @@ -40,7 +40,7 @@ def __init__(self, identifier): self.representation = PROFILES[identifier][0] def can_decode(self, path): - return Path(path).suffix.lower() == ".vmesh" and probe_codec(path) == self.id + return contains_codec(path, self.id) def encode(self, sequence, destination, **options): return encode_native(sequence, destination, codec=self.id, **options) diff --git a/open4d/codec/_npz.py b/open4d/codec/_npz.py index cbe95ef9..12fdd9f9 100644 --- a/open4d/codec/_npz.py +++ b/open4d/codec/_npz.py @@ -23,9 +23,11 @@ from ._protocol import CodecError from ._metadata import _json_value -from ._vmesh_format import pack_vmesh, probe_codec, unpack_vmesh +from ._vmesh_format import contains_codec, pack_vmesh, probe_codec, unpack_vmesh _MAX_ARRAY_BYTES = 256 * 1024 * 1024 +# Worst-case RLE doubles its input, plus its length prefix and the NPY header. +_MAX_RLE_MEMBER_BYTES = 2 * _MAX_ARRAY_BYTES + 8 + 4096 _FIELDS = ("positions", "triangles", "colors", "normals", "texture_coordinates") @@ -48,13 +50,13 @@ def _array_from_bytes(payload: bytes) -> np.ndarray: return np.load(BytesIO(payload), allow_pickle=False, max_header_size=4096) -def _read_array(path: Path, name: str, codec: "NumPyZipCodec") -> np.ndarray: +def _read_array(archive: ZipFile, name: str, codec: "NumPyZipCodec") -> np.ndarray: try: - with ZipFile(path) as archive: - member = archive.getinfo(f"{name}.npy") - if not 0 < member.file_size <= _MAX_ARRAY_BYTES: - raise CodecError("array payload outside limits") - value = _array_from_bytes(archive.read(member)) + member = archive.getinfo(f"{name}.npy") + limit = _MAX_RLE_MEMBER_BYTES if codec.rle else _MAX_ARRAY_BYTES + if not 0 < member.file_size <= limit: + raise CodecError("array payload outside limits") + value = _array_from_bytes(archive.read(member)) if codec.rle: if value.dtype != np.uint8 or value.ndim != 1: raise CodecError("RLE array payload must contain encoded bytes") @@ -98,15 +100,20 @@ def get_frame(self, index: int) -> Frame: except IndexError as error: raise IndexError("frame index out of range") from error arrays = record["arrays"] - values = { - name: _read_array(self.native / f"frame_{index:06d}.npz", path, self.codec) - for name, path in arrays.items() - if name != "attributes" - } - attributes = { - name: _read_array(self.native / f"frame_{index:06d}.npz", path, self.codec) - for name, path in arrays.get("attributes", {}).items() - } + try: + archive = ZipFile(self.native / f"frame_{index:06d}.npz") + except BadZipFile as error: + raise CodecError(f"invalid array payload frame {index}: {error}") from error + with archive: + values = { + name: _read_array(archive, path, self.codec) + for name, path in arrays.items() + if name != "attributes" + } + attributes = { + name: _read_array(archive, path, self.codec) + for name, path in arrays.get("attributes", {}).items() + } return Frame( frame_index=record["frame_index"], timestamp=record["timestamp"], @@ -162,12 +169,25 @@ def unpack(self, payload: bytes) -> bytes: expected = int.from_bytes(payload[:8], "little") pairs = np.frombuffer(payload[8:], dtype=np.uint8).reshape(-1, 2) if (expected > _MAX_ARRAY_BYTES or np.any(pairs[:, 0] == 0) - or sum(int(count) for count in pairs[:, 0]) != expected): + or int(pairs[:, 0].sum(dtype=np.int64)) != expected): raise CodecError("RLE payload length does not match its bounded header") return np.repeat(pairs[:, 1], pairs[:, 0]).tobytes() + def _member(self, value: np.ndarray, label: str) -> bytes: + # Enforce the decoder's bound here so encode never publishes an + # artifact that its own decoder would reject. + payload = _array_bytes(value) + if len(payload) > _MAX_ARRAY_BYTES: + raise CodecError( + f"{self.id} array {label} is {len(payload)} bytes; the limit is {_MAX_ARRAY_BYTES}" + ) + payload = self.pack(payload) + if self.rle: + payload = _array_bytes(np.frombuffer(payload, dtype=np.uint8)) + return payload + def can_decode(self, source: Path) -> bool: - return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id + return contains_codec(source, self.id) def encode( self, @@ -207,18 +227,12 @@ def encode( for name in _FIELDS: value = getattr(frame.geometry, name) if value is not None: - payload = self.pack(_array_bytes(value)) - if self.rle: - payload = _array_bytes(np.frombuffer(payload, dtype=np.uint8)) - archive.writestr(f"{name}.npy", payload) + archive.writestr(f"{name}.npy", self._member(value, f"frame {ordinal} {name}")) arrays[name] = name attributes = {} for number, (name, value) in enumerate(frame.geometry.attributes.items()): key = f"attribute_{number:04d}" - payload = self.pack(_array_bytes(value)) - if self.rle: - payload = _array_bytes(np.frombuffer(payload, dtype=np.uint8)) - archive.writestr(f"{key}.npy", payload) + archive.writestr(f"{key}.npy", self._member(value, f"frame {ordinal} attribute {name!r}")) attributes[name] = key arrays["attributes"] = attributes manifest["frames"].append({ diff --git a/open4d/codec/_protocol.py b/open4d/codec/_protocol.py index d6ca8883..04ace865 100644 --- a/open4d/codec/_protocol.py +++ b/open4d/codec/_protocol.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from open4d.gaussians import GaussianSplats, NeuralGaussianFrame + from open4d.native import NativeSequence class CodecError(RuntimeError): @@ -26,5 +27,7 @@ class Codec(Protocol): def encode(self, sequence: Sequence | Iterable[GaussianSplats], destination: Path, **options) -> Path: """Encode *sequence* into *destination*.""" - def decode(self, source: Path, **options) -> Sequence | tuple[NeuralGaussianFrame, ...]: + def decode( + self, source: Path, **options + ) -> Sequence | NativeSequence | tuple[NeuralGaussianFrame, ...]: """Open a decoded sequence backed by *source*.""" diff --git a/open4d/codec/_qndf.py b/open4d/codec/_qndf.py index 66d6288c..97ac691a 100644 --- a/open4d/codec/_qndf.py +++ b/open4d/codec/_qndf.py @@ -21,7 +21,7 @@ from ._protocol import CodecError from ._research import research_module from ._torch import torch_device -from ._vmesh_format import pack_vmesh, probe_codec, unpack_vmesh +from ._vmesh_format import contains_codec, pack_vmesh, probe_codec, unpack_vmesh _MAX_CONTEXT_BYTES = 64 * 1024 * 1024 _MAX_GRAPH_ELEMENTS = 64 * 1024 * 1024 @@ -203,7 +203,7 @@ def __init__(self, *, int8: bool = False) -> None: self.suffixes = (".vmesh",) def can_decode(self, source: Path) -> bool: - return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id + return contains_codec(source, self.id) def encode( self, sequence: Sequence, destination: Path, *, overwrite: bool = False, diff --git a/open4d/codec/_temporal.py b/open4d/codec/_temporal.py index c1498b2b..e44938b5 100644 --- a/open4d/codec/_temporal.py +++ b/open4d/codec/_temporal.py @@ -14,7 +14,7 @@ from ._metadata import _json_value from ._npz import NumPyZipCodec, _read_array -from ._vmesh_format import pack_vmesh, probe_codec, unpack_vmesh +from ._vmesh_format import contains_codec, pack_vmesh, probe_codec, unpack_vmesh from ._protocol import CodecError @@ -100,7 +100,7 @@ def __init__(self, identifier: str) -> None: self.suffixes = (".vmesh",) def can_decode(self, source: Path) -> bool: - return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id + return contains_codec(source, self.id) def encode( self, sequence: Sequence, destination: Path, *, overwrite: bool = False, @@ -164,8 +164,8 @@ def decode(self, source: Path, *, device: str | None = None) -> Sequence: if (len(members) != len(set(members)) or set(members) != {f"{name}.npy" for name in _temporal_fields(self.id)}): raise CodecError("unexpected temporal native arrays") - artifact = {name: _read_array(native / "sequence.npz", name, _PLAIN_ARRAYS) - for name in _temporal_fields(self.id)} + artifact = {name: _read_array(archive, name, _PLAIN_ARRAYS) + for name in _temporal_fields(self.id)} reference, faces = artifact["reference"], artifact["triangles"] reference_mesh = TriangleMesh(reference, faces) count, vertices = len(manifest["frames"]), len(reference_mesh.positions) diff --git a/open4d/codec/_tracked.py b/open4d/codec/_tracked.py index b3ae1bbf..e983d97f 100644 --- a/open4d/codec/_tracked.py +++ b/open4d/codec/_tracked.py @@ -17,7 +17,7 @@ from ._metadata import _json_value, _validate_manifest, require_vmesh_destination from ._protocol import CodecError from ._native import run as _run_native -from ._vmesh_format import pack_vmesh, probe_codec, unpack_vmesh +from ._vmesh_format import contains_codec, pack_vmesh, probe_codec, unpack_vmesh def _executable(value, label: str) -> str: @@ -97,7 +97,7 @@ def __init__(self, identifier): self.suffixes = (".vmesh",) def can_decode(self, source: Path) -> bool: - return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id + return contains_codec(source, self.id) def _settings(self, backend, python, encoder, decoder, *, encoding): variable = f"OPEN4D_{self.id.upper()}_ROOT" diff --git a/open4d/codec/_vmesh.py b/open4d/codec/_vmesh.py index a8cfe6bb..66e889ee 100644 --- a/open4d/codec/_vmesh.py +++ b/open4d/codec/_vmesh.py @@ -19,7 +19,7 @@ from ._metadata import _json_value, _validate_manifest, require_vmesh_destination from ._protocol import CodecError from ._native import run as _run -from ._vmesh_format import pack_vmesh, probe_codec, unpack_vmesh +from ._vmesh_format import contains_codec, pack_vmesh, probe_codec, unpack_vmesh _POSITION_BIT_DEPTH = 12 _DEFAULT_RAW_FPS = 30.0 @@ -171,7 +171,7 @@ def __init__(self, identifier: str) -> None: self._environment = identifier.upper() def can_decode(self, source: Path) -> bool: - return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id + return contains_codec(source, self.id) def encode( self, diff --git a/open4d/codec/_vmesh_format.py b/open4d/codec/_vmesh_format.py index 3f89ab96..bc495d11 100644 --- a/open4d/codec/_vmesh_format.py +++ b/open4d/codec/_vmesh_format.py @@ -18,7 +18,7 @@ from ._metadata import _validate_manifest from ._protocol import CodecError -from ._native_profiles import MAX_FRAMES, PROFILES, layout as _layout +from ._native_profiles import MAX_FRAMES, NEURAL_CODECS, PROFILES, layout as _layout _SCHEMA = "vmesh/1" _CHUNK = 1024 * 1024 @@ -164,10 +164,23 @@ def probe_codec(source: str | Path) -> str | None: return descriptor["codec"] +def contains_codec(source: str | Path, codec: str) -> bool: + """Boolean probe for ``Codec.can_decode``; never raises for foreign input. + + Callers that need the reason a file is rejected use probe_codec instead. + """ + if Path(source).suffix.lower() != ".vmesh": + return False + try: + return probe_codec(source) == codec + except (CodecError, OSError): + return False + + def is_mesh_profile(source: str | Path) -> bool: """Classify VMESH for mesh-only tools without loading a decoder.""" codec = probe_codec(source) - if codec in ("vega", "queen", "3dgstream", "rerf"): + if codec in NEURAL_CODECS: return False if codec == "frames": with Path(source).open("rb") as stream: diff --git a/open4d/codec/tests/test_artifact_validation.py b/open4d/codec/tests/test_artifact_validation.py index bc461188..29847aa9 100644 --- a/open4d/codec/tests/test_artifact_validation.py +++ b/open4d/codec/tests/test_artifact_validation.py @@ -106,6 +106,16 @@ def test_codec_detection_returns_false_for_malformed_manifest(tmp_path, codec, m assert codec.can_decode(artifact) is False +@pytest.mark.parametrize("payload", [b"PK\x03\x04legacy", b"VMESH\x00\x01\x00", b"VMESH\x00\x02\x00"]) +def test_codec_detection_returns_false_for_foreign_vmesh(tmp_path, payload): + from open4d.codec._api import _CODECS, _codec + artifact = tmp_path / "foreign.vmesh" + artifact.write_bytes(payload) + assert all(codec.can_decode(artifact) is False for codec in _CODECS.values()) + with pytest.raises(CodecError, match="invalid VMESH"): + _codec(None, artifact) + + @pytest.mark.parametrize("frame", [{"frame_index": -1, "timestamp": 0}, {"frame_index": 0, "timestamp": float("nan")}, {"frame_index": 0, "timestamp": 0, "metadata": []}]) diff --git a/open4d/codec/tests/test_codec_review.py b/open4d/codec/tests/test_codec_review.py index c5918bc3..34559ea2 100644 --- a/open4d/codec/tests/test_codec_review.py +++ b/open4d/codec/tests/test_codec_review.py @@ -19,6 +19,42 @@ def test_native_timeout_is_bounded_and_explained(monkeypatch): _vmesh._run([sys.executable, "-c", "import time; time.sleep(1)"], "native") +def test_native_timeout_is_opt_in(monkeypatch): + from open4d.codec import _native + + monkeypatch.delenv("OPEN4D_NATIVE_TIMEOUT", raising=False) + assert _native._timeout() is None + + +@pytest.mark.parametrize("value", ["soon", "0", "-5", "inf", "nan"]) +def test_invalid_native_timeout_is_a_codec_error(monkeypatch, value): + import sys + from open4d.codec import CodecError, _vmesh + + monkeypatch.setenv("OPEN4D_NATIVE_TIMEOUT", value) + with pytest.raises(CodecError, match="OPEN4D_NATIVE_TIMEOUT"): + _vmesh._run([sys.executable, "-c", "pass"], "native") + + +@pytest.mark.parametrize("identifier", ["npz", "rle"]) +def test_array_codec_refuses_to_write_arrays_it_cannot_read(tmp_path, monkeypatch, identifier): + from open4d import Frame, MemoryFrameProvider, Sequence, TriangleMesh + from open4d.codec import CodecError, _npz + + codec = {c.id: c for c in _npz.REFERENCE_CODECS}[identifier] + mesh = TriangleMesh(np.random.default_rng(0).random((64, 3)), [[0, 1, 2]]) + source = Sequence(MemoryFrameProvider([Frame(0, 0., mesh)])) + monkeypatch.setattr(_npz, "_MAX_ARRAY_BYTES", 512) + destination = tmp_path / "large.vmesh" + with pytest.raises(CodecError, match="limit"): + codec.encode(source, destination) + assert not destination.exists() + monkeypatch.setattr(_npz, "_MAX_ARRAY_BYTES", 4096) + monkeypatch.setattr(_npz, "_MAX_RLE_MEMBER_BYTES", 2 * 4096 + 8 + 4096) + with codec.decode(codec.encode(source, destination)) as decoded: + np.testing.assert_array_equal(decoded[0].geometry.positions, mesh.positions) + + def test_n4mc_reconstruction_uses_tsdf_sampling_extent(): pytest.importorskip("torch") pytest.importorskip("skimage") diff --git a/open4d/codec/tests/test_n4mc_vmesh.py b/open4d/codec/tests/test_n4mc_vmesh.py index 768001af..76518715 100644 --- a/open4d/codec/tests/test_n4mc_vmesh.py +++ b/open4d/codec/tests/test_n4mc_vmesh.py @@ -13,17 +13,17 @@ pytestmark = pytest.mark.cpu -def legacy_archive(path, *, omit=None, duplicate=False, normalization=None): +def legacy_archive(path, *, omit=None, duplicate=False, normalization=None, frames=2): manifest = dict(schema="open4d.n4mc-sequence/v1", codec="n4mc", normalization=normalization or dict(center=[1, 2, 3], scale=2.), metadata={"scene": "fixture"}, allow_nonmonotonic_timestamps=False, frames=[dict(frame_index=7, timestamp=.125, metadata={"key": "first"}), - dict(frame_index=19, timestamp=.875, metadata={"key": "last"})]) + dict(frame_index=19, timestamp=.875, metadata={"key": "last"})][:frames]) torch = pytest.importorskip("torch") checkpoint = BytesIO() torch.save({"model": {}, "model_config": {}, "schema": "open4d.n4mc/v1"}, checkpoint) - payloads = {"checkpoint.pt": checkpoint.getvalue(), "frame_000000.npz": b"opaque latent 0", - "frame_000001.npz": b"opaque latent 1"} + payloads = {"checkpoint.pt": checkpoint.getvalue(), + **{f"frame_{i:06d}.npz": f"opaque latent {i}".encode() for i in range(frames)}} with ZipFile(path, "w") as archive: archive.writestr("manifest.json", json.dumps(manifest)) for name, data in payloads.items(): @@ -65,6 +65,14 @@ def test_n4d_migration_preserves_native_payloads_without_loading_models(tmp_path open4d.load(artifact, codec="tsmc") +def test_single_frame_n4d_migrates(tmp_path, monkeypatch): + source = tmp_path / "single.n4d" + manifest, _ = legacy_archive(source, frames=1) + monkeypatch.setattr(_n4mc, "_backend", lambda: pytest.fail("repacking must not load a model")) + artifact = migrate_legacy(source, tmp_path / "single.vmesh") + assert inspect_vmesh(artifact)["sequence"]["frames"] == manifest["frames"] + + @pytest.mark.parametrize("options", [dict(omit="checkpoint.pt"), dict(omit="frame_000001.npz"), dict(duplicate=True), dict(normalization={"center": [0, 0, 0], "scale": 0})]) def test_invalid_n4d_never_publishes_vmesh(tmp_path, options): diff --git a/open4d/gaussians.py b/open4d/gaussians.py index e7a89b46..0450ed1a 100644 --- a/open4d/gaussians.py +++ b/open4d/gaussians.py @@ -10,13 +10,16 @@ import tempfile from dataclasses import dataclass, field from pathlib import Path -from typing import Sequence +from typing import TYPE_CHECKING, Sequence import numpy as np from numpy.typing import NDArray from ._files import publish_directory +if TYPE_CHECKING: + from .native import NativeSequence + @dataclass(frozen=True, eq=False) class _GaussianGeometry: @@ -474,8 +477,8 @@ class _VegaCodec: preserves = ("positions", "scales", "rotations", "opacities", "neural_appearance") def can_decode(self, source): - from .codec._vmesh_format import probe_codec - return Path(source).suffix.lower() == ".vmesh" and probe_codec(source) == self.id + from .codec._vmesh_format import contains_codec + return contains_codec(source, self.id) def encode(self, sequence, destination: Path, **options) -> Path: from .codec._metadata import require_vmesh_destination @@ -493,7 +496,8 @@ def encode(self, sequence, destination: Path, **options) -> Path: with import_native(encoded, **timeline) as native: return save_native(native, destination, overwrite=overwrite) - def decode(self, source: Path, **options): + def decode(self, source: Path, **options) -> NativeSequence: + """Open the owned native state; call ``.decode()`` to evaluate frames.""" from .codec._native_temporal import NativeTemporalCodec return NativeTemporalCodec("vega").decode(source, **options) diff --git a/open4d/io/_native_usd.py b/open4d/io/_native_usd.py index d283ecca..14175885 100644 --- a/open4d/io/_native_usd.py +++ b/open4d/io/_native_usd.py @@ -37,6 +37,16 @@ def _open(path): def is_native_usd(path): + # Layer metadata first: ordinary mesh USD is rejected without parsing its + # scene description, so load() opens it once and reports its own errors. + Sdf, _, _, _ = _pxr() + from pxr import Tf + try: + layer = Sdf.Layer.OpenAsAnonymous(str(Path(path).absolute()), metadataOnly=True) + except Tf.ErrorException: + return False + if layer is None or layer.defaultPrim != _PRIM.lstrip("/"): + return False _, prim = _open(path) return bool(prim and prim.HasAttribute("vmesh:schema")) diff --git a/open4d/io/tests/test_usd_sequence.py b/open4d/io/tests/test_usd_sequence.py index 52891f6b..30e8e6f6 100644 --- a/open4d/io/tests/test_usd_sequence.py +++ b/open4d/io/tests/test_usd_sequence.py @@ -217,6 +217,15 @@ def test_public_usd_load_is_lazy(tmp_path, monkeypatch): decoded.close() +def test_mesh_usd_load_does_not_fully_open_the_layer_twice(tmp_path, monkeypatch): + from open4d.io import _native_usd + + path = open4d.save(rich_sequence(), tmp_path / "capture.usdc") + monkeypatch.setattr(_native_usd, "_open", lambda path: pytest.fail("mesh USD opened as native")) + with open4d.load(path) as decoded: + assert len(decoded) == 2 + + def test_usd_empty_geometry_omits_extent_and_round_trips(tmp_path): empty = TriangleMesh( np.empty((0, 3), dtype=np.float32),