diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 300f6d67..501fe278 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,7 @@ jobs: - name: Run every Open3D-backed test run: >- python -m pytest -m open3d - integrations/open3d/tests open4d/torch_ops/tests open4d/streaming/tests + integrations/open3d/tests open4d/torch_ops/tests open4d/reconstruction/rgbd/tests open4d/codec/tests/test_notebook.py research-codecs: @@ -219,7 +219,8 @@ jobs: - name: Compile supported Python sources run: >- python -m compileall -q - open4d/*.py open4d/streaming/*.py open4d/codec open4d/core open4d/io + open4d/*.py open4d/reconstruction/*.py open4d/reconstruction/rgbd/*.py + open4d/transport open4d/codec open4d/core open4d/io open4d/torch_ops open4d/visualization integrations/__init__.py integrations/open3d examples/visualization examples/streaming_demo.py scripts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bece1873..695429eb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,15 +18,16 @@ For research and reconstruction tests on Python 3.12: ```bash python -m pip install -e '.[dev,klt,n4mc,qndf,open3d,gaussians]' python -m pytest open4d/codec/tests/test_research_cpu.py -m 'not gpu' -python -m pytest open4d/streaming/tests integrations/open3d/tests +python -m pytest open4d/reconstruction/rgbd/tests integrations/open3d/tests ``` ## Code - Keep the base install NumPy-only and import optional dependencies when used. - Put public mesh codec adapters in `open4d/codec`, Gaussian APIs in - `open4d/gaussians.py`, and transport/reconstruction in `open4d/streaming`. -- Preserve the research implementations under `open4d/codecs` and + `open4d/gaussians.py`, RGB-D reconstruction in `open4d/reconstruction/rgbd`, + and TCP frame transport in `open4d/transport`. +- Preserve the research implementations under `open4d/codecs` and the rest of `open4d/reconstruction`. Connect them through small adapters. - A decoded artifact must work without the original input or hidden encoder files. Keep native backends separate from the installed Python package. @@ -38,7 +39,7 @@ python -m pytest open4d/streaming/tests integrations/open3d/tests ```bash python -m pytest -python -m compileall -q open4d/*.py open4d/streaming/*.py open4d/codec open4d/core open4d/io open4d/torch_ops open4d/visualization integrations/__init__.py integrations/open3d examples/visualization scripts +python -m compileall -q open4d/*.py open4d/reconstruction/*.py open4d/reconstruction/rgbd/*.py open4d/transport open4d/codec open4d/core open4d/io open4d/torch_ops open4d/visualization integrations/__init__.py integrations/open3d examples/visualization scripts python scripts/check_markdown_links.py python scripts/check_provenance.py python scripts/check_release_gate.py --expect-blocked diff --git a/MANIFEST.in b/MANIFEST.in index ad6fd2ff..78a5cb8d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -25,9 +25,12 @@ global-exclude __pycache__ global-exclude *.py[cod] global-exclude .DS_Store -recursive-exclude open4d/streaming/tests * -prune open4d/streaming/app -prune open4d/streaming/src -prune open4d/streaming/include -prune open4d/streaming/python -prune open4d/streaming/tools +include open4d/reconstruction/__init__.py +recursive-include open4d/reconstruction/rgbd *.py +recursive-exclude open4d/reconstruction/rgbd/tests * +prune open4d/reconstruction/rgbd/app +prune open4d/reconstruction/rgbd/src +prune open4d/reconstruction/rgbd/include +prune open4d/reconstruction/rgbd/python +prune open4d/reconstruction/rgbd/tools +recursive-exclude open4d/transport/tests * diff --git a/README.md b/README.md index fbbe5cba..71e335f3 100644 --- a/README.md +++ b/README.md @@ -169,18 +169,18 @@ network or SSH tunnel; this protocol has no authentication or encryption. Use `realtime=False` to transfer a recorded sequence as fast as possible. The camera capture and native reconstruction programs are in -[open4d/streaming](open4d/streaming/README.md), formerly `reconstruction/rgbd`. +[open4d/reconstruction/rgbd](open4d/reconstruction/rgbd/README.md). ## Compare methods in a user study -[open4d/webclients](open4d/webclients/README.md) is a browser app that runs one +[open4d/streamer/study](open4d/streamer/study/README.md) is a browser app that runs one participant through several streaming methods — Ours, ViVo, NAVA and Vega — under the same network trace, from the same start view, along the same camera path, and asks them to rate each clip. The trace can be uploaded in the browser; the results export as CSV. ```bash -cd open4d/webclients/system/WebClient && npm install && node build.js +cd open4d/streamer/study/system/WebClient && npm install && node build.js cd ../.. && PYTHONPATH=/path/to/4DVideoStreaming PYTHON_BIN= scripts/run_web_demo.sh # then open http://:3000/web/ ``` diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index bd55a776..b89e868b 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -21,7 +21,7 @@ all required notices to be assembled, and a recorded maintainer approval. | Package | Basis | Candidate decision | | --- | --- | --- | -| `open4d`, `open4d.codec`, `open4d.core`, `open4d.io`, `open4d.visualization`, `open4d.streaming` | Project-authored, root MIT; NumPy base | Candidate after final review; research implementations remain excluded | +| `open4d`, `open4d.codec`, `open4d.core`, `open4d.io`, `open4d.visualization`, `open4d.reconstruction.rgbd`, `open4d.transport` | Project-authored, root MIT; NumPy base | Candidate after final review; research implementations remain excluded | | `open4d.torch_ops` | Project-authored source; Torch optional and not redistributed | Candidate after final review | | `integrations`, `integrations.open3d` | Project-authored adapter source; Open3D optional and not redistributed | Candidate after final review | @@ -46,7 +46,7 @@ distribution and still needs separate review before any redistribution. | `open4d/codecs/tsmc` | Research pipeline, copied trees, Draco `47238930f698250f474163e1a29d77858aa5c158`, SAM3 `5dd401d1c5c1d5c3eedff06d41b77af824517619`, tracked datasets/paper. Local change: entropy coding supports constant displacement matrices and rejects values outside the int32/ANS range | No top-level component license; copied subtrees have separate terms | `BLOCK`; inventory source, patches, paper/media, data rights, SAM3 terms, and notices | | `open4d/codecs/vdmc` | MPEG reference submodule `ecffe4212e5e956761c4fa14a17c453ae916b0b1` | Not audited in the uninitialized tree | `BLOCK`; initialize, record license/notices, and review intended source/binary distribution | | `open4d/codecs/faster_vdmc` | Fork `cicm4/mpeg-vdmc-tm` at `93cdd5e1367b0f9f81c251ef89255bcb2f0d2d3f`, derived from `MPEGGroup/mpeg-vdmc-tm` at `ecffe4212e5e956761c4fa14a17c453ae916b0b1` with eight encoder/decoder performance commits | Fork `COPYING` contains the ISO/IEC BSD 3-Clause terms and expressly excludes patent and other third-party rights; build-fetched dependencies need separate review | `BLOCK`; preserve the fork history and modification record, inventory dependency licenses/notices and patent implications, and review source/binary distribution before release | -| `open4d/streaming` | Project reconstruction and protocol experiments | Root MIT is current repository evidence; camera SDK/native dependencies external | `EXCLUDED` for native/capture code; project-authored top-level Python API only is in the candidate package; camera/native dependency review remains open | +| `open4d/reconstruction/rgbd` | Project reconstruction and protocol experiments | Root MIT is current repository evidence; camera SDK/native dependencies external | `EXCLUDED` for native/capture code; project-authored top-level Python API only is in the candidate package; camera/native dependency review remains open | | `open4d/reconstruction/3dgstream` | Imported research tree; consolidated upstream revision/patch list absent | Top-level MIT plus separate SIBR/rasterizer/Gaussian terms | `BLOCK`; record all upstream revisions, patches, subtree licenses, data/model rights, and allowed artifact boundary | | `open4d/reconstruction/queen` | Imported research tree including MiDaS, SIBR, and rasterizers | Top-level NVIDIA non-commercial license plus multiple subtree licenses | `BLOCK`; record revisions, patches, all notices, model/data rights, and redistribution limits | | `open4d/reconstruction/gs_tools` | Consolidated Gaussian tooling tree containing SIBR viewers, GLM, and three rasterizer imports; one immutable upstream/patch manifest is absent | Component-local SIBR, GLM, and rasterizer license files exist with differing terms | `BLOCK`; inventory exact upstream revisions and patches, preserve every notice, and determine compatible source/binary distribution terms | diff --git a/docs/api.md b/docs/api.md index e2e7ace2..c02b93b8 100644 --- a/docs/api.md +++ b/docs/api.md @@ -97,7 +97,7 @@ saying how to install it. For several clips in one bundle, a constrained link, or the delivered-quality metrics, use that package directly — `streamer.Bundle`, `streamer.Link`, `streamer.serve`. -Separately, [`open4d/webclients`](../open4d/webclients) is the vendored +Separately, [`open4d/streamer/study`](../open4d/streamer/study) is the vendored browser-client research tree that compares five delivery systems against each other. It is not this API and shares no code with it. diff --git a/docs/components.md b/docs/components.md index f90121e8..e4ad8f8f 100644 --- a/docs/components.md +++ b/docs/components.md @@ -21,14 +21,14 @@ hardware requirements to the shared Python baseline. See | Module | | |---|---| -| [**RGB-D**](../open4d/streaming/README.md) | Synchronized multi-camera RGB-D ingestion, calibrated point-cloud fusion, CUDA TSDF mesh reconstruction, and live browser playback. Includes both the original native reconstruction code and the Python two-camera streaming pipeline | +| [**RGB-D**](../open4d/reconstruction/rgbd/README.md) | Synchronized multi-camera RGB-D ingestion, calibrated point-cloud fusion, CUDA TSDF mesh reconstruction, and live browser playback. Includes both the original native reconstruction code and the Python two-camera streaming pipeline | | [**QUEEN**](../open4d/reconstruction/queen/README.md) | Quantized efficient encoding of dynamic Gaussians for streaming free-viewpoint video (NeurIPS 2024) | | [**3DGStream**](../open4d/reconstruction/3dgstream/README.md) | On-the-fly training of 3D Gaussians for streaming photo-realistic free-viewpoint video (CVPR 2024) | | [**Vega**](../open4d/reconstruction/vega/README.md) | An ORBIT adaptation of Vega (MobiCom 2025): mobile volumetric video streaming with 3D Gaussian splatting | | [**ReRF**](../open4d/reconstruction/rerf/README.md) | Neural residual radiance fields (CVPR 2023) as a streamable compression method | | [**gs-tools**](../open4d/reconstruction/gs_tools/README.md) | The one environment, rasterizers, and viewer the Gaussian methods share, plus `gs-tools view` for putting several methods under one camera | | [**streamer**](../open4d/streamer/README.md) | Streaming and playback for 4D reconstructions, whatever their representation. Producers import it to describe and serve their output; it imports none of them | -| [**webclients**](../open4d/webclients/README.md) | A browser user study: one participant, several streaming methods, the same network trace and camera path for each, then a questionnaire and CSV export. Runs Ours, ViVo, NAVA and Vega; needs the 4DVideoStreaming research repo at run time | +| [**streamer/study**](../open4d/streamer/study/README.md) | A browser user study: one participant, several streaming methods, the same network trace and camera path for each, then a questionnaire and CSV export. Runs Ours, ViVo, NAVA and Vega; needs the 4DVideoStreaming research repo at run time | ## Integrations diff --git a/docs/requirements.md b/docs/requirements.md index a34b71ea..ea2d6fe5 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -136,5 +136,5 @@ with `Hardware MFT failed to start`. 5 synchronized pairs/s held over Wi-Fi and VPN; 15 did not. Calibration layout and the step-by-step session walkthrough are in -[`open4d/streaming/README.md`](../open4d/streaming/README.md), +[`open4d/reconstruction/rgbd/README.md`](../open4d/reconstruction/rgbd/README.md), which covers how to run the pipeline and leaves requirements to this page. diff --git a/open4d/__init__.py b/open4d/__init__.py index 895cf6b7..ca80cc65 100644 --- a/open4d/__init__.py +++ b/open4d/__init__.py @@ -26,7 +26,7 @@ from ._api import load, save, unload, reconstruct, stream from ._streamer import StreamerDependencyError from .codec import available_codecs, decode_sequence as decode, encode_sequence as encode -from .streaming import receive, send +from .transport import receive, send from .gaussians import GaussianSplats, GaussianRun, NeuralGaussianFrame, load_gaussians from .metrics import compare_meshes, compare_sequences from .visualization import visualize diff --git a/open4d/_api.py b/open4d/_api.py index 27d40704..51968eae 100644 --- a/open4d/_api.py +++ b/open4d/_api.py @@ -153,7 +153,7 @@ def reconstruct(source, output=None, *, method="rgbd", **options): if method == "rgbd": if output is not None: raise TypeError("RGB-D reconstruction returns a Sequence; omit output") - from .streaming import reconstruct as reconstruct_rgbd + from .reconstruction.rgbd import reconstruct as reconstruct_rgbd return reconstruct_rgbd(source, **options) if method in ("queen", "3dgstream"): @@ -176,7 +176,7 @@ def stream(source, *address, **options): browser_options = {"out_dir", "name", "title", "rungs", "fps", "open_browser", "block"} if address or (not isinstance(source, (str, os.PathLike)) and not browser_options.intersection(options)): - from .streaming import send + from .transport import send return send(source, *address, **options) from ._streamer import stream as stream_to_browser diff --git a/open4d/reconstruction/__init__.py b/open4d/reconstruction/__init__.py new file mode 100644 index 00000000..f521b8cb --- /dev/null +++ b/open4d/reconstruction/__init__.py @@ -0,0 +1 @@ +"""Reconstruction methods. Only rgbd is part of the package; the rest is vendored research code.""" diff --git a/open4d/streaming/.gitignore b/open4d/reconstruction/rgbd/.gitignore similarity index 100% rename from open4d/streaming/.gitignore rename to open4d/reconstruction/rgbd/.gitignore diff --git a/open4d/streaming/CMakeLists.txt b/open4d/reconstruction/rgbd/CMakeLists.txt similarity index 100% rename from open4d/streaming/CMakeLists.txt rename to open4d/reconstruction/rgbd/CMakeLists.txt diff --git a/open4d/reconstruction/rgbd/README.md b/open4d/reconstruction/rgbd/README.md new file mode 100644 index 00000000..056594ab --- /dev/null +++ b/open4d/reconstruction/rgbd/README.md @@ -0,0 +1,132 @@ +# RGB-D reconstruction + +Two synchronized RGB-D cameras send frames to an Ubuntu GPU machine. That +machine fuses them into a live point cloud and CUDA meshes, and you watch the +result in a browser. This module was formerly called `MeshReduce`. + +```text +2 cameras -> Windows capture host -> SSH tunnel -> Ubuntu receiver (fuse + CUDA TSDF) -> Open3D WebRTC -> browser +``` + +The point cloud updates on every camera pair; the mesh is rebuilt +periodically. From Python, `open4d.reconstruct(source, method="rgbd")` runs +the same reconstruction on saved captures. + +## Setup + +Hardware and the tested configuration are in the [repository README](../../../README.md). + +```bash +python3 -m venv .venv && source .venv/bin/activate +python -m pip install numpy opencv-python zstandard +# plus a CUDA-enabled Open3D build (the pip wheel may not include CUDA) +python -c "import open3d as o3d; print(o3d.__version__, o3d.core.cuda.is_available())" # want True +``` + +**Calibration.** Camera 1 is the reference frame. Export each camera's +factory calibration, calibrate camera 2 against camera 1, and lay out the files +like this (`EY` = camera 1 and `J3` = camera 2, legacy names). This module only +reads calibration; it doesn't create it. Recalibrate if you move either camera. + +```text +/ +├── source/work/calibration_stepwise/factory/{ey,j3}_factory_calibration.json +└── final_validated_fusion/j3_depth_to_ey_depth_refined.txt +``` + +## Run a live session + +Start these in order. + +**1. Ubuntu receiver**, run from this directory: + +```bash +export FOURD_CALIBRATION_DIR=/abs/path/to/calibration +export PYTHON=/path/to/python-with-open3d +export FOURD_CUDA_DEVICE=0 +export FOURD_CAMERA1_SERIAL=... FOURD_CAMERA2_SERIAL=... +DISPLAY_MODE=pointcloud MESH_WINDOW=1 ./tools/run_browser_viewer.sh +# -> listening on 127.0.0.1:17000, viewer on 127.0.0.1:8888 +``` + +**2. Windows data tunnel.** Close Orbbec Viewer first. + +```powershell +ssh -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -N ` + -L 127.0.0.1:17000:127.0.0.1:17000 user@gpu-host +Test-NetConnection 127.0.0.1 -Port 17000 # TcpTestSucceeded : True +``` + +**3. Windows camera sender.** Use any sender that speaks OBP1 (packet format +in `python/protocol.py`). The tested sender is a separate companion script, so +copy it to the capture host and set your camera serials in it. + +```powershell +& C:\path\to\python.exe C:\path\to\windows_sender.py ` + --sdk-bin "C:\path\to\OrbbecSDK-K4A-Wrapper\bin" ` + --host 127.0.0.1 --port 17000 --fps 5 --report sender_report.json +``` + +Keep the default 5 FPS: at higher rates the receiver can't prepare point clouds +fast enough and replaces frames. The Ubuntu log should show `Sender connected` +and a rising `processed_pairs`. + +**4. Browser tunnel**, from the machine you're watching on: + +```bash +ssh -N -o ExitOnForwardFailure=yes -L 127.0.0.1:18888:127.0.0.1:8888 user@gpu-host +``` + +Then open . If the page loads but nothing moves, the +sender isn't connected. To stop, press Ctrl+C in the sender first, then close +the tunnels and the receiver. If everything runs on one machine, skip both +tunnels and open . + +## Options + +| Variable | Values | +|---|---| +| `DISPLAY_MODE` | `pointcloud` (motion), `mesh` (still scenes), `auto` (points until the first mesh), `both` (debugging) | +| `MESH_WINDOW` | how many recent camera pairs go into each mesh: `1` for live, `7` for a still scene | +| `MESH_FUSION_MODE` | `independent-merge` (default; one TSDF per camera, then merge) or `shared-tsdf` (one volume; usually cleaner) | +| `MESH_MERGE_MODE` | `concatenate` (keep every triangle) or `weld` (merge vertices within `MESH_WELD_RADIUS`, metres) | + +Merging does no cropping, decimation or hole filling, so overlapping views can +show doubled surfaces. The MeshReduce paper's overlap removal and stitching +step isn't implemented yet. + +Output goes to `output/two-camera-fusion/`: +`latest_live_full_scene_{pointcloud,mesh}.ply`, `live_fusion_report.json` and +`live_browser.log`. A healthy mesh log shows `"backend": "CUDA:0"`. + +## Without the live cameras + +```bash +# replay saved pairs through the live receiver (start step 1 first) +"$PYTHON" tools/replay_obp1_sender.py --captures-root /abs/path/to/saved-pairs --fps 2 + +# bounded run from the remote sender, or one-shot reconstruction of a saved sequence +MAX_PAIRS=30 ./tools/run_remote_two_camera_fusion.sh +FOURD_CAPTURE_ROOT=/path/to/capture-data ./tools/reconstruct_saved_two_camera.py +``` + +## Native C++ pipeline + +For K4A-compatible cameras plugged directly into this machine, or recorded +`.mkv` files. It writes PLY, textured OBJ, Draco and stage metrics. + +```bash +sudo apt install -y build-essential cmake ninja-build git \ + libopencv-dev libeigen3-dev libjsoncpp-dev libdraco-dev draco +cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release +cmake --build build --target rgbd_streamer dual_camera_fusion -j"$(nproc)" +./build/app/rgbd_streamer .json # one camera or .mkv +./build/app/dual_camera_fusion config.dual.json +ctest --test-dir build --output-on-failure +``` + +## Files + +- `python/protocol.py`: OBP1 packet definitions +- `tools/receive_mesh_frame.py`, `tools/receive_live_stream.py`: reference receivers for MRD1/2 and MRD3 +- [`docs/artifacts.md`](../../../docs/artifacts.md): generated-data policy (outputs are git-ignored) diff --git a/open4d/reconstruction/rgbd/__init__.py b/open4d/reconstruction/rgbd/__init__.py new file mode 100644 index 00000000..26478374 --- /dev/null +++ b/open4d/reconstruction/rgbd/__init__.py @@ -0,0 +1,5 @@ +"""Reconstruct meshes from calibrated RGB-D captures.""" + +from ._reconstruction import reconstruct + +__all__ = ["reconstruct"] diff --git a/open4d/streaming/_reconstruction.py b/open4d/reconstruction/rgbd/_reconstruction.py similarity index 99% rename from open4d/streaming/_reconstruction.py rename to open4d/reconstruction/rgbd/_reconstruction.py index 473af6dd..a1f693c5 100644 --- a/open4d/streaming/_reconstruction.py +++ b/open4d/reconstruction/rgbd/_reconstruction.py @@ -7,7 +7,7 @@ import numpy as np -from ..core import Frame, Sequence, TopologyMode, TriangleMesh +from ...core import Frame, Sequence, TopologyMode, TriangleMesh def _positive(value, name): diff --git a/open4d/streaming/app/CMakeLists.txt b/open4d/reconstruction/rgbd/app/CMakeLists.txt similarity index 100% rename from open4d/streaming/app/CMakeLists.txt rename to open4d/reconstruction/rgbd/app/CMakeLists.txt diff --git a/open4d/streaming/app/dual_camera_fusion.cpp b/open4d/reconstruction/rgbd/app/dual_camera_fusion.cpp similarity index 100% rename from open4d/streaming/app/dual_camera_fusion.cpp rename to open4d/reconstruction/rgbd/app/dual_camera_fusion.cpp diff --git a/open4d/streaming/app/register_meshes.cpp b/open4d/reconstruction/rgbd/app/register_meshes.cpp similarity index 100% rename from open4d/streaming/app/register_meshes.cpp rename to open4d/reconstruction/rgbd/app/register_meshes.cpp diff --git a/open4d/streaming/app/rgbd_live_streamer.cpp b/open4d/reconstruction/rgbd/app/rgbd_live_streamer.cpp similarity index 100% rename from open4d/streaming/app/rgbd_live_streamer.cpp rename to open4d/reconstruction/rgbd/app/rgbd_live_streamer.cpp diff --git a/open4d/streaming/app/rgbd_streamer.cpp b/open4d/reconstruction/rgbd/app/rgbd_streamer.cpp similarity index 100% rename from open4d/streaming/app/rgbd_streamer.cpp rename to open4d/reconstruction/rgbd/app/rgbd_streamer.cpp diff --git a/open4d/streaming/app/sensor_client.cpp b/open4d/reconstruction/rgbd/app/sensor_client.cpp similarity index 100% rename from open4d/streaming/app/sensor_client.cpp rename to open4d/reconstruction/rgbd/app/sensor_client.cpp diff --git a/open4d/streaming/config.dual.json b/open4d/reconstruction/rgbd/config.dual.json similarity index 100% rename from open4d/streaming/config.dual.json rename to open4d/reconstruction/rgbd/config.dual.json diff --git a/open4d/streaming/config.json b/open4d/reconstruction/rgbd/config.json similarity index 100% rename from open4d/streaming/config.json rename to open4d/reconstruction/rgbd/config.json diff --git a/open4d/streaming/config.live.json b/open4d/reconstruction/rgbd/config.live.json similarity index 100% rename from open4d/streaming/config.live.json rename to open4d/reconstruction/rgbd/config.live.json diff --git a/open4d/streaming/config.rgbd-tcp.json b/open4d/reconstruction/rgbd/config.rgbd-tcp.json similarity index 100% rename from open4d/streaming/config.rgbd-tcp.json rename to open4d/reconstruction/rgbd/config.rgbd-tcp.json diff --git a/open4d/streaming/config.rgbd.json b/open4d/reconstruction/rgbd/config.rgbd.json similarity index 100% rename from open4d/streaming/config.rgbd.json rename to open4d/reconstruction/rgbd/config.rgbd.json diff --git a/open4d/streaming/config.single-camera-1.json b/open4d/reconstruction/rgbd/config.single-camera-1.json similarity index 100% rename from open4d/streaming/config.single-camera-1.json rename to open4d/reconstruction/rgbd/config.single-camera-1.json diff --git a/open4d/streaming/include/capture_thread.hpp b/open4d/reconstruction/rgbd/include/capture_thread.hpp similarity index 100% rename from open4d/streaming/include/capture_thread.hpp rename to open4d/reconstruction/rgbd/include/capture_thread.hpp diff --git a/open4d/streaming/include/data_type.hpp b/open4d/reconstruction/rgbd/include/data_type.hpp similarity index 100% rename from open4d/streaming/include/data_type.hpp rename to open4d/reconstruction/rgbd/include/data_type.hpp diff --git a/open4d/streaming/include/pipe.hpp b/open4d/reconstruction/rgbd/include/pipe.hpp similarity index 100% rename from open4d/streaming/include/pipe.hpp rename to open4d/reconstruction/rgbd/include/pipe.hpp diff --git a/open4d/streaming/include/reconstruction/reconstruction.hpp b/open4d/reconstruction/rgbd/include/reconstruction/reconstruction.hpp similarity index 100% rename from open4d/streaming/include/reconstruction/reconstruction.hpp rename to open4d/reconstruction/rgbd/include/reconstruction/reconstruction.hpp diff --git a/open4d/streaming/include/reconstruction/texture_mapping.hpp b/open4d/reconstruction/rgbd/include/reconstruction/texture_mapping.hpp similarity index 100% rename from open4d/streaming/include/reconstruction/texture_mapping.hpp rename to open4d/reconstruction/rgbd/include/reconstruction/texture_mapping.hpp diff --git a/open4d/streaming/include/reconstruction/texture_mapping_cuda.hpp b/open4d/reconstruction/rgbd/include/reconstruction/texture_mapping_cuda.hpp similarity index 100% rename from open4d/streaming/include/reconstruction/texture_mapping_cuda.hpp rename to open4d/reconstruction/rgbd/include/reconstruction/texture_mapping_cuda.hpp diff --git a/open4d/streaming/include/sensor/cv_convert_util.hpp b/open4d/reconstruction/rgbd/include/sensor/cv_convert_util.hpp similarity index 100% rename from open4d/streaming/include/sensor/cv_convert_util.hpp rename to open4d/reconstruction/rgbd/include/sensor/cv_convert_util.hpp diff --git a/open4d/streaming/include/sensor/kinect_cameras.hpp b/open4d/reconstruction/rgbd/include/sensor/kinect_cameras.hpp similarity index 100% rename from open4d/streaming/include/sensor/kinect_cameras.hpp rename to open4d/reconstruction/rgbd/include/sensor/kinect_cameras.hpp diff --git a/open4d/streaming/include/sensor/kinect_capture.hpp b/open4d/reconstruction/rgbd/include/sensor/kinect_capture.hpp similarity index 100% rename from open4d/streaming/include/sensor/kinect_capture.hpp rename to open4d/reconstruction/rgbd/include/sensor/kinect_capture.hpp diff --git a/open4d/streaming/include/streaming/network_stream.hpp b/open4d/reconstruction/rgbd/include/streaming/network_stream.hpp similarity index 100% rename from open4d/streaming/include/streaming/network_stream.hpp rename to open4d/reconstruction/rgbd/include/streaming/network_stream.hpp diff --git a/open4d/streaming/include/streaming/send_image.hpp b/open4d/reconstruction/rgbd/include/streaming/send_image.hpp similarity index 100% rename from open4d/streaming/include/streaming/send_image.hpp rename to open4d/reconstruction/rgbd/include/streaming/send_image.hpp diff --git a/open4d/streaming/include/streaming/sender_to_unity.hpp b/open4d/reconstruction/rgbd/include/streaming/sender_to_unity.hpp similarity index 100% rename from open4d/streaming/include/streaming/sender_to_unity.hpp rename to open4d/reconstruction/rgbd/include/streaming/sender_to_unity.hpp diff --git a/open4d/streaming/include/util.hpp b/open4d/reconstruction/rgbd/include/util.hpp similarity index 100% rename from open4d/streaming/include/util.hpp rename to open4d/reconstruction/rgbd/include/util.hpp diff --git a/open4d/streaming/python/live_two_camera_fusion.py b/open4d/reconstruction/rgbd/python/live_two_camera_fusion.py similarity index 100% rename from open4d/streaming/python/live_two_camera_fusion.py rename to open4d/reconstruction/rgbd/python/live_two_camera_fusion.py diff --git a/open4d/streaming/python/live_two_camera_webrtc.py b/open4d/reconstruction/rgbd/python/live_two_camera_webrtc.py similarity index 100% rename from open4d/streaming/python/live_two_camera_webrtc.py rename to open4d/reconstruction/rgbd/python/live_two_camera_webrtc.py diff --git a/open4d/streaming/python/protocol.py b/open4d/reconstruction/rgbd/python/protocol.py similarity index 100% rename from open4d/streaming/python/protocol.py rename to open4d/reconstruction/rgbd/python/protocol.py diff --git a/open4d/streaming/src/CMakeLists.txt b/open4d/reconstruction/rgbd/src/CMakeLists.txt similarity index 100% rename from open4d/streaming/src/CMakeLists.txt rename to open4d/reconstruction/rgbd/src/CMakeLists.txt diff --git a/open4d/streaming/src/reconstruction/CMakeLists.txt b/open4d/reconstruction/rgbd/src/reconstruction/CMakeLists.txt similarity index 100% rename from open4d/streaming/src/reconstruction/CMakeLists.txt rename to open4d/reconstruction/rgbd/src/reconstruction/CMakeLists.txt diff --git a/open4d/streaming/src/reconstruction/reconstruction.cpp b/open4d/reconstruction/rgbd/src/reconstruction/reconstruction.cpp similarity index 100% rename from open4d/streaming/src/reconstruction/reconstruction.cpp rename to open4d/reconstruction/rgbd/src/reconstruction/reconstruction.cpp diff --git a/open4d/streaming/src/reconstruction/texture_mapping.cpp b/open4d/reconstruction/rgbd/src/reconstruction/texture_mapping.cpp similarity index 100% rename from open4d/streaming/src/reconstruction/texture_mapping.cpp rename to open4d/reconstruction/rgbd/src/reconstruction/texture_mapping.cpp diff --git a/open4d/streaming/src/reconstruction/texture_mapping_cuda.cu b/open4d/reconstruction/rgbd/src/reconstruction/texture_mapping_cuda.cu similarity index 100% rename from open4d/streaming/src/reconstruction/texture_mapping_cuda.cu rename to open4d/reconstruction/rgbd/src/reconstruction/texture_mapping_cuda.cu diff --git a/open4d/streaming/src/sensor/CMakeLists.txt b/open4d/reconstruction/rgbd/src/sensor/CMakeLists.txt similarity index 100% rename from open4d/streaming/src/sensor/CMakeLists.txt rename to open4d/reconstruction/rgbd/src/sensor/CMakeLists.txt diff --git a/open4d/streaming/src/sensor/kinect_cameras.cpp b/open4d/reconstruction/rgbd/src/sensor/kinect_cameras.cpp similarity index 100% rename from open4d/streaming/src/sensor/kinect_cameras.cpp rename to open4d/reconstruction/rgbd/src/sensor/kinect_cameras.cpp diff --git a/open4d/streaming/src/sensor/kinect_capture.cpp b/open4d/reconstruction/rgbd/src/sensor/kinect_capture.cpp similarity index 100% rename from open4d/streaming/src/sensor/kinect_capture.cpp rename to open4d/reconstruction/rgbd/src/sensor/kinect_capture.cpp diff --git a/open4d/streaming/src/streaming/CMakeLists.txt b/open4d/reconstruction/rgbd/src/streaming/CMakeLists.txt similarity index 100% rename from open4d/streaming/src/streaming/CMakeLists.txt rename to open4d/reconstruction/rgbd/src/streaming/CMakeLists.txt diff --git a/open4d/streaming/src/streaming/network_stream.cpp b/open4d/reconstruction/rgbd/src/streaming/network_stream.cpp similarity index 100% rename from open4d/streaming/src/streaming/network_stream.cpp rename to open4d/reconstruction/rgbd/src/streaming/network_stream.cpp diff --git a/open4d/streaming/src/streaming/send_image.cpp b/open4d/reconstruction/rgbd/src/streaming/send_image.cpp similarity index 100% rename from open4d/streaming/src/streaming/send_image.cpp rename to open4d/reconstruction/rgbd/src/streaming/send_image.cpp diff --git a/open4d/streaming/src/streaming/sender_to_unity.cpp b/open4d/reconstruction/rgbd/src/streaming/sender_to_unity.cpp similarity index 100% rename from open4d/streaming/src/streaming/sender_to_unity.cpp rename to open4d/reconstruction/rgbd/src/streaming/sender_to_unity.cpp diff --git a/open4d/streaming/src/util.cpp b/open4d/reconstruction/rgbd/src/util.cpp similarity index 100% rename from open4d/streaming/src/util.cpp rename to open4d/reconstruction/rgbd/src/util.cpp diff --git a/open4d/streaming/tests/network_stream_test.cpp b/open4d/reconstruction/rgbd/tests/network_stream_test.cpp similarity index 100% rename from open4d/streaming/tests/network_stream_test.cpp rename to open4d/reconstruction/rgbd/tests/network_stream_test.cpp diff --git a/open4d/streaming/tests/pipe_test.cpp b/open4d/reconstruction/rgbd/tests/pipe_test.cpp similarity index 100% rename from open4d/streaming/tests/pipe_test.cpp rename to open4d/reconstruction/rgbd/tests/pipe_test.cpp diff --git a/open4d/streaming/tests/test_capture.py b/open4d/reconstruction/rgbd/tests/test_capture.py similarity index 100% rename from open4d/streaming/tests/test_capture.py rename to open4d/reconstruction/rgbd/tests/test_capture.py diff --git a/open4d/streaming/tests/test_native.py b/open4d/reconstruction/rgbd/tests/test_native.py similarity index 100% rename from open4d/streaming/tests/test_native.py rename to open4d/reconstruction/rgbd/tests/test_native.py diff --git a/open4d/reconstruction/rgbd/tests/test_reconstruction.py b/open4d/reconstruction/rgbd/tests/test_reconstruction.py new file mode 100644 index 00000000..24af7c15 --- /dev/null +++ b/open4d/reconstruction/rgbd/tests/test_reconstruction.py @@ -0,0 +1,90 @@ +import importlib +import sys +from types import SimpleNamespace + +import numpy as np +import pytest + +from open4d.reconstruction.rgbd import reconstruct + + +def test_reconstruct_reports_missing_open3d(monkeypatch): + original = importlib.import_module + + def missing(name): + if name == "open3d": + raise ModuleNotFoundError("No module named 'open3d'", name="open3d") + return original(name) + + monkeypatch.setattr(importlib, "import_module", missing) + with pytest.raises(ModuleNotFoundError, match=r"open4d\[open3d\]"): + reconstruct(np.ones((1, 4, 4)), intrinsics=(10, 10, 2, 2)) + + +@pytest.mark.parametrize("kwargs,match", [ + ({"fps": 0}, "fps"), + ({"intrinsics": (-1, 20, 16, 16)}, "intrinsics"), + ({"camera_poses": np.zeros((4, 4))}, "rigid"), + ({"color": np.zeros((1, 32, 32, 3))}, "uint8"), +]) +def test_reconstruct_validates_before_loading_optional_dependencies(kwargs, match): + settings = {"intrinsics": (20, 20, 16, 16), **kwargs} + with pytest.raises(ValueError, match=match): + reconstruct(np.ones((1, 32, 32)), **settings) + + +@pytest.mark.parametrize("version", ["0.18.0", "0.20.0"]) +def test_reconstruct_rejects_unsupported_open3d_before_returning_a_sequence(monkeypatch, version): + monkeypatch.setitem(sys.modules, "open3d", SimpleNamespace(__version__=version)) + with pytest.raises(RuntimeError, match=r"Open3D 0\.19\.x.*open3d>=0\.19,<0\.20"): + reconstruct(np.ones((1, 32, 32)), intrinsics=(20, 20, 16, 16)) + + +@pytest.mark.open3d +def test_reconstruct_depth_planes_preserves_motion_units_and_camera_pose(): + pytest.importorskip("open3d") + depth = np.stack([np.full((48, 48), 1000), np.full((48, 48), 1500)]) + color = np.zeros((*depth.shape, 3), dtype=np.uint8) + color[..., 0] = 255 + pose = np.eye(4) + pose[0, 3] = 2 + with reconstruct(depth, color, intrinsics=(60, 60, 24, 24), camera_poses=pose, + voxel_size=0.025, truncation=0.1, fps=5) as sequence: + assert sequence.timestamps == (0, 0.2) + for index, frame in enumerate(sequence): + mesh = frame.geometry + assert len(mesh.triangles) > 100 + assert mesh.positions[:, 2] == pytest.approx(1 + index * 0.5, abs=0.015) + assert np.median(mesh.positions[:, 0]) == pytest.approx(2, abs=0.08) + assert np.mean(mesh.colors[:, 0]) > 0.95 + assert np.max(mesh.colors[:, 1:]) < 0.05 + + +@pytest.mark.open3d +def test_reconstruct_two_cameras_and_empty_depth(): + pytest.importorskip("open3d") + depth = np.full((1, 2, 48, 48), 1.0) + poses = np.stack([np.eye(4), np.eye(4)]) + poses[1, 0, 3] = 0.5 + with reconstruct(depth, intrinsics=(60, 60, 24, 24), camera_poses=poses, + depth_scale=1, voxel_size=0.025, truncation=0.1) as sequence: + mesh = sequence[0].geometry + assert mesh.positions[:, 0].max() > 0.8 + assert mesh.positions[:, 0].min() < -0.3 + assert mesh.positions[:, 2] == pytest.approx(1, abs=0.015) + assert mesh.colors is None + assert mesh.normals is None # Geometry-only output can go straight to a mesh codec. + with reconstruct(np.zeros((1, 48, 48)), intrinsics=(60, 60, 24, 24)) as sequence: + assert len(sequence[0].geometry.positions) == 0 + + +@pytest.mark.open3d +@pytest.mark.parametrize("scale", [1e-50, 1e50]) +def test_reconstruct_scales_depth_before_float32_conversion(scale): + pytest.importorskip("open3d") + depth = np.full((1, 48, 48), scale) + with reconstruct(depth, intrinsics=(60, 60, 24, 24), depth_scale=scale, + voxel_size=0.025, truncation=0.1) as sequence: + mesh = sequence[0].geometry + assert len(mesh.triangles) > 100 + assert mesh.positions[:, 2] == pytest.approx(1, abs=0.015) diff --git a/open4d/streaming/tests/texture_mapping_cuda_test.cpp b/open4d/reconstruction/rgbd/tests/texture_mapping_cuda_test.cpp similarity index 100% rename from open4d/streaming/tests/texture_mapping_cuda_test.cpp rename to open4d/reconstruction/rgbd/tests/texture_mapping_cuda_test.cpp diff --git a/open4d/streaming/tools/receive_live_stream.py b/open4d/reconstruction/rgbd/tools/receive_live_stream.py similarity index 100% rename from open4d/streaming/tools/receive_live_stream.py rename to open4d/reconstruction/rgbd/tools/receive_live_stream.py diff --git a/open4d/streaming/tools/receive_mesh_frame.py b/open4d/reconstruction/rgbd/tools/receive_mesh_frame.py similarity index 100% rename from open4d/streaming/tools/receive_mesh_frame.py rename to open4d/reconstruction/rgbd/tools/receive_mesh_frame.py diff --git a/open4d/streaming/tools/reconstruct_saved_two_camera.py b/open4d/reconstruction/rgbd/tools/reconstruct_saved_two_camera.py similarity index 100% rename from open4d/streaming/tools/reconstruct_saved_two_camera.py rename to open4d/reconstruction/rgbd/tools/reconstruct_saved_two_camera.py diff --git a/open4d/streaming/tools/replay_obp1_sender.py b/open4d/reconstruction/rgbd/tools/replay_obp1_sender.py similarity index 100% rename from open4d/streaming/tools/replay_obp1_sender.py rename to open4d/reconstruction/rgbd/tools/replay_obp1_sender.py diff --git a/open4d/streaming/tools/run_browser_viewer.sh b/open4d/reconstruction/rgbd/tools/run_browser_viewer.sh similarity index 100% rename from open4d/streaming/tools/run_browser_viewer.sh rename to open4d/reconstruction/rgbd/tools/run_browser_viewer.sh diff --git a/open4d/streaming/tools/run_remote_two_camera_fusion.sh b/open4d/reconstruction/rgbd/tools/run_remote_two_camera_fusion.sh similarity index 97% rename from open4d/streaming/tools/run_remote_two_camera_fusion.sh rename to open4d/reconstruction/rgbd/tools/run_remote_two_camera_fusion.sh index 57d76ecc..3e0d7474 100755 --- a/open4d/streaming/tools/run_remote_two_camera_fusion.sh +++ b/open4d/reconstruction/rgbd/tools/run_remote_two_camera_fusion.sh @@ -2,7 +2,7 @@ # Bounded two-camera reconstruction from a live remote capture host. # # Requires the capture host's sender to be running and its data tunnel to map -# the sender's 127.0.0.1:17000 to this machine. See REMOTE_TWO_CAMERA.md. +# the sender's 127.0.0.1:17000 to this machine. See README.md. # # Configuration (all overridable): # FOURD_CAPTURE_ROOT directory holding your calibration set (required) diff --git a/open4d/streamer/README.md b/open4d/streamer/README.md index 59ba8b64..2b626b02 100644 --- a/open4d/streamer/README.md +++ b/open4d/streamer/README.md @@ -1,30 +1,22 @@ # streamer -Streaming and playback for 4D reconstructions — whatever their representation. +Serves 4D reconstructions (meshes, point clouds, Gaussians or rendered pixels) +to a browser for playback. [`../reconstruction`](../reconstruction) makes the content and +this package delivers it. It imports none of the methods that produce it. -Beside `../reconstruction` rather than inside it, because it produces no -reconstruction: those modules make the content and this one delivers it. It is -specific to none of them, and was never specific to Gaussians. The dependency runs one way: producers import this -to describe and serve their output, and this imports none of them. +[`study/`](study/README.md) next to this package is a separate browser user study, +not part of the pip package. -``` -streamer/ - codecs.py what is on the wire, and who can decode it - representations.py what a decoded frame is, and whether it renders here - bundle.py the view.json contract, shared by both sides - monitor.py what actually went over the wire - export.py any open4d.Sequence as a bundle - session.py several clips, a rung ladder, one index - live.py clips rendered as they are watched - server/ sending - client/ playback, the scheduler, and the decode worker +## Install + +```bash +pip install -e open4d/streamer ``` -## Building one +`open4d.stream()` uses this package. Without it, `open4d.stream` raises +`StreamerDependencyError`. -`export.from_sequence` writes one clip and hands it back; `bundle.write` turns -a list of clips into the `view.json` that makes them playable. `session.Bundle` -holds the list so the two cannot come apart: +## Quick start ```python import open4d @@ -34,689 +26,123 @@ with open4d.load("capture.usdc") as sequence: with Bundle("out/", title="Capture") as clips: clips.add(sequence, name="capture", rungs=["draco", "draco@11"]) -serve("out/") -``` - -The index is written when the block ends, and **not** written if it raises: -frames with no manifest is a half-export you can finish, a manifest listing -frames that were never written is one a client reports as missing frames. - -Rungs are how a clip becomes adaptable. The first is the rendition a reader -that knows nothing about variants plays; the rest become `bundle.Variant` -entries with their sizes measured off disk, and their `quality` left empty -because this knows what a rung cost and `metrics` is what says what it was -worth. On the ten-frame basketball sequence the three rungs measure 7.6 MB -(`ply`), 591 kB (`draco`, 14-bit) and 471 kB (`draco@11`) — the middle one -12.9x smaller than interchange, matching what `export.DRACO_FORMAT` records, -and the bottom one a further 20% for eight times the position error. - -Open4D's public API wraps the whole of the above in one verb, for the common -case of one sequence: - -```python -open4d.stream("capture.usdc") -``` - -It imports this package on the call rather than at load, because the -dependency runs one way -- this imports `open4d`, and `open4d` must not import -it back. Without this package installed, `open4d.stream` raises -`StreamerDependencyError` saying how to install it. - -Decoding happens in a worker (`client/worker.js`), not on the page's thread. -Parsing is the one expensive synchronous step in playback — 16.5 ms for a 3DGS -PLY, 32.1 ms for a 439k-Gaussian `.splat`, 17.8 ms for a Draco mesh — and on -the main thread each of those is a missed `requestAnimationFrame` for *every* -pane, because there is only one main thread. Four Gaussian panes blocked it for -66 ms a frame, two whole budgets at 30 fps; now they block it for about 1 ms -and parse while the previous frame draws. - -The codecs live in the worker and nowhere else. Sharing them with the page would -mean either duplicating them, which drifts, or a build step, which this client -does not have. Results come back with their buffers **transferred**, since a -4.3 MB frame parses to several megabytes of typed arrays and copying those back -would return much of what the worker saved. - -`decodeImage` is the exception and stays on the page: it needs `Image`, and a -browser already decodes an image off the main thread, so there is nothing to -move. - -## The codec axis - -Two questions, deliberately separate. A **representation** is what a decoded -frame *is* — `mesh`, `points`, `gaussians`, `pixels`. A **codec** is the format -it travels in and whether the client can decode it: - -| representation | suffix | codec | decodes | | -| --- | --- | --- | --- | --- | -| mesh | `.ply` | `mesh-ply` | client | interchange | -| mesh | `.drc` | `mesh-draco` | client | lossy, 12.9x | -| points | `.ply` / `.drc` | `points-ply` / `points-draco` | client | | -| gaussians | `.ply` | `3dgs-ply` | client | interchange | -| gaussians | `.splat` | `splat` | client | lossy, degree-0 only | -| pixels | `.jpg` / `.png` | `jpeg` / `png` | client | fixed viewpoint | -| pixels | `.rerf` | `rerf` | **server** | undecodable in a browser | - -**The key is (representation, suffix), not suffix.** `.ply` appears three times -above and needs two different parsers — a 3DGS PLY and a mesh PLY share an -extension and nothing else. Keying on the extension alone is what previously -forced the client to carry a hand-written dispatcher per representation. - -`decodes` is what makes the platform honest about heterogeneous modules. -`codecs.client_decodable(representation)` answers "can I stream this to a -browser at all" as a lookup rather than by reading the client's source. For -ReRF the answer is permanently no — its entropy coder ships only as a CPython -3.8 binary — so it is registered as producing `pixels` server-side, which is -the representation it actually delivers. - -A lossy codec **must** state its cost; the constructor refuses one that does -not, because that line is what reaches the person looking at the render. - -Adding a codec is one registration plus a parser: - -```python -codecs.register(codecs.CodecSpec( - name="mesh-vdmc", suffix=".v4d", representation=Representation.MESH, - lossy=True, cost="V-DMC is lossy at any rate worth using", -)) +serve("out/") # open the printed URL ``` -The server's extension map, each representation's `media_types`, and the -client's `CODECS` table all follow from this. The Python registry and the -client's table are two hand-maintained lists in two languages, so -`gs_tools_tests/test_representation_vocabulary.py` asserts they agree — a -promise on one side with no parser on the other is a blank pane. - -## Two transports, because the modules genuinely differ - -Measured on this repository's own content, per frame and at 30 fps: - -| on the wire | per frame | at 30 fps | -| --- | --- | --- | -| QUEEN `.splat`, 439k gaussians | 14 MB | 422 MB/s | -| Vega decoded PLY | 4.31 MB | 129 MB/s | -| Vega's own bitstream | 2.44 MB | 73 MB/s | -| mesh PLY, 20k verts | 760 kB | 23 MB/s | -| mesh through **Draco**, decoded in the browser | **59 kB** | **1.8 MB/s** | -| ReRF bitstream | 533 kB | 16 MB/s | -| **ReRF rendered, one view** | **47 kB** | **1.4 MB/s** | +For one sequence, `open4d.stream("capture.usdc")` does all of this. -Sending geometry buys a free camera and costs LAN-class bandwidth. Sending -pixels costs two orders of magnitude less and fixes the viewpoint. Neither is -the right answer for every module, so both exist: +A **bundle** is a directory of frames plus a `view.json` index. The index is +written only if the `with` block finishes without an error. Each **rung** is +the same clip at another quality. The first rung is the default. -* **Client-decode** — a frame list, fetched and parsed by the `Scheduler`. - Free camera, `mesh`/`points`/`gaussians`. For meshes this is a genuinely - *compressed* wire format: see Draco below. -* **Server-render** — `live.mjpeg(url, ...)`, a URL instead of a frame list. - The only transport ReRF has at all, its entropy coder being a sourceless - CPython 3.8 binary, and the only one here that works over a tunnel. +## Modules -A live clip gets **its own scene**, because a live renderer chooses its own -camera and putting one beside a rig pose would break the guarantee Compare -exists to make. It has no timeline either, so the transport bar disables itself -rather than leaving a scrubber that does nothing. +| Module | Does | +|---|---| +| `export` | `from_sequence` / `from_source`: anything `open4d.load` reads becomes a clip | +| `bundle`, `session` | the `view.json` format; `Bundle` collects clips into one index | +| `codecs`, `representations` | the wire formats and what they decode to | +| `server` | `serve()`: static HTTP/1.1 file server, counters at `/stats.json` | +| `transfer` | `fetch()`: download a bundle; interrupted downloads resume | +| `live` | `live.mjpeg()`: server-rendered clips (e.g. ReRF, Vega live) | +| `link` | `Link` / `Trace`: simulated bandwidth and latency, applied inside the server | +| `metrics` | PSNR/SSIM of each clip against the captured reference | +| `policy` | picks a rung per clip for a bandwidth budget | +| `sequence` | packs a clip's frames into one `.seq` file | +| `adopt` | imports frames exported from another Python (e.g. ReRF on 3.8) | +| `client/` | the browser player; decoding runs in `client/worker.js` | -### Live is not the same as replay +## Formats -MJPEG carries a decode-and-render-on-demand loop and a slideshow of files -equally well, and this repository has both, so `live.mjpeg` requires an -`origin` and has no default — a default would let a slideshow be presented as -live by saying nothing: +The codec depends on both the representation and the suffix, since `.ply` can +hold either a mesh or 3DGS. -| origin | means | example here | -| --- | --- | --- | -| `rendered` | decoded and drawn per frame, on demand | Vega's wall demo | -| `replay` | frames rendered earlier, looped over the same transport | NeVo's viewer | - -`rendered` does not claim the *encode* is happening live — it isn't, any more -than it is for a video. It claims the decode is. NeVo is `replay` because a -NeRF frame takes about half a second to ray-march, which is not a playback -rate; its own status page has always said so. - -### Running the live demo - -The streaming loop is Vega's, driven from this tree: - -```bash -cd open4d/reconstruction/vega -PYTHONPATH=. python -m orbitvega.wall_demo \ - --bitstream-dir \ - --objects basketball dancer mitch thomas \ - --chunk-port 8801 --mjpeg-port 8800 -``` +| Representation | Suffix | Codec | Decoded in | +|---|---|---|---| +| mesh | `.ply` / `.drc` | `mesh-ply` / `mesh-draco` | browser | +| points | `.ply` / `.drc` | `points-ply` / `points-draco` | browser | +| gaussians | `.ply` / `.splat` | `3dgs-ply` / `splat` | browser | +| pixels | `.jpg` / `.png` | `jpeg` / `png` | browser (fixed view) | +| pixels | `.rerf` | `rerf` | server | -Then register it and serve: - -```python -from streamer import bundle, live, serve -clip = live.mjpeg("http://127.0.0.1:8800/stream", name="vega-live", - origin="rendered", scene="Vega live") -bundle.write(out, title="live", source="vega", clips=[clip]) -serve(out, port=8770) -``` - -The chunk server's access log **is** the streaming loop: one `GET -//frame_NNNN.pt` per object per frame, reassembled by -`vega.player.StreamingPlayer`, colour-decoded from the hash grid, culled and -scheduled per `vega.pipeline`, then rasterised. Measured at about 14 frames in -6 seconds for four objects at 360 px tiles on one RTX 4090 — which is the -honest rate for decoding four Gaussian sequences per frame, not a target. - -## The one abstraction that matters - -A **representation** is what a decoded frame *is* — `mesh`, `points`, -`gaussians`, `pixels` — and it is deliberately not the codec that produced it. A -triangle mesh is a mesh whether it arrived as OBJ, as a Draco payload, or out of -a V-DMC bitstream, so one renderer serves every codec of a given shape. The -vocabulary is `open4d.core.Representation`'s, not this module's, so a bundle and -Open4D's own sequence model name the same things the same way. - -Adding one is a registration: - -```python -from open4d.core import Representation -from streamer.representations import RepresentationSpec, register - -register(RepresentationSpec( - representation=Representation.MESH, - media_types={".drc": "application/octet-stream"}, -)) -``` - -That is all. The server builds its extension map from the registry, so frames -get the right `Content-Type` without an edit, and the client keys its renderers -off the same names. - -Every representation core defines is playable today, so nothing passes -`playable=False`. The flag stays because the next representation will arrive -before its renderer does, and a bundle declaring something the client cannot -draw should report "no renderer for X" rather than show an empty pane. - -A spec answers only what transport actually asks. It does **not** carry -`has_geometry` — whether a free camera is meaningful is -`open4d.core.Representation`'s to answer, and a second copy would be a second -opinion. It does not describe decoding, which is the client's business and for -some representations cannot be ours at all: ReRF's entropy coder exists only as -a CPython 3.8 binary, which is why `pixels` is a first-class representation -rather than a fallback. - -## Playing Open4D's own sequences - -`export.from_source` hands anything `open4d.load` reads to the client, so a mesh -sequence gets the same viewport, camera and scrubbing a Gaussian one does: +Draco cuts a mesh frame from 761 kB to 59 kB (12.9×) at 14-bit positions. +It needs `open4d[draco]` to encode; the browser needs nothing installed. ```python from streamer import from_source -from_source("captures/basketball_player", "~/bundles/basketball", fps=10) +from_source("captures/basketball_player", "out/", fps=10, frame_format="draco") ``` -Whether a clip is `mesh` or `points` is read from the geometry, not the file -extension -- both write `.ply`. Frames come from `open4d.io.write_sequence`, -which already emits one `frame_NNNNNN.ply` per frame, so this chooses the -representation, records bounds over the whole clip and hands the rest to code -that already existed. - -The renderer is flat-shaded from screen-space derivatives rather than from -stored normals: Open4D's PLY writer refuses to store normals, and a normal -derived in the fragment shader is right for whatever geometry actually arrived. -Point clouds are shaded as small spheres from the point sprite's own -coordinates. A mesh frame carrying no faces falls through to points rather than -drawing nothing. - -Verified on the 10-frame `basketball_player` OBJ sequence the TVMC codec -vendors -- ~20k vertices and ~39k triangles a frame, loaded through -`open4d.load`, parsed by the shipped client under Node in -`streamer_tests/test_client_parsers.py`. - -## Draco: a compressed format, decoded in the client - -The first thing here that puts a *compression* of geometry on the wire rather -than an interchange dump of it. Measured on the mesh sequence the TVMC codec -vendors, at Draco's 14-bit position quantisation: - -| | per frame | 10-frame clip | at 30 fps | -| --- | --- | --- | --- | -| PLY | 761 kB | 7.4 MB | 23 MB/s | -| Draco | **59 kB** | **0.61 MB** | **1.8 MB/s** | - -12.9x, and it decodes *faster* — 17.8 ms against 20.1 ms for parsing the PLY in -JavaScript, because the WASM decoder does the work the JS parser was doing by -hand. +To add a codec, register it and add a parser to the client's `CODECS` table. +A test checks that the two lists match: ```python -from streamer import from_source -from_source("captures/basketball_player", out, fps=10, frame_format="draco") +codecs.register(codecs.CodecSpec( + name="mesh-vdmc", suffix=".v4d", representation=Representation.MESH, + lossy=True, cost="V-DMC is lossy at any rate worth using", +)) ``` -The decoder is Google's, vendored under `client/vendor/draco` and served by the -bundle server from this origin — never a CDN, which is what keeps the page free -of external dependencies. `open4d[draco]` is needed on the encoding side only; -the client needs nothing installed. - -Lossy in two bounded ways, both in the clip's notes and measured in -`streamer_tests/test_draco.py`: +Lossy codecs must state their `cost`. -* **Positions are quantised.** At 14 bits the worst vertex moved 0.0046% of the - model's diagonal on that sequence. 11 bits saves a further 20% for eight - times the error, which is why 14 is the default. -* **Duplicate vertices are merged**, so the decoded count is lower than the - encoder was given — 20,672 to 19,747 there, because the source OBJ splits - vertices at seams. Deduplication, not quantisation; it does not change the - surface. - -A `.drc` frame is a delivery form. The PLY stays the source of truth, and -re-encoding from a `.drc` would compound the quantisation. - -## Sending, receiving, measuring +## Serving, fetching, shaping ```python -from streamer import serve, fetch - -server = serve("~/bundles/basketball", host="0.0.0.0") # send -fetch("http://gpu-box:8770", "~/local-copy") # receive -server.monitor.snapshot() # measure -``` +from streamer import serve, fetch, Link, Trace -`serve` is the simplest transport the model admits: every frame is a file, -reachable in one request, in any order. It binds loopback unless told otherwise -and has **no authentication of any kind**. - -It speaks HTTP/1.1 with keep-alive, which is not a detail. The default in -Python's `http.server` is 1.0, closing after every response: one handshake and -one fresh slow-start per frame. On loopback that is invisible, which is why it -survived a long time; over a 20 ms link a 59 kB Draco frame then costs a setup -round trip plus about three more while the congestion window opens — roughly -80 ms a frame, a ~12 fps ceiling *regardless of bandwidth*. Any rate measured -through it would have been measuring that rather than the network. - -Keep-alive needs `TCP_NODELAY` to be worth having. Without it a response's two -writes — headers, then body — hit Nagle's algorithm against the client's -delayed-ACK timer, and 30 frames took **1.20 s instead of 0.01 s**, 40 ms -each. Closing the connection had been masking it, so the stall only appeared -once keep-alive worked. There is a timed regression test. - -`fetch` pulls a bundle, or `only=[...]` some clips of one, onto the machine you -are sitting at — a 30-frame Gaussian clip is over 100 MB as PLY and nobody -wants that twice over a tunnel. Complete files are skipped and a partial one is -**continued from where it stopped** with a byte range, so a tunnel that drops -mid-frame costs what was missed rather than the whole frame. It checks that the -server actually answered `206` before appending: a `200` means the range was -ignored, and appending then would build a corrupt file of exactly the size the -completeness check accepts. - -`Monitor` counts requests, bytes, errors and a per-clip rollup, exposed at -`/stats.json`. It is counters and nothing more, deliberately: a monitor that -decided things would be a second scheduler. - -## Quality rungs - -A clip can carry the same content at several rates: +server = serve("bundle/", host="0.0.0.0") # no authentication +fetch("http://gpu-box:8770", "local-copy/") # resumes partial files +server.monitor.snapshot() # requests, bytes, errors -```python -bundle.Clip( - name="thomas-rerf-cam00", representation="pixels", - frames=[...], # the default rendition - variants=[bundle.Variant(name="low", frames=[...], bytes=91_000, - quality={"psnr": 40.45, "ssim": 0.9723}).as_dict()], -) +serve("bundle/", link=Link(capacity=20e6, latency=0.020)) +serve("bundle/", link=Link(trace=Trace.read("walk.trace"))) ``` -Measured on `g_thomas`, from one export at `--rungs high,medium,low`: - -| rung | resolution | kB/frame | Mbit/s @30 | PSNR | SSIM | -| --- | --- | --- | --- | --- | --- | -| default | 1280×960 q92 | 33.5 | 8.04 | 45.91 | 0.9902 | -| medium | 640×480 q88 | 9.9 | 2.38 | 43.72 | 0.9853 | -| low | 320×240 q80 | 3.0 | 0.73 | 40.45 | 0.9723 | +`serve` binds to loopback unless you pass `host`. `Link` shapes traffic in the +server and needs no root. Measured, it delivers 2.6–15% below its target +capacity, with latency within 1 ms. +It does not model TCP congestion control. -Four things about the shape, each of which was a choice: +The browser measures its own throughput and splits it evenly across panes. +Each pane plays the best rung its share affords. -**Rungs live inside a clip, not as sibling clips.** A clip is one method's -output for one subject at one station; a variant is one of the ways to get it. -Three sibling clips would be three panes with nothing able to switch between -them mid-playback. - -**`frames` stays the default rendition** rather than moving into the variant -list, so a reader that has never heard of variants plays the clip correctly and -needs no change. That is why this did not bump `VERSION` — it is additive. - -**Rate and quality are measured, not predicted.** A system that chooses before -encoding has to model them; a bundle holds content that already exists, so -bytes come from the files and quality from `metrics`. A prediction here would -be strictly worse data than the measurement it stands in for. - -**A lower rung is resampled, not re-rendered.** Re-rendering at half -resolution samples the volume differently and gives a slightly *different* -picture — fine as an image, wrong as a rendition, because switching between two -renditions has to be a rate change and not a visible cut. - -`write` refuses a ladder a consumer would trip over: renditions must share the -frame count (something switching at frame *n* has to land on frame *n*), rung -names must be unique, and a live clip cannot have rungs because it has no frame -list to offer at another quality. - -## Measuring what arrived +## Command line ```bash -python -m streamer.metrics ~/bundles/basketball # per scene and rung -python -m streamer.metrics ~/bundles/basketball --per-clip --json +python -m streamer.metrics bundle/ [--per-clip] [--json] [--write] +python -m streamer.policy bundle/ --scene thomas +python -m streamer.sequence bundle/ [--clip NAME] [--keep-frames] +python -m streamer.adopt exported/ --bundle bundle/ [--replace] ``` -`Monitor` says how many bytes moved; this says how good the picture was, which -is the other half of any comparison. It works on **a bundle, not a method**: a -scene's `captured` clip at a station is the reference, and every other clip at -that station is scored against it. Same code, same reference, so two numbers -are comparable — a metric shipped inside each method would be nine metrics. - -It reports what it *could not* measure rather than omitting it. Geometry clips -have no pixels until something renders them from the reference's pose; live -clips have no frame list; a depth map is not an attempt to reproduce a -photograph, which a clip says with `detail["depicts"]`. That last one exists -because scoring depth against colour yields 0.4 dB — a number that reads as -catastrophic failure rather than as a comparison that was never meaningful. - -SSIM is implemented here rather than imported, so this package still needs only -`open4d`. That shortcut is checked, not trusted: the tests hold it to -scikit-image's implementation, and on a real frame the two agree to 2.4×10⁻⁸. -scipy is used for the blur when present, with a numpy fallback — and the test -that compares the two backends is what caught them disagreeing, since scipy's -`reflect` repeats the edge sample and numpy's does not. - -## A free camera for a method that renders on a GPU - -Vega ships explicit geometry, so the browser rasterises whatever viewpoint the -mouse asks for. ReRF is also free-viewpoint — that is the paper's title, and -`rerf_render.py --render_360` is the path `rerf_stream.export --orbit` drives — -but its ray-march runs on CUDA, so the pixels are made offline and the browser -receives a set of rendered views rather than something it can re-aim. - -There is also a third option, which is the best of them where it applies: -`rerf_stream.geometry` reads the density field out as a coloured point cloud, -and a `points` clip gets the same genuinely free camera Vega does — the browser -rasterises it, nothing is pre-rendered, nothing snaps. It costs about 20 dB -against the ray-march of the same frame -- roughly 14 from thresholding the -density field and 6 from freezing one colour per point -- so the bundle carries -both and the comparison is the point. - -For the pixel clips that remain, the viewer treats a drag as the same gesture -rather than a different mode, and snaps to the nearest rendered view: - -- `shellOf` decides whether a rig can stand in for an orbit at all. Its - stations have to lie at one radius about a common look-at point — otherwise - this is an arbitrary cloud of capture positions, and snapping a free camera - onto it would move the camera somewhere nobody pointed it. Rigs that fail the - test keep the old behaviour: pixels are compare-only. -- The centre is solved from the poses' own view rays, not taken as the centroid - of their positions. For one ring those agree, which is why the centroid was - good enough at first; stack rings at different elevations and the centroid - sits off the axis and every radius measured from it differs. -- `nearestStation` compares *directions* from the subject via a dot product, - so azimuth and elevation are handled together, the ±180° seam needs no - special case, and there is no need to weigh a degree of azimuth against a - degree of elevation. -- Following is debounced by 140 ms. Crossing a station changes which clip the - pane plays, so re-resolving per `pointermove` would fetch a container per - station — a spin across 216 of them is 330 MB of pictures nobody stopped on. - The geometry panes are unaffected: for them the camera is the only input. - -The pane label and the notes state the caveats, because a pane that quietly -shows the nearest view looks like one that followed exactly — and someone -comparing it against a rasterised pane would read the offset as a -reconstruction error. On `g_basketball`, 216 stations: ±2.5° azimuth, 3 -elevations (-25°, 0°, 25°). - -## Packing a clip into one file - -```bash -python -m streamer.sequence ~/bundles/basketball # every packable clip -python -m streamer.sequence ~/bundles/basketball --clip basketball-vega -``` - -A clip's frames are separate files, and fetching them costs a round trip each. -On loopback that is free — 30 frames in 87 ms — which is why per-frame fetching -survived so long; over a 20 ms link it is 30 round trips, 600 ms of pure -latency before anything can play. - -`streamer.sequence` packs a clip into one `.seq` container: a header naming the -frames and their absolute offsets, then their bytes end to end. Deliberately -not a zip — the frames are already compressed, so an archive's own compression -would spend CPU to save nothing, and a client would need a decoder for it. - -The offsets being absolute is what makes it useful in flight: a client with the -first few hundred bytes already knows how many frames are coming, and can slice -any one of them out without walking the ones before it. Measured through -`streamer.server`, the Vega clip is 62.9 MB behind a 265-byte header, and a -`Range: bytes=0-511` request is enough to read the whole frame table. - -This is the on-demand shape and it suits a free camera exactly. A player -fetches a few frames ahead and discards them behind the playhead, because it -only ever draws the frame it is showing. A viewer that can be spun around needs -the geometry resident to redraw it from a new angle — so the frame has to stay, -and if it has to stay there was no reason to fetch it late. - -Packing removes the frame files it replaced, since a client handed a container -never asks for the pieces; `--keep-frames` leaves both. Live clips are skipped -rather than refused — there is no end to pack — and so are clips already packed, -so the command is safe to re-run. - -## Adopting frames from another interpreter - -```bash -python -m streamer.adopt ~/rerf-clips --bundle ~/bundles/basketball -``` - -Some methods cannot be driven from this package at all: ReRF's entropy coder is -a prebuilt Python 3.8 binary, and this needs 3.10, so neither side can import -the other. Rather than force one interpreter on both, the exporting side writes -its frames plus a small `clips.json`, and `adopt` reads that, copies the frames -in and adds the clips. Every frame the sidecar names is checked to exist first, -because a clip whose frames are half there plays for two seconds and then 404s -while the manifest insists nothing is wrong. - -`adopt` also reads a *bundle* rather than a sidecar, because a bundle is a -superset of one: `gs-tools export` writes whole bundles, and merging two of them -used to need a hand-rolled copy-and-merge. Clips keep their own -`representation` when read this way — a sidecar states one for the whole -export, and taking that for a bundle would relabel a point cloud as pixels and -send it to the image decoder. - -One ordering hazard is worth knowing about, and is now refused rather than -survived. An export installs a rig only into a scene that has none, so -whichever lands first wins — and a Vega export brings the ORBIT corpus's -8-camera capture rig while a ReRF orbit export brings its own 216 stations. -Adopt them the wrong way round and the orbit clips are numbered against a rig -that stops at 7, so most Compare stations show nothing at all, silently, -because a pane with no clip is a legitimate state. `adopt.check_cameras` now -rejects that before writing anything. - -## Building one subject end to end - -Every subject in the demo bundle is built the same way, and the order is the -point: - -```bash -SUBJECT=dancer -RUN=/media/frozzzen/LocalDisk/nevo_runs/g_$SUBJECT - -# 1. ReRF orbit rings — 216 pre-rendered stations, and the rig they are -# numbered against. Adopted first, so that rig is the one installed. -python -m rerf_stream.export --config $RUN/config.py \ - --compression-path $RUN/rerf --out /tmp/rings \ - --name g_$SUBJECT --scene $SUBJECT --orbit 72 --elevations 0,25,-25 - -# 2. ReRF point cloud — geometry, so Explore gets a real free camera -python -m rerf_stream.geometry --config $RUN/config.py \ - --compression-path $RUN/rerf --out /tmp/points \ - --name g_$SUBJECT --scene $SUBJECT - -# 3. Vega Gaussians, as .splat -gs-tools export --method vega --frame-format splat \ - -i ~/4DVideoStreaming/results/vega-gaussian/prepared-final \ - --objects $SUBJECT -o /tmp/vega - -# 4. into the bundle, rings first, then pack each clip into one file -for part in rings points vega; do - python -m streamer.adopt /tmp/$part --bundle ~/open4d-view --replace -done -python -m streamer.sequence ~/open4d-view -``` - -Costs per subject, measured: the rings are ~7 minutes of GPU and 334 MB, the -point cloud is ~1 second and 50 MB, and Vega is ~1 minute and 55 MB. The rings -dominate both, and they are what Compare uses; Explore never touches them. - -## Planning a seek - -A clip declares how its frames depend on one another, in -`open4d.core.Dependency`'s vocabulary: `independent`, `gop` (frames need the -group's key frame first, as in Vega's group-of-volumes) or `sequential` (the -decode stream does not rewind at all, as in ReRF). The manifest carries it per -clip, absent meaning independent, and the client's `Scheduler` plans the chain -rather than requesting a frame and hoping. - -The asymmetry is the point. A forward step reuses where the decoder already -sits; a backward seek in a sequential stream comes back as a full replay from -zero, because that stream cannot be rewound. Both bitstreams here declare a -single group spanning every frame, so a cold seek to the last of 30 costs 30 -decodes — the kind of thing that was invisible before it was modelled. - -`chain` exists twice, in Python and transcribed into the client, because the -browser cannot run Python. Two implementations of one rule drift, so -`streamer_tests/test_scheduler.py` runs both over the same 56 cases and asserts -they agree. Change one and that test tells you about the other. - -The Scheduler also owns caching and look-ahead, which the two clip sources used -to keep separately with different policies — and only one of which evicted, so -scrubbing a long pixel clip kept every frame it had ever shown. Buffer -occupancy, decode count and bytes are in the viewer's stats panel; decode order -and replay count appear only when something is not independent, since a replay -count of zero says nothing about an independent clip. - -**Every exporter here writes independent frames**, because they all decode -before they write. The `gop` and `sequential` paths are therefore tested rather -than exercised: they are what a producer serving a bitstream *as* its frames -would need, and building them afterwards would mean changing the client at the -same time as trusting a new encoder. - -## Choosing a rung, and a link to choose against - -```bash -python -m streamer.metrics ~/bundles/basketball --write # fill in what rungs buy -python -m streamer.policy ~/bundles/basketball --scene thomas -``` - -``` - budget spent mean cam00 cam01 cam02 cam03 cam04 cam05 - 6.4M 6.4M 40.47 low low low low low low - 38.8M 34.0M 44.43 medium default medium medium medium medium - 71.3M 71.3M 46.16 default default default default default default -``` - -The choice is a **multiple-choice knapsack**, solved exactly by dynamic -programming. Greedy "best quality per byte" is the usual approximation and it -is wrong in a way that matters here: with rungs 3.2× and 10.6× apart it spends -everything on the first pane it considers and leaves the rest at their floor, -which is precisely the lopsided allocation a comparison view must not have. -Nine clips by three rungs makes exactness free. - -Weights decide who gets headroom, and who is dropped when even the floor does -not fit — there is no rung cheaper than the cheapest, so something has to go, -and it is named rather than silently missing. `switch_penalty` charges churn -against a previous selection. - -Rate and quality are **measured, not predicted**, which is what removes the -trained regressors a system choosing before encoding would need. -`metrics --write` puts the quality half into the manifest beside the bytes. - -### The link +## Live clips ```python -from streamer import Link, Trace, serve - -serve(bundle_dir, link=Link(capacity=20e6, latency=0.020)) -serve(bundle_dir, link=Link(trace=Trace.read("walk.trace"))) +from streamer import bundle, live, serve +clip = live.mjpeg("http://127.0.0.1:8800/stream", name="vega-live", + origin="rendered", scene="Vega live") +bundle.write("out/", title="live", source="vega", clips=[clip]) +serve("out/", port=8770) ``` -Until this existed every transport here ran on loopback, which made the budget -`policy` needs unmeasurable and any throughput figure a statement about the -disk. `Link` is a single-queue bottleneck: bytes leave in request order, at the -capacity in force, after a propagation delay, and queueing emerges from -contention — which is the behaviour that matters when panes share a pipe. It is -shared by every connection on purpose. - -Shaped in the server rather than with `tc`. Kernel shaping is more faithful and -was rejected because it needs root, perturbs the whole machine, and cannot be -exercised by a test — which for a measurement instrument is disqualifying. -Measured accuracy: **2.6% at 5 Mbit/s, 7.4% at 20, 15% at 50**, always -*under*-delivering, and latency within 1 ms. What it does not reproduce is TCP: -no congestion window, no slow start. Loss is charged as the delay a -retransmission costs, because that is what an application above TCP sees. - -### The viewer adapts - -The page measures the link from its own fetches (`RateMeter`, an exponentially -weighted mean over recent transfers, byte-gated so a cache hit cannot read as a -gigabit link), reads each clip's ladder, and picks a rung per pane. There is a -checkbox to pin every pane to its default rendition, and the stats panel shows -the measured rate, the per-pane share, and which rung each pane is on. - -**The client's rule is deliberately not `policy`'s.** `policy` maximises total -weighted quality, and the way to maximise a sum is to make the panes *unequal* -— spending on whichever gains most per byte. A viewer whose purpose is judging -two methods against each other must not decide that one of them gets the -bandwidth. So the client splits the budget evenly and each pane takes the best -rung its share affords. `policy` remains what answers "what would an optimising -client do", from Python, where it can be measured — and not transcribing it -also means there is no second implementation of it to drift. - -Switching rung keeps the cache. Renditions share a timeline, so a decoded frame -is still the right picture for its index; dropping the cache would re-fetch what -is already in hand and stall at exactly the moment the link is under pressure. - -Tracking a trace end to end, through a link that steps 25 → 3 → 12 Mbit/s: +`origin` is required: `rendered` means frames are drawn on demand, `replay` +means pre-rendered frames on a loop. For the Vega source, run +`orbitvega.wall_demo` from `open4d/reconstruction/vega`. -``` - t ewma share rung mix - 0 7.0M 0.87M low=8 - 26 9.9M 1.23M low=6 medium=2 - 33 6.2M 0.78M low=7 dropped 1 - 42 3.4M 0.43M low=4 dropped 4 - 51 6.1M 0.76M low=7 dropped 1 -``` +## Limits -The estimate lags the cliff by a few seconds, which is what a half-life of -eight samples means and is the interesting part of an adaptive client rather -than a defect. It tops out near 10 Mbit/s on a 25 Mbit/s link because serial -requests at 15 ms each cannot fill it — the honest budget for that request -pattern. - -`link.observed()` reports what actually arrived, split into the rate while it -was *delivering* and how much of the time it was busy at all. That split is not -cosmetic: dividing bytes by wall clock reported **0.3 Mbit/s for a 25 Mbit/s -pipe** when the client fetched a few frames and then waited, and a chooser -handed that drops every pane. It is also **cumulative since the link started**, -so it summarises a run rather than reading a current rate — a client tracking a -changing link needs the recent-window estimate the page uses instead. - -## What is deliberately missing - -**A buffer model.** Per-clip buffer occupancy, a deliberate freeze -told apart from a stall, churn charged across segments. `policy` allocates for -a budget at an instant; none of it plays continuously against a trace and -reacts. `switch_penalty` is the only part of that dynamics testable today, and -the rest needs a client that streams for minutes rather than a function that -returns a selection. - -**A held-out camera.** Every rig camera was a training view for both Vega and -ReRF, so the numbers `metrics` reports measure reconstruction rather than -generalisation. That caveat has to travel with them. +- No buffer model: `policy` picks rungs for one moment in time and does not + simulate playback over a trace. +- Every rig camera was a training view for Vega and ReRF, so `metrics` measures + reconstruction quality, not how well a method generalizes to new views. +- Every exporter writes independent frames. Seeking in `gop` and `sequential` + clips is tested but no real content uses it yet. ## Tests ```bash -python -m pytest streamer_tests -q +cd open4d/streamer && python -m pytest -q ``` -No GPU, no display, no dataset. The client's frame parsers are run as shipped, -cut out of `viewer.html` and executed under Node — those cases skip if `node` is -absent, which is stated rather than silently covering less. +No GPU, display or dataset needed. The client parser tests need `node` and are +skipped without it. diff --git a/open4d/webclients/.gitignore b/open4d/streamer/study/.gitignore similarity index 100% rename from open4d/webclients/.gitignore rename to open4d/streamer/study/.gitignore diff --git a/open4d/webclients/PROVENANCE b/open4d/streamer/study/PROVENANCE similarity index 100% rename from open4d/webclients/PROVENANCE rename to open4d/streamer/study/PROVENANCE diff --git a/open4d/streamer/study/README.md b/open4d/streamer/study/README.md new file mode 100644 index 00000000..6fa87552 --- /dev/null +++ b/open4d/streamer/study/README.md @@ -0,0 +1,99 @@ +# study + +A browser user study for volumetric video streaming. One participant watches +several methods under the **same network trace, start view and camera path**, +then rates each clip. It's a separate Node app: it doesn't use the `streamer` +Python package next to it and isn't part of its wheel. The methods themselves +are in [`../../reconstruction`](../../reconstruction). + +| Method | Content | In the study | +|---|---|---| +| Ours | textured meshes, viewpoint-aware ladder | yes | +| ViVo, NAVA | point clouds | yes (needs the tile corpus) | +| Vega | 3D Gaussian splats | yes, for objects it has exported (`dancer`, `thomas`) | +| NeVo | ReRF, pre-rendered | no: fixed camera, can't follow a path | +| LiVo, MetaStream, DeltaStream | RGB-D | no: need source RGB-D data that isn't available | + +## Run it + +It needs a checkout of the +[4DVideoStreaming](https://github.com/frozzzen3/4DVideoStreaming) research repo, +which has the ladder solver, the baseline servers and the datasets. + +```bash +cd system/WebClient && npm install && node build.js +cd ../.. && PYTHONPATH=/path/to/4DVideoStreaming PYTHON_BIN= scripts/run_web_demo.sh +``` + +Open the printed `http://:3000/web/`. Ctrl-C stops everything. Each +method also has its own page (`mesh.html`, `baseline.html`, `vega.html`, +`nevo.html`); see [`system/WebClient`](system/WebClient/README.md). + +Serve the H.264 corpus (the default): every browser can decode it, but HEVC +only plays in Safari and hardware-enabled Chrome. + +## A session + +1. **Setup:** participant code (never a name), methods, objects, trial length + and network trace. Pick a built-in trace from `system/Client/traces/` or + upload a `time,bandwidth` CSV (seconds, Mbps). +2. **Record the path:** the participant drives one unrated practice pass. Every + trial replays that same camera path. +3. **Trials:** blinded as "Clip A", "Clip B"…, in Williams order. Each one + restarts the trace from its beginning. +4. **Questionnaire:** C1–C4 on five stars, plus up to two artifacts. +5. **Results:** a table and `export.csv`, one row per trial. + +Each trial also records the GPU the browser rendered with. The setup page +warns if that's a software renderer (for example, headless Chrome's +SwiftShader), because methods play far below their frame rate there. Vega +loads its whole clip *before* the trace starts, so its trial starts at +playback like every other method (`preloadSeconds` records the load time). + +Sessions are saved in `system/Server/server_results/study/`. That directory +holds participant data and is git-ignored. + +The trace is applied inside the server (`system/Server/study/shaper.js`, and +`follower.js` for the ViVo/NAVA bridge), so it needs no root and doesn't +throttle anything else on the machine. The CSV's `shaped_by` and +`shaped_bytes` columns show which shaper carried each trial's bytes and how +many; the bridge reports its own at `GET /shaping`. Outside the study, +`sudo scripts/shape_web_demo.sh ` shapes the method pages with `tc`, +which throttles the whole machine. + +## Adding a method + +1. **Page:** if `study.studyParams()` is set, call `study.loadStudyContext`, + `study.enterStudyChrome` and `study.createStudyDriver`. The renderer then + calls `driver.apply(camera, controls, info)` once per frame. + `src/vega-main.js` is the smallest example. +2. **Register it** in `METHODS` (`system/Server/study/store.js`), in + `STUDY_PAGES` (`src/study/study-page.js`), in `build.js` and in `public/`. +3. **Shape its traffic:** every download must go through the study's shaper. +4. **If it loads a whole clip before playing**, add it to `PRELOAD` in + `src/study/study-page.js`. With `?preload=1`, the page posts `preloaded`, + then waits for `study.awaitGo()` before it starts its driver. + +If the method adapts in the browser, implement the +[`ClientPlatform`](system/ClientCore/README.md) contract and use +`StreamingClient`. If it adapts on the server, or not at all, write a plain page. + +## Layout and tests + +| Path | | +|---|---| +| `system/ClientCore` | browser-independent streaming logic: ABR, bandwidth estimation, metrics | +| `system/WebClient` | the study and the method pages, plus the WebSocket↔TCP bridge | +| `system/Server` | serves the ladder, media and study sessions | +| `tile_ladder.py` | lets ViVo/NAVA serve prepared tiles without the RGB-D source | + +The `system/` level mirrors the research repo so relative imports still work. +This directory started as a copy of that repo (see `PROVENANCE`) and is now +developed here. + +```bash +node --test tests/*.js +``` + +Two tests compare the JS decoders against the Python ones, so they need +`PYTHONPATH=/path/to/4DVideoStreaming` and `VS4D_TEST_PYTHON=`. diff --git a/open4d/webclients/scene_layout.json b/open4d/streamer/study/scene_layout.json similarity index 100% rename from open4d/webclients/scene_layout.json rename to open4d/streamer/study/scene_layout.json diff --git a/open4d/webclients/scripts/run_web_demo.sh b/open4d/streamer/study/scripts/run_web_demo.sh similarity index 100% rename from open4d/webclients/scripts/run_web_demo.sh rename to open4d/streamer/study/scripts/run_web_demo.sh diff --git a/open4d/webclients/scripts/serve_pointcloud_baseline.sh b/open4d/streamer/study/scripts/serve_pointcloud_baseline.sh similarity index 100% rename from open4d/webclients/scripts/serve_pointcloud_baseline.sh rename to open4d/streamer/study/scripts/serve_pointcloud_baseline.sh diff --git a/open4d/webclients/scripts/shape_web_demo.sh b/open4d/streamer/study/scripts/shape_web_demo.sh similarity index 100% rename from open4d/webclients/scripts/shape_web_demo.sh rename to open4d/streamer/study/scripts/shape_web_demo.sh diff --git a/open4d/webclients/system/Client/traces/amazonfcc.csv b/open4d/streamer/study/system/Client/traces/amazonfcc.csv similarity index 100% rename from open4d/webclients/system/Client/traces/amazonfcc.csv rename to open4d/streamer/study/system/Client/traces/amazonfcc.csv diff --git a/open4d/webclients/system/Client/traces/cascade-20.csv b/open4d/streamer/study/system/Client/traces/cascade-20.csv similarity index 100% rename from open4d/webclients/system/Client/traces/cascade-20.csv rename to open4d/streamer/study/system/Client/traces/cascade-20.csv diff --git a/open4d/webclients/system/Client/traces/cascade-5.csv b/open4d/streamer/study/system/Client/traces/cascade-5.csv similarity index 100% rename from open4d/webclients/system/Client/traces/cascade-5.csv rename to open4d/streamer/study/system/Client/traces/cascade-5.csv diff --git a/open4d/webclients/system/Client/traces/good-wifi.csv b/open4d/streamer/study/system/Client/traces/good-wifi.csv similarity index 100% rename from open4d/webclients/system/Client/traces/good-wifi.csv rename to open4d/streamer/study/system/Client/traces/good-wifi.csv diff --git a/open4d/webclients/system/Client/traces/lte.csv b/open4d/streamer/study/system/Client/traces/lte.csv similarity index 100% rename from open4d/webclients/system/Client/traces/lte.csv rename to open4d/streamer/study/system/Client/traces/lte.csv diff --git a/open4d/webclients/system/Client/traces/nuwave-0-20.csv b/open4d/streamer/study/system/Client/traces/nuwave-0-20.csv similarity index 100% rename from open4d/webclients/system/Client/traces/nuwave-0-20.csv rename to open4d/streamer/study/system/Client/traces/nuwave-0-20.csv diff --git a/open4d/webclients/system/Client/traces/nuwave_trace.csv b/open4d/streamer/study/system/Client/traces/nuwave_trace.csv similarity index 100% rename from open4d/webclients/system/Client/traces/nuwave_trace.csv rename to open4d/streamer/study/system/Client/traces/nuwave_trace.csv diff --git a/open4d/webclients/system/Client/traces/nuwins-60-80.csv b/open4d/streamer/study/system/Client/traces/nuwins-60-80.csv similarity index 100% rename from open4d/webclients/system/Client/traces/nuwins-60-80.csv rename to open4d/streamer/study/system/Client/traces/nuwins-60-80.csv diff --git a/open4d/webclients/system/Client/traces/nuwins_trace.csv b/open4d/streamer/study/system/Client/traces/nuwins_trace.csv similarity index 100% rename from open4d/webclients/system/Client/traces/nuwins_trace.csv rename to open4d/streamer/study/system/Client/traces/nuwins_trace.csv diff --git a/open4d/webclients/system/Client/traces/poor-wifi.csv b/open4d/streamer/study/system/Client/traces/poor-wifi.csv similarity index 100% rename from open4d/webclients/system/Client/traces/poor-wifi.csv rename to open4d/streamer/study/system/Client/traces/poor-wifi.csv diff --git a/open4d/streamer/study/system/ClientCore/README.md b/open4d/streamer/study/system/ClientCore/README.md new file mode 100644 index 00000000..434d298f --- /dev/null +++ b/open4d/streamer/study/system/ClientCore/README.md @@ -0,0 +1,72 @@ +# ClientCore + +Streaming logic with no platform code, shared by the browser client +(`system/WebClient`) and the research repo's Node client. + +**Rule:** nothing here imports `fs`, `http`, `https`, `path`, `os` or +`process`, or calls `console`, `Date.now`, `setInterval` or `fetch` +directly. All of that goes through the `ClientPlatform` contract in +[platform.js](platform.js), and `tests/test_client_core_purity.js` enforces +the rule. That way, a difference between the two clients reflects the system +under test, not the client code. + +## Files + +| File | Role | +|---|---| +| [streaming-client.js](streaming-client.js) | `StreamingClient`: segment loop, playback clock, downloads, reporting | +| [platform.js](platform.js) | the contract and `validatePlatform` | +| [abr.js](abr.js) | MCKP rung selection and per-object buffers | +| [bandwidth.js](bandwidth.js), [bandwidth-estimator.js](bandwidth-estimator.js), [link-throughput.js](link-throughput.js) | throughput estimate and spend budget | +| [download-plan.js](download-plan.js) | what a segment needs, and whether it arrived | +| [metrics.js](metrics.js), [stalls.js](stalls.js), [payloads.js](payloads.js) | run metrics, stall tracking, server request bodies | +| [stream-config.js](stream-config.js) | validated timing from `/api/config` | +| [testing/fake-platform.js](testing/fake-platform.js) | in-memory platform with a virtual clock, for tests | + +## The platform contract + +| Capability | Methods | +|---|---| +| `transport` | `getJson`, `postJson`, `assetUrl`, `fetchAsset` | +| `storage` | `handle`, `createScratch`, `release`, `writeText`, `writeResult`, `openAppendStream` | +| `viewpoints` | `list` | +| `clock` | `now`, `every`, `cancel`, `delay` | +| `logger` | `emit` | +| `renderer` | `start`, `stop`, `stageSegment`, `setPlaybackState`, `setCallbacks`, `latestCamera`; may be `null` | +| `lifecycle` | `exit`, `onShutdownRequest` | + +Call `validatePlatform(platform, { requireRenderer })` at startup. It lists +every missing method at once. Storage handles are opaque, so pass them along +and never parse them. + +A new platform must: + +1. make `fetchAsset` uncacheable (`cache: 'no-store'`); +2. never navigate away in `lifecycle.exit`, because results are POSTed during + shutdown; +3. never block in `openAppendStream.write` (telemetry arrives at 30 Hz, so + buffer it); +4. keep `renderer: null` working, since simulated mode depends on it. + +## Testing + +`createFakePlatform()` has a virtual clock, so a 40-segment run takes about a +millisecond. + +```js +const { createFakePlatform } = require('../system/ClientCore/testing/fake-platform'); +const platform = createFakePlatform(); +platform.transport.onGet('/api/config', () => ({ segmentDuration: 2, framesPerSegment: 60, + segmentIntervalMs: 2000, totalSegments: 20, updateIntervalSegments: 1 })); +platform.transport.onGet('/api/manifest', () => menu); +await platform.clock.advance(20 * 2000); +assert.ok(platform.storage.result); +``` + +Assert on state (`storage.files`, `transport.requests`, `logger.at('WARN')`), +not on call expectations. + +```bash +node --test tests/test_client_core*.js tests/test_bandwidth_estimator.js \ + tests/test_client_platform.js tests/test_streaming_client.js +``` diff --git a/open4d/webclients/system/ClientCore/abr.js b/open4d/streamer/study/system/ClientCore/abr.js similarity index 100% rename from open4d/webclients/system/ClientCore/abr.js rename to open4d/streamer/study/system/ClientCore/abr.js diff --git a/open4d/webclients/system/ClientCore/bandwidth-estimator.js b/open4d/streamer/study/system/ClientCore/bandwidth-estimator.js similarity index 100% rename from open4d/webclients/system/ClientCore/bandwidth-estimator.js rename to open4d/streamer/study/system/ClientCore/bandwidth-estimator.js diff --git a/open4d/webclients/system/ClientCore/bandwidth.js b/open4d/streamer/study/system/ClientCore/bandwidth.js similarity index 100% rename from open4d/webclients/system/ClientCore/bandwidth.js rename to open4d/streamer/study/system/ClientCore/bandwidth.js diff --git a/open4d/webclients/system/ClientCore/download-plan.js b/open4d/streamer/study/system/ClientCore/download-plan.js similarity index 100% rename from open4d/webclients/system/ClientCore/download-plan.js rename to open4d/streamer/study/system/ClientCore/download-plan.js diff --git a/open4d/webclients/system/ClientCore/link-throughput.js b/open4d/streamer/study/system/ClientCore/link-throughput.js similarity index 100% rename from open4d/webclients/system/ClientCore/link-throughput.js rename to open4d/streamer/study/system/ClientCore/link-throughput.js diff --git a/open4d/webclients/system/ClientCore/metrics.js b/open4d/streamer/study/system/ClientCore/metrics.js similarity index 100% rename from open4d/webclients/system/ClientCore/metrics.js rename to open4d/streamer/study/system/ClientCore/metrics.js diff --git a/open4d/webclients/system/ClientCore/payloads.js b/open4d/streamer/study/system/ClientCore/payloads.js similarity index 100% rename from open4d/webclients/system/ClientCore/payloads.js rename to open4d/streamer/study/system/ClientCore/payloads.js diff --git a/open4d/webclients/system/ClientCore/platform.js b/open4d/streamer/study/system/ClientCore/platform.js similarity index 100% rename from open4d/webclients/system/ClientCore/platform.js rename to open4d/streamer/study/system/ClientCore/platform.js diff --git a/open4d/webclients/system/ClientCore/stalls.js b/open4d/streamer/study/system/ClientCore/stalls.js similarity index 100% rename from open4d/webclients/system/ClientCore/stalls.js rename to open4d/streamer/study/system/ClientCore/stalls.js diff --git a/open4d/webclients/system/ClientCore/stream-config.js b/open4d/streamer/study/system/ClientCore/stream-config.js similarity index 100% rename from open4d/webclients/system/ClientCore/stream-config.js rename to open4d/streamer/study/system/ClientCore/stream-config.js diff --git a/open4d/webclients/system/ClientCore/streaming-client.js b/open4d/streamer/study/system/ClientCore/streaming-client.js similarity index 100% rename from open4d/webclients/system/ClientCore/streaming-client.js rename to open4d/streamer/study/system/ClientCore/streaming-client.js diff --git a/open4d/webclients/system/ClientCore/testing/fake-platform.js b/open4d/streamer/study/system/ClientCore/testing/fake-platform.js similarity index 100% rename from open4d/webclients/system/ClientCore/testing/fake-platform.js rename to open4d/streamer/study/system/ClientCore/testing/fake-platform.js diff --git a/open4d/webclients/system/Server/package-lock.json b/open4d/streamer/study/system/Server/package-lock.json similarity index 100% rename from open4d/webclients/system/Server/package-lock.json rename to open4d/streamer/study/system/Server/package-lock.json diff --git a/open4d/webclients/system/Server/package.json b/open4d/streamer/study/system/Server/package.json similarity index 100% rename from open4d/webclients/system/Server/package.json rename to open4d/streamer/study/system/Server/package.json diff --git a/open4d/webclients/system/Server/public/index.html b/open4d/streamer/study/system/Server/public/index.html similarity index 100% rename from open4d/webclients/system/Server/public/index.html rename to open4d/streamer/study/system/Server/public/index.html diff --git a/open4d/webclients/system/Server/quest-launch.json b/open4d/streamer/study/system/Server/quest-launch.json similarity index 100% rename from open4d/webclients/system/Server/quest-launch.json rename to open4d/streamer/study/system/Server/quest-launch.json diff --git a/open4d/webclients/system/Server/quest-trace-player.js b/open4d/streamer/study/system/Server/quest-trace-player.js similarity index 100% rename from open4d/webclients/system/Server/quest-trace-player.js rename to open4d/streamer/study/system/Server/quest-trace-player.js diff --git a/open4d/webclients/system/Server/server.js b/open4d/streamer/study/system/Server/server.js similarity index 99% rename from open4d/webclients/system/Server/server.js rename to open4d/streamer/study/system/Server/server.js index e82e1080..c402d796 100644 --- a/open4d/webclients/system/Server/server.js +++ b/open4d/streamer/study/system/Server/server.js @@ -510,8 +510,20 @@ app.use("/api/study", createStudyRouter({ shaper: studyShaper, tracesDir: path.resolve(__dirname, "../Client/traces"), layoutFile: path.resolve(__dirname, "../../scene_layout.json"), + bridgeShaping: readBridgeShaping, })); +// What the bridge carrying a point-cloud baseline shaped its bytes to, read +// from its /shaping endpoint when a trial finishes. Null for any other method. +async function readBridgeShaping(method) { + const entry = POINTCLOUD_BASELINES.find(value => value.id === method); + if (!entry) return null; + const response = await fetch(`http://127.0.0.1:${entry.bridgePort}/shaping`, + { cache: "no-store", signal: AbortSignal.timeout(2000) }); + if (!response.ok) throw new Error(`bridge /shaping: HTTP ${response.status}`); + return response.json(); +} + // All per-run artifacts (metrics, bitrate counts, QoE, download telemetry) // live in server_results// so each run stays self-contained. function broadcastResultsDir(broadcastId) { diff --git a/open4d/webclients/system/Server/study/follower.js b/open4d/streamer/study/system/Server/study/follower.js similarity index 100% rename from open4d/webclients/system/Server/study/follower.js rename to open4d/streamer/study/system/Server/study/follower.js diff --git a/open4d/webclients/system/Server/study/order.js b/open4d/streamer/study/system/Server/study/order.js similarity index 100% rename from open4d/webclients/system/Server/study/order.js rename to open4d/streamer/study/system/Server/study/order.js diff --git a/open4d/webclients/system/Server/study/questionnaire.js b/open4d/streamer/study/system/Server/study/questionnaire.js similarity index 100% rename from open4d/webclients/system/Server/study/questionnaire.js rename to open4d/streamer/study/system/Server/study/questionnaire.js diff --git a/open4d/webclients/system/Server/study/routes.js b/open4d/streamer/study/system/Server/study/routes.js similarity index 85% rename from open4d/webclients/system/Server/study/routes.js rename to open4d/streamer/study/system/Server/study/routes.js index 4fad5dbd..e97c7274 100644 --- a/open4d/webclients/system/Server/study/routes.js +++ b/open4d/streamer/study/system/Server/study/routes.js @@ -77,8 +77,11 @@ function builtInTraces(directory) { * @param {import('./shaper').TraceShaper} args.shaper * @param {string} [args.tracesDir] built-in traces offered beside uploads * @param {string} [args.layoutFile] scene_layout.json, for the start pose + * @param {(method: string) => Promise} [args.bridgeShaping] the + * shaper status of the bridge carrying `method`, or null if none does */ -function createStudyRouter({ resultsRoot, shaper, tracesDir, layoutFile }) { +function createStudyRouter({ resultsRoot, shaper, tracesDir, layoutFile, + bridgeShaping = async () => null }) { const store = new StudyStore(resultsRoot); const router = express.Router(); router.use(express.json({ limit: '8mb' })); @@ -152,15 +155,28 @@ function createStudyRouter({ resultsRoot, shaper, tracesDir, layoutFile }) { return { trial, shaping: shaper.status(), trace: session.trace.points }; })); - router.post('/sessions/:id/trials/:position/finish', wrap(req => { - const shaping = shaper.status(); - const result = store.finishTrial(req.params.id, req.params.position, { - ...(req.body || {}), shaping - }); - shaper.disarm(); - live = null; - return result; - })); + router.post('/sessions/:id/trials/:position/finish', async (req, res) => { + try { + // Read before disarming: the bridge follows this shaper, and once + // it sees the link released it resets its own counters. + const shaping = shaper.status(); + const trial = store.session(req.params.id).trials[Number(req.params.position)]; + const bridge = trial && await bridgeShaping(trial.method) + .catch(error => ({ error: error.message })); + if (bridge) { + shaping.bridge = { ...bridge, + matchesTrial: bridge.label === shaping.label && bridge.label !== null }; + } + const result = store.finishTrial(req.params.id, req.params.position, { + ...(req.body || {}), shaping + }); + shaper.disarm(); + live = null; + res.set('Cache-Control', 'no-store').json(result); + } catch (error) { + res.status(error.statusCode || 400).json({ error: error.message }); + } + }); /** Release the link after an abandoned trial, without recording it. */ router.post('/abort', wrap(() => { diff --git a/open4d/webclients/system/Server/study/shaper.js b/open4d/streamer/study/system/Server/study/shaper.js similarity index 100% rename from open4d/webclients/system/Server/study/shaper.js rename to open4d/streamer/study/system/Server/study/shaper.js diff --git a/open4d/webclients/system/Server/study/store.js b/open4d/streamer/study/system/Server/study/store.js similarity index 94% rename from open4d/webclients/system/Server/study/store.js rename to open4d/streamer/study/system/Server/study/store.js index 6cc04fe5..927674f8 100644 --- a/open4d/webclients/system/Server/study/store.js +++ b/open4d/streamer/study/system/Server/study/store.js @@ -337,7 +337,7 @@ class StudyStore { 'session', 'participant', 'position', 'label', 'method', 'trace', 'trace_sha256', 'trajectory_sha256', ...questionnaire.RATINGS.map(rating => rating.id), 'artifacts', - 'questionnaire_seconds', + 'questionnaire_seconds', 'shaped_by', 'shaped_bytes', 'paced_seconds', ...metricKeys.map(key => `m_${key}`) ]; const cell = value => { @@ -351,10 +351,26 @@ class StudyStore { ...questionnaire.RATINGS.map(rating => record.questionnaire.ratings[rating.id]), record.questionnaire.artifacts.join(';'), record.questionnaire.timing.totalSeconds, + ...shapingCells(record.shaping), ...metricKeys.map(key => record.metrics[key]) ].map(cell).join(',')); return `${columns.join(',')}\n${rows.join('\n')}${rows.length ? '\n' : ''}`; } } +/** + * Which shaper carried a trial's bytes, and how many. The point-cloud + * baselines' bytes cross the bridge's shaper, never the server's, so a trial + * record carries the bridge's status beside the server's own. + */ +function shapingCells(shaping) { + if (!shaping) return [null, null, null]; + const bridge = shaping.bridge && !shaping.bridge.error ? shaping.bridge : null; + const bytes = (shaping.deliveredBytes || 0) + (bridge?.deliveredBytes || 0); + const paced = (shaping.pacedSeconds || 0) + (bridge?.pacedSeconds || 0); + const by = [shaping.deliveredBytes ? 'server' : null, bridge?.deliveredBytes ? 'bridge' : null] + .filter(Boolean).join('+') || 'none'; + return [by, bytes, Number(paced.toFixed(3))]; +} + module.exports = { StudyStore, METHODS, validateTrajectory, TRAJECTORY_SPACE, sha256 }; diff --git a/open4d/webclients/system/Server/study/trace.js b/open4d/streamer/study/system/Server/study/trace.js similarity index 100% rename from open4d/webclients/system/Server/study/trace.js rename to open4d/streamer/study/system/Server/study/trace.js diff --git a/open4d/webclients/system/Server/test_offline_trajectory.js b/open4d/streamer/study/system/Server/test_offline_trajectory.js similarity index 100% rename from open4d/webclients/system/Server/test_offline_trajectory.js rename to open4d/streamer/study/system/Server/test_offline_trajectory.js diff --git a/open4d/webclients/system/Server/test_quest_trace_player.js b/open4d/streamer/study/system/Server/test_quest_trace_player.js similarity index 100% rename from open4d/webclients/system/Server/test_quest_trace_player.js rename to open4d/streamer/study/system/Server/test_quest_trace_player.js diff --git a/open4d/streamer/study/system/WebClient/README.md b/open4d/streamer/study/system/WebClient/README.md new file mode 100644 index 00000000..8e931a43 --- /dev/null +++ b/open4d/streamer/study/system/WebClient/README.md @@ -0,0 +1,111 @@ +# WebClient + +The user study and one browser page per method, served by `system/Server` at `/web`. + +| Page | Method | Adapts | +|---|---|---| +| `/web/` | the user study | — | +| `/web/mesh.html` | Ours, adaptive mesh ladder | in the browser, via `ClientCore` | +| `/web/baseline.html` | ViVo / NAVA point clouds (`?bridge=` picks which) | on the server | +| `/web/vega.html` | Vega Gaussian splats | no, fixed quality | +| `/web/nevo.html` | NeVo (ReRF), pre-rendered | no | + +Only `mesh.html` uses `ClientCore`, the same core the research repo's Node +client runs. The baselines push frames and adapt on the server, and wrapping +them in a segment loop would measure the wrapper instead of the method. + +## Build and run + +```bash +npm install && node build.js # --watch to rebuild on change +cd ../.. && PYTHON_BIN= scripts/run_web_demo.sh +``` + +This starts the server on the H.264 corpus with 300 segments, plus ViVo and +NAVA. To see adaptation, shape the link (needs root; affects the whole host): + +```bash +sudo scripts/shape_web_demo.sh cascade-20 # 12.5 → 175 Mbps staircase +``` + +## Page parameters + +**`mesh.html`**: `server` (page origin), `mode` (`interactive` | `simulated`), +`storage` (`memory` | `opfs`), `viewpoints` (required for `simulated`), +`decodeBudget` (512 MB), `concurrency` / `inflight` (10 / 2), `label`, +`objects` (comma list; default is the full catalog). + +Keep `objects` small. Every object needs at least one representation, so all +nine ORBIT objects need **116 Mbps** before any quality choice, while three need +about 21 Mbps. + +**`baseline.html`**: `bridge` (`ws://:8790`), `pointSize` (0.012 m), +`strict` (1 = abort on a frame gap). Override the ports with +`VS4D_POINTCLOUD_PORTS=vivo:8790:12345,nava:8791:12346`. + +**`vega.html`**: `assets`, `objects`, `frame` (`object` | `all`), +`splatMode` (`isotropic` | `anisotropic`; anisotropic is untested), `splatScale`. +The whole clip loads before playback, then loops. Streaming it live would need +about 509 Mbps for two objects. + +**`nevo.html`**: `assets`, `object` (`g_dancer`), `fps` (8), `nevoOnly`. + +## Serving the baselines + +```bash +# point clouds: a supervised server + WebSocket bridge per baseline +PYTHON_BIN= scripts/serve_pointcloud_baseline.sh vivo dancer,thomas & # :8790 +PYTHON_BIN= scripts/serve_pointcloud_baseline.sh nava dancer,thomas & # :8791 + +# vega: export once to VGS (location: VS4D_VEGA_WEB_ROOT) +python -m baselines.Vega.orbitvega.export_quest \ + --prepared-dir results/vega-gaussian/prepared-final --output-dir results/vega-web \ + --dataset-root --objects dancer thomas + +# nevo: renders from orbitnevo/render_frames.py (location: VS4D_NEVO_WEB_ROOT) +``` + +## Main files + +| File | Role | +|---|---| +| `src/browser-platform.js` | the browser `ClientPlatform` | +| `src/webgl-renderer.js`, `decode-cache.js`, `texture-decoder.js`, `draco-worker.js` | mesh rendering and decoding | +| `src/camera-pose.js` | Three.js camera → Open3D pose | +| `src/study/` | study page, questionnaire, trial driver and camera path | +| `bridge/v4ds-bridge.js`, `src/v4ds-protocol.js` | WebSocket↔TCP bridge and the V4DS wire format | +| `src/vgs-format.js`, `splat-renderer.js` | Vega splat decoding and rendering | +| `src/nevo-*.js`, `src/{baseline,vega}-client.js` | per-page logic | + +## Pitfalls + +Most of these fail silently. + +- **No HTTP caching.** Media and API fetches use `cache: 'no-store'`; + otherwise bandwidth measurements are wrong and manifests go stale. +- **Stop must not navigate away.** Results are POSTed during shutdown. +- **Viewpoints must be Open3D `PinholeCameraParameters`** (Y-down, +Z forward, + millimetres). Anything else silently gives every object equal weight. +- **Byte order and number types:** V4DS is big-endian with nanosecond BigInt + timestamps. VGS1 is little-endian, and its quaternions are *signed* bytes. +- **Matrix order:** `camera_to_world` arrives row-major, but Three.js is + column-major. +- **Size `SplatObject` from the catalogue.** Three.js caches the instance count + on first bind, so growing it later draws one splat. +- **HEVC** only decodes in Safari and hardware-enabled Chrome, and where it + can't decode, pages fall back to untextured meshes. Serve H.264 and never + transcode at serve time. + +Known gaps: WebCodecs texture decode and mouse camera controls are untested +here, and NeVo has only a single view. + +## Tests and debugging + +```bash +node build.js && node --test ../../tests/*.js # the bundle test needs dist/ +``` + +In the browser, `window.__vs4d.inspect()` shows renderer state. For headless +runs, use puppeteer-core with real Chrome: `--virtual-time-budget` deadlocks +the Draco worker. Screenshots come out black under SwiftShader, so call +`gl.readPixels()` in the same task as `renderer.render()`. diff --git a/open4d/webclients/system/WebClient/bridge/v4ds-bridge.js b/open4d/streamer/study/system/WebClient/bridge/v4ds-bridge.js similarity index 91% rename from open4d/webclients/system/WebClient/bridge/v4ds-bridge.js rename to open4d/streamer/study/system/WebClient/bridge/v4ds-bridge.js index 4ce13883..0e338779 100644 --- a/open4d/webclients/system/WebClient/bridge/v4ds-bridge.js +++ b/open4d/streamer/study/system/WebClient/bridge/v4ds-bridge.js @@ -30,6 +30,10 @@ * `accept()` exactly once. A second browser is refused with a clear reason * rather than being silently queued behind the first. * + * `GET /shaping` on the same port reports this bridge's shaper, so the study + * server can record what a point-cloud trial was actually shaped to: those + * bytes never pass the server's own shaper. + * * Usage: * node bridge/v4ds-bridge.js --baseline-port 12345 [options] * @@ -41,6 +45,7 @@ * --verbose log every message */ +const http = require('http'); const net = require('net'); const { WebSocketServer } = require('ws'); const { ShapingFollower } = require('../../Server/study/follower'); @@ -149,11 +154,6 @@ const SHAPED_QUEUE_HIGH_BYTES = 4 * 1024 * 1024; const SHAPED_QUEUE_LOW_BYTES = 1024 * 1024; function createBridge(options) { - const server = new WebSocketServer({ - host: options.listenHost, - port: options.listenPort - }); - let active = null; // Shared by every connection: it is one link. Unarmed it passes everything // straight through, so outside a study trial the bridge is the dumb proxy // it always was. @@ -161,7 +161,26 @@ function createBridge(options) { : options.studyServer ? new ShapingFollower({ serverUrl: options.studyServer, log }).start() : null; - server.on('close', () => follower?.stop()); + // Our own HTTP server rather than ws's, whose only answer to a plain + // request is 426, so the shaper can be read on the WebSocket's port. + const web = http.createServer((request, response) => { + if (request.method === 'GET' && request.url === '/shaping') { + response.writeHead(200, { 'Content-Type': 'application/json', + 'Cache-Control': 'no-store' }); + return response.end(JSON.stringify(follower + ? { following: true, ...follower.shaper.status() } + : { following: false })); + } + response.writeHead(426, { 'Content-Type': 'text/plain' }); + response.end('Upgrade Required'); + }); + const server = new WebSocketServer({ server: web }); + web.listen(options.listenPort, options.listenHost); + let active = null; + server.on('close', () => { + follower?.stop(); + web.close(); + }); server.on('listening', () => { log('INFO', `WebSocket on ws://${options.listenHost}:${options.listenPort}` diff --git a/open4d/webclients/system/WebClient/build.js b/open4d/streamer/study/system/WebClient/build.js similarity index 100% rename from open4d/webclients/system/WebClient/build.js rename to open4d/streamer/study/system/WebClient/build.js diff --git a/open4d/webclients/system/WebClient/package-lock.json b/open4d/streamer/study/system/WebClient/package-lock.json similarity index 100% rename from open4d/webclients/system/WebClient/package-lock.json rename to open4d/streamer/study/system/WebClient/package-lock.json diff --git a/open4d/webclients/system/WebClient/package.json b/open4d/streamer/study/system/WebClient/package.json similarity index 100% rename from open4d/webclients/system/WebClient/package.json rename to open4d/streamer/study/system/WebClient/package.json diff --git a/open4d/webclients/system/WebClient/public/baseline.html b/open4d/streamer/study/system/WebClient/public/baseline.html similarity index 100% rename from open4d/webclients/system/WebClient/public/baseline.html rename to open4d/streamer/study/system/WebClient/public/baseline.html diff --git a/open4d/webclients/system/WebClient/public/index.html b/open4d/streamer/study/system/WebClient/public/index.html similarity index 100% rename from open4d/webclients/system/WebClient/public/index.html rename to open4d/streamer/study/system/WebClient/public/index.html diff --git a/open4d/webclients/system/WebClient/public/mesh.html b/open4d/streamer/study/system/WebClient/public/mesh.html similarity index 100% rename from open4d/webclients/system/WebClient/public/mesh.html rename to open4d/streamer/study/system/WebClient/public/mesh.html diff --git a/open4d/webclients/system/WebClient/public/nevo.html b/open4d/streamer/study/system/WebClient/public/nevo.html similarity index 100% rename from open4d/webclients/system/WebClient/public/nevo.html rename to open4d/streamer/study/system/WebClient/public/nevo.html diff --git a/open4d/webclients/system/WebClient/public/vega.html b/open4d/streamer/study/system/WebClient/public/vega.html similarity index 100% rename from open4d/webclients/system/WebClient/public/vega.html rename to open4d/streamer/study/system/WebClient/public/vega.html diff --git a/open4d/webclients/system/WebClient/src/baseline-client.js b/open4d/streamer/study/system/WebClient/src/baseline-client.js similarity index 100% rename from open4d/webclients/system/WebClient/src/baseline-client.js rename to open4d/streamer/study/system/WebClient/src/baseline-client.js diff --git a/open4d/webclients/system/WebClient/src/baseline-main.js b/open4d/streamer/study/system/WebClient/src/baseline-main.js similarity index 100% rename from open4d/webclients/system/WebClient/src/baseline-main.js rename to open4d/streamer/study/system/WebClient/src/baseline-main.js diff --git a/open4d/webclients/system/WebClient/src/browser-platform.js b/open4d/streamer/study/system/WebClient/src/browser-platform.js similarity index 100% rename from open4d/webclients/system/WebClient/src/browser-platform.js rename to open4d/streamer/study/system/WebClient/src/browser-platform.js diff --git a/open4d/webclients/system/WebClient/src/camera-pose.js b/open4d/streamer/study/system/WebClient/src/camera-pose.js similarity index 100% rename from open4d/webclients/system/WebClient/src/camera-pose.js rename to open4d/streamer/study/system/WebClient/src/camera-pose.js diff --git a/open4d/webclients/system/WebClient/src/decode-cache.js b/open4d/streamer/study/system/WebClient/src/decode-cache.js similarity index 100% rename from open4d/webclients/system/WebClient/src/decode-cache.js rename to open4d/streamer/study/system/WebClient/src/decode-cache.js diff --git a/open4d/webclients/system/WebClient/src/draco-worker.js b/open4d/streamer/study/system/WebClient/src/draco-worker.js similarity index 100% rename from open4d/webclients/system/WebClient/src/draco-worker.js rename to open4d/streamer/study/system/WebClient/src/draco-worker.js diff --git a/open4d/webclients/system/WebClient/src/link-rate.js b/open4d/streamer/study/system/WebClient/src/link-rate.js similarity index 100% rename from open4d/webclients/system/WebClient/src/link-rate.js rename to open4d/streamer/study/system/WebClient/src/link-rate.js diff --git a/open4d/webclients/system/WebClient/src/main.js b/open4d/streamer/study/system/WebClient/src/main.js similarity index 100% rename from open4d/webclients/system/WebClient/src/main.js rename to open4d/streamer/study/system/WebClient/src/main.js diff --git a/open4d/webclients/system/WebClient/src/nevo-client.js b/open4d/streamer/study/system/WebClient/src/nevo-client.js similarity index 100% rename from open4d/webclients/system/WebClient/src/nevo-client.js rename to open4d/streamer/study/system/WebClient/src/nevo-client.js diff --git a/open4d/webclients/system/WebClient/src/nevo-main.js b/open4d/streamer/study/system/WebClient/src/nevo-main.js similarity index 100% rename from open4d/webclients/system/WebClient/src/nevo-main.js rename to open4d/streamer/study/system/WebClient/src/nevo-main.js diff --git a/open4d/webclients/system/WebClient/src/nevo-manifest.js b/open4d/streamer/study/system/WebClient/src/nevo-manifest.js similarity index 100% rename from open4d/webclients/system/WebClient/src/nevo-manifest.js rename to open4d/streamer/study/system/WebClient/src/nevo-manifest.js diff --git a/open4d/webclients/system/WebClient/src/point-reconstruction.js b/open4d/streamer/study/system/WebClient/src/point-reconstruction.js similarity index 100% rename from open4d/webclients/system/WebClient/src/point-reconstruction.js rename to open4d/streamer/study/system/WebClient/src/point-reconstruction.js diff --git a/open4d/webclients/system/WebClient/src/point-renderer.js b/open4d/streamer/study/system/WebClient/src/point-renderer.js similarity index 100% rename from open4d/webclients/system/WebClient/src/point-renderer.js rename to open4d/streamer/study/system/WebClient/src/point-renderer.js diff --git a/open4d/webclients/system/WebClient/src/splat-renderer.js b/open4d/streamer/study/system/WebClient/src/splat-renderer.js similarity index 100% rename from open4d/webclients/system/WebClient/src/splat-renderer.js rename to open4d/streamer/study/system/WebClient/src/splat-renderer.js diff --git a/open4d/webclients/system/WebClient/src/study/camera.js b/open4d/streamer/study/system/WebClient/src/study/camera.js similarity index 100% rename from open4d/webclients/system/WebClient/src/study/camera.js rename to open4d/streamer/study/system/WebClient/src/study/camera.js diff --git a/open4d/webclients/system/WebClient/src/study/driver.js b/open4d/streamer/study/system/WebClient/src/study/driver.js similarity index 100% rename from open4d/webclients/system/WebClient/src/study/driver.js rename to open4d/streamer/study/system/WebClient/src/study/driver.js diff --git a/open4d/webclients/system/WebClient/src/study/harness.js b/open4d/streamer/study/system/WebClient/src/study/harness.js similarity index 84% rename from open4d/webclients/system/WebClient/src/study/harness.js rename to open4d/streamer/study/system/WebClient/src/study/harness.js index 472738ba..cef4383f 100644 --- a/open4d/webclients/system/WebClient/src/study/harness.js +++ b/open4d/streamer/study/system/WebClient/src/study/harness.js @@ -5,6 +5,8 @@ * * ?study=&trial= replay the shared path, as trial n * ?study=&record=1 the unrated pass that records the path + * ...&preload=1 load the whole clip, say 'preloaded', + * and play only when the host says 'go' * * Everything the page needs comes from the server by session id -- the object * subset, the trial length, the path -- so a trial URL is complete on its own @@ -34,7 +36,8 @@ function studyParams(search = globalThis.location?.search || '') { throw new Error('?study needs either &trial= or &record=1'); } // Which method the recording pass runs on, so the path records it. - return { session, record, trial: record ? null : Number(trial), method: params.get('method') }; + return { session, record, trial: record ? null : Number(trial), method: params.get('method'), + preload: !record && params.get('preload') === '1' }; } async function getJson(serverUrl, route) { @@ -62,6 +65,7 @@ async function loadStudyContext({ serverUrl, params }) { mode: params.record ? 'record' : 'replay', method: params.record ? params.method : trial.method, durationSeconds: session.durationSeconds, + preload: Boolean(params.preload), startPose: stagePose(layout, session.objects, { aspect: STUDY_ASPECT }) }; } @@ -90,6 +94,23 @@ function post(type, payload = {}) { globalThis.location.origin); } +/** + * Resolve when the hosting study page says 'go'. A preloading page waits on + * this between loading its clip and playing it: the host arms the trace in + * between, so the load is not shaped and the trial starts at playback. + */ +function awaitGo() { + return new Promise(resolve => { + const listener = event => { + if (event.origin !== globalThis.location.origin) return; + if (event.data?.source !== MESSAGE_SOURCE || event.data.type !== 'go') return; + globalThis.removeEventListener('message', listener); + resolve(); + }; + globalThis.addEventListener('message', listener); + }); +} + /** * Build the driver for this pass, and report its progress to the host. * @@ -134,6 +155,6 @@ function createStudyDriver({ context, method, onEnded }) { } module.exports = { - studyParams, loadStudyContext, createStudyDriver, post, enterStudyChrome, + studyParams, loadStudyContext, createStudyDriver, post, awaitGo, enterStudyChrome, STUDY_ASPECT, MESSAGE_SOURCE }; diff --git a/open4d/webclients/system/WebClient/src/study/questionnaire-form.js b/open4d/streamer/study/system/WebClient/src/study/questionnaire-form.js similarity index 100% rename from open4d/webclients/system/WebClient/src/study/questionnaire-form.js rename to open4d/streamer/study/system/WebClient/src/study/questionnaire-form.js diff --git a/open4d/webclients/system/WebClient/src/study/study-page.js b/open4d/streamer/study/system/WebClient/src/study/study-page.js similarity index 86% rename from open4d/webclients/system/WebClient/src/study/study-page.js rename to open4d/streamer/study/system/WebClient/src/study/study-page.js index 3c784417..8bf18208 100644 --- a/open4d/webclients/system/WebClient/src/study/study-page.js +++ b/open4d/streamer/study/system/WebClient/src/study/study-page.js @@ -30,6 +30,15 @@ const { parseTrace, windowTrace, summarize } = require('../../../Server/study/tr const STUDY_PAGES = new Set(['mesh', 'vivo', 'nava', 'vega']); /** Baselines served by a restartable Python process, with its own object set. */ const POINTCLOUD = new Set(['vivo', 'nava']); +/** + * Methods that load their whole clip before playing. Loaded before the trace + * is armed, behind the same cover as everything else: under the trace, Vega's + * 64 MB took 41 s at 12.5 Mbps, which made its trial the one with a long + * wait -- identifiable, and several times longer than the rest. + */ +const PRELOAD = new Set(['vega']); +/** How long a preload may take before the trial is declared stuck. */ +const PRELOAD_TIMEOUT_SECONDS = 300; /** * Why a method that speaks the same protocol still cannot take part here. * Stated rather than leaving it off the list, so a missing method reads as a @@ -41,6 +50,24 @@ const UNAVAILABLE = { deltastream: 'needs the RGB-D dataset from baselines.DeltaStream.orbitstream.converter' }; +/** + * WebGL implementations that rasterise on the CPU. Headless Chrome falls back + * to SwiftShader, where the splat and mesh pages run at 2-9 fps; a trial + * rendered that way measures the machine, not the method. + */ +const SOFTWARE_GL = /swiftshader|llvmpipe|softpipe|software rasterizer|microsoft basic render/i; + +/** The GPU this browser renders WebGL with, as the driver names it. */ +function glRenderer() { + try { + const gl = document.createElement('canvas').getContext('webgl2'); + const info = gl.getExtension('WEBGL_debug_renderer_info'); + return String(gl.getParameter(info ? info.UNMASKED_RENDERER_WEBGL : gl.RENDERER)); + } catch { + return null; + } +} + /** How long past its length a trial may run before it is declared stuck. */ const TRIAL_GRACE_SECONDS = 180; @@ -48,7 +75,7 @@ const $ = id => document.getElementById(id); const state = { config: null, systems: [], session: null, trajectory: null, - traceText: null, traceName: null, listener: null + traceText: null, traceName: null, listener: null, glRenderer: null }; // ------------------------------------------------------------------- http --- @@ -153,12 +180,34 @@ function awaitPass(durationSeconds, onProgress, onLoading) { }); } +/** Wait for one message of `type` from the hosted page; rejects on its error. */ +function awaitMessage(type, timeoutSeconds) { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + reject(new Error(`the method page did not report ${type} within ${timeoutSeconds} s`)); + }, timeoutSeconds * 1000); + const listener = event => { + if (event.origin !== window.location.origin) return; + const data = event.data; + if (!data || data.source !== MESSAGE_SOURCE) return; + if (data.type === type) { cleanup(); resolve(data); } + else if (data.type === 'error') { cleanup(); reject(new Error(data.message)); } + }; + const cleanup = () => { + clearTimeout(timeout); + window.removeEventListener('message', listener); + }; + window.addEventListener('message', listener); + }); +} + function progress(barId, fraction) { $(barId).style.width = `${Math.max(0, Math.min(1, fraction)) * 100}%`; } /** Where a method's page lives, carrying what it needs to run one pass. */ -function methodUrl(method, { trial = null, record = false, bridge = null } = {}) { +function methodUrl(method, { trial = null, record = false, bridge = null, preload = false } = {}) { const system = state.systems.find(entry => entry.id === method); if (!system) throw new Error(`the server does not offer ${method}`); const url = new URL(system.page, window.location.origin); @@ -170,6 +219,7 @@ function methodUrl(method, { trial = null, record = false, bridge = null } = {}) url.searchParams.set('method', method); } else url.searchParams.set('trial', String(trial)); + if (preload) url.searchParams.set('preload', '1'); return url.toString(); } @@ -306,6 +356,12 @@ async function boot() { ]); state.config = config; state.systems = systems; + state.glRenderer = glRenderer(); + if (state.glRenderer && SOFTWARE_GL.test(state.glRenderer)) { + fail(new Error(`This browser renders WebGL in software (${state.glRenderer}), so ` + + 'methods will play far below their frame rate. Use a browser with GPU ' + + 'acceleration for real sessions; trials run here are flagged.')); + } renderMethods(); const select = $('trace-builtin'); @@ -482,6 +538,19 @@ async function runTrial(trial) { { id: trial.method, objects: session.objects }); bridge = started.bridge; } + // A preloading method loads its clip now, unshaped and untimed, like the + // point-cloud restart above; its trial starts when it can play. + let frame = null; + let preload = null; + if (PRELOAD.has(trial.method)) { + $('trial-status').textContent = 'preparing…'; + const loaded = awaitMessage('preloaded', PRELOAD_TIMEOUT_SECONDS); + const began = performance.now(); + frame = mount('trial-stage', methodUrl(trial.method, { trial: trial.position, preload: true })); + const report = await loaded; + preload = { seconds: Number(((performance.now() - began) / 1000).toFixed(2)), + bytes: report.bytes ?? null }; + } // Arming the trace is the trial's t=0: every method is shaped by the same // stretch of it, startup included. await api('POST', `/api/study/sessions/${encodeURIComponent(session.id)}/trials/${trial.position}/start`); @@ -490,7 +559,11 @@ async function runTrial(trial) { progress('trial-bar', fraction); $('trial-status').textContent = 'playing'; }, ({ seconds }) => cover('trial-stage', `Loading… ${seconds} s`)); - mount('trial-stage', methodUrl(trial.method, { trial: trial.position, bridge })); + if (frame) { + frame.contentWindow.postMessage({ source: MESSAGE_SOURCE, type: 'go' }, window.location.origin); + } else { + mount('trial-stage', methodUrl(trial.method, { trial: trial.position, bridge })); + } const result = await done; unmount('trial-stage', 'Clip finished.'); $('trial-status').textContent = ''; @@ -501,7 +574,12 @@ async function runTrial(trial) { ...scalars(result.summary), ...flatten('native_', result.native), broadcastId: result.broadcastId ?? null, - exitCode: result.exitCode ?? null + exitCode: result.exitCode ?? null, + glRenderer: state.glRenderer ?? null, + softwareRendering: state.glRenderer ? SOFTWARE_GL.test(state.glRenderer) : null, + preloadedBeforeTrial: preload !== null, + preloadSeconds: preload?.seconds ?? null, + preloadBytes: preload?.bytes ?? null }; const finished = await api('POST', `/api/study/sessions/${encodeURIComponent(session.id)}/trials/${trial.position}/finish`, diff --git a/open4d/webclients/system/WebClient/src/texture-decoder.js b/open4d/streamer/study/system/WebClient/src/texture-decoder.js similarity index 100% rename from open4d/webclients/system/WebClient/src/texture-decoder.js rename to open4d/streamer/study/system/WebClient/src/texture-decoder.js diff --git a/open4d/webclients/system/WebClient/src/v4ds-protocol.js b/open4d/streamer/study/system/WebClient/src/v4ds-protocol.js similarity index 100% rename from open4d/webclients/system/WebClient/src/v4ds-protocol.js rename to open4d/streamer/study/system/WebClient/src/v4ds-protocol.js diff --git a/open4d/webclients/system/WebClient/src/vega-client.js b/open4d/streamer/study/system/WebClient/src/vega-client.js similarity index 97% rename from open4d/webclients/system/WebClient/src/vega-client.js rename to open4d/streamer/study/system/WebClient/src/vega-client.js index bd7931e0..ab726a76 100644 --- a/open4d/webclients/system/WebClient/src/vega-client.js +++ b/open4d/streamer/study/system/WebClient/src/vega-client.js @@ -69,7 +69,13 @@ class VegaClient { _emit(type, detail = {}) { this._onEvent?.({ type, ...detail }); } - async start(objectNames = null) { + /** + * @param {string[]} [objectNames] + * @param {object} [options] + * @param {() => Promise} [options.beforePlay] awaited between the + * preload and the first frame advance; a study trial arms its trace here + */ + async start(objectNames = null, { beforePlay = null } = {}) { this._three = this._buildScene(); this._loop(); @@ -118,6 +124,11 @@ class VegaClient { this._three.scene.add(splat.mesh); this.objects.set(entry.name, { entry, splat, frames: new Map() }); } + // The objects exist only now, after the scene applied its viewport, and + // a study pass never auto-frames (which applies it again). Left at its + // (1, 1) placeholder, every splat is drawn hundreds of times too large: + // 70% of the canvas covered instead of 3%, at 8 fps instead of 60. + this._applyViewport(); this._emit('catalog', { fps: this.catalog.fps, @@ -149,6 +160,7 @@ class VegaClient { // does with its renders. await this._preload(); this._showFrame(0); + if (beforePlay) await beforePlay(); this.playing = true; this._lastAdvance = performance.now(); } diff --git a/open4d/webclients/system/WebClient/src/vega-main.js b/open4d/streamer/study/system/WebClient/src/vega-main.js similarity index 84% rename from open4d/webclients/system/WebClient/src/vega-main.js rename to open4d/streamer/study/system/WebClient/src/vega-main.js index 2093a534..fa66f717 100644 --- a/open4d/webclients/system/WebClient/src/vega-main.js +++ b/open4d/streamer/study/system/WebClient/src/vega-main.js @@ -156,26 +156,36 @@ async function main() { ].join('\n'); }, 500); + // A preloading trial builds its driver only at 'go', so the trial clock and + // the startup delay start at playback, not at the unshaped load before it. + const makeDriver = () => { + driver = study.createStudyDriver({ + context: studyContext, method: 'vega', + onEnded: () => { + if (studyOver) return; + studyOver = true; + client.stop(); + const native = Object.fromEntries(Object.entries(client.stats).filter(([, value]) => + value === null || ['number', 'string', 'boolean'].includes(typeof value))); + study.post('ended', { method: 'vega', exitCode: 0, broadcastId: null, + summary: driver.summary(), native: { stats: native }, + trajectory: driver.trajectory() }); + } + }); + client.setCameraDriver(driver); + }; try { - if (studyContext) { - driver = study.createStudyDriver({ - context: studyContext, method: 'vega', - onEnded: () => { - if (studyOver) return; - studyOver = true; - client.stop(); - const native = Object.fromEntries(Object.entries(client.stats).filter(([, value]) => - value === null || ['number', 'string', 'boolean'].includes(typeof value))); - study.post('ended', { method: 'vega', exitCode: 0, broadcastId: null, - summary: driver.summary(), native: { stats: native }, - trajectory: driver.trajectory() }); - } - }); - client.setCameraDriver(driver); - } - await client.start(config.objects); + if (studyContext && !studyContext.preload) makeDriver(); + await client.start(config.objects, !studyContext?.preload ? {} : { + beforePlay: async () => { + study.post('preloaded', { method: 'vega', frames: client.frameCount, + bytes: client.stats.bytesFetched }); + await study.awaitGo(); + makeDriver(); + } + }); } catch (error) { - if (driver) study.post('error', { method: 'vega', message: error.message }); + if (studyContext) study.post('error', { method: 'vega', message: error.message }); status.textContent = `could not start: ${error.message}`; log('error', error.message); log('info', 'Export assets first: python -m baselines.Vega.orbitvega.' diff --git a/open4d/webclients/system/WebClient/src/vgs-format.js b/open4d/streamer/study/system/WebClient/src/vgs-format.js similarity index 100% rename from open4d/webclients/system/WebClient/src/vgs-format.js rename to open4d/streamer/study/system/WebClient/src/vgs-format.js diff --git a/open4d/webclients/system/WebClient/src/webgl-renderer.js b/open4d/streamer/study/system/WebClient/src/webgl-renderer.js similarity index 100% rename from open4d/webclients/system/WebClient/src/webgl-renderer.js rename to open4d/streamer/study/system/WebClient/src/webgl-renderer.js diff --git a/open4d/webclients/system/WebClient/vendor/draco/draco_decoder.js b/open4d/streamer/study/system/WebClient/vendor/draco/draco_decoder.js similarity index 100% rename from open4d/webclients/system/WebClient/vendor/draco/draco_decoder.js rename to open4d/streamer/study/system/WebClient/vendor/draco/draco_decoder.js diff --git a/open4d/webclients/system/WebClient/vendor/draco/draco_decoder.wasm b/open4d/streamer/study/system/WebClient/vendor/draco/draco_decoder.wasm similarity index 100% rename from open4d/webclients/system/WebClient/vendor/draco/draco_decoder.wasm rename to open4d/streamer/study/system/WebClient/vendor/draco/draco_decoder.wasm diff --git a/open4d/webclients/tests/fixtures/view_00.json b/open4d/streamer/study/tests/fixtures/view_00.json similarity index 100% rename from open4d/webclients/tests/fixtures/view_00.json rename to open4d/streamer/study/tests/fixtures/view_00.json diff --git a/open4d/webclients/tests/fixtures_nevo_manifest.json b/open4d/streamer/study/tests/fixtures_nevo_manifest.json similarity index 100% rename from open4d/webclients/tests/fixtures_nevo_manifest.json rename to open4d/streamer/study/tests/fixtures_nevo_manifest.json diff --git a/open4d/webclients/tests/fixtures_recon_golden.gen.py b/open4d/streamer/study/tests/fixtures_recon_golden.gen.py similarity index 100% rename from open4d/webclients/tests/fixtures_recon_golden.gen.py rename to open4d/streamer/study/tests/fixtures_recon_golden.gen.py diff --git a/open4d/webclients/tests/fixtures_recon_golden.json b/open4d/streamer/study/tests/fixtures_recon_golden.json similarity index 100% rename from open4d/webclients/tests/fixtures_recon_golden.json rename to open4d/streamer/study/tests/fixtures_recon_golden.json diff --git a/open4d/webclients/tests/fixtures_v4ds_golden.gen.py b/open4d/streamer/study/tests/fixtures_v4ds_golden.gen.py similarity index 100% rename from open4d/webclients/tests/fixtures_v4ds_golden.gen.py rename to open4d/streamer/study/tests/fixtures_v4ds_golden.gen.py diff --git a/open4d/webclients/tests/fixtures_v4ds_golden.json b/open4d/streamer/study/tests/fixtures_v4ds_golden.json similarity index 100% rename from open4d/webclients/tests/fixtures_v4ds_golden.json rename to open4d/streamer/study/tests/fixtures_v4ds_golden.json diff --git a/open4d/webclients/tests/fixtures_vgs_golden.gen.py b/open4d/streamer/study/tests/fixtures_vgs_golden.gen.py similarity index 100% rename from open4d/webclients/tests/fixtures_vgs_golden.gen.py rename to open4d/streamer/study/tests/fixtures_vgs_golden.gen.py diff --git a/open4d/webclients/tests/fixtures_vgs_golden.json b/open4d/streamer/study/tests/fixtures_vgs_golden.json similarity index 100% rename from open4d/webclients/tests/fixtures_vgs_golden.json rename to open4d/streamer/study/tests/fixtures_vgs_golden.json diff --git a/open4d/webclients/tests/test_bandwidth_estimator.js b/open4d/streamer/study/tests/test_bandwidth_estimator.js similarity index 100% rename from open4d/webclients/tests/test_bandwidth_estimator.js rename to open4d/streamer/study/tests/test_bandwidth_estimator.js diff --git a/open4d/webclients/tests/test_browser_platform.js b/open4d/streamer/study/tests/test_browser_platform.js similarity index 100% rename from open4d/webclients/tests/test_browser_platform.js rename to open4d/streamer/study/tests/test_browser_platform.js diff --git a/open4d/webclients/tests/test_camera_pose.js b/open4d/streamer/study/tests/test_camera_pose.js similarity index 100% rename from open4d/webclients/tests/test_camera_pose.js rename to open4d/streamer/study/tests/test_camera_pose.js diff --git a/open4d/webclients/tests/test_client_core.js b/open4d/streamer/study/tests/test_client_core.js similarity index 100% rename from open4d/webclients/tests/test_client_core.js rename to open4d/streamer/study/tests/test_client_core.js diff --git a/open4d/webclients/tests/test_client_core_parity.js b/open4d/streamer/study/tests/test_client_core_parity.js similarity index 100% rename from open4d/webclients/tests/test_client_core_parity.js rename to open4d/streamer/study/tests/test_client_core_parity.js diff --git a/open4d/webclients/tests/test_client_core_purity.js b/open4d/streamer/study/tests/test_client_core_purity.js similarity index 100% rename from open4d/webclients/tests/test_client_core_purity.js rename to open4d/streamer/study/tests/test_client_core_purity.js diff --git a/open4d/webclients/tests/test_client_platform.js b/open4d/streamer/study/tests/test_client_platform.js similarity index 100% rename from open4d/webclients/tests/test_client_platform.js rename to open4d/streamer/study/tests/test_client_platform.js diff --git a/open4d/webclients/tests/test_decode_cache.js b/open4d/streamer/study/tests/test_decode_cache.js similarity index 100% rename from open4d/webclients/tests/test_decode_cache.js rename to open4d/streamer/study/tests/test_decode_cache.js diff --git a/open4d/webclients/tests/test_link_rate.js b/open4d/streamer/study/tests/test_link_rate.js similarity index 100% rename from open4d/webclients/tests/test_link_rate.js rename to open4d/streamer/study/tests/test_link_rate.js diff --git a/open4d/webclients/tests/test_nevo_manifest.js b/open4d/streamer/study/tests/test_nevo_manifest.js similarity index 100% rename from open4d/webclients/tests/test_nevo_manifest.js rename to open4d/streamer/study/tests/test_nevo_manifest.js diff --git a/open4d/webclients/tests/test_point_reconstruction.js b/open4d/streamer/study/tests/test_point_reconstruction.js similarity index 100% rename from open4d/webclients/tests/test_point_reconstruction.js rename to open4d/streamer/study/tests/test_point_reconstruction.js diff --git a/open4d/webclients/tests/test_streaming_client.js b/open4d/streamer/study/tests/test_streaming_client.js similarity index 100% rename from open4d/webclients/tests/test_streaming_client.js rename to open4d/streamer/study/tests/test_streaming_client.js diff --git a/open4d/webclients/tests/test_study_bridge.js b/open4d/streamer/study/tests/test_study_bridge.js similarity index 100% rename from open4d/webclients/tests/test_study_bridge.js rename to open4d/streamer/study/tests/test_study_bridge.js diff --git a/open4d/webclients/tests/test_study_camera.js b/open4d/streamer/study/tests/test_study_camera.js similarity index 100% rename from open4d/webclients/tests/test_study_camera.js rename to open4d/streamer/study/tests/test_study_camera.js diff --git a/open4d/webclients/tests/test_study_session.js b/open4d/streamer/study/tests/test_study_session.js similarity index 80% rename from open4d/webclients/tests/test_study_session.js rename to open4d/streamer/study/tests/test_study_session.js index d31d7462..0864bf86 100644 --- a/open4d/webclients/tests/test_study_session.js +++ b/open4d/streamer/study/tests/test_study_session.js @@ -175,7 +175,7 @@ test('a trajectory is refused if time runs backwards', () => { // -------------------------------------------------------------- routes --- -async function serve() { +async function serve({ bridgeShaping } = {}) { const root = tempDir(); const files = tempDir(); // 2 MB at the trace's opening 40 Mbps is about 0.4 s; unshaped it is instant. @@ -185,7 +185,7 @@ async function serve() { app.use(express.json({ limit: '200mb' })); app.use('/files', shapedStatic(files, shaper)); app.use('/files', express.static(files)); - app.use('/api/study', createStudyRouter({ resultsRoot: root, shaper })); + app.use('/api/study', createStudyRouter({ resultsRoot: root, shaper, bridgeShaping })); const server = await new Promise(resolve => { const s = app.listen(0, '127.0.0.1', () => resolve(s)); }); @@ -291,3 +291,72 @@ test('abort frees the link without recording a result', { skip: NO_EXPRESS }, as server.close(); } }); + +test('a point-cloud trial records what the bridge shaped, not the server', { skip: NO_EXPRESS }, async () => { + const net = require('node:net'); + const { createBridge } = require('../system/WebClient/bridge/v4ds-bridge'); + const MESSAGE_BYTES = 64 * 1024; + const MESSAGES = 16; + const baseline = await new Promise(resolve => { + const server = net.createServer(socket => { + for (let index = 0; index < MESSAGES; index++) { + const prefix = Buffer.alloc(4); + prefix.writeUInt32BE(MESSAGE_BYTES); + socket.write(Buffer.concat([prefix, Buffer.alloc(MESSAGE_BYTES, index)])); + } + }); + server.listen(0, '127.0.0.1', () => resolve(server)); + }); + let bridgePort = null; + const bridgeStatus = () => fetch(`http://127.0.0.1:${bridgePort}/shaping`).then(r => r.json()); + const { server, base, call, shaper } = await serve({ + bridgeShaping: async method => (method === 'vivo' ? bridgeStatus() : null) + }); + const bridge = createBridge({ + listenHost: '127.0.0.1', listenPort: 0, baselineHost: '127.0.0.1', + baselinePort: baseline.address().port, maxMessageBytes: 1 << 24, + verbose: false, studyServer: base + }); + await new Promise(resolve => bridge.on('listening', resolve)); + bridgePort = bridge.address().port; + try { + const session = (await call('POST', '/api/study/sessions', { + participant: 'p07', methods: ['vivo'], durationSeconds: 20, trace: { text: TRACE } + })).json; + await call('PUT', `/api/study/sessions/${session.id}/trajectory`, trajectory()); + await call('POST', `/api/study/sessions/${session.id}/trials/0/start`); + + // The follower polls every 500 ms; wait for it to pick the trial up. + for (let tries = 0; tries < 40 && !(await bridgeStatus()).shaped; tries++) { + await new Promise(resolve => setTimeout(resolve, 50)); + } + const socket = new WebSocket(`ws://127.0.0.1:${bridgePort}`); + await new Promise((resolve, reject) => { + let got = 0; + socket.onmessage = () => { if (++got === MESSAGES) resolve(); }; + socket.onerror = reject; + }); + socket.close(); + + const finished = await call('POST', `/api/study/sessions/${session.id}/trials/0/finish`, + { metrics: {}, questionnaire: RESPONSE }); + assert.strictEqual(finished.status, 200, finished.text); + const shaping = finished.json.record.shaping; + assert.strictEqual(shaping.deliveredBytes, 0, 'none of it crossed the server'); + assert.strictEqual(shaping.bridge.following, true); + assert.strictEqual(shaping.bridge.matchesTrial, true); + assert.strictEqual(shaping.bridge.deliveredBytes, MESSAGES * MESSAGE_BYTES); + + const csv = await call('GET', `/api/study/sessions/${session.id}/export.csv`); + const [header, row] = csv.text.trim().split('\n'); + const columns = header.split(','); + const cells = row.split(','); + assert.strictEqual(cells[columns.indexOf('shaped_by')], 'bridge'); + assert.strictEqual(Number(cells[columns.indexOf('shaped_bytes')]), MESSAGES * MESSAGE_BYTES); + } finally { + shaper.disarm(); + await new Promise(resolve => bridge.close(resolve)); + baseline.close(); + server.close(); + } +}); diff --git a/open4d/webclients/tests/test_study_shaper.js b/open4d/streamer/study/tests/test_study_shaper.js similarity index 100% rename from open4d/webclients/tests/test_study_shaper.js rename to open4d/streamer/study/tests/test_study_shaper.js diff --git a/open4d/webclients/tests/test_tile_ladder.py b/open4d/streamer/study/tests/test_tile_ladder.py similarity index 100% rename from open4d/webclients/tests/test_tile_ladder.py rename to open4d/streamer/study/tests/test_tile_ladder.py diff --git a/open4d/webclients/tests/test_v4ds_bridge.js b/open4d/streamer/study/tests/test_v4ds_bridge.js similarity index 100% rename from open4d/webclients/tests/test_v4ds_bridge.js rename to open4d/streamer/study/tests/test_v4ds_bridge.js diff --git a/open4d/webclients/tests/test_v4ds_protocol.js b/open4d/streamer/study/tests/test_v4ds_protocol.js similarity index 100% rename from open4d/webclients/tests/test_v4ds_protocol.js rename to open4d/streamer/study/tests/test_v4ds_protocol.js diff --git a/open4d/webclients/tests/test_vgs_format.js b/open4d/streamer/study/tests/test_vgs_format.js similarity index 100% rename from open4d/webclients/tests/test_vgs_format.js rename to open4d/streamer/study/tests/test_vgs_format.js diff --git a/open4d/webclients/tests/test_web_client_bundle.js b/open4d/streamer/study/tests/test_web_client_bundle.js similarity index 100% rename from open4d/webclients/tests/test_web_client_bundle.js rename to open4d/streamer/study/tests/test_web_client_bundle.js diff --git a/open4d/webclients/tile_ladder.py b/open4d/streamer/study/tile_ladder.py similarity index 100% rename from open4d/webclients/tile_ladder.py rename to open4d/streamer/study/tile_ladder.py diff --git a/open4d/streaming/README.md b/open4d/streaming/README.md deleted file mode 100644 index 05b5474a..00000000 --- a/open4d/streaming/README.md +++ /dev/null @@ -1,336 +0,0 @@ -# RGB-D streaming - -Two synchronized RGB-D cameras stream to an Ubuntu GPU machine for live -point-cloud fusion and CUDA mesh reconstruction. The result is viewed in a web -browser. - -This module was formerly named `MeshReduce`. The original C++ reconstruction -applications are still included. - -## Data path - -```text -Camera 1 + Camera 2 - -> capture host (RGB JPEG + compressed depth) - -> SSH tunnel - -> Ubuntu receiver - - decode both frames - - transform camera 2 into camera 1 coordinates - - display a live fused point cloud - - reconstruct and merge CUDA meshes - -> Open3D WebRTC - -> browser -``` - -The point cloud updates for every processed camera pair. The mesh is rebuilt -periodically and is not a video stream. - -Networking and frame preparation run on the CPU. CUDA handles TSDF integration -and mesh extraction. This is not a GPUDirect or RDMA pipeline. - -## Setup - -Hardware, operating systems, and the tested configuration are in the repository -[README](../../README.md). Any camera works if the sender -produces the same RGB-D packet format and the receiver constants match its -resolutions. - -Create a Python environment on the reconstruction host: - -```bash -python3 -m venv .venv -source .venv/bin/activate -python -m pip install numpy opencv-python zstandard -``` - -Install a CUDA-enabled Open3D build in that environment. The standard pip wheel -may not include CUDA. Verify the installation: - -```bash -python -c \ - "import open3d as o3d; print(o3d.__version__, o3d.core.cuda.is_available())" -``` - -The second value should be `True` for GPU meshes. - -## Use your own cameras - -1. Choose camera 1 as the reference coordinate system. -2. Configure the sender with your two serial numbers and sync roles. -3. Export the depth/color factory calibration for each camera. -4. Calibrate camera 2 relative to camera 1. -5. Store the files in this layout: - -```text -/ -├── source/work/calibration_stepwise/factory/ -│ ├── ey_factory_calibration.json -│ └── j3_factory_calibration.json -└── final_validated_fusion/ - └── j3_depth_to_ey_depth_refined.txt -``` - -`EY` and `J3` are legacy labels for camera 1 and camera 2. They do not need to -match your serial numbers. This module consumes calibration files but does not -currently generate them. Recalibrate after moving either camera. - -## Start a live session - -Start these components in order. - -### 1. Ubuntu receiver - -```bash -cd /path/to/Open4D/open4d/streaming - -export FOURD_CALIBRATION_DIR=/absolute/path/to/calibration -export PYTHON=/path/to/python-with-open3d -export FOURD_CUDA_DEVICE=0 -export FOURD_CAMERA1_SERIAL=your-camera-1-serial -export FOURD_CAMERA2_SERIAL=your-camera-2-serial - -DISPLAY_MODE=pointcloud \ -MESH_WINDOW=1 \ -./tools/run_browser_viewer.sh -``` - -Expected: - -```text -Live fusion receiver listening on 127.0.0.1:17000 -Browser viewer ready through Ubuntu 127.0.0.1:8888 -``` - -Leave this terminal open. - -### 2. Windows data tunnel - -Close Orbbec Viewer and anything else using the cameras. In PowerShell: - -```powershell -$GpuUser = "your-user" -$GpuHost = "192.168.1.50" - -ssh -o ExitOnForwardFailure=yes ` - -o ServerAliveInterval=30 ` - -o ServerAliveCountMax=3 ` - -N ` - -L 127.0.0.1:17000:127.0.0.1:17000 ` - "$GpuUser@$GpuHost" -``` - -The command stays quiet after login. Leave it running. - -In another PowerShell window, check the tunnel: - -```powershell -Test-NetConnection 127.0.0.1 -Port 17000 -``` - -Expected: - -```text -TcpTestSucceeded : True -``` - -### 3. Windows camera sender - -Start an OBP1-compatible sender and point it at the local end of the tunnel. -The receiver and protocol are included in this module; the tested Windows -capture sender is currently a separate companion script. Copy that script and -`protocol.py` to the capture host, or provide another sender that implements -OBP1. For the tested Python sender: - -```powershell -$Python = "C:\path\to\python.exe" -$Sender = "C:\path\to\windows_sender.py" -$Report = "C:\path\to\sender_report.json" - -& $Python $Sender ` - --sdk-bin "C:\path\to\OrbbecSDK-K4A-Wrapper\bin" ` - --host 127.0.0.1 ` - --port 17000 ` - --fps 5 ` - --report $Report -``` - -The companion sender must be configured with your camera serial numbers before -use. A healthy sender prints: - -```text -{"produced_pairs": 30, "sync_error_us": ..., "queue_dropped": 0, ...} -``` - -Expected on Ubuntu: - -```text -Sender connected from 127.0.0.1 -{"processed_pairs": 10, "latest_pair": 10, "live_points": ...} -``` - -The sender defaults to 5 FPS. Higher input rates currently cause frame -replacement during CPU point-cloud preparation. - -### 4. Browser tunnel - -On the viewing machine: - -```bash -GPU_USER=your-user -GPU_HOST=192.168.1.50 - -ssh -N \ - -o ExitOnForwardFailure=yes \ - -L 127.0.0.1:18888:127.0.0.1:8888 \ - "$GPU_USER@$GPU_HOST" -``` - -Open [http://127.0.0.1:18888/](http://127.0.0.1:18888/). - -If the page opens but the scene does not move, the Windows sender is not -connected. The Ubuntu log must show `Sender connected` and increasing -`processed_pairs`. - -To stop, press `Ctrl+C` in the Windows sender first. Then stop the Windows data -tunnel and Ubuntu receiver. - -If capture, reconstruction, and viewing all run on the same host, skip both SSH -tunnels. Send to `127.0.0.1:17000` and open -`http://127.0.0.1:8888/`. - -## Viewer modes - -Set `DISPLAY_MODE` when starting the browser viewer: - -| Value | Display | -|---|---| -| `pointcloud` | Latest fused point cloud; use this for motion | -| `auto` | Point cloud until the first mesh, then mesh only | -| `mesh` | Mesh only; use this for stationary scenes | -| `both` | Point cloud and mesh together; useful for debugging but can look doubled | - -Recommended live view: - -```bash -DISPLAY_MODE=pointcloud MESH_WINDOW=1 ./tools/run_browser_viewer.sh -``` - -Recommended stationary mesh: - -```bash -DISPLAY_MODE=mesh MESH_WINDOW=7 ./tools/run_browser_viewer.sh -``` - -## Fusion modes - -The live point cloud always transforms J3 points into EY coordinates and -concatenates the two aligned point sets. - -The default mesh path is `independent-merge`: - -1. Reconstruct EY in its own CUDA TSDF volume. -2. Reconstruct J3 in a second CUDA TSDF volume. -3. Transform the J3 mesh into EY coordinates. -4. Merge the partial meshes. - -Controls: - -```bash -MESH_FUSION_MODE=independent-merge # separate per-camera meshes -MESH_FUSION_MODE=shared-tsdf # both cameras in one TSDF volume - -MESH_MERGE_MODE=concatenate # keep every triangle -MESH_MERGE_MODE=weld # merge nearby vertices -MESH_WELD_RADIUS=0.003 # metres -``` - -`concatenate` does not crop, decimate, remove components, or fill holes. It can -show two nearby surfaces where the camera views overlap. `shared-tsdf` usually -looks cleaner. - -The full MeshReduce paper merger removes redundant overlap with raycasting and -then stitches boundaries. That step is not implemented yet. - -## Output - -The browser launcher writes to `output/two-camera-fusion/`: - -| File | Contents | -|---|---| -| `latest_live_full_scene_pointcloud.ply` | latest fused point cloud | -| `latest_live_full_scene_mesh.ply` | latest mesh | -| `live_fusion_report.json` | receiver and synchronization summary | -| `live_browser.log` | startup, frame, and mesh logs | - -A successful mesh log includes: - -```text -"backend": "CUDA:0" -"fusion_mode": "independent-merge" -"partial_vertices": [, ] -"vertices": -``` - -For `concatenate`, the final vertex count equals the sum of the two partial -counts. - -## Replay without cameras - -Start the Ubuntu browser receiver as above. In a second Ubuntu terminal: - -```bash -cd /path/to/Open4D/open4d/streaming - -"$PYTHON" \ - tools/replay_obp1_sender.py \ - --captures-root /absolute/path/to/saved-pairs \ - --fps 2 -``` - -This exercises transport, reconstruction, and browser rendering with recorded -frames. It is not a live camera feed. - -For a one-shot saved reconstruction: - -```bash -export FOURD_CAPTURE_ROOT=/path/to/capture-data -./tools/reconstruct_saved_two_camera.py -``` - -## Original C++ pipeline - -The original application accepts a live K4A-compatible camera or recorded -`.mkv`, then writes PLY, textured OBJ, Draco geometry, and stage metrics. - -```bash -sudo apt install -y build-essential cmake ninja-build git \ - libopencv-dev libeigen3-dev libjsoncpp-dev libdraco-dev draco - -cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -cmake --build build --target rgbd_streamer -j"$(nproc)" -./build/app/rgbd_streamer config.playback.json -``` - -Expected final output: - -```text -mesh faces before_qem=... after_qem=... vertices=... -wrote output/playback/mesh.ply, .obj, .mtl, and _texture.png -MESHREDUCE_METRICS {...} -``` - -Run tests: - -```bash -ctest --test-dir build --output-on-failure -``` - - -## Related files - -- [`docs/artifacts.md`](../../docs/artifacts.md): generated-data policy -- `python/protocol.py`: OBP1 packet definitions -- `tools/receive_mesh_frame.py`: MRD1/MRD2 reference receiver -- `tools/receive_live_stream.py`: MRD3 reference receiver - -Generated datasets and output directories are git-ignored. diff --git a/open4d/streaming/REMOTE_TWO_CAMERA.md b/open4d/streaming/REMOTE_TWO_CAMERA.md deleted file mode 100644 index 62b132f9..00000000 --- a/open4d/streaming/REMOTE_TWO_CAMERA.md +++ /dev/null @@ -1,53 +0,0 @@ -# Remote two-camera Femto Bolt reconstruction - -The two Femto Bolt cameras are USB-connected to the Windows capture host, not -to this GPU machine. The original `dual_camera_fusion` executable opens local -K4A-compatible USB devices and therefore cannot capture this installation -directly. - -`tools/run_remote_two_camera_fusion.sh` is the module entry point for the -installed topology: - -``` -Windows synchronized RGB-D capture - -> OBP1 TCP transport - -> calibrated J3-depth to EY-depth transform - -> shared TSDF reconstruction - -> full-scene point cloud and triangle mesh -``` - -Run a bounded reconstruction: - -```bash -cd /home/ryan/Open4D/open4d/streaming -MAX_PAIRS=30 ./tools/run_remote_two_camera_fusion.sh -``` - -Outputs: - -- `output/two-camera-fusion/latest_live_full_scene_pointcloud.ply` -- `output/two-camera-fusion/latest_live_full_scene_mesh.ply` -- `output/two-camera-fusion/live_fusion_report.json` - -The reconstruction excludes only invalid or out-of-range depth measurements. -It performs no spatial crop, connected-component filtering, geometry -decimation, or scene cleanup. - -When the Windows sender is unavailable, reproduce a module-owned result from -the last saved synchronized sequence: - -```bash -./tools/reconstruct_saved_two_camera.py -``` - -This writes `output/two-camera-fusion/saved_sequence_two_camera_fused_mesh.ply` -and `saved_sequence_fusion_report.json`. - -If both cameras are later USB-connected directly to this GPU host, build and -run the native CUDA path: - -```bash -cmake -S . -B build -G Ninja -cmake --build build --target dual_camera_fusion -j2 -./build/app/dual_camera_fusion config.dual.json -``` diff --git a/open4d/streaming/__init__.py b/open4d/streaming/__init__.py deleted file mode 100644 index fe04cc9b..00000000 --- a/open4d/streaming/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Reconstruct RGB-D captures and send decoded mesh frames over TCP.""" - -from ._reconstruction import reconstruct -from ._transport import receive, send - -__all__ = ["reconstruct", "receive", "send"] diff --git a/open4d/streaming/tests/test_python.py b/open4d/streaming/tests/test_python.py deleted file mode 100644 index 2002a57f..00000000 --- a/open4d/streaming/tests/test_python.py +++ /dev/null @@ -1,180 +0,0 @@ -from concurrent.futures import ThreadPoolExecutor -import importlib -import json -import socket -import sys -from types import SimpleNamespace - -import numpy as np -import pytest - -from open4d.core import Frame, TriangleMesh -from open4d.demo import mesh_sequence -from open4d.streaming import receive, reconstruct, send -from open4d.streaming import _transport - - -def test_stream_preserves_frames_and_attributes(): - mesh = TriangleMesh( - [[0., 0., 0.], [1., 0., 0.], [0., 1., 0.]], [[0, 1, 2]], - colors=np.full((3, 3), 0.5), normals=np.tile([0., 0., 1.], (3, 1)), - texture_coordinates=np.array([[0., 0.], [1., 0.], [0., 1.]]), - attributes={"confidence": np.array([0.1, 0.5, 0.9])}, - ) - expected = [Frame(8, 1.25, mesh, {"camera": "left", "tags": [1, "rgbd"]}), - Frame(9, 1.5, mesh)] - with receive(port=0, timeout=3) as receiver, ThreadPoolExecutor() as pool: - future = pool.submit(send, expected, *receiver.address, realtime=False) - actual = list(receiver) - assert future.result(timeout=3) == 2 - assert [frame.frame_index for frame in actual] == [8, 9] - assert [frame.timestamp for frame in actual] == [1.25, 1.5] - assert actual[0].metadata == expected[0].metadata - for name in ("positions", "triangles", "colors", "normals", "texture_coordinates"): - np.testing.assert_array_equal(getattr(actual[0].geometry, name), getattr(mesh, name)) - np.testing.assert_array_equal(actual[0].geometry.attributes["confidence"], - mesh.attributes["confidence"]) - - -def test_stream_synthetic_sequence_and_empty_mesh(): - sample = mesh_sequence(side=4, frames=3) - empty = Frame(3, 1, TriangleMesh(np.empty((0, 3)), np.empty((0, 3), dtype=np.uint32))) - expected = [*sample, empty] - with receive(port=0, timeout=3) as receiver, ThreadPoolExecutor() as pool: - future = pool.submit(send, expected, *receiver.address, realtime=False) - actual = list(receiver) - assert future.result(timeout=3) == 4 - for original, decoded in zip(expected, actual): - np.testing.assert_array_equal(original.geometry.positions, decoded.geometry.positions) - np.testing.assert_array_equal(original.geometry.triangles, decoded.geometry.triangles) - - -def test_empty_stream_finishes(): - with receive(port=0, timeout=3) as receiver, ThreadPoolExecutor() as pool: - future = pool.submit(send, [], *receiver.address) - assert list(receiver) == [] - assert future.result(timeout=3) == 0 - - -def test_receiver_closes_early_and_releases_port(): - with receive(port=0, timeout=3) as receiver: - address = receiver.address - with receive(*address, timeout=0.01) as second: - with pytest.raises(TimeoutError): - next(second) - assert list(receiver) == [] - - -@pytest.mark.parametrize("payload,error", [ - (_transport._HEADER.pack(b"BAD!", 0, 0), ValueError), - (_transport._HEADER.pack(_transport._MAGIC, 10, 1000), ValueError), - (_transport._HEADER.pack(_transport._MAGIC, 10, 10) + b"{}", EOFError), -]) -def test_receiver_rejects_invalid_or_truncated_messages(payload, error): - with receive(port=0, timeout=3, max_frame_bytes=100) as receiver: - with socket.create_connection(receiver.address, timeout=3) as sender: - sender.sendall(payload) - with pytest.raises(error): - next(receiver) - - -def test_receiver_rejects_object_arrays(): - header = json.dumps({"frame_index": 0, "timestamp": 0, "metadata": {}, - "arrays": [{"name": "positions", "dtype": "O", "shape": [1, 3]}]}).encode() - with receive(port=0, timeout=3) as receiver: - with socket.create_connection(receiver.address, timeout=3) as sender: - sender.sendall(_transport._HEADER.pack(_transport._MAGIC, len(header), 0) + header) - with pytest.raises(ValueError, match="invalid array"): - next(receiver) - - -def test_sender_rejects_large_frames_without_hanging_receiver(): - with receive(port=0, timeout=3) as receiver, ThreadPoolExecutor() as pool: - future = pool.submit(send, mesh_sequence(side=4, frames=1), *receiver.address, - max_frame_bytes=1) - with pytest.raises(ValueError, match="max_frame_bytes"): - future.result(timeout=3) - with pytest.raises(EOFError): - next(receiver) - - -def test_reconstruct_reports_missing_open3d(monkeypatch): - original = importlib.import_module - - def missing(name): - if name == "open3d": - raise ModuleNotFoundError("No module named 'open3d'", name="open3d") - return original(name) - - monkeypatch.setattr(importlib, "import_module", missing) - with pytest.raises(ModuleNotFoundError, match=r"open4d\[open3d\]"): - reconstruct(np.ones((1, 4, 4)), intrinsics=(10, 10, 2, 2)) - - -@pytest.mark.parametrize("kwargs,match", [ - ({"fps": 0}, "fps"), - ({"intrinsics": (-1, 20, 16, 16)}, "intrinsics"), - ({"camera_poses": np.zeros((4, 4))}, "rigid"), - ({"color": np.zeros((1, 32, 32, 3))}, "uint8"), -]) -def test_reconstruct_validates_before_loading_optional_dependencies(kwargs, match): - settings = {"intrinsics": (20, 20, 16, 16), **kwargs} - with pytest.raises(ValueError, match=match): - reconstruct(np.ones((1, 32, 32)), **settings) - - -@pytest.mark.parametrize("version", ["0.18.0", "0.20.0"]) -def test_reconstruct_rejects_unsupported_open3d_before_returning_a_sequence(monkeypatch, version): - monkeypatch.setitem(sys.modules, "open3d", SimpleNamespace(__version__=version)) - with pytest.raises(RuntimeError, match=r"Open3D 0\.19\.x.*open3d>=0\.19,<0\.20"): - reconstruct(np.ones((1, 32, 32)), intrinsics=(20, 20, 16, 16)) - - -@pytest.mark.open3d -def test_reconstruct_depth_planes_preserves_motion_units_and_camera_pose(): - pytest.importorskip("open3d") - depth = np.stack([np.full((48, 48), 1000), np.full((48, 48), 1500)]) - color = np.zeros((*depth.shape, 3), dtype=np.uint8) - color[..., 0] = 255 - pose = np.eye(4) - pose[0, 3] = 2 - with reconstruct(depth, color, intrinsics=(60, 60, 24, 24), camera_poses=pose, - voxel_size=0.025, truncation=0.1, fps=5) as sequence: - assert sequence.timestamps == (0, 0.2) - for index, frame in enumerate(sequence): - mesh = frame.geometry - assert len(mesh.triangles) > 100 - assert mesh.positions[:, 2] == pytest.approx(1 + index * 0.5, abs=0.015) - assert np.median(mesh.positions[:, 0]) == pytest.approx(2, abs=0.08) - assert np.mean(mesh.colors[:, 0]) > 0.95 - assert np.max(mesh.colors[:, 1:]) < 0.05 - - -@pytest.mark.open3d -def test_reconstruct_two_cameras_and_empty_depth(): - pytest.importorskip("open3d") - depth = np.full((1, 2, 48, 48), 1.0) - poses = np.stack([np.eye(4), np.eye(4)]) - poses[1, 0, 3] = 0.5 - with reconstruct(depth, intrinsics=(60, 60, 24, 24), camera_poses=poses, - depth_scale=1, voxel_size=0.025, truncation=0.1) as sequence: - mesh = sequence[0].geometry - assert mesh.positions[:, 0].max() > 0.8 - assert mesh.positions[:, 0].min() < -0.3 - assert mesh.positions[:, 2] == pytest.approx(1, abs=0.015) - assert mesh.colors is None - assert mesh.normals is None # Geometry-only output can go straight to a mesh codec. - with reconstruct(np.zeros((1, 48, 48)), intrinsics=(60, 60, 24, 24)) as sequence: - assert len(sequence[0].geometry.positions) == 0 - - -@pytest.mark.open3d -@pytest.mark.parametrize("scale", [1e-50, 1e50]) -def test_reconstruct_scales_depth_before_float32_conversion(scale): - pytest.importorskip("open3d") - depth = np.full((1, 48, 48), scale) - with reconstruct(depth, intrinsics=(60, 60, 24, 24), depth_scale=scale, - voxel_size=0.025, truncation=0.1) as sequence: - mesh = sequence[0].geometry - assert len(mesh.triangles) > 100 - assert mesh.positions[:, 2] == pytest.approx(1, abs=0.015) diff --git a/open4d/transport/__init__.py b/open4d/transport/__init__.py new file mode 100644 index 00000000..4430070a --- /dev/null +++ b/open4d/transport/__init__.py @@ -0,0 +1,5 @@ +"""Send decoded mesh frames over TCP and receive them on the other side.""" + +from ._tcp import receive, send + +__all__ = ["receive", "send"] diff --git a/open4d/streaming/_transport.py b/open4d/transport/_tcp.py similarity index 100% rename from open4d/streaming/_transport.py rename to open4d/transport/_tcp.py diff --git a/open4d/transport/tests/test_tcp.py b/open4d/transport/tests/test_tcp.py new file mode 100644 index 00000000..6c0e2e20 --- /dev/null +++ b/open4d/transport/tests/test_tcp.py @@ -0,0 +1,94 @@ +from concurrent.futures import ThreadPoolExecutor +import json +import socket + +import numpy as np +import pytest + +from open4d.core import Frame, TriangleMesh +from open4d.demo import mesh_sequence +from open4d.transport import _tcp, receive, send + + +def test_stream_preserves_frames_and_attributes(): + mesh = TriangleMesh( + [[0., 0., 0.], [1., 0., 0.], [0., 1., 0.]], [[0, 1, 2]], + colors=np.full((3, 3), 0.5), normals=np.tile([0., 0., 1.], (3, 1)), + texture_coordinates=np.array([[0., 0.], [1., 0.], [0., 1.]]), + attributes={"confidence": np.array([0.1, 0.5, 0.9])}, + ) + expected = [Frame(8, 1.25, mesh, {"camera": "left", "tags": [1, "rgbd"]}), + Frame(9, 1.5, mesh)] + with receive(port=0, timeout=3) as receiver, ThreadPoolExecutor() as pool: + future = pool.submit(send, expected, *receiver.address, realtime=False) + actual = list(receiver) + assert future.result(timeout=3) == 2 + assert [frame.frame_index for frame in actual] == [8, 9] + assert [frame.timestamp for frame in actual] == [1.25, 1.5] + assert actual[0].metadata == expected[0].metadata + for name in ("positions", "triangles", "colors", "normals", "texture_coordinates"): + np.testing.assert_array_equal(getattr(actual[0].geometry, name), getattr(mesh, name)) + np.testing.assert_array_equal(actual[0].geometry.attributes["confidence"], + mesh.attributes["confidence"]) + + +def test_stream_synthetic_sequence_and_empty_mesh(): + sample = mesh_sequence(side=4, frames=3) + empty = Frame(3, 1, TriangleMesh(np.empty((0, 3)), np.empty((0, 3), dtype=np.uint32))) + expected = [*sample, empty] + with receive(port=0, timeout=3) as receiver, ThreadPoolExecutor() as pool: + future = pool.submit(send, expected, *receiver.address, realtime=False) + actual = list(receiver) + assert future.result(timeout=3) == 4 + for original, decoded in zip(expected, actual): + np.testing.assert_array_equal(original.geometry.positions, decoded.geometry.positions) + np.testing.assert_array_equal(original.geometry.triangles, decoded.geometry.triangles) + + +def test_empty_stream_finishes(): + with receive(port=0, timeout=3) as receiver, ThreadPoolExecutor() as pool: + future = pool.submit(send, [], *receiver.address) + assert list(receiver) == [] + assert future.result(timeout=3) == 0 + + +def test_receiver_closes_early_and_releases_port(): + with receive(port=0, timeout=3) as receiver: + address = receiver.address + with receive(*address, timeout=0.01) as second: + with pytest.raises(TimeoutError): + next(second) + assert list(receiver) == [] + + +@pytest.mark.parametrize("payload,error", [ + (_tcp._HEADER.pack(b"BAD!", 0, 0), ValueError), + (_tcp._HEADER.pack(_tcp._MAGIC, 10, 1000), ValueError), + (_tcp._HEADER.pack(_tcp._MAGIC, 10, 10) + b"{}", EOFError), +]) +def test_receiver_rejects_invalid_or_truncated_messages(payload, error): + with receive(port=0, timeout=3, max_frame_bytes=100) as receiver: + with socket.create_connection(receiver.address, timeout=3) as sender: + sender.sendall(payload) + with pytest.raises(error): + next(receiver) + + +def test_receiver_rejects_object_arrays(): + header = json.dumps({"frame_index": 0, "timestamp": 0, "metadata": {}, + "arrays": [{"name": "positions", "dtype": "O", "shape": [1, 3]}]}).encode() + with receive(port=0, timeout=3) as receiver: + with socket.create_connection(receiver.address, timeout=3) as sender: + sender.sendall(_tcp._HEADER.pack(_tcp._MAGIC, len(header), 0) + header) + with pytest.raises(ValueError, match="invalid array"): + next(receiver) + + +def test_sender_rejects_large_frames_without_hanging_receiver(): + with receive(port=0, timeout=3) as receiver, ThreadPoolExecutor() as pool: + future = pool.submit(send, mesh_sequence(side=4, frames=1), *receiver.address, + max_frame_bytes=1) + with pytest.raises(ValueError, match="max_frame_bytes"): + future.result(timeout=3) + with pytest.raises(EOFError): + next(receiver) diff --git a/open4d/webclients/README.md b/open4d/webclients/README.md deleted file mode 100644 index 5df9331d..00000000 --- a/open4d/webclients/README.md +++ /dev/null @@ -1,165 +0,0 @@ -# webclients - -A browser user study for volumetric streaming methods, and the clients it -runs them in. [`../reconstruction`](../reconstruction) holds the methods; this -holds the app that delivers them to a participant under one network trace and -one camera path, and asks them to rate each. Not to be confused with -[`../streamer`](../streamer), the Python bundle server, or -[`../streaming`](../streaming), RGB-D reconstruction sent over TCP. - -The methods it knows: - -| system | representation | -|---|---| -| Ours | textured meshes, viewpoint-aware ladder | -| ViVo | point clouds, 4×4×4 tiles | -| NAVA | point clouds | -| Vega | 3D Gaussian splats | -| NeVo | ReRF neural volumetric | - -## Quick start - -```bash -cd system/WebClient && npm install && node build.js -cd ../.. && PYTHON_BIN= scripts/run_web_demo.sh -``` - -Open the URL it prints (`…:3000/web/`): that is the study. Each method's own -page is still there on its own (`mesh.html`, `baseline.html`, `vega.html`, -`nevo.html`) for looking at one without running a session. Ctrl-C stops -everything. - -A study trial shapes the link itself, from the trace chosen in setup. Outside -the study, `scripts/shape_web_demo.sh` still replays a trace with `tc` for the -method pages, with the caveats in that script: it needs root, and it shapes -every flow on the host. - -**Needs the research repo.** The ladder solver (`vstream/`), the baseline -servers (`baselines/`) and the corpora live in `4DVideoStreaming`, so -`run_web_demo.sh` expects a checkout of it. Without one you can still build the -clients and run the tests. - -## Adding a method - -A method takes part in the study once it has a page that can run a study pass -and the study knows about it: - -1. **The page.** Read `study.studyParams()`; when set, load the context with - `study.loadStudyContext`, hide the page chrome with `study.enterStudyChrome`, - and hand the renderer's camera to `study.createStudyDriver` -- the renderer - calls `driver.apply(camera, controls, info)` once per frame, just before its - orbit update. See `src/study/harness.js`; `src/vega-main.js` is the smallest - example. -2. **The study.** Add the method to `METHODS` in - [`system/Server/study/store.js`](system/Server/study/store.js) and to - `STUDY_PAGES` in `src/study/study-page.js`, and give it a page in `build.js` - and `public/`. `/api/systems` in `system/Server/server.js` says whether it - is available and which objects it has. -3. **Its bytes.** Whatever it downloads must pass the study's shaper: `/files` - and `/vega-assets` do in the server, and the WebSocket bridge does for the - V4DS baselines. - -If a client **adapts in the browser**, implement the `ClientPlatform` contract in -[`system/ClientCore/platform.js`](system/ClientCore/platform.js) and hand it to -`StreamingClient` for the segment loop, MCKP selector, bandwidth estimation and -metrics. If it **adapts server-side or not at all**, write a plain page and do -*not* wrap it in `ClientCore`: that models an HTTP segment loop over a published -ladder, and forcing a push-based or fixed-quality method into it measures the -wrapper rather than the method. - -## User study - -`…:3000/web/study.html` runs one participant through every chosen method under -the **same network trace, from the same start view, along the same camera path**, -then asks them to rate each clip. It is the browser counterpart of the Quest -study (`scripts/user_study.py` in the research repo), and uses the same -questionnaire wording and the same counterbalancing. - -1. **Setup.** A pseudonymous participant code (never a name), the methods, the - objects on stage, the trial length, and a network trace: a built-in one from - `system/Client/traces/` or an uploaded `time,bandwidth` CSV in seconds and - Mbps, optionally windowed and scaled. -2. **Record the path.** The participant drives one unrated pass on Ours, from - a fixed start that frames every object on stage. That path is the one every - trial replays, including the first, so no method is rated from active - viewing while the others are rated from passive viewing. It can be - downloaded and reused for later participants. -3. **Trials.** In Williams order keyed on the participant number, each shown only - as "Clip A", "Clip B"… Each trial re-arms the trace at its start, replays the - path on the content clock, and ends after the trial length. -4. **Questionnaire.** C1–C4 on five stars, then up to two artifacts ("None of - the above" is exclusive), with time on each page recorded. -5. **Results.** A table, and `export.csv`: one row per trial with the ratings, - artifacts, and the same playback metrics for every method. - -```bash -PYTHONPATH=/path/to/4DVideoStreaming PYTHON_BIN= scripts/run_web_demo.sh -# then open http://:3000/web/ -``` - -Sessions are written under `system/Server/server_results/study/`, one -self-contained directory each: the trace exactly as uploaded with its hash, the -path, and one `trial.json` per trial. That directory is ignored by git, since -it holds participant data. - -What the study does **differently from the Quest** is how it shapes the link, -because this one must accept a trace uploaded from a browser: - -- The trace is applied **in-process**, not with `tc`. `/files` responses pass - through a token bucket in the server (`system/Server/study/shaper.js`) and - the ViVo/NAVA WebSocket bridge follows the same trace from the same instant - (`follower.js`). It needs no root and throttles nothing else on the host; - what it gives up is a kernel queue under TCP, so congestion control sees - pacing rather than loss. The burst and step semantics match - `quest-trace-player.js`. - -Which methods can take part, and what the rest are missing: - -| method | in the study | notes | -|---|---|---| -| Ours (mesh) | yes | adapts in the browser | -| ViVo, NAVA | yes | restarted on the session's objects before each trial; need the tile corpus | -| Vega | yes | only for sessions of objects it has exported (here: `dancer`, `thomas`); preloads its 30-frame clip, then loops | -| NeVo | no | pre-rendered from one capture camera with no camera control, so it cannot follow a shared path | -| LiVo | no | the browser client does not decode LiVo's RGB-D segments, and no LiVo package is prepared | -| MetaStream, DeltaStream | no | need the RGB-D dataset from `baselines.DeltaStream.orbitstream.converter` | - -The setup page lists all of them, and says why a method cannot be chosen — -including when it lacks one of the session's objects. - -**Which method records the path is a setup choice.** Any that can take part -will do, but it matters: the participant has seen that method before rating -it. Ours in particular may not show the whole stage at startup — it begins with -a 5 Mbps estimate, and the five stage objects need about 89 Mbps even on the -H.264 corpus — so the practice pass holds the start view until every object is -visible, for up to 60 s, and flags a path recorded over a partial stage. - -Serve the **H.264** corpus (`ORBIT_datasets_compressed_h264`, the one -`run_web_demo.sh` defaults to) where possible: every browser decodes it. The -HEVC corpus decodes only in Safari, and in Chrome with suitable hardware. - -## Tests - -```bash -node --test tests/*.js # 347 of 349 -``` - -The other two cross-check the JS decoders against the Python ones byte for -byte, so they need `PYTHONPATH=/path/to/4DVideoStreaming` and -`VS4D_TEST_PYTHON=`. - -## Layout - -`system/ClientCore` is the platform-free logic; `system/WebClient` the study, the method -pages and the WebSocket↔TCP bridge; `system/Server` publishes the ladder and -serves media; `tile_ladder.py` lets ViVo/NAVA serve prepared tiles without the -RGB-D source. The `system/` level is kept so relative imports resolve unchanged -against the research repo — load-bearing, not a leftover. - -Pitfalls worth reading before changing a client are in -[`system/WebClient/README.md`](system/WebClient/README.md); most fail silently. - -This directory began as a copy of the research repo's clients (see -`PROVENANCE`) and is developed here now; it is not kept in sync with that repo. -It still needs a checkout of it at run time, for the ladder solver and the -baseline servers. diff --git a/open4d/webclients/system/ClientCore/README.md b/open4d/webclients/system/ClientCore/README.md deleted file mode 100644 index e5b91fb8..00000000 --- a/open4d/webclients/system/ClientCore/README.md +++ /dev/null @@ -1,119 +0,0 @@ -# ClientCore - -Platform-free streaming logic, shared by the Node desktop client -(`system/Client`) and the browser client (`system/WebClient`). - -The rule: **nothing here may import `fs`, `http`, `https`, `path`, `os` or -`process`, or touch `console`, `Date.now`, `setInterval` or `fetch` directly.** -Everything platform-shaped goes through the `ClientPlatform` contract in -[platform.js](platform.js), and `tests/test_client_core_purity.js` enforces it. -If both clients run the same core, a difference between their results is a real -difference in the system under test rather than a difference between two -hand-written clients. - -## Contents - -| File | Role | -|---|---| -| [streaming-client.js](streaming-client.js) | `StreamingClient` — segment loop, playback clock, download orchestration, server reporting, shutdown | -| [platform.js](platform.js) | The contract: typedefs, `PLATFORM_CONTRACT` as data, `validatePlatform` | -| [testing/fake-platform.js](testing/fake-platform.js) | In-memory platform — virtual clock, routed transport, Map-backed storage. Test substrate and reference implementation | -| [abr.js](abr.js) | The MCKP selector and per-object buffers | -| [link-throughput.js](link-throughput.js) | `LinkThroughputMeter` — byte/busy-time accounting across overlapping downloads | -| [bandwidth.js](bandwidth.js) | `BandwidthEstimator` + `computeBudget` — harmonic estimate and health-scaled spend budget | -| [bandwidth-estimator.js](bandwidth-estimator.js) | The causal harmonic-mean throughput estimate itself | -| [download-plan.js](download-plan.js) | What a segment needs, and how to judge the outcome | -| [metrics.js](metrics.js) | The run's metrics record and its builders | -| [payloads.js](payloads.js) | Server request bodies (wire formats read by `system/Server`) | -| [stalls.js](stalls.js) | Per-segment stall collection and stall-event folding | -| [stream-config.js](stream-config.js) | `resolveStreamConfig` — validated timing from `/api/config` | - -## The seven capabilities - -| Capability | Methods | -|---|---| -| `transport` | `getJson`, `postJson`, `assetUrl`, `fetchAsset` | -| `storage` | `handle`, `createScratch`, `release`, `writeText`, `writeResult`, `openAppendStream` | -| `viewpoints` | `list` | -| `clock` | `now`, `every`, `cancel`, `delay` | -| `logger` | `emit` | -| `renderer` | `start`, `stop`, `stageSegment`, `setPlaybackState`, `setCallbacks`, `latestCamera` (nullable) | -| `lifecycle` | `exit`, `onShutdownRequest` | - -Nothing here is speculative: the set was derived by auditing every platform -touchpoint in the pre-refactor `client.js` (10 fetch sites, 9 distinct `fs` -calls, 5 `process` calls, 4 timer sites, the renderer child process, console -logging), and each method backs at least one real call site. The two -implementations are [`Client/node-platform.js`](../Client/node-platform.js) and -[`WebClient/src/browser-platform.js`](../WebClient/src/browser-platform.js). - -Call `validatePlatform(platform, { requireRenderer })` once at startup. It -reports *every* missing method in one error, because the alternative is -`undefined is not a function` twelve segments into a run — which leaves a -truncated metrics file that looks like a legitimate result. - -## Asset handles - -The core never learns where a downloaded byte lives. `storage.handle(...)` -mints an **opaque** handle, `transport.fetchAsset` fills it, -`renderer.stageSegment` consumes it. In Node a handle is a filesystem path; in -the browser it is an OPFS path or a key into a buffer map. Treat handles as -values to pass along, never to parse. This is why -[download-plan.js](download-plan.js) takes `destinationFor` injected: it slots -assets by frame index without knowing what a destination is. - -## Constraints a new platform must respect - -1. **`fetchAsset` must not be cacheable** (`cache: 'no-store'`). The segment - loop has to keep generating real network load or the shaped-bandwidth - experiment stops meaning anything — the same reason - `system/Client/decode_cache.py` caches decodes but never downloads. -2. **`lifecycle.exit` must not navigate away.** The final `POST /api/results` - happens during shutdown. Stop timers, resolve the run promise, leave the - page alive. -3. **`openAppendStream.write` must not block.** Render telemetry arrives at - 30 Hz; in Node a synchronous write per frame filled the renderer's stdout - pipe and blocked the loop calling `poll_events()`, so client disk I/O was - directly stalling the Open3D window. Buffer and flush. (OPFS - `createSyncAccessHandle` is worker-only.) -4. **Keep the `renderer: null` path working.** Simulated mode credits segments - straight from the download result; interactive mode waits for - `object_ready`. They measure different things and both must survive. - -## Testing against the fake platform - -`createFakePlatform()` returns a contract-valid platform with a **virtual -clock**, so a 40-segment run costs a millisecond of test time instead of -80 seconds. `tests/test_streaming_client.js` drives complete runs this way. - -```js -const { createFakePlatform } = require('../system/ClientCore/testing/fake-platform'); - -const platform = createFakePlatform(); -platform.transport.onGet('/api/config', () => ({ - segmentDuration: 2, framesPerSegment: 60, segmentIntervalMs: 2000, - totalSegments: 20, updateIntervalSegments: 1 -})); -platform.transport.onGet('/api/manifest', () => menu); -platform.transport.failAsset('dancer_fr0003'); // one bad geometry frame - -await platform.clock.advance(20 * 2000); // 20 segments, instantly - -assert.ok(platform.storage.result); // the run wrote its metrics -assert.strictEqual(platform.clock.pending, 0); // no timer leaked -``` - -Assert on observable state — `storage.files`, `transport.requests`, -`renderer.staged`, `logger.at('WARN')` — rather than on call expectations. - -```bash -node --test tests/test_bandwidth_estimator.js tests/test_client_core.js \ - tests/test_client_core_parity.js tests/test_client_platform.js \ - tests/test_client_core_purity.js tests/test_streaming_client.js -``` - -`node --test ` does not work in this environment; name the files. -`test_client_core_parity.js` runs the extracted download planner against the -pre-refactor implementation across 12 manifest shapes — keep it until the -end-to-end parity harness (same trace through both clients, compared with -`vstream/evaluation/QoE_full.py`) is running. diff --git a/open4d/webclients/system/WebClient/README.md b/open4d/webclients/system/WebClient/README.md deleted file mode 100644 index 949ae1ec..00000000 --- a/open4d/webclients/system/WebClient/README.md +++ /dev/null @@ -1,344 +0,0 @@ -# WebClient - -The user study, and one browser page per system under comparison, served by -`system/Server` at `/web`. - -Start at **`/web/`** — the study (`public/index.html`, `src/study/`). It asks -`/api/systems` which systems have their assets present and which objects each -has, says why any it cannot run is unavailable, and runs each method's page in -a frame for its trials. Every method page carries a `study` link back to it. - -| Page | System | Runs the ABR? | -|---|---|---| -| `/web/` | the user study — setup, practice, blinded trials, ratings, CSV | — | -| `/web/mesh.html` | **ours** — adaptive mesh ladder | yes, in the browser | -| `/web/baseline.html` | ViVo / NAVA point clouds (`?bridge=` selects which) | yes, server-side | -| `/web/vega.html` | Vega (3D Gaussian splatting) | no, fixed quality | -| `/web/nevo.html` | NeVo (ReRF neural volumetric) | no, pre-rendered | - -The first page is the important one: it runs the **same** `system/ClientCore` -as the Node desktop client — same MCKP ABR, same segment loop, same bandwidth -estimator, same metrics — behind a browser implementation of the -`ClientPlatform` contract. So a desktop/browser difference is a real difference -in the system under test, not a difference between two hand-written clients. - -The other three do **not** use ClientCore, deliberately. ClientCore models an -HTTP segment loop over a published ladder; the baselines push frames at their -own cadence and adapt server-side. Wrapping them in a segment loop would -measure the wrapper. - -## Build and run - -```bash -cd system/WebClient && npm install && node build.js # --watch to rebuild on change -cd ../.. && PYTHON_BIN= scripts/run_web_demo.sh -``` - -That starts everything — the server on the **H.264** corpus plus a supervised -ViVo and NAVA — and prints the URLs. Start at -`http://:3000/web/`. Ctrl-C stops all of it. - -Two defaults in that script are deliberate. It serves H.264 because HEVC is -Firefox-no and Chrome-only-with-hardware, and it sets 300 segments because the -server's own default of 10 is a *trial* length: the page reaches "run complete" -in 20 s, before you have finished looking at it. - -To make adaptation visible rather than merely running, replay a trace in -another shell (needs root, and shapes the whole host): - -```bash -sudo scripts/shape_web_demo.sh cascade-20 -``` - -## Page parameters - -Query strings, mirroring how the Node client takes environment variables. - -**`/web/mesh.html` (ours)** - -| parameter | default | meaning | -|---|---|---| -| `server` | page origin | server base URL, for a cross-host run | -| `mode` | `interactive` | `interactive` \| `simulated` | -| `storage` | `memory` | `memory` \| `opfs` asset store | -| `viewpoints` | — | viewpoint-index JSON; **required** for `simulated` | -| `decodeBudget` | `512` | decoded-clip cache budget, MB | -| `concurrency` / `inflight` | `10` / `2` | parallel asset requests / concurrent segments | -| `label` | `web-client` | run label recorded on the server | -| `objects` | server's full catalog | comma-separated object subset | - -`simulated` has no default viewpoint file on purpose: solving the first ladder -against an arbitrary pose would silently invalidate the viewpoint-aware -comparison. - -`objects` is the parameter that decides whether a run demonstrates adaptation -or a permanent deficit. The ladder publishes at least one representation per -object in the scene, so the scene size sets an irreducible floor: all nine -ORBIT objects floor at **116 Mbps**, which no ordinary link carries, and the -MCKP then buys the few highest-weighted objects at their cheapest rung and -freezes the rest for the whole run. Three objects floor near **21 Mbps**. An -absent list resets the server to its full catalog, so a run never inherits the -previous one's scene. The Node client reads the same subset from -`VS4D_SCENE_OBJECTS`. - -**`/web/baseline.html`** — `bridge` (`ws://:8790`), `pointSize` (`0.012` m), -`strict` (`0`; `1` aborts on a frame gap instead of resynchronising). - -Which point-cloud baselines can actually run depends on the corpus, and the two -cases need opposite responses: - -`?bridge=` is what selects the baseline: the page is identical either way, and -`/api/systems` probes each port so the study marks a baseline ready only when -something is actually listening on it (which is different from the tile corpus -merely existing -- the two need opposite fixes). Override the port map with -`VS4D_POINTCLOUD_PORTS=vivo:8790:12345,nava:8791:12346`. - -- **ViVo and NAVA** run from the prepared tile corpus with - `--tile-catalog-ladder`. Everything they read at serve time is in - `catalog.json`, and the cameras they put in the connection header are - synthetic per-tile identities, so the absent RGB-D source costs them nothing. - See [`baselines/ViVo/orbitvivo/tile_ladder.py`](../../baselines/ViVo/orbitvivo/tile_ladder.py). -- **MetaStream, DeltaStream and LiVo** index the real capture rig while serving - (`obj.cameras[camera_index]`) and read the source RGB-D frames. For them the - absent corpus is missing *data*, and no adapter can substitute for it. - -**`/web/vega.html`** — `assets` (`/vega-assets`), `objects` (all nine), -`frame` (`object`\|`all`), `splatMode` (`isotropic`\|`anisotropic`), -`splatScale` (`1.0`). - -The Vega page **loads the whole clip before playing** and then loops it from -memory; expect ~15 s of visible progress on a 35 Mbps link for the 64 MB -two-object export. It is not streamed, and cannot be: a frame is ~1.1 MB at -30 fps, so two objects in real time would need **509 Mbps**. The full -nine-object export is ~345 MB, so use `?objects=` (the study passes the -session's objects) rather than opening all of them. The earlier -sliding-window prefetch only worked when served from the same machine — over a -real link it issued requests faster than they completed and, because it checked -only the decoded cache, re-issued each pending frame every 33 ms tick until -Chrome's socket pool gave out and every fetch returned a bare -`TypeError: Failed to fetch` with no backoff. Fetches are now deduplicated by -`(object, frame)`, bounded to 6 in flight, and a failed frame is remembered -rather than retried. - -**`/web/nevo.html`** — `assets` (`/nevo-assets`), `object` (`g_dancer`), -`fps` (`8`), `nevoOnly` (`0`). - -### Getting the baselines to serve something - -```bash -# point clouds: raw TCP, which a browser cannot open, so bridge it. This runs -# the bridge and supervises the baseline server, because a baseline serves one -# connection from frame zero and then exits -- correct for a measured trial, -# but it means an unsupervised server survives exactly one page load. -# Run both: they take different default ports (vivo 8790, nava 8791), so the -# study can run either and switching is a click rather than a restart. -PYTHON_BIN= scripts/serve_pointcloud_baseline.sh vivo dancer,thomas & -PYTHON_BIN= scripts/serve_pointcloud_baseline.sh nava dancer,thomas & -# -> /web/baseline.html?bridge=ws://:8790 (vivo) -# -> /web/baseline.html?bridge=ws://:8791 (nava) - -# vega: export the trained bitstream to the portable VGS format once -python -m baselines.Vega.orbitvega.export_quest \ - --prepared-dir results/vega-gaussian/prepared-final --output-dir results/vega-web \ - --dataset-root --objects dancer thomas -# -> /web/vega.html?objects=dancer (override location: VS4D_VEGA_WEB_ROOT) - -# nevo: renders come from orbitnevo/render_frames.py -# -> /web/nevo.html?object=g_dancer (override location: VS4D_NEVO_WEB_ROOT) -``` - -## Files - -| File | Role | -|---|---| -| `src/browser-platform.js` | The `ClientPlatform` capabilities: transport, storage, viewpoints, clock, logger, lifecycle | -| `src/webgl-renderer.js` | Renderer capability: Three.js scene, frame presentation, decode orchestration | -| `src/decode-cache.js` | Decoded-clip LRU under a byte budget, keyed `(objectName, repId)` | -| `src/texture-decoder.js` | MP4 demux (mp4box) + WebCodecs `VideoDecoder` | -| `src/draco-worker.js` | Draco mesh and point-cloud decode, off the main thread | -| `src/camera-pose.js` | Three.js camera → Open3D `PinholeCameraParameters` | -| `src/main.js` | Entry point for `/web/` | -| `src/study/study-page.js` | The study: setup, practice pass, blinded trials, results | -| `src/study/questionnaire-form.js` | The two-page C1–C5 questionnaire, as on the Quest | -| `src/study/driver.js`, `camera.js`, `harness.js` | Trial clock, start pose and camera path, and the page-side glue | -| `bridge/v4ds-bridge.js` | WebSocket ↔ TCP proxy for the point-cloud baselines; reframes only | -| `src/v4ds-protocol.js` | V4DS CONNECTION/FRAME decode, FEEDBACK encode | -| `src/point-reconstruction.js` | MetaStream/DeltaStream delta reconstruction | -| `src/point-renderer.js` | `THREE.Points` per object | -| `src/vgs-format.js` | VGS1 decoder — positions, scales, rotations, opacity, colour | -| `src/splat-renderer.js` | Instanced splat quads, depth sort, GLSL3 shaders | -| `src/nevo-manifest.js` | Conditions, frame filenames, panel layout, captions (pure) | -| `src/nevo-client.js` | Preload, subject crop, 2D-canvas compositing | -| `src/{baseline,vega,nevo}-client.js`, `*-main.js` | Per-page logic and wiring | -| `public/*.html` | Canvas, status line, log pane, artifact links | -| `vendor/draco/` | Official Draco JS/WASM decoder, unmodified | - -## Pitfalls - -Each of these cost real debugging time. Most fail *silently* — plausible output -rather than an error — which is why they are written down. - -**Measurement integrity** - -- Media and API fetches are `cache: 'no-store'`. A segment served from the HTTP - cache turns a shaped-bandwidth measurement into fiction; a cached manifest - pins the client to a stale ladder with no error. -- Stop finalizes, it does not navigate. The final `POST /api/results` happens - during shutdown, so unloading would cancel it. `pagehide` finalizes too. -- Telemetry writes never block: 30 Hz render telemetry through a synchronous - write per frame starved the renderer in Node. -- Only decodes are cached, never downloads. A 1920-wide frame is `w*h*1.5` = - 5.5 MB decoded however it was stored, so a 60-frame clip × 5 objects is - 1.66 GB. The cache pins the clip on screen, since evicting *that* looks like - a frame reverting mid-playback rather than an error. -- Bandwidth shaping stays `tc` on the Linux host, exactly as - `system/Client/run-client-experiments.sh` does it — the browser is just - another process behind the same qdisc. DevTools throttling is not scriptable - enough to be an experimental control. -- **Without shaping the demo cannot show adaptation**, only run it. On an - unshaped LAN every adaptive system settles on one operating point: NAVA held - quality level 5 for thirteen consecutive segments while its DP solved each - one. Replay a trace with `sudo scripts/shape_web_demo.sh cascade-20` (a - 12.5 → 25 → 50 → 100 → 175 Mbps staircase in 20 s steps) and both `/web/` and - `/web/baseline.html` show the enforced rate on their `link:` line beside the - client's own estimate, so a representation switch reads as cause and effect. - `GET /api/shaping` is the unprivileged source — reading `tc` needs no root, - only installing rules does. Note the trace player installs a **root** token - bucket, so it shapes every outbound flow on the host, not just the demo; - that is inherited from the Quest methodology because a destination `u32` - filter silently misses traffic on a multiqueue NIC, and these NICs are - multiqueue. - -**Formats and coordinates** - -- The pose POSTed to `/api/viewpoint` must be Open3D `PinholeCameraParameters`; - the server writes it straight to disk and reads it with - `o3d.io.read_pinhole_camera_parameters`. Anything else leaves the ladder - unable to parse it, and it returns **equal** object weights with no error — - silently disabling the whole point of the system. `camera-pose.js` handles - both conversions: Three.js is Y-up/−Z-forward, Open3D is Y-down/+Z-forward, - and translations are in **millimetres** (`VIEW_RAYCAST_UNITS_PER_METER`). -- V4DS is **big-endian**; VGS1 is **little-endian**. Same client decodes both. -- Streamed baseline points are in **camera** space; `camera_to_world` arrives - **row-major**, and `THREE.Matrix4.fromArray` is column-major, so handing it - over directly transposes every camera. -- V4DS timestamps are epoch **nanoseconds** (~1.8×10¹⁸, ~200× `MAX_SAFE_INTEGER`) - and need BigInt. Frame ids stay Numbers; `sourceTimestampMs` is derived. -- VGS1 quaternion components are **signed** bytes. Read unsigned, every - rotation mirrors into a still-plausible blob of Gaussians. - -**Rendering** - -- The camera auto-frames the content. ORBIT subjects are baked into venue world - coordinates with three floor tiers (`scene_layout.json`), so a mesh can sit - metres above the origin — the first `dancer` frame centres at y=3.03. Framing - stops as soon as the viewer touches the camera; their pose is the input. -- **Fitting the whole scene is the wrong default for a two-object viewer.** The - same venue layout puts `dancer` at z −1.8..−0.9 and `thomas` at z +8.0..+8.4, - ten metres apart, so a camera that fits both retreats to z 18.5 and each - subject covers **0.43%** of the canvas — indistinguishable from an empty - player. Vega therefore frames one object (**3.6–5.1%** coverage), names the - others in its log, and offers a `Focus` button to cycle; `?frame=all` - restores the whole-venue fit for inspecting the layout itself. -- Geometry gaps hold, they do not shift: a null frame re-presents the previous - one, which is why `ClientCore/download-plan.js` preallocates a null slot per - frame rather than collecting only successes. -- Splats: data lives in a texture and only a 16-bit-depth counting sort over an - index attribute is reordered (230 KB instead of ~3 MB per sort). Depth test - and depth write are both off — in 3DGS the ordering *is* the depth resolution. -- **A `SplatObject` must be constructed at its sequence's real splat count.** - Three caches `_maxInstanceCount` from the instanced attributes the first time - it binds a geometry, and draws - `min(geometry.instanceCount, _maxInstanceCount)`. Swapping in a bigger - `splatIndex` attribute afterwards does not reliably reset that cache, so a - geometry first bound at a placeholder capacity keeps drawing **one** instance - however large `instanceCount` is — 4 triangles instead of 213,224, an - empty-looking canvas with every other diagnostic reading healthy. Whether it - happened depended on whether the render loop bound the mesh before the first - frame arrived, so it failed on roughly three page loads in four while the - stats panel still reported 108,342 splats and zero decode failures. - `_allocate` now throws on post-draw growth instead of capping silently, the - mesh stays hidden until it has data, and `VegaClient` takes each capacity - from the catalogue. -- Vega colour is **baked**, not view-dependent: `export_quest.py` quantises a - view-independent sample per splat. Geometry and opacity are exact; appearance - is the same approximation the offline evaluator uses, so it is comparable - with those numbers but is not the paper's full appearance. -- NeVo has no camera control because it cannot run client-side: a frame takes - ~0.5 s to ray-march on a workstation GPU and its entropy decoder is CUDA + - Python 3.8. The page plays pre-rendered frames — plain ReRF, NeVo's - visibility-filtered output, and the captured camera at the same instant and - viewpoint, each captioned with its kept voxel fraction. All renders preload - before playback (a condition one frame behind would be indistinguishable from - a real difference) and the crop is measured once and applied to every panel. - -## Known limitations - -- **Texture codec.** The decoder is codec-agnostic: it reads `track.codec` and - whichever of avcC/hvcC the file carries, so it decodes H.264 - (`avc1.64001f`) and HEVC (`hvc1.1.6.L90.90`) through the same path. **Serve - the H.264 corpus for a browser audience** — HEVC is Safari-yes, Firefox-no, - Chrome-only-where-the-OS-provides-hardware-decode, and - `analysis/compare_codec_rd.py` measures H.264 at just **1.003x** the bitrate - of HEVC at matched quality across all nine objects, so the ladder's floor and - ceiling barely move. Where the served codec cannot be decoded the page - renders untextured geometry and logs the reason once per object - (`texture-decoder.js: probeCodecSupport`). Never transcode at serve time: - the quality/bitrate models are codec-specific, so re-encoding changes - delivered bitrate out from under the ladder. - - -- **`splatMode=anisotropic` is untested.** Isotropic is the default because - Vega's Gaussians are near-isotropic (log scales within ~0.3) and because a - vertex shader that merely *contains* the covariance projection draws nothing - under headless SwiftShader, even when `gl_Position` ignores the result. Every - input was verified independently, which points at the driver. Try it on real - GPU hardware. -- **MetaStream, DeltaStream and LiVo are not in the demo.** They index the real - capture rig while serving (`obj.cameras[camera_index]`) and read the source - RGB-D frames, which are absent with no surviving copy — missing data, not - missing metadata, so no adapter can substitute. LiVo additionally needs - client-side depth unprojection for its `LIVO_SEGMENT` colour+depth messages; - the protocol decoder recognises the type and says so rather than failing - silently. -- **WebCodecs texture decode has never succeeded here**, for the HEVC reason - above; only the degradation path is exercised. OrbitControls interaction is - untested (headless has no pointer). -- **The multi-view NeVo sprite path** (`export_quest.py`, 8 yaw views/frame) - needs `rerf_render.py --render_views 8` under the ReRF CUDA environment. - Those renders do not exist, so the viewer is single-view. - -## Tests - -```bash -node --test tests/test_browser_platform.js tests/test_decode_cache.js \ - tests/test_camera_pose.js tests/test_v4ds_protocol.js \ - tests/test_v4ds_bridge.js tests/test_point_reconstruction.js \ - tests/test_vgs_format.js tests/test_nevo_manifest.js \ - tests/test_web_client_bundle.js -``` - -The binary decoders are checked against the authoritative Python -implementations on real data: `test_v4ds_protocol.js` against golden bytes from -the Python encoder plus a live round-trip, `test_point_reconstruction.js` -point-for-point, `test_vgs_format.js` on a real exported frame, -`test_camera_pose.js` against a real captured viewpoint file. The -`fixtures_*.gen.py` scripts regenerate the golden files. -`test_v4ds_bridge.js` covers adversarial TCP chunk boundaries and a 3 MiB -message through real sockets. `test_web_client_bundle.js` loads the **built** -`dist/main.js` in a stubbed DOM and drives a complete simulated run, so run -`node build.js` first. - -### Debugging in a real browser - -`window.__vs4d.inspect()` returns live renderer state — object visibility, -vertex counts, camera pose, decode-cache stats — which is the fastest way to -tell "nothing is drawn" from "drawn off-screen". Two headless traps: - -1. `--virtual-time-budget` deadlocks the Draco worker: `importScripts` is - synchronous and the worker goes silent with no error. Drive the page with - puppeteer-core against the installed Chrome instead. -2. `page.screenshot()` comes out black — SwiftShader does not composite the - WebGL surface. Call `renderer.render()` and `gl.readPixels()` in the *same* - synchronous task, since `preserveDrawingBuffer: false` clears after - compositing. diff --git a/pyproject.toml b/pyproject.toml index 2360eddd..7a9ab6d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,9 @@ packages = [ "open4d.codec", "open4d.core", "open4d.io", - "open4d.streaming", + "open4d.reconstruction", + "open4d.reconstruction.rgbd", + "open4d.transport", "open4d.torch_ops", "open4d.visualization", "integrations", @@ -55,7 +57,8 @@ testpaths = [ "open4d/core/tests", "open4d/codec/tests", "open4d/io/tests", - "open4d/streaming/tests", + "open4d/reconstruction/rgbd/tests", + "open4d/transport/tests", "open4d/torch_ops/tests", "open4d/visualization/tests", "examples/visualization/tests", diff --git a/scripts/check_provenance.py b/scripts/check_provenance.py index a8bd3beb..057aa6ba 100644 --- a/scripts/check_provenance.py +++ b/scripts/check_provenance.py @@ -20,14 +20,15 @@ ) EXPLICIT_REQUIRED_LEDGER_PATHS = ( "integrations/unity", - "open4d/streaming", ) ALLOWED_PACKAGES = { "open4d", "open4d.codec", "open4d.core", "open4d.io", - "open4d.streaming", + "open4d.reconstruction", + "open4d.reconstruction.rgbd", + "open4d.transport", "open4d.torch_ops", "open4d.visualization", "integrations", @@ -156,8 +157,9 @@ def main() -> int: manifest = (ROOT / "MANIFEST.in").read_text(encoding="utf-8") for path in ( "open4d/codecs", "open4d/reconstruction", "integrations/unity", - "open4d/streaming/app", "open4d/streaming/src", "open4d/streaming/include", - "open4d/streaming/python", "open4d/streaming/tools", + "open4d/reconstruction/rgbd/app", "open4d/reconstruction/rgbd/src", + "open4d/reconstruction/rgbd/include", "open4d/reconstruction/rgbd/python", + "open4d/reconstruction/rgbd/tools", ): if f"prune {path}" not in manifest: errors.append(f"MANIFEST.in must prune {path}") diff --git a/scripts/check_wheel_contents.py b/scripts/check_wheel_contents.py index 2679a645..6f9f6bb2 100644 --- a/scripts/check_wheel_contents.py +++ b/scripts/check_wheel_contents.py @@ -22,7 +22,9 @@ "open4d.codec": ROOT / "open4d/codec", "open4d.core": ROOT / "open4d/core", "open4d.io": ROOT / "open4d/io", - "open4d.streaming": ROOT / "open4d/streaming", + "open4d.reconstruction": ROOT / "open4d/reconstruction", + "open4d.reconstruction.rgbd": ROOT / "open4d/reconstruction/rgbd", + "open4d.transport": ROOT / "open4d/transport", "open4d.torch_ops": ROOT / "open4d/torch_ops", "open4d.visualization": ROOT / "open4d/visualization", "integrations": ROOT / "integrations", diff --git a/scripts/tests/test_check_provenance.py b/scripts/tests/test_check_provenance.py index f7127d3a..3aaf4246 100644 --- a/scripts/tests/test_check_provenance.py +++ b/scripts/tests/test_check_provenance.py @@ -49,7 +49,6 @@ def test_component_discovery_covers_new_directories_and_submodules(tmp_path: Pat assert discover_required_ledger_paths(tmp_path) == { "integrations/unity", - "open4d/streaming", "open4d/codecs/existing", "open4d/codecs/faster_vdmc", "open4d/reconstruction/capture", @@ -70,7 +69,6 @@ def test_nested_submodules_are_covered_by_their_top_level_component(tmp_path: Pa assert discover_required_ledger_paths(tmp_path) == { "integrations/unity", - "open4d/streaming", "open4d/codecs/qndf", } @@ -82,7 +80,6 @@ def test_component_discovery_ignores_generated_and_hidden_directories(tmp_path: assert discover_required_ledger_paths(tmp_path) == { "integrations/unity", - "open4d/streaming", "open4d/codecs/qndf", }