From b4c5386c10348d2c75a4d830c8eddb197464f37c Mon Sep 17 00:00:00 2001 From: ryanmkim Date: Sun, 27 Sep 2026 00:07:39 -0400 Subject: [PATCH] Make the browser user study the webclients app --- README.md | 17 + docs/components.md | 1 + open4d/webclients/.gitignore | 11 + open4d/webclients/PROVENANCE | 16 +- open4d/webclients/README.md | 150 ++++- open4d/webclients/scene_layout.json | 291 +++++++++ open4d/webclients/scripts/check-sync.sh | 58 -- open4d/webclients/scripts/run_web_demo.sh | 8 +- .../system/Client/traces/amazonfcc.csv | 181 ++++++ .../system/Client/traces/cascade-20.csv | 181 ++++++ .../system/Client/traces/cascade-5.csv | 181 ++++++ .../system/Client/traces/good-wifi.csv | 181 ++++++ .../webclients/system/Client/traces/lte.csv | 181 ++++++ .../system/Client/traces/nuwave-0-20.csv | 22 + .../system/Client/traces/nuwave_trace.csv | 121 ++++ .../system/Client/traces/nuwins-60-80.csv | 22 + .../system/Client/traces/nuwins_trace.csv | 121 ++++ .../system/Client/traces/poor-wifi.csv | 181 ++++++ open4d/webclients/system/Server/server.js | 77 ++- .../system/Server/study/follower.js | 97 +++ .../webclients/system/Server/study/order.js | 73 +++ .../system/Server/study/questionnaire.js | 104 ++++ .../webclients/system/Server/study/routes.js | 199 ++++++ .../webclients/system/Server/study/shaper.js | 255 ++++++++ .../webclients/system/Server/study/store.js | 360 +++++++++++ .../webclients/system/Server/study/trace.js | 102 +++ open4d/webclients/system/WebClient/README.md | 32 +- .../system/WebClient/bridge/v4ds-bridge.js | 48 +- open4d/webclients/system/WebClient/build.js | 8 +- .../system/WebClient/public/baseline.html | 4 +- .../system/WebClient/public/compare.html | 62 -- .../system/WebClient/public/index.html | 263 +++++--- .../system/WebClient/public/mesh.html | 85 +++ .../system/WebClient/public/nevo.html | 4 +- .../system/WebClient/public/vega.html | 4 +- .../system/WebClient/src/baseline-client.js | 7 +- .../system/WebClient/src/baseline-main.js | 43 +- .../system/WebClient/src/chooser.js | 227 ------- .../system/WebClient/src/link-rate.js | 6 +- .../webclients/system/WebClient/src/main.js | 55 +- .../system/WebClient/src/point-renderer.js | 43 +- .../system/WebClient/src/study/camera.js | 252 ++++++++ .../system/WebClient/src/study/driver.js | 182 ++++++ .../system/WebClient/src/study/harness.js | 139 +++++ .../WebClient/src/study/questionnaire-form.js | 151 +++++ .../system/WebClient/src/study/study-page.js | 589 ++++++++++++++++++ .../system/WebClient/src/vega-client.js | 36 +- .../system/WebClient/src/vega-main.js | 33 + .../system/WebClient/src/webgl-renderer.js | 35 +- open4d/webclients/tests/test_study_bridge.js | 119 ++++ open4d/webclients/tests/test_study_camera.js | 247 ++++++++ open4d/webclients/tests/test_study_session.js | 293 +++++++++ open4d/webclients/tests/test_study_shaper.js | 165 +++++ 53 files changed, 5804 insertions(+), 519 deletions(-) create mode 100644 open4d/webclients/scene_layout.json delete mode 100755 open4d/webclients/scripts/check-sync.sh create mode 100644 open4d/webclients/system/Client/traces/amazonfcc.csv create mode 100644 open4d/webclients/system/Client/traces/cascade-20.csv create mode 100644 open4d/webclients/system/Client/traces/cascade-5.csv create mode 100644 open4d/webclients/system/Client/traces/good-wifi.csv create mode 100644 open4d/webclients/system/Client/traces/lte.csv create mode 100644 open4d/webclients/system/Client/traces/nuwave-0-20.csv create mode 100644 open4d/webclients/system/Client/traces/nuwave_trace.csv create mode 100644 open4d/webclients/system/Client/traces/nuwins-60-80.csv create mode 100644 open4d/webclients/system/Client/traces/nuwins_trace.csv create mode 100644 open4d/webclients/system/Client/traces/poor-wifi.csv create mode 100644 open4d/webclients/system/Server/study/follower.js create mode 100644 open4d/webclients/system/Server/study/order.js create mode 100644 open4d/webclients/system/Server/study/questionnaire.js create mode 100644 open4d/webclients/system/Server/study/routes.js create mode 100644 open4d/webclients/system/Server/study/shaper.js create mode 100644 open4d/webclients/system/Server/study/store.js create mode 100644 open4d/webclients/system/Server/study/trace.js delete mode 100644 open4d/webclients/system/WebClient/public/compare.html create mode 100644 open4d/webclients/system/WebClient/public/mesh.html delete mode 100644 open4d/webclients/system/WebClient/src/chooser.js create mode 100644 open4d/webclients/system/WebClient/src/study/camera.js create mode 100644 open4d/webclients/system/WebClient/src/study/driver.js create mode 100644 open4d/webclients/system/WebClient/src/study/harness.js create mode 100644 open4d/webclients/system/WebClient/src/study/questionnaire-form.js create mode 100644 open4d/webclients/system/WebClient/src/study/study-page.js create mode 100644 open4d/webclients/tests/test_study_bridge.js create mode 100644 open4d/webclients/tests/test_study_camera.js create mode 100644 open4d/webclients/tests/test_study_session.js create mode 100644 open4d/webclients/tests/test_study_shaper.js diff --git a/README.md b/README.md index 0431e039..fbbe5cba 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,23 @@ Use `realtime=False` to transfer a recorded sequence as fast as possible. The camera capture and native reconstruction programs are in [open4d/streaming](open4d/streaming/README.md), formerly `reconstruction/rgbd`. +## Compare methods in a user study + +[open4d/webclients](open4d/webclients/README.md) is a browser app that runs one +participant through several streaming methods — Ours, ViVo, NAVA and Vega — +under the same network trace, from the same start view, along the same camera +path, and asks them to rate each clip. The trace can be uploaded in the browser; +the results export as CSV. + +```bash +cd open4d/webclients/system/WebClient && npm install && node build.js +cd ../.. && PYTHONPATH=/path/to/4DVideoStreaming PYTHON_BIN= scripts/run_web_demo.sh +# then open http://:3000/web/ +``` + +It needs a checkout of the 4DVideoStreaming research repo for the ladder solver +and the baseline servers. + ## Gaussian splats Read splat frames, then encode them with the local Vega adaptation: diff --git a/docs/components.md b/docs/components.md index 973316e1..f90121e8 100644 --- a/docs/components.md +++ b/docs/components.md @@ -28,6 +28,7 @@ hardware requirements to the shared Python baseline. See | [**ReRF**](../open4d/reconstruction/rerf/README.md) | Neural residual radiance fields (CVPR 2023) as a streamable compression method | | [**gs-tools**](../open4d/reconstruction/gs_tools/README.md) | The one environment, rasterizers, and viewer the Gaussian methods share, plus `gs-tools view` for putting several methods under one camera | | [**streamer**](../open4d/streamer/README.md) | Streaming and playback for 4D reconstructions, whatever their representation. Producers import it to describe and serve their output; it imports none of them | +| [**webclients**](../open4d/webclients/README.md) | A browser user study: one participant, several streaming methods, the same network trace and camera path for each, then a questionnaire and CSV export. Runs Ours, ViVo, NAVA and Vega; needs the 4DVideoStreaming research repo at run time | ## Integrations diff --git a/open4d/webclients/.gitignore b/open4d/webclients/.gitignore index 9cc5fe3a..2d76679b 100644 --- a/open4d/webclients/.gitignore +++ b/open4d/webclients/.gitignore @@ -2,3 +2,14 @@ # Scoped here rather than in the repo root so this directory stays self-contained. node_modules/ dist/ + +# Written by the server at run time: per-broadcast results (including every +# study session and its questionnaires), the viewpoints clients post, and the +# ladder's selections. Mirrors the research repo's root .gitignore; these hold +# participant data and must never be committed. +system/Server/server_results/ +system/Server/server_viewpoints/ +system/Server/server_selections/ +# Per-corpus packaged media caches, as upstream. +files/ +files-*/ diff --git a/open4d/webclients/PROVENANCE b/open4d/webclients/PROVENANCE index f45dc5dc..a52b3a02 100644 --- a/open4d/webclients/PROVENANCE +++ b/open4d/webclients/PROVENANCE @@ -1,11 +1,17 @@ -# Where this directory came from, so drift is detectable. -# Check with: VS4D_REPO=/path/to/4DVideoStreaming scripts/check-sync.sh -vendored-from: c3df4f9b2ef4ce04af8a0ea08dce94441c9bafde -source-repo: github.com/frozzzen3/4DVideoStreaming -source-paths: +# Where this directory came from. It is developed in Open4D now and is not +# kept in sync with its origin; this records the starting point for licensing +# and for anyone tracing a file back. +origin-repo: github.com/frozzzen3/4DVideoStreaming +origin-commit: c3df4f9b2ef4ce04af8a0ea08dce94441c9bafde +origin-paths: system/ClientCore -> system/ClientCore system/WebClient -> system/WebClient system/Server -> system/Server baselines/ViVo/orbitvivo/tile_ladder.py -> tile_ladder.py tests/ -> tests/ scripts/{run_web_demo,serve_pointcloud_baseline,shape_web_demo}.sh -> scripts/ + scene_layout.json -> scene_layout.json + system/Client/traces -> system/Client/traces +# Added in Open4D after that commit: the browser user study (system/Server/study, +# system/WebClient/src/study, public/index.html) and the study hooks in each +# method page and renderer. diff --git a/open4d/webclients/README.md b/open4d/webclients/README.md index 50ad3765..5df9331d 100644 --- a/open4d/webclients/README.md +++ b/open4d/webclients/README.md @@ -1,10 +1,13 @@ # webclients -Browser clients for volumetric adaptive streaming. A sibling of -[`../reconstruction`](../reconstruction): that holds the methods, this holds the -clients that deliver and play them. Named for what it is rather than for what -it does, so that `streamer` -- the Python module in `../reconstruction` that -serves bundles -- and this are not two things both called streaming. Five systems on one page, side by side: +A browser user study for volumetric streaming methods, and the clients it +runs them in. [`../reconstruction`](../reconstruction) holds the methods; this +holds the app that delivers them to a participant under one network trace and +one camera path, and asks them to rate each. Not to be confused with +[`../streamer`](../streamer), the Python bundle server, or +[`../streaming`](../streaming), RGB-D reconstruction sent over TCP. + +The methods it knows: | system | representation | |---|---| @@ -21,14 +24,15 @@ cd system/WebClient && npm install && node build.js cd ../.. && PYTHON_BIN= scripts/run_web_demo.sh ``` -Open the URL it prints (`…:3000/web/compare.html`) and pick a system. Ctrl-C -stops everything. To *see* adaptation rather than just run it, replay a trace in -another shell — without it every adaptive system settles on one operating point -and looks static: +Open the URL it prints (`…:3000/web/`): that is the study. Each method's own +page is still there on its own (`mesh.html`, `baseline.html`, `vega.html`, +`nevo.html`) for looking at one without running a session. Ctrl-C stops +everything. -```bash -sudo scripts/shape_web_demo.sh cascade-20 # 12.5 → 175 Mbps staircase -``` +A study trial shapes the link itself, from the trace chosen in setup. Outside +the study, `scripts/shape_web_demo.sh` still replays a trace with `tc` for the +method pages, with the caveats in that script: it needs root, and it shapes +every flow on the host. **Needs the research repo.** The ladder solver (`vstream/`), the baseline servers (`baselines/`) and the corpora live in `4DVideoStreaming`, so @@ -37,24 +41,107 @@ clients and run the tests. ## Adding a method -Write a page, then add one line to `DESCRIPTION` in `src/chooser.js`, one entry -to `build.js`, and one `public/.html`. The chooser lists whatever -`/api/systems` reports, one row each, so it stays readable as methods pile up. +A method takes part in the study once it has a page that can run a study pass +and the study knows about it: + +1. **The page.** Read `study.studyParams()`; when set, load the context with + `study.loadStudyContext`, hide the page chrome with `study.enterStudyChrome`, + and hand the renderer's camera to `study.createStudyDriver` -- the renderer + calls `driver.apply(camera, controls, info)` once per frame, just before its + orbit update. See `src/study/harness.js`; `src/vega-main.js` is the smallest + example. +2. **The study.** Add the method to `METHODS` in + [`system/Server/study/store.js`](system/Server/study/store.js) and to + `STUDY_PAGES` in `src/study/study-page.js`, and give it a page in `build.js` + and `public/`. `/api/systems` in `system/Server/server.js` says whether it + is available and which objects it has. +3. **Its bytes.** Whatever it downloads must pass the study's shaper: `/files` + and `/vega-assets` do in the server, and the WebSocket bridge does for the + V4DS baselines. + +If a client **adapts in the browser**, implement the `ClientPlatform` contract in +[`system/ClientCore/platform.js`](system/ClientCore/platform.js) and hand it to +`StreamingClient` for the segment loop, MCKP selector, bandwidth estimation and +metrics. If it **adapts server-side or not at all**, write a plain page and do +*not* wrap it in `ClientCore`: that models an HTTP segment loop over a published +ladder, and forcing a push-based or fixed-quality method into it measures the +wrapper rather than the method. + +## User study + +`…:3000/web/study.html` runs one participant through every chosen method under +the **same network trace, from the same start view, along the same camera path**, +then asks them to rate each clip. It is the browser counterpart of the Quest +study (`scripts/user_study.py` in the research repo), and uses the same +questionnaire wording and the same counterbalancing. + +1. **Setup.** A pseudonymous participant code (never a name), the methods, the + objects on stage, the trial length, and a network trace: a built-in one from + `system/Client/traces/` or an uploaded `time,bandwidth` CSV in seconds and + Mbps, optionally windowed and scaled. +2. **Record the path.** The participant drives one unrated pass on Ours, from + a fixed start that frames every object on stage. That path is the one every + trial replays, including the first, so no method is rated from active + viewing while the others are rated from passive viewing. It can be + downloaded and reused for later participants. +3. **Trials.** In Williams order keyed on the participant number, each shown only + as "Clip A", "Clip B"… Each trial re-arms the trace at its start, replays the + path on the content clock, and ends after the trial length. +4. **Questionnaire.** C1–C4 on five stars, then up to two artifacts ("None of + the above" is exclusive), with time on each page recorded. +5. **Results.** A table, and `export.csv`: one row per trial with the ratings, + artifacts, and the same playback metrics for every method. -If your client **adapts in the browser**, implement the `ClientPlatform` -contract in [`system/ClientCore/platform.js`](system/ClientCore/platform.js) -and hand it to `StreamingClient` — you inherit the segment loop, MCKP selector, -bandwidth estimation and metrics. `src/browser-platform.js` is the reference. +```bash +PYTHONPATH=/path/to/4DVideoStreaming PYTHON_BIN= scripts/run_web_demo.sh +# then open http://:3000/web/ +``` -If it **adapts server-side or not at all**, write a plain page and do *not* wrap -it in `ClientCore`: that models an HTTP segment loop over a published ladder, -and forcing a push-based or fixed-quality method into it measures the wrapper -rather than the method. `src/vega-main.js` is the smallest example. +Sessions are written under `system/Server/server_results/study/`, one +self-contained directory each: the trace exactly as uploaded with its hash, the +path, and one `trial.json` per trial. That directory is ignored by git, since +it holds participant data. + +What the study does **differently from the Quest** is how it shapes the link, +because this one must accept a trace uploaded from a browser: + +- The trace is applied **in-process**, not with `tc`. `/files` responses pass + through a token bucket in the server (`system/Server/study/shaper.js`) and + the ViVo/NAVA WebSocket bridge follows the same trace from the same instant + (`follower.js`). It needs no root and throttles nothing else on the host; + what it gives up is a kernel queue under TCP, so congestion control sees + pacing rather than loss. The burst and step semantics match + `quest-trace-player.js`. + +Which methods can take part, and what the rest are missing: + +| method | in the study | notes | +|---|---|---| +| Ours (mesh) | yes | adapts in the browser | +| ViVo, NAVA | yes | restarted on the session's objects before each trial; need the tile corpus | +| Vega | yes | only for sessions of objects it has exported (here: `dancer`, `thomas`); preloads its 30-frame clip, then loops | +| NeVo | no | pre-rendered from one capture camera with no camera control, so it cannot follow a shared path | +| LiVo | no | the browser client does not decode LiVo's RGB-D segments, and no LiVo package is prepared | +| MetaStream, DeltaStream | no | need the RGB-D dataset from `baselines.DeltaStream.orbitstream.converter` | + +The setup page lists all of them, and says why a method cannot be chosen — +including when it lacks one of the session's objects. + +**Which method records the path is a setup choice.** Any that can take part +will do, but it matters: the participant has seen that method before rating +it. Ours in particular may not show the whole stage at startup — it begins with +a 5 Mbps estimate, and the five stage objects need about 89 Mbps even on the +H.264 corpus — so the practice pass holds the start view until every object is +visible, for up to 60 s, and flags a path recorded over a partial stage. + +Serve the **H.264** corpus (`ORBIT_datasets_compressed_h264`, the one +`run_web_demo.sh` defaults to) where possible: every browser decodes it. The +HEVC corpus decodes only in Safari, and in Chrome with suitable hardware. ## Tests ```bash -node --test tests/*.js # 292 of 294 +node --test tests/*.js # 347 of 349 ``` The other two cross-check the JS decoders against the Python ones byte for @@ -63,8 +150,8 @@ byte, so they need `PYTHONPATH=/path/to/4DVideoStreaming` and ## Layout -`system/ClientCore` is the platform-free logic; `system/WebClient` the pages, -chooser and WebSocket↔TCP bridge; `system/Server` publishes the ladder and +`system/ClientCore` is the platform-free logic; `system/WebClient` the study, the method +pages and the WebSocket↔TCP bridge; `system/Server` publishes the ladder and serves media; `tile_ladder.py` lets ViVo/NAVA serve prepared tiles without the RGB-D source. The `system/` level is kept so relative imports resolve unchanged against the research repo — load-bearing, not a leftover. @@ -72,10 +159,7 @@ against the research repo — load-bearing, not a leftover. Pitfalls worth reading before changing a client are in [`system/WebClient/README.md`](system/WebClient/README.md); most fail silently. -This directory is vendored from the research repo, the same way -`../reconstruction` holds its methods. `PROVENANCE` records the commit it came -from, and `scripts/check-sync.sh` reports whether the two have drifted: - -```bash -VS4D_REPO=/path/to/4DVideoStreaming scripts/check-sync.sh -``` +This directory began as a copy of the research repo's clients (see +`PROVENANCE`) and is developed here now; it is not kept in sync with that repo. +It still needs a checkout of it at run time, for the ladder solver and the +baseline servers. diff --git a/open4d/webclients/scene_layout.json b/open4d/webclients/scene_layout.json new file mode 100644 index 00000000..7ab87c85 --- /dev/null +++ b/open4d/webclients/scene_layout.json @@ -0,0 +1,291 @@ +{ + "_comment": [ + "World frame for the composed ORBIT scene: metres, Y-up, right-handed.", + "The background GLB is already metric and defines the frame, so it is placed at identity;", + "every human sequence is native millimetre and is scaled by 0.001.", + "", + "The stage faces +Z (backdrop wall sits at -Z, lighting truss and access stairs at +Z),", + "so performers on the deck face yaw~0 and the audience on the ground faces yaw~180.", + "", + "yaw_deg is the WORLD direction the subject should face, measured from +Z, turning toward +X.", + "native_facing_deg is the direction the subject faces in its OWN raw coordinates, same", + "convention. The composer rotates each subject by (yaw_deg - native_facing_deg).", + "native_facing_deg values were read off rendered orbit sheets, not computed - these subjects", + "turn while they move, so they are a best-fit starting orientation, not an exact quantity.", + "", + "anchor_* come from scripts/calibrate_sequences.py: floor_y_mm is the 10th percentile of", + "per-frame min-Y over the sequence (its contact-with-floor level) and center_x/z_mm is the", + "mean horizontal bbox centre over the sequence. Subtracting the anchor puts the subject's", + "feet at the origin while preserving its motion relative to that anchor." + ], + "world_units": "meters", + "mm_to_world": 0.001, + "background": { + "path": "Background/outdoor_music_stage_venue.glb", + "units": "meters", + "front_axis": "+Z" + }, + "surfaces": { + "_comment": "Measured from upward-facing horizontal faces in the background GLB.", + "ground": { + "y": 0.0, + "note": "truss feet / venue base plane; no floor mesh is modelled at this height" + }, + "stage": { + "y": 1.55, + "x_range": [ + -6.54, + 6.54 + ], + "z_range": [ + -4.47, + 4.47 + ], + "area_m2": 117.1, + "note": "main deck" + }, + "riser": { + "y": 2.11, + "x_range": [ + -1.82, + 1.84 + ], + "z_range": [ + -3.2, + 0.16 + ], + "area_m2": 12.35, + "note": "raised platform at back-centre of the deck" + } + }, + "objects": [ + { + "name": "dancer", + "glob": "dancer/dancer_fr*.obj", + "anchor_floor_y_mm": -483.7, + "anchor_center_x_mm": -110.3, + "anchor_center_z_mm": 84.2, + "native_facing_deg": 180.0, + "surface": "riser", + "x": 0, + "z": -1.5, + "yaw_deg": 10.0 + }, + { + "name": "mitch", + "glob": "mitch/mitch_fr*.obj", + "anchor_floor_y_mm": 7.1, + "anchor_center_x_mm": 67.3, + "anchor_center_z_mm": 81.2, + "native_facing_deg": 0.0, + "surface": "ground", + "x": -11, + "z": -2.5, + "yaw_deg": 0 + }, + { + "name": "thomas", + "glob": "thomas/thomas_fr*.obj", + "anchor_floor_y_mm": -3.2, + "anchor_center_x_mm": 21.4, + "anchor_center_z_mm": 6.4, + "native_facing_deg": 0.0, + "surface": "ground", + "x": 0.2, + "z": 8.2, + "yaw_deg": 180 + }, + { + "name": "basketball", + "glob": "basketball/basketball_player_fr*.obj", + "anchor_floor_y_mm": -481.0, + "anchor_center_x_mm": 1.6, + "anchor_center_z_mm": 79.6, + "native_facing_deg": 180.0, + "surface": "ground", + "x": 3, + "z": 15, + "yaw_deg": 175.0 + }, + { + "name": "UMA_0", + "glob": "UMA_0/meshes_60k/UMA0_*.obj", + "anchor_floor_y_mm": -15.0, + "anchor_center_x_mm": 571.3, + "anchor_center_z_mm": 537.5, + "native_facing_deg": 90.0, + "surface": "stage", + "x": 0, + "z": 3.5, + "yaw_deg": 185.0 + }, + { + "name": "UMA_1", + "glob": "UMA_1/meshes_60k/UMA1_*.obj", + "anchor_floor_y_mm": -16.1, + "anchor_center_x_mm": 633.2, + "anchor_center_z_mm": 537.1, + "native_facing_deg": 0.0, + "surface": "stage", + "x": 4.5, + "z": 3, + "yaw_deg": 0.0 + }, + { + "name": "UMA_2", + "glob": "UMA_2/meshes_60k/UMA2_*.obj", + "anchor_floor_y_mm": -15.1, + "anchor_center_x_mm": 607.8, + "anchor_center_z_mm": 539.6, + "native_facing_deg": 315.0, + "surface": "stage", + "x": -4.6, + "z": 2.4, + "yaw_deg": 0 + }, + { + "name": "UMA_3", + "glob": "UMA_3/meshes_60k/UMA3_*.obj", + "anchor_floor_y_mm": -14.4, + "anchor_center_x_mm": 590.5, + "anchor_center_z_mm": 623.0, + "native_facing_deg": 0.0, + "surface": "stage", + "x": 2.4, + "z": 3.1, + "yaw_deg": 0 + }, + { + "name": "UMA_4", + "glob": "UMA_4/meshes_60k/UMA4_*.obj", + "anchor_floor_y_mm": -16.1, + "anchor_center_x_mm": 626.5, + "anchor_center_z_mm": 503.6, + "native_facing_deg": 0.0, + "surface": "stage", + "x": -2.2, + "z": 2.8, + "yaw_deg": 0 + }, + { + "name": "thomas_01", + "glob": "thomas/thomas_fr06[2-7][0-9].obj", + "anchor_floor_y_mm": -3.2, + "anchor_center_x_mm": 21.4, + "anchor_center_z_mm": 6.4, + "native_facing_deg": 0.0, + "surface": "ground", + "x": -6.0, + "z": 8.0, + "yaw_deg": 143.1 + }, + { + "name": "thomas_02", + "glob": "thomas/thomas_fr06[2-7][0-9].obj", + "anchor_floor_y_mm": -3.2, + "anchor_center_x_mm": 21.4, + "anchor_center_z_mm": 6.4, + "native_facing_deg": 0.0, + "surface": "ground", + "x": -3.5, + "z": 8.0, + "yaw_deg": 156.4 + }, + { + "name": "thomas_03", + "glob": "thomas/thomas_fr06[2-7][0-9].obj", + "anchor_floor_y_mm": -3.2, + "anchor_center_x_mm": 21.4, + "anchor_center_z_mm": 6.4, + "native_facing_deg": 0.0, + "surface": "ground", + "x": -1.5, + "z": 8.0, + "yaw_deg": 169.4 + }, + { + "name": "thomas_04", + "glob": "thomas/thomas_fr06[2-7][0-9].obj", + "anchor_floor_y_mm": -3.2, + "anchor_center_x_mm": 21.4, + "anchor_center_z_mm": 6.4, + "native_facing_deg": 0.0, + "surface": "ground", + "x": 2.0, + "z": 8.0, + "yaw_deg": 194.0 + }, + { + "name": "thomas_05", + "glob": "thomas/thomas_fr06[2-7][0-9].obj", + "anchor_floor_y_mm": -3.2, + "anchor_center_x_mm": 21.4, + "anchor_center_z_mm": 6.4, + "native_facing_deg": 0.0, + "surface": "ground", + "x": 4.5, + "z": 8.0, + "yaw_deg": 209.4 + }, + { + "name": "thomas_06", + "glob": "thomas/thomas_fr06[2-7][0-9].obj", + "anchor_floor_y_mm": -3.2, + "anchor_center_x_mm": 21.4, + "anchor_center_z_mm": 6.4, + "native_facing_deg": 0.0, + "surface": "ground", + "x": -5.0, + "z": 10.5, + "yaw_deg": 154.5 + }, + { + "name": "thomas_07", + "glob": "thomas/thomas_fr06[2-7][0-9].obj", + "anchor_floor_y_mm": -3.2, + "anchor_center_x_mm": 21.4, + "anchor_center_z_mm": 6.4, + "native_facing_deg": 0.0, + "surface": "ground", + "x": -2.5, + "z": 10.5, + "yaw_deg": 166.6 + }, + { + "name": "thomas_08", + "glob": "thomas/thomas_fr06[2-7][0-9].obj", + "anchor_floor_y_mm": -3.2, + "anchor_center_x_mm": 21.4, + "anchor_center_z_mm": 6.4, + "native_facing_deg": 0.0, + "surface": "ground", + "x": 0.0, + "z": 10.5, + "yaw_deg": 180.0 + }, + { + "name": "thomas_09", + "glob": "thomas/thomas_fr06[2-7][0-9].obj", + "anchor_floor_y_mm": -3.2, + "anchor_center_x_mm": 21.4, + "anchor_center_z_mm": 6.4, + "native_facing_deg": 0.0, + "surface": "ground", + "x": 2.5, + "z": 10.5, + "yaw_deg": 193.4 + }, + { + "name": "thomas_10", + "glob": "thomas/thomas_fr06[2-7][0-9].obj", + "anchor_floor_y_mm": -3.2, + "anchor_center_x_mm": 21.4, + "anchor_center_z_mm": 6.4, + "native_facing_deg": 0.0, + "surface": "ground", + "x": -1.0, + "z": 13.0, + "yaw_deg": 175.6 + } + ] +} diff --git a/open4d/webclients/scripts/check-sync.sh b/open4d/webclients/scripts/check-sync.sh deleted file mode 100755 index 026ebe47..00000000 --- a/open4d/webclients/scripts/check-sync.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -# Is this vendored copy still identical to the research repo it came from? -# -# The client code lives in two places: here, and in 4DVideoStreaming where it -# is developed. Nothing enforces that they match — Open4D vendors its methods -# as plain files (see reconstruction/), and that is deliberate, because the two -# repos have unrelated histories and one is private. The cost is silent drift, -# so this makes it a command instead of a discovery. -# -# VS4D_REPO=/path/to/4DVideoStreaming scripts/check-sync.sh -# -# Exits non-zero if anything differs, so CI can call it. -set -uo pipefail - -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -SRC="${VS4D_REPO:-$HOME/4DVideoStreaming}" -RECORDED="$(sed -n 's/^vendored-from: *//p' "$HERE/PROVENANCE" 2>/dev/null)" - -if [[ ! -d "$SRC" ]]; then - echo "no research repo at $SRC; set VS4D_REPO" >&2 - exit 2 -fi - -echo "vendored from : ${RECORDED:-unrecorded}" -if have=$(git -C "$SRC" rev-parse HEAD 2>/dev/null); then - echo "that repo now : $have" - [[ -n "$RECORDED" && "$have" != "$RECORDED" ]] && - echo " (it has moved on; differences below may be intended)" -fi -echo - -# Generated output and dependencies are not part of the comparison. -EXCLUDES=(-x node_modules -x dist -x __pycache__ -x .gitignore -x PROVENANCE - -x README.md -x check-sync.sh) -status=0 -compare() { # - local out - out=$(diff -r "${EXCLUDES[@]}" "$HERE/$1" "$SRC/$2" 2>&1 \ - | grep -vE "^Only in $HERE" || true) - if [[ -n "$out" ]]; then - echo "DIFFERS $1" - echo "$out" | sed 's/^/ /' | head -12 - status=1 - else - echo "same $1" - fi -} - -compare system/ClientCore system/ClientCore -compare system/WebClient/src system/WebClient/src -compare system/WebClient/public system/WebClient/public -compare system/WebClient/bridge system/WebClient/bridge -compare system/Server/server.js system/Server/server.js -compare tile_ladder.py baselines/ViVo/orbitvivo/tile_ladder.py - -echo -[[ $status -eq 0 ]] && echo "in sync" || echo "DRIFTED — reconcile before relying on either copy" -exit $status diff --git a/open4d/webclients/scripts/run_web_demo.sh b/open4d/webclients/scripts/run_web_demo.sh index 3cd59772..72be29f1 100755 --- a/open4d/webclients/scripts/run_web_demo.sh +++ b/open4d/webclients/scripts/run_web_demo.sh @@ -81,12 +81,12 @@ fi HOST="$(hostname -I | awk '{print $1}')" cat < String(value ?? "") .replace(/(? res.set("Cache-Control", "no-store") })); @@ -276,6 +286,10 @@ app.get("/api/geometry-bundle", async (req, res) => { }); const PORT = process.env.PORT || 3000; +// Point-cloud bridges this server spawns (via /api/pointcloud/start) inherit +// the environment; this is how they find the study trace to follow, so a ViVo +// or NAVA trial is shaped exactly as a mesh trial is. See study/follower.js. +process.env.VS4D_STUDY_SERVER ||= `http://127.0.0.1:${PORT}`; function numberFromEnv(name, fallback) { const value = Number(process.env[name]); @@ -491,6 +505,12 @@ const STATE_DIR = FILES_ROOT; // media tree. This mount makes the generated PNGs viewable without exposing // arbitrary host paths returned by the debugging API. app.use("/server-results", express.static(SERVER_RESULTS_DIR)); +app.use("/api/study", createStudyRouter({ + resultsRoot: path.join(SERVER_RESULTS_DIR, "study"), + shaper: studyShaper, + tracesDir: path.resolve(__dirname, "../Client/traces"), + layoutFile: path.resolve(__dirname, "../../scene_layout.json"), +})); // All per-run artifacts (metrics, bitrate counts, QoE, download telemetry) // live in server_results// so each run stays self-contained. @@ -910,7 +930,33 @@ app.get("/api/manifest", async (req, res) => { // request. `?dir=` selects a sibling set (the directory holds several). // Is something listening? Used to tell "the corpus exists" from "a server is // actually up", which need opposite responses from whoever is reading the -// chooser. Short timeout: this runs inline in a request handler. +// study page. Short timeout: this runs inline in a request handler. +/** + * Whether something is listening on `port`, found WITHOUT connecting to it. + * + * The point-cloud baselines accept exactly one connection and exit when it + * closes. Probing one with a connect is therefore not a probe: the baseline + * serves the probe as its session, exits, and is restarting a second later + * when the real client dials -- ECONNREFUSED for a page that navigates + * straight away. Reading the kernel's listening-socket table consumes nothing. + * Returns null where there is no /proc to read, so callers can fall back. + */ +function isListeningQuietly(port) { + const suffix = `:${port.toString(16).toUpperCase().padStart(4, "0")}`; + let readable = false; + for (const file of ["/proc/net/tcp", "/proc/net/tcp6"]) { + let text; + try { text = fs.readFileSync(file, "utf8"); } catch { continue; } + readable = true; + for (const line of text.split("\n").slice(1)) { + const columns = line.trim().split(/\s+/); + // local_address is column 1, st is column 3; 0A is TCP_LISTEN. + if (columns[3] === "0A" && columns[1]?.endsWith(suffix)) return true; + } + } + return readable ? false : null; +} + function portIsOpen(port, host = "127.0.0.1", timeoutMs = 250) { return new Promise(resolve => { const socket = new net.Socket(); @@ -930,7 +976,7 @@ function portIsOpen(port, host = "127.0.0.1", timeoutMs = 250) { // They index the real capture rig while serving (`obj.cameras[camera_index]`) // and read the source RGB-D frames, so the prepared tiles cannot substitute // and the absent corpus is missing data rather than missing metadata. Listing -// them as permanently unavailable was noise in a chooser whose job is to say +// them as permanently unavailable was noise on a setup page whose job is to say // what you can look at. // // The point-cloud baselines that can run from the prepared tiles, and the @@ -1040,8 +1086,11 @@ app.post("/api/pointcloud/start", async (req, res) => { // crashed and is being restarted in a loop, so probing it alone reports // "serving" over a dead baseline -- observed when the spawned process had // the wrong interpreter. - if (await portIsOpen(entry.bridgePort) - && await portIsOpen(entry.serverPort)) { + // The baseline's port is checked quietly (see isListeningQuietly): a + // connect would spend the one session it is waiting to give the page. + const serverListening = isListeningQuietly(entry.serverPort) + ?? await portIsOpen(entry.serverPort); + if (await portIsOpen(entry.bridgePort) && serverListening) { logInfo(id.toUpperCase(), `serving ${requested.join(", ")}`); return res.json({ id, objects: requested, bridgePort: entry.bridgePort, @@ -1073,6 +1122,16 @@ for (const signal of ["SIGINT", "SIGTERM"]) { } app.get("/api/shaping", (req, res) => { + // A study trial owns the link: report the rate it is enforcing, not tc's. + if (studyShaper.armed) { + res.set("Cache-Control", "no-store"); + const status = studyShaper.status(); + return res.json({ + interface: "in-process", shaped: status.shaped, rateMbps: status.rateMbps, + detail: `study trial ${status.label}: in-process token bucket, ` + + `${status.elapsedSeconds}s into a ${status.traceSeconds}s trace`, + }); + } execFile("tc", ["qdisc", "show", "dev", SHAPED_INTERFACE], { timeout: 2000 }, (error, stdout) => { res.set("Cache-Control", "no-store"); @@ -1113,7 +1172,7 @@ app.get("/api/shaping", (req, res) => { // needs an export, NeVo needs pre-rendered frames, the point-cloud baselines // need a Python server plus a WebSocket bridge. Without this the pages fail at // fetch time with a 404, which reads as a bug in the viewer rather than as -// missing input. The chooser asks here first and says which is which. +// missing input. The study page asks here first and says which is which. // // The mesh entry also carries the per-object ladder cost, because the scene // size sets an irreducible bitrate floor (the ladder must publish at least one @@ -1158,7 +1217,7 @@ app.get("/api/systems", async (req, res) => { const priced = objects.filter(o => o.floorMbps !== null); const floorMbps = priced.reduce((sum, o) => sum + o.floorMbps, 0); - // Vega's own catalogue, read for the same reason: so the chooser can offer + // Vega's own catalogue, read for the same reason: so the study page can offer // its objects instead of the page taking all of them by default. const vega = (() => { try { @@ -1181,7 +1240,7 @@ app.get("/api/systems", async (req, res) => { // be up at once on different ports, so choosing between ViVo and NAVA is a // click rather than a server restart. const haveTiles = fs.existsSync(path.join(VIVO_TILES_ROOT, "catalog.json")); - // The tile catalogue's object list, so the chooser can offer a selection. + // The tile catalogue's object list, so the study page can offer a selection. // Unlike the other systems this is not a URL parameter: a baseline fixes its // object set at startup, so choosing one means restarting the process. const tileObjects = (() => { @@ -1211,7 +1270,7 @@ app.get("/api/systems", async (req, res) => { adaptsServerSide: true, ready: live, objects: tileObjects, - // What the running process was started with, so the chooser can tell a + // What the running process was started with, so the study page can tell a // restart is needed rather than silently connecting to the wrong scene. servingObjects: runningPointcloud.get(entry.id) || null, restartable: haveTiles, @@ -1232,7 +1291,7 @@ app.get("/api/systems", async (req, res) => { { id: "mesh", name: "Ours", - page: "/web/", + page: "/web/mesh.html", adaptive: true, ready: objects.length > 0, detail: objects.length diff --git a/open4d/webclients/system/Server/study/follower.js b/open4d/webclients/system/Server/study/follower.js new file mode 100644 index 00000000..f7903555 --- /dev/null +++ b/open4d/webclients/system/Server/study/follower.js @@ -0,0 +1,97 @@ +'use strict'; + +/** + * Keep a shaper in another process on the study server's trace. + * + * The point-cloud baselines do not come through the Node server: their bytes + * go Python server -> TCP -> v4ds-bridge.js -> WebSocket -> browser. So the + * bridge shapes them itself, and this is how it knows what to shape to. It + * polls /api/study/shaping and arms a local TraceShaper with the *same* trace + * from the *same* wall-clock instant the server armed, so a baseline trial and + * a mesh trial see an identical link. + * + * Polled rather than pushed, because the bridge is a separate short-lived + * process with no channel to the server otherwise, and half a second of + * arming lag does not shift the trace: the rate is a function of wall-clock + * time since the server's start, not of when the bridge heard about it. + */ + +const { TraceShaper } = require('./shaper'); + +class ShapingFollower { + /** + * @param {object} args + * @param {string} args.serverUrl the study server, e.g. http://127.0.0.1:3000 + * @param {TraceShaper} [args.shaper] + * @param {number} [args.intervalMs=500] + * @param {typeof fetch} [args.fetchImpl] + * @param {(level: string, message: string) => void} [args.log] + */ + constructor({ serverUrl, shaper = new TraceShaper(), intervalMs = 500, + fetchImpl = globalThis.fetch, log = () => {} }) { + this.serverUrl = String(serverUrl).replace(/\/+$/, ''); + this.shaper = shaper; + this.intervalMs = intervalMs; + this._fetch = fetchImpl; + this._log = log; + this._startedAt = null; + this._timer = null; + this._failures = 0; + } + + start() { + if (this._timer) return this; + const tick = () => this.poll().catch(() => {}); + tick(); + this._timer = setInterval(tick, this.intervalMs); + this._timer.unref?.(); + return this; + } + + stop() { + if (this._timer) clearInterval(this._timer); + this._timer = null; + this.shaper.disarm(); + this._startedAt = null; + } + + /** One look at the server's link, and the local shaper brought into line. */ + async poll() { + let status; + try { + const response = await this._fetch(`${this.serverUrl}/api/study/shaping`, + { cache: 'no-store' }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + status = await response.json(); + this._failures = 0; + } catch (error) { + // Losing the server mid-trial must not freeze the link at whatever + // it last was: after a few misses, stop shaping and say so. + this._failures += 1; + if (this._failures === 3 && this.shaper.armed) { + this._log('WARN', `study server unreachable (${error.message}); unshaping`); + this.shaper.disarm(); + this._startedAt = null; + } + throw error; + } + const armed = status.startedAt !== null && Array.isArray(status.points) + && status.points.length > 0; + if (armed && status.startedAt !== this._startedAt) { + this.shaper.arm(status.points, { + hold: status.hold !== false, startedAt: status.startedAt, + label: status.label ?? null + }); + this._startedAt = status.startedAt; + this._log('INFO', `following study trial ${status.label ?? ''} ` + + `(${status.rateMbps ?? '?'} Mbps now)`); + } else if (!armed && this.shaper.armed) { + this.shaper.disarm(); + this._startedAt = null; + this._log('INFO', 'study trial over; link unshaped'); + } + return status; + } +} + +module.exports = { ShapingFollower }; diff --git a/open4d/webclients/system/Server/study/order.js b/open4d/webclients/system/Server/study/order.js new file mode 100644 index 00000000..cac68baa --- /dev/null +++ b/open4d/webclients/system/Server/study/order.js @@ -0,0 +1,73 @@ +'use strict'; + +/** + * Trial order: a Williams balanced Latin square, keyed on the participant. + * + * The same design the Quest study uses (`scripts/user_study.py`), for the same + * reason. Rating a method is not independent of what was watched just before + * it, so the order is counterbalanced: across a full set of participants every + * method appears in every position once, and every method is immediately + * preceded by every other exactly once, which balances first-order carryover. + * An even number of methods needs n rows; an odd number needs 2n, the rows and + * their reverses. + * + * Participants see only a position label, never the method, so a rating + * cannot be anchored on a name. + */ + +/** The first row of the design: 0, 1, n-1, 2, n-2, ... */ +function firstRow(n) { + const row = [0]; + for (let k = 1; k < n; k++) row.push(k % 2 ? (k + 1) / 2 : n - k / 2); + return row; +} + +/** Every row of the design for `n` conditions. */ +function williamsRows(n) { + if (!Number.isInteger(n) || n < 1) throw new Error(`need at least one condition, got ${n}`); + const base = firstRow(n); + const rows = []; + for (let shift = 0; shift < n; shift++) rows.push(base.map(value => (value + shift) % n)); + if (n % 2) for (let index = 0; index < n; index++) rows.push([...rows[index]].reverse()); + return rows; +} + +/** + * The digits in a participant code, as the index that picks a row. "p07" and + * "p7" are the same participant, which is the point of reading the number + * rather than hashing the text. + */ +function participantIndex(code) { + const digits = String(code || '').match(/\d+/g); + return digits ? Number(digits.join('')) : 0; +} + +/** Position labels shown to the participant: A, B, ... Z, AA, AB, ... */ +function positionLabel(index) { + let label = ''; + let n = index + 1; + while (n > 0) { + const remainder = (n - 1) % 26; + label = String.fromCharCode(65 + remainder) + label; + n = Math.floor((n - 1) / 26); + } + return label; +} + +/** + * @param {string[]} methods condition ids, in a fixed canonical order + * @param {number} index participant index + * @returns {{row: number, order: {position: number, label: string, method: string}[]}} + */ +function trialOrder(methods, index) { + const rows = williamsRows(methods.length); + const row = ((index % rows.length) + rows.length) % rows.length; + return { + row, + order: rows[row].map((condition, position) => ({ + position, label: positionLabel(position), method: methods[condition] + })) + }; +} + +module.exports = { williamsRows, participantIndex, positionLabel, trialOrder }; diff --git a/open4d/webclients/system/Server/study/questionnaire.js b/open4d/webclients/system/Server/study/questionnaire.js new file mode 100644 index 00000000..cda53bd8 --- /dev/null +++ b/open4d/webclients/system/Server/study/questionnaire.js @@ -0,0 +1,104 @@ +'use strict'; + +/** + * The post-clip questionnaire, shared by the browser page and the server. + * + * Wording is copied from `QuestQuestionnairePanel.cs` rather than written + * afresh, so that a rating collected in a browser answers the same question as + * one collected in the headset and the two can be pooled or compared. Change it + * there and here together, or not at all. + * + * Pure data and one validator, with no DOM and no Node, because both sides + * require it: the page to build the form, the server to refuse a response the + * form should not have been able to produce. + */ + +const RATING_MIN = 1; +const RATING_MAX = 5; +const MAXIMUM_ARTIFACT_SELECTIONS = 2; + +const RATINGS = Object.freeze([ + { + id: 'C1', title: 'Visual quality', + prompt: "How sharp, detailed, and accurate were the colors on the performers' surfaces?" + }, + { + id: 'C2', title: 'Geometry & depth fidelity', + prompt: 'How solid, complete, and correctly shaped did performers look as you moved around them?' + }, + { + id: 'C3', title: 'Temporal smoothness', + prompt: 'How smooth was playback—without freezing, stuttering, or visible jumps in quality?' + }, + { + id: 'C4', title: 'Overall experience', + prompt: 'Overall, how would you rate your experience watching this clip?' + } +]); + +const ARTIFACTS = Object.freeze([ + { id: 'surface_geometry_degradation', label: "Blurry surfaces or blocky / 'melted' body shapes" }, + { id: 'missing_parts', label: 'Holes or missing parts of a person' }, + { id: 'quality_flicker', label: 'Flickering or popping between quality levels' }, + { id: 'freezing', label: 'Freezing or a person standing still / disappearing' }, + { id: 'mixed_quality_parts', label: 'Different parts of the same performer showed different quality' }, + { id: 'client_low_fps', label: 'Low frame rate or the whole view stuttering / getting stuck' }, + { id: 'view_dependent_degradation', label: 'Quality got worse when I moved closer or changed angle' }, + { id: 'none', label: 'None of the above' } +]); + +const ARTIFACT_PROMPT = + 'Select up to TWO most severe issues. Choose None if nothing applied.'; + +/** + * Throw unless `response` is one the form could have produced. + * + * @param {object} response + * @param {Object} response.ratings C1..C4 -> 1..5 + * @param {string[]} response.artifacts ids; "none" is exclusive + * @returns {object} a normalized copy + */ +function validateResponse(response) { + if (!response || typeof response !== 'object') { + throw new Error('questionnaire response must be an object'); + } + const ratings = {}; + for (const { id } of RATINGS) { + const value = response.ratings?.[id]; + if (!Number.isInteger(value) || value < RATING_MIN || value > RATING_MAX) { + throw new Error(`${id} must be an integer ${RATING_MIN}-${RATING_MAX}, got ${value}`); + } + ratings[id] = value; + } + const known = new Set(ARTIFACTS.map(artifact => artifact.id)); + const chosen = [...new Set(Array.isArray(response.artifacts) ? response.artifacts : [])]; + if (!chosen.length) throw new Error('choose at least one artifact, or "none"'); + for (const id of chosen) { + if (!known.has(id)) throw new Error(`unknown artifact "${id}"`); + } + if (chosen.includes('none') && chosen.length > 1) { + throw new Error('"none" cannot be combined with other artifacts'); + } + if (chosen.length > MAXIMUM_ARTIFACT_SELECTIONS) { + throw new Error(`choose at most ${MAXIMUM_ARTIFACT_SELECTIONS} artifacts`); + } + const timing = response.timing && typeof response.timing === 'object' ? response.timing : {}; + const seconds = value => (Number.isFinite(value) && value >= 0 ? Number(value) : null); + return { + ratings, + artifacts: chosen, + timing: { + totalSeconds: seconds(timing.totalSeconds), + ratingsPageSeconds: seconds(timing.ratingsPageSeconds), + artifactsPageSeconds: seconds(timing.artifactsPageSeconds), + ratingSelections: Number.isInteger(timing.ratingSelections) ? timing.ratingSelections : null, + artifactSelections: Number.isInteger(timing.artifactSelections) ? timing.artifactSelections : null + } + }; +} + +module.exports = { + RATINGS, ARTIFACTS, ARTIFACT_PROMPT, + RATING_MIN, RATING_MAX, MAXIMUM_ARTIFACT_SELECTIONS, + validateResponse +}; diff --git a/open4d/webclients/system/Server/study/routes.js b/open4d/webclients/system/Server/study/routes.js new file mode 100644 index 00000000..4fad5dbd --- /dev/null +++ b/open4d/webclients/system/Server/study/routes.js @@ -0,0 +1,199 @@ +'use strict'; + +/** + * HTTP surface of the browser study, mounted by server.js at /api/study. + * + * One trial runs at a time, because there is one link: the shaper is the + * process-wide bottleneck every shaped response passes through, and a second + * trial arming it would reset the first one's trace mid-run. Starting a trial + * while another is live is therefore refused rather than allowed to corrupt it. + */ + +const express = require('express'); +const fs = require('fs'); +const path = require('path'); +const mime = require('mime-types'); + +const questionnaire = require('./questionnaire'); +const { StudyStore, METHODS } = require('./store'); +const { summarize, parseTrace } = require('./trace'); + +/** + * `/files` through the shaper, while a trial is armed. + * + * Unarmed, it steps aside and express.static serves exactly as before, so the + * demo pages are untouched by this existing. Armed, it serves the file itself + * so every byte crosses the bucket. A Range header is ignored and the whole + * entity sent, which HTTP permits; the segment client never asks for ranges, + * and serving one unshaped would be a hole in the link. + */ +function shapedStatic(root, shaper) { + const base = path.resolve(root); + return (req, res, next) => { + if (!shaper.armed || !['GET', 'HEAD'].includes(req.method)) return next(); + let relative; + try { relative = decodeURIComponent(req.path); } catch { return next(); } + const file = path.resolve(base, `.${relative}`); + if (file !== base && !file.startsWith(base + path.sep)) return res.status(403).end(); + fs.stat(file, (error, stat) => { + if (error || !stat.isFile()) return next(); + res.status(200); + res.set({ + 'Content-Type': mime.lookup(file) || 'application/octet-stream', + 'Content-Length': String(stat.size), + // The segment loop must keep generating real load; a cached + // segment is a free one, and the link would stop meaning anything. + 'Cache-Control': 'no-store', + 'X-Study-Shaped': '1' + }); + if (req.method === 'HEAD') return res.end(); + const source = fs.createReadStream(file); + const paced = shaper.pace(); + const abort = () => { source.destroy(); paced.destroy(); }; + res.on('close', abort); + source.on('error', abort); + source.pipe(paced).pipe(res); + }); + }; +} + +function builtInTraces(directory) { + if (!directory || !fs.existsSync(directory)) return []; + return fs.readdirSync(directory) + .filter(name => name.endsWith('.csv')) + .sort() + .map(name => { + try { + return { name, ...summarize(parseTrace(fs.readFileSync(path.join(directory, name), 'utf8'))) }; + } catch (error) { + return { name, error: error.message }; + } + }); +} + +/** + * @param {object} args + * @param {string} args.resultsRoot where sessions are stored + * @param {import('./shaper').TraceShaper} args.shaper + * @param {string} [args.tracesDir] built-in traces offered beside uploads + * @param {string} [args.layoutFile] scene_layout.json, for the start pose + */ +function createStudyRouter({ resultsRoot, shaper, tracesDir, layoutFile }) { + const store = new StudyStore(resultsRoot); + const router = express.Router(); + router.use(express.json({ limit: '8mb' })); + // Which trial owns the link right now, if any. + let live = null; + + const wrap = handler => (req, res) => { + try { + const out = handler(req, res); + if (out !== undefined) res.set('Cache-Control', 'no-store').json(out); + } catch (error) { + res.status(error.statusCode || 400).json({ error: error.message }); + } + }; + + router.get('/config', wrap(() => ({ + methods: METHODS, + questionnaire: { + ratings: questionnaire.RATINGS, + artifacts: questionnaire.ARTIFACTS, + artifactPrompt: questionnaire.ARTIFACT_PROMPT, + min: questionnaire.RATING_MIN, + max: questionnaire.RATING_MAX, + maxArtifacts: questionnaire.MAXIMUM_ARTIFACT_SELECTIONS + }, + traces: builtInTraces(tracesDir) + }))); + + // Where every object stands, so a page can frame the whole stage before any + // of it has downloaded -- the start pose must not depend on what arrived. + router.get('/layout', (req, res) => { + if (!layoutFile || !fs.existsSync(layoutFile)) { + return res.status(404).json({ error: 'no scene_layout.json on this server' }); + } + res.set('Cache-Control', 'no-store').type('application/json') + .send(fs.readFileSync(layoutFile, 'utf8')); + }); + + router.get('/traces/:name', (req, res) => { + const name = path.basename(String(req.params.name)); + const file = tracesDir && path.join(tracesDir, name); + if (!file || !name.endsWith('.csv') || !fs.existsSync(file)) { + return res.status(404).json({ error: `no built-in trace ${name}` }); + } + res.type('text/csv').set('Cache-Control', 'no-store').send(fs.readFileSync(file, 'utf8')); + }); + + router.get('/sessions', wrap(() => ({ sessions: store.listSessions() }))); + router.post('/sessions', wrap(req => store.createSession(req.body || {}))); + router.get('/sessions/:id', wrap(req => store.session(req.params.id))); + router.get('/sessions/:id/trajectory', wrap(req => store.trajectory(req.params.id))); + router.get('/sessions/:id/trials', wrap(req => ({ trials: store.trialRecords(req.params.id) }))); + router.put('/sessions/:id/trajectory', + wrap(req => store.saveTrajectory(req.params.id, req.body))); + + router.post('/sessions/:id/trials/:position/start', wrap(req => { + if (live && !(live.session === req.params.id + && live.position === Number(req.params.position))) { + throw Object.assign(new Error( + `trial ${live.label} of ${live.session} owns the link; finish or abort it first`), + { statusCode: 409 }); + } + const { session, trial } = store.startTrial(req.params.id, req.params.position); + // Armed at the start of *every* trial, so each method plays the same + // stretch of trace from its first second -- the Quest study re-arms + // the trace player per trial for the same reason. + shaper.arm(session.trace.points, { + hold: session.trace.hold, label: `${session.id} ${trial.label}` + }); + live = { session: session.id, position: trial.position, label: trial.label }; + return { trial, shaping: shaper.status(), trace: session.trace.points }; + })); + + router.post('/sessions/:id/trials/:position/finish', wrap(req => { + const shaping = shaper.status(); + const result = store.finishTrial(req.params.id, req.params.position, { + ...(req.body || {}), shaping + }); + shaper.disarm(); + live = null; + return result; + })); + + /** Release the link after an abandoned trial, without recording it. */ + router.post('/abort', wrap(() => { + shaper.disarm(); + const was = live; + live = null; + return { aborted: was, shaping: shaper.status() }; + })); + + /** + * What the link is doing, for anything else that must follow it. The + * ViVo/NAVA bridge polls this so its socket follows the same trace from + * the same instant as the HTTP responses. + */ + router.get('/shaping', wrap(() => ({ + ...shaper.status(), + startedAt: shaper.armed ? shaper._startedAt : null, + points: shaper.armed ? shaper._points : null, + live + }))); + + router.get('/sessions/:id/export.csv', (req, res) => { + try { + const csv = store.exportCsv(req.params.id); + res.type('text/csv') + .set('Content-Disposition', `attachment; filename="${req.params.id}.csv"`) + .send(csv); + } catch (error) { + res.status(error.statusCode || 400).json({ error: error.message }); + } + }); + + return router; +} + +module.exports = { createStudyRouter, shapedStatic }; diff --git a/open4d/webclients/system/Server/study/shaper.js b/open4d/webclients/system/Server/study/shaper.js new file mode 100644 index 00000000..79f41927 --- /dev/null +++ b/open4d/webclients/system/Server/study/shaper.js @@ -0,0 +1,255 @@ +'use strict'; + +/** + * A trace-driven bottleneck, applied to responses rather than to the NIC. + * + * `quest-trace-player.js` shapes with `tc`: a root token bucket on the egress + * interface, which is the real kernel queue and the most faithful option. It + * also needs root, and it shapes every flow on the host -- on a shared box that + * is every other user's traffic and the viewer's own connection to the page. + * A study where the participant uploads a trace from a browser cannot run + * `sudo`, so this does the same job one layer up: a token bucket that the + * bytes of every shaped response pass through, following the trace from the + * moment a trial is armed. + * + * What it keeps from the tc path, so the two are comparable: + * + * - the rate is the trace's step function, rebased to the trial start; + * - the burst is `max(128 KiB, 1 ms of rate)`, the tbf bucket the trace + * player installs; + * - one bucket per process, served first-in first-out, so concurrent + * requests contend for one link rather than each getting their own. + * + * What it gives up: there is no kernel queue underneath, so TCP's congestion + * control never sees loss or queueing delay -- the sender is simply paced. For + * a segment client that times whole downloads that is the quantity it + * measures; for anything that reacts to RTT it is not the same experiment. + * + * Time is wall-clock seconds (`Date.now`), not a process-local clock, so that + * a second process -- the ViVo/NAVA WebSocket bridge -- can follow the same + * trace from the same armed instant. + */ + +const { Transform } = require('stream'); +const { rateAt, traceDuration } = require('./trace'); + +const BURST_FLOOR_BYTES = 128 * 1024; +/** Longest single sleep, so disarming or re-arming takes effect promptly. */ +const MAX_SLEEP_SECONDS = 0.05; +/** Shortest, so a wait always moves the clock. */ +const MIN_SLEEP_SECONDS = 1e-6; +const TOKEN_EPSILON = 1; + +function burstBytes(mbps) { + return Math.max(BURST_FLOOR_BYTES, Math.ceil(mbps * 125)); +} + +function bytesPerSecond(mbps) { + return (mbps * 1e6) / 8; +} + +const wallClock = () => Date.now() / 1000; +const realSleep = seconds => new Promise(resolve => setTimeout(resolve, seconds * 1000)); + +class TraceShaper { + /** + * @param {object} [deps] + * @param {() => number} [deps.now] seconds; injectable so a test can drive it + * @param {(seconds: number) => Promise} [deps.sleep] + */ + constructor({ now = wallClock, sleep = realSleep } = {}) { + this._now = now; + this._sleep = sleep; + this._queue = Promise.resolve(); + this.disarm(); + } + + /** + * Start following `points` now (or at `startedAt`). + * + * @param {{time: number, bandwidth: number}[]} points rebased trace + * @param {object} [options] + * @param {boolean} [options.hold=true] keep the last rate after the trace + * ends, as the study's `--hold` does; otherwise the link goes unshaped + * @param {number} [options.startedAt] wall-clock seconds + * @param {string} [options.label] what to call it in `status()` + */ + arm(points, { hold = true, startedAt = this._now(), label = null } = {}) { + if (!Array.isArray(points) || !points.length) { + throw new Error('cannot arm a shaper with an empty trace'); + } + this._points = points; + this._hold = hold; + this._startedAt = startedAt; + this._label = label; + // A fresh trial starts with a full bucket, as a freshly installed tbf + // does, and forgets what the previous trial sent. + this._tokens = burstBytes(points[0].bandwidth); + this._at = startedAt; + this._delivered = 0; + this._waited = 0; + this._generation = (this._generation || 0) + 1; + } + + disarm() { + this._points = null; + this._startedAt = null; + this._label = null; + this._generation = (this._generation || 0) + 1; + } + + get armed() { return this._points !== null; } + + /** Seconds since the trial was armed, or null when unarmed. */ + elapsed(at = this._now()) { + return this.armed ? at - this._startedAt : null; + } + + /** The rate in force at wall-clock `at`, in Mbps, or null when unshaped. */ + rateMbps(at = this._now()) { + if (!this.armed) return null; + const t = Math.max(0, at - this._startedAt); + if (t > traceDuration(this._points) && !this._hold) return null; + return rateAt(this._points, t); + } + + status() { + const now = this._now(); + const rate = this.rateMbps(now); + return { + shaped: rate !== null, + mechanism: 'in-process', + label: this._label, + rateMbps: rate === null ? null : Number(rate.toFixed(2)), + elapsedSeconds: this.armed ? Number(this.elapsed(now).toFixed(3)) : null, + traceSeconds: this.armed ? traceDuration(this._points) : null, + hold: this.armed ? this._hold : null, + deliveredBytes: this.armed ? this._delivered : 0, + pacedSeconds: this.armed ? Number(this._waited.toFixed(3)) : 0 + }; + } + + /** + * Resolve once `bytes` may leave. Callers are served in the order they + * asked, which is what makes several responses share one link. + */ + take(bytes) { + const turn = this._queue.then(() => this._take(bytes)); + // A failed take must not wedge everyone queued behind it. + this._queue = turn.catch(() => {}); + return turn; + } + + async _take(bytes) { + let remaining = bytes; + while (remaining > 0) { + if (!this.armed) return; + const generation = this._generation; + const now = this._now(); + const rate = this.rateMbps(now); + if (rate === null) { this._delivered += remaining; return; } + this._refill(now); + const piece = Math.min(remaining, burstBytes(rate)); + // Within a byte counts as enough: the deficit is otherwise a + // rounding residue whose wait is too small to move the clock. + if (this._tokens + TOKEN_EPSILON >= piece) { + this._tokens = Math.max(0, this._tokens - piece); + this._delivered += piece; + remaining -= piece; + continue; + } + const wait = Math.min(MAX_SLEEP_SECONDS, Math.max(MIN_SLEEP_SECONDS, + this._secondsUntil(piece - this._tokens, now))); + this._waited += wait; + await this._sleep(wait); + // Re-armed while asleep: the bucket belongs to a new trial now, and + // bytes owed to the old one are not the new one's to pay. + if (generation !== this._generation) return; + } + } + + /** The rate at trace time `t` (seconds since arming), or null. */ + _rateAtTraceTime(t) { + if (t > traceDuration(this._points) && !this._hold) return null; + return rateAt(this._points, Math.max(0, t)); + } + + /** + * Integrate the trace from the last refill up to wall-clock `to`. + * + * In trace time, not wall time. The boundaries are the trace's own sample + * times; converting each back to wall time and subtracting again does not + * round-trip in floating point, and a step that lands exactly where it + * began loops for ever. + */ + _refill(to) { + let t = this._at - this._startedAt; + const end = to - this._startedAt; + while (t < end) { + const rate = this._rateAtTraceTime(t); + if (rate === null) break; + const next = Math.min(end, this._nextBoundary(t)); + if (!(next > t)) break; + this._tokens += bytesPerSecond(rate) * (next - t); + t = next; + } + this._at = to; + const rate = this.rateMbps(to); + if (rate !== null) this._tokens = Math.min(this._tokens, burstBytes(rate)); + } + + /** Seconds from wall-clock `now` until `deficit` more bytes have accumulated. */ + _secondsUntil(deficit, now) { + const start = now - this._startedAt; + let t = start; + let owed = deficit; + for (let guard = 0; guard < 100000; guard++) { + const rate = this._rateAtTraceTime(t); + if (rate === null) return t - start; + const perSecond = bytesPerSecond(rate); + const boundary = this._nextBoundary(t); + const span = boundary - t; + if (!Number.isFinite(span) || !(span > 0) || perSecond * span >= owed) { + return t - start + owed / perSecond; + } + owed -= perSecond * span; + t = boundary; + } + return MAX_SLEEP_SECONDS; + } + + /** Trace time of the next rate change after `t`, or Infinity. */ + _nextBoundary(t) { + for (const point of this._points) { + if (point.time > t) return point.time; + } + const end = traceDuration(this._points); + return !this._hold && t < end ? end : Infinity; + } + + /** + * A stream stage that holds each chunk until the link can carry it. Chunks + * are split to the burst size first: a 64 KiB read waiting on a bucket + * that can only ever hold less would never leave. + */ + pace() { + const shaper = this; + return new Transform({ + transform(chunk, _encoding, done) { + (async () => { + let offset = 0; + while (offset < chunk.length) { + const rate = shaper.rateMbps(); + const size = rate === null ? chunk.length - offset : burstBytes(rate); + const piece = chunk.subarray(offset, offset + size); + await shaper.take(piece.length); + this.push(piece); + offset += piece.length; + } + })().then(() => done(), done); + } + }); + } +} + +module.exports = { TraceShaper, burstBytes, BURST_FLOOR_BYTES }; diff --git a/open4d/webclients/system/Server/study/store.js b/open4d/webclients/system/Server/study/store.js new file mode 100644 index 00000000..6cc04fe5 --- /dev/null +++ b/open4d/webclients/system/Server/study/store.js @@ -0,0 +1,360 @@ +'use strict'; + +/** + * Study sessions on disk: one directory per session, self-contained. + * + * //session.json who, what order, which trace, which path + * //trace.csv the trace exactly as uploaded + * //trajectory.json the camera path every method replays + * //trials/-