diff --git a/.github/workflows/sidecar-release.yml b/.github/workflows/sidecar-release.yml new file mode 100644 index 0000000..957165e --- /dev/null +++ b/.github/workflows/sidecar-release.yml @@ -0,0 +1,37 @@ +name: Sidecar release gate + +on: + pull_request: + paths: + - "sidecar/**" + - "src/lib/mocap/**" + - "src/app/api/mocap/**/sidecar/**" + - "src/app/mocap/page.tsx" + - "tests/sidecar*.test.ts" + - "tests/freemocapSidecarSource.test.ts" + - "tests/e2e/mocap-capture.spec.ts" + - "package.json" + - "package-lock.json" + - ".github/workflows/sidecar-release.yml" + workflow_dispatch: + +jobs: + sidecar: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "22.12.0" + cache: npm + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - run: npm ci + - run: python -m pip install --upgrade pip + - run: python -m pip install -e sidecar + - run: rowing-tracker-sidecar --help + - run: npm run test:sidecar diff --git a/docs/adr/0005-freemocap-sidecar-contract.md b/docs/adr/0005-freemocap-sidecar-contract.md index 6e196ac..fbb540a 100644 --- a/docs/adr/0005-freemocap-sidecar-contract.md +++ b/docs/adr/0005-freemocap-sidecar-contract.md @@ -70,7 +70,7 @@ interface FrameQuality { The sidecar is a local Python process exposing: - `ws://localhost:8765/pose-stream` — streams `KeypointFrame` JSON, one message per frame -- `GET http://localhost:8765/health` — returns `{ "status": "ready", "fps": 30, "cameras": 3, "schemaVersion": 2 }` +- `GET http://localhost:8765/health` — returns `{ "status": "ready", "fps": 30, "cameras": 3, "schemaVersion": 2, "contractVersion": 1, "sidecarVersion": "0.1.0" }` - `POST http://localhost:8765/session/start` — arms capture, returns `{ "sessionId": "", "calibrationId": "" }` - `POST http://localhost:8765/session/stop` — flushes, closes stream diff --git a/docs/sidecar-hardware-smoke.md b/docs/sidecar-hardware-smoke.md new file mode 100644 index 0000000..ad480b1 --- /dev/null +++ b/docs/sidecar-hardware-smoke.md @@ -0,0 +1,169 @@ +# FreeMoCap sidecar hardware smoke test + +This is a manual, opt-in smoke test for a real FreeMoCap camera rig connected +through the Rowing Tracker sidecar. It is hardware-gated and must not block +normal CI when a camera rig, calibration, or live provider is unavailable. + +The default `--source freemocap` path probes for the optional FreeMoCap Python +package and fails closed when no supported live API is available. The current +live hardware path is an injected provider loaded with `--freemocap-provider`. +Synthetic mode and recorded-data mode are useful for contract checks, but they +do not count as the real camera-rig smoke test. + +## Pass criteria + +- The sidecar is installed from this repository, not public PyPI. +- A live FreeMoCap provider reports a ready rig on localhost with + `schemaVersion = 2`, `contractVersion = 1`, a nonzero fps, the expected camera + count, and a real calibration id. +- Rowing Tracker records a short session from `/mocap` with **Multi-camera + sidecar** enabled. +- The stored capture is a sidecar capture: `source = sidecar`, + `capturePerspective = sidecar-3d`, `coordinateSpace = world-mm-3d`, and it has + pose frames. +- Replay opens from **View replay** and renders the captured sidecar session. +- Post-session analysis completes. `analysisMode` should be `pose-segmented` + for an unlinked mocap capture or `csv-aligned` after linking to a rowing + session. `strokeMetricCount` should be greater than zero; `faultCount` may be + zero for a clean or short smoke capture. +- Raw video, raw 3D keypoints, and reconstructed body geometry stay local unless + the operator explicitly uses a separate Rowing Tracker sharing/export path. + +## Preconditions + +- Python 3.10 through 3.12 for `sidecar[freemocap]`, matching current FreeMoCap + release metadata. +- Rowing Tracker dependencies installed and the app able to run locally. +- A calibrated FreeMoCap-compatible camera rig. +- An operator-provided provider factory for the sidecar, for example + `my_runtime_module:create_provider`. The provider owns camera discovery, + calibration readiness, `start_session`, `stop_session`, and frame capture + behind the stable ADR-0005 sidecar contract. +- Local port `8765` available, or `UserSettings.sidecarPort` configured to the + port used by the sidecar. + +## Install + +Install the repo-local sidecar package from the repository root: + +```bash +python3 -m venv .venv-sidecar +source .venv-sidecar/bin/activate +python -m pip install --upgrade pip +python -m pip install -e "sidecar[freemocap]" +rowing-tracker-sidecar --help +``` + +Do not use `pip install rowing-tracker-sidecar` unless a release note explicitly +says public publishing has happened. + +## Start the live sidecar + +Start the sidecar with the injected live provider: + +```bash +rowing-tracker-sidecar \ + --source freemocap \ + --freemocap-provider my_runtime_module:create_provider \ + --port 8765 +``` + +Leave the default local-only binding in place. Do not expose the sidecar port to +the network for this smoke test. + +Verify health from another shell: + +```bash +curl -s http://localhost:8765/health +``` + +Expected health shape: + +```json +{ + "status": "ready", + "fps": 30, + "cameras": 3, + "schemaVersion": 2, + "contractVersion": 1, + "sidecarVersion": "0.1.0", + "source": "freemocap", + "calibrationId": "rig-calibration-id" +} +``` + +Do not proceed if `status` is not `ready`, fps is zero, the camera count is +wrong, schema or contract versions are missing, or the calibration id is absent. + +## Record through Rowing Tracker + +1. Start the app locally: + + ```bash + npm run dev + ``` + +2. Open `http://localhost:3000/mocap` and sign in if needed. +3. Enable **Multi-camera sidecar**. +4. Wait for the app to show `Sidecar ready`. +5. Click **Start sidecar capture**. +6. Row a short controlled segment, usually 15 to 30 seconds. +7. Click **Stop**. +8. Open **View replay**. + +The capture must go through Rowing Tracker. A sidecar-only recording, a +FreeMoCap GUI recording, or a recorded-data replay is not enough to pass this +hardware smoke test. + +## Verify replay and analysis + +Use the replay page and session details to verify: + +- The replay opens without sidecar connection errors. +- The capture is not `record-only`. +- Session metadata identifies a sidecar capture with `sidecar-3d` perspective, + schema v2, the expected camera count, and the provider calibration id. +- The pose stream contains frames for the rowed interval. +- Analysis completes with `analysisMode = pose-segmented` for an unlinked + capture, or `analysisMode = csv-aligned` if the capture is linked to a rowing + session. +- `strokeMetricCount > 0`. `faultCount = 0` is acceptable if no posture fault is + detected. +- Reanalysis, if manually triggered from the session page, completes without + changing the capture out of sidecar mode. + +Record the sidecar command, provider identifier, sanitized `/health` response, +Rowing Tracker session id, replay result, and analysis summary in the issue or +release checklist. Do not attach raw video, raw frame dumps, or full 3D pose +streams to public issues. + +## Privacy expectations + +- The sidecar should bind to `127.0.0.1` by default. +- The sidecar should make no cloud calls. +- Rowing Tracker capture and replay should keep raw video and raw 3D pose frames + local to the app storage configured for the operator. +- Cloud AI is not part of this smoke test. Do not enable or verify cloud AI with + raw frames or reconstructable body geometry. +- If an operator later enables detailed AI sharing, only the documented Rowing + Tracker AI payload tiers apply; that is separate from this hardware smoke + procedure. + +## CI status + +This procedure is not an automated CI gate. CI should continue to use synthetic +and deterministic contract tests for hardware-free coverage. Mark the hardware +smoke test as skipped when no real rig and provider are available; do not fail a +normal PR or release check solely because the hardware smoke test was skipped. + +An opt-in Playwright helper exercises the same Rowing Tracker UI path when a +ready sidecar is already running on port `8765`: + +```bash +ROWING_TRACKER_SIDECAR_HARDWARE_SMOKE=1 npm run test:e2e:sidecar-smoke +``` + +By default the helper is skipped. For local wiring checks without hardware, it +can be run against the synthetic sidecar with +`ROWING_TRACKER_SIDECAR_HARDWARE_ALLOW_SYNTHETIC=1`; that does not count as the +real camera-rig smoke result. diff --git a/docs/sidecar-local-setup.md b/docs/sidecar-local-setup.md index ba9731a..fe763bc 100644 --- a/docs/sidecar-local-setup.md +++ b/docs/sidecar-local-setup.md @@ -13,6 +13,10 @@ unless a release note says public publishing has happened. - Python 3.10+ with `venv` - The app running locally with `npm run dev` +The optional `sidecar[freemocap]` extra follows current FreeMoCap releases and +requires Python 3.10 through 3.12. Synthetic mode and recorded-data mode can run +without that extra. + ## Install from this repository ```bash @@ -49,6 +53,8 @@ curl http://localhost:8765/health "fps": 30.0, "cameras": 3, "schemaVersion": 2, + "contractVersion": 1, + "sidecarVersion": "0.1.0", "source": "synthetic", "calibrationId": "synthetic-calibration" } @@ -63,7 +69,7 @@ against real `world-mm-3d` coordinates before a live camera runtime is wired in. ```bash rowing-tracker-sidecar \ --source freemocap \ - --freemocap-data /path/to/freemocap/output/mediapipe_body_3d_xyz.npy \ + --freemocap-data /path/to/freemocap/output/skeleton_3d.npy \ --camera-count 3 \ --fps 30 \ --port 8765 @@ -76,13 +82,79 @@ Supported input formats: - `.npy` containing `(frames, 33, 4)` FreeMoCap body keypoints; this requires `numpy` in the sidecar environment - a directory containing a known FreeMoCap output file such as + `skeleton_3d.npy`, `3dData_numFrames_numTrackedPoints_spatialXYZ.npy`, or `mediapipe_body_3d_xyz.npy` -If you select `--source freemocap` without `--freemocap-data`, health reports -`status: "error"` with diagnostics. That failure is intentional: the current -repo-owned sidecar has a stable adapter boundary for FreeMoCap data, while live -camera capture depends on the FreeMoCap runtime available in the user's -environment. +When a directory also contains +`3dData_numFrames_numTrackedPoints_reprojectionError.npy` or +`3dData_numCams_numFrames_numTrackedPoints_reprojectionError.npy`, the sidecar +maps the per-frame finite mean into `quality.reprojection_error_mm`. +Rowing Tracker preserves that signal in captured v2 frame quality flags as +`high-reprojection-error` when it exceeds 50 mm. + +## FreeMoCap headless recording-processing mode + +If you have a FreeMoCap recording folder with synchronized videos, the sidecar +can ask the optional FreeMoCap package to process that folder headlessly before +streaming the generated `output_data` through the ADR-0005 contract. This uses +FreeMoCap's batch/headless processor; it does not open the FreeMoCap GUI and it +does not turn the sidecar into a live camera runtime. + +Install the optional FreeMoCap extra in the sidecar environment: + +```bash +python -m pip install -e "sidecar[freemocap]" +``` + +Then run: + +```bash +rowing-tracker-sidecar \ + --source freemocap \ + --freemocap-recording /path/to/freemocap/recording-folder \ + --freemocap-calibration-toml /path/to/calibration.toml \ + --camera-count 3 \ + --fps 30 \ + --port 8765 +``` + +The sidecar disables Blender export, notebook generation, and tqdm progress for +this path. While processing runs, `/health` reports `status: "initializing"` +with a `processing FreeMoCap recording ...` diagnostic and `/session/start` +returns `409`. On success, health diagnostics include `processed FreeMoCap +recording ...`, then the generated `output_data/skeleton_3d.npy` or raw 3D +output is loaded exactly like recorded-data mode. If processing fails, `/health` +reports `status: "error"` with the FreeMoCap processing exception and the +recording is not armed. + +If you select `--source freemocap` without `--freemocap-data` or +`--freemocap-recording` or `--freemocap-provider`, the sidecar probes for the +optional FreeMoCap Python package and then fails closed with `status: "error"`. +Missing-package diagnostics explain how to install `sidecar[freemocap]`; +installed-package diagnostics explain that upstream FreeMoCap exposes +batch/headless processing and recorded outputs, but not a supported one-call +live calibrated camera API for this sidecar to invoke directly. + +## FreeMoCap live-runtime provider hook + +The sidecar can load a live runtime adapter with `--freemocap-provider`: + +```bash +rowing-tracker-sidecar \ + --source freemocap \ + --freemocap-provider my_runtime_module:create_provider \ + --port 8765 +``` + +The provider factory must return an object that owns real rig health, +`start_session`, `stop_session`, and frame capture. The sidecar keeps the +localhost HTTP/WebSocket contract stable while the provider owns camera/runtime +details. If the provider cannot be loaded or the runtime cannot start/stop, the +sidecar reports structured diagnostics instead of pretending capture is ready. + +The repository includes fake-provider contract coverage for this boundary. +Treat SkellyCam, SkellyTracker, calibration, and live triangulation glue as +provider-owned runtime code until upstream exposes a stable live API. ## Using the sidecar in the app @@ -107,6 +179,12 @@ comes from the sidecar's `calibrationId`. | `POST` | `/session/stop` | Stop capture and flush/close the stream | | `WS` | `/pose-stream` | Stream one schema-v2 `sidecar-3d` keypoint frame per message | +If `/session/start` refuses capture with `409`, the response repeats the +readiness fields from `/health`: `status`, `fps`, `cameras`, `schemaVersion`, +`contractVersion`, `sidecarVersion`, `source`, `calibrationId`, and +`diagnostics`. Use that body in logs and bug reports; it is the source of truth +for why the sidecar would not arm capture. + The app also proxies lifecycle calls through: | Method | Path | Purpose | @@ -116,8 +194,23 @@ The app also proxies lifecycle calls through: | `POST` | `/api/mocap/sessions/:id/sidecar/stop` | Stop the sidecar session | All app routes require an authenticated session and an owned `MocapSession`. -`connect` also requires the app-side session to be in `capturing` status and the -sidecar schema to match `keypointSchemaVersion = 2`. +`connect` also requires the app-side session to be in `capturing` status, the +sidecar schema to match `keypointSchemaVersion = 2`, and the sidecar contract to +match `contractVersion = 1`. + +## Version compatibility + +`/health` reports both the frame schema and sidecar package contract: + +- `schemaVersion`: pose-frame payload schema. Rowing Tracker currently accepts + only `2` for `sidecar-3d`. +- `contractVersion`: HTTP/WebSocket contract version. Rowing Tracker currently + accepts only `1`. +- `sidecarVersion`: installed Python package version, stored in new app-side + captures as part of `captureModelVersion`. + +If any of these fields are missing or incompatible, the app refuses capture +before the Start button is enabled. ## Troubleshooting @@ -127,11 +220,17 @@ sidecar schema to match `keypointSchemaVersion = 2`. and confirm `curl http://localhost:8765/health` works. - **Wrong port**: run the sidecar with `--port ` and configure `UserSettings.sidecarPort` to the same value. -- **`status: "error"` with FreeMoCap diagnostics**: the FreeMoCap source is not - configured or the data path cannot be read. Pass `--freemocap-data`. +- **`status: "error"` with FreeMoCap diagnostics**: read the first diagnostic + line. The optional package may be missing, the installed package may not + expose a supported live API, or the data path may be unreadable. Use + `--freemocap-data` for recorded output or `--freemocap-provider` for a live + camera adapter. - **Incompatible schema**: Rowing Tracker expects schema version `2`. - **Missing cameras or calibration**: health should remain `initializing` or `error`; do not start capture until it is `ready`. +- **`/session/start` returns `409`**: inspect the response body. It mirrors the + health payload so camera count, fps, source, version, calibration, and + diagnostic details are still visible even when capture cannot arm. - **Low or zero fps**: reduce camera load, check USB bandwidth, and verify the source reports a stable fps before recording. - **Stream errors during capture**: stop the app capture, stop the sidecar, and @@ -140,19 +239,11 @@ sidecar schema to match `keypointSchemaVersion = 2`. ## Hardware-gated smoke test -Use this manual path when a real camera rig and FreeMoCap output are available: - -1. Capture or locate a FreeMoCap `(frames, 33, 4)` output file. -2. Start the sidecar in recorded-data mode with that file. -3. Confirm `/health` is `ready`, `schemaVersion` is `2`, and `cameras` matches - the rig. -4. Record a Rowing Tracker sidecar session from `/mocap`. -5. Stop capture and open the replay. -6. Confirm the stored session uses `source=sidecar`, - `capturePerspective=sidecar-3d`, has pose frames, and runs post-session - analysis. - -This smoke test is hardware/data-gated and should not block normal CI. +For the manual, opt-in real camera-rig procedure, see +[FreeMoCap sidecar hardware smoke test](sidecar-hardware-smoke.md). It uses the +current injected `--freemocap-provider` live path, records through Rowing +Tracker, verifies replay and analysis output, and must not block normal CI when +hardware is unavailable. ## Privacy and licensing @@ -166,6 +257,7 @@ dependency boundary before distributing bundled artifacts. ## Tests ```bash +npm run test:sidecar npx tsx --test tests/sidecarCliContract.test.ts npx tsx --test tests/sidecarPackageInstall.test.ts npx tsx --test tests/sidecarTracer.test.ts @@ -173,3 +265,7 @@ npx tsx --test tests/freemocapSidecarSource.test.ts npx tsx --test tests/sidecarMockContract.test.ts npm run test:e2e -- tests/e2e/mocap-capture.spec.ts ``` + +The GitHub Actions `Sidecar release gate` workflow runs the install/startup +check and `npm run test:sidecar` for PRs that touch the sidecar contract, +package, app-side sidecar client, or focused sidecar tests. diff --git a/package.json b/package.json index 0bc00a5..4a8345e 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,9 @@ "db:reset": "npx prisma migrate reset", "admin:promote": "npx tsx scripts/promote-admin.ts", "test": "npx tsx --test tests/*.test.ts", + "test:sidecar": "npx tsx --test tests/sidecarPackageInstall.test.ts tests/sidecarCliContract.test.ts tests/sidecarPoseSource.test.ts tests/freemocapSidecarSource.test.ts tests/sidecarMockContract.test.ts tests/sidecarTracer.test.ts", "test:e2e": "npx playwright test", + "test:e2e:sidecar-smoke": "npx playwright test tests/e2e/sidecar-hardware-smoke.spec.ts", "preinstall": "node scripts/check-node-version.mjs", "postinstall": "prisma generate" }, diff --git a/sidecar/README.md b/sidecar/README.md index d0e8f7d..83cea06 100644 --- a/sidecar/README.md +++ b/sidecar/README.md @@ -23,12 +23,34 @@ Run recorded FreeMoCap-style output through the same live contract: ```bash rowing-tracker-sidecar \ --source freemocap \ - --freemocap-data /path/to/mediapipe_body_3d_xyz.npy \ + --freemocap-data /path/to/skeleton_3d.npy \ --camera-count 3 \ --fps 30 \ --port 8765 ``` +Process an existing FreeMoCap recording folder headlessly, then stream the +generated `output_data` through the same contract: + +```bash +python -m pip install -e "sidecar[freemocap]" +rowing-tracker-sidecar \ + --source freemocap \ + --freemocap-recording /path/to/freemocap/recording-folder \ + --freemocap-calibration-toml /path/to/calibration.toml \ + --camera-count 3 \ + --fps 30 \ + --port 8765 +``` + +This uses FreeMoCap's batch/headless recording processor with Blender export, +notebook generation, and tqdm progress disabled. It does not claim live +camera-runtime support; use `--freemocap-provider` for operator-owned live +camera adapters. While processing runs, `/health` reports +`status: "initializing"` and `/session/start` returns `409`; successful +processing changes health to `ready`, and processing failures change health to +`error` with diagnostics. + The service binds to `127.0.0.1` by default and exposes: - `GET /health` @@ -36,9 +58,46 @@ The service binds to `127.0.0.1` by default and exposes: - `POST /session/stop` - `ws://localhost:/pose-stream` +`GET /health` includes `schemaVersion`, `contractVersion`, and +`sidecarVersion`. The app currently accepts schema version `2` and contract +version `1`; missing or incompatible versions are rejected before capture +starts. +When `POST /session/start` returns `409`, its JSON body repeats the same health +fields plus diagnostics so operators can see the camera count, fps, source, +calibration id, and version state that blocked capture. + Synthetic mode is hardware-free and intended for development and CI. FreeMoCap recorded-data mode supports JSON, JSONL, and NPY `(frames, 33, 4)` sources. -Selecting `--source freemocap` without `--freemocap-data` fails readiness with a -clear diagnostic instead of pretending the camera rig is ready. +Directory mode also loads known FreeMoCap reprojection-error NPY files and maps +finite per-frame means into `quality.reprojection_error_mm` when present. +Rowing Tracker records values above 50 mm as `high-reprojection-error` quality +flags on the captured v2 frames. +Selecting `--source freemocap` without `--freemocap-data`, +`--freemocap-recording`, or `--freemocap-provider` fails readiness with a clear +diagnostic instead of pretending the camera rig is ready. The default FreeMoCap +runtime path probes for the optional `freemocap` package, but current upstream +FreeMoCap releases expose stable batch/headless processing and recorded outputs, +not a supported one-call live calibrated camera API for this sidecar to invoke +directly. + +Advanced live runtime adapters can be loaded with: + +```bash +rowing-tracker-sidecar \ + --source freemocap \ + --freemocap-provider my_runtime_module:create_provider +``` + +The provider owns real camera health, session start/stop, calibration ids, and +frame capture behind the stable ADR-0005 sidecar contract. Treat SkellyCam, +SkellyTracker, calibration, and live triangulation glue as provider-owned +runtime code until upstream exposes a stable live API. + +Run the focused release gate before changing package or contract behavior: + +```bash +npm run test:sidecar +``` -The sidecar makes no cloud calls and is local-only by default. +The sidecar makes no cloud calls and is local-only by default. Keep FreeMoCap as +a separate local process when adding live camera runtime support. diff --git a/sidecar/src/rowing_tracker_sidecar/cli.py b/sidecar/src/rowing_tracker_sidecar/cli.py index fdeb988..36a4c2f 100644 --- a/sidecar/src/rowing_tracker_sidecar/cli.py +++ b/sidecar/src/rowing_tracker_sidecar/cli.py @@ -23,10 +23,23 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--fps", type=float, default=30.0) parser.add_argument("--camera-count", type=int, default=3) parser.add_argument("--calibration-id") - parser.add_argument( + freemocap_input = parser.add_mutually_exclusive_group() + freemocap_input.add_argument( "--freemocap-data", help="FreeMoCap-style JSON/JSONL/NPY data file or output directory.", ) + freemocap_input.add_argument( + "--freemocap-recording", + help="FreeMoCap recording folder to process headlessly before streaming output_data.", + ) + parser.add_argument( + "--freemocap-calibration-toml", + help="Calibration TOML passed to FreeMoCap headless recording processing.", + ) + parser.add_argument( + "--freemocap-provider", + help="Advanced live FreeMoCap runtime provider as module:factory.", + ) parser.add_argument( "--log-level", default="INFO", @@ -47,6 +60,9 @@ def main(argv: list[str] | None = None) -> int: camera_count=args.camera_count, calibration_id=args.calibration_id, freemocap_data=args.freemocap_data, + freemocap_recording=args.freemocap_recording, + freemocap_calibration_toml=args.freemocap_calibration_toml, + freemocap_provider=args.freemocap_provider, ) server = SidecarServer(args.host, args.port, source) try: diff --git a/sidecar/src/rowing_tracker_sidecar/contract.py b/sidecar/src/rowing_tracker_sidecar/contract.py index 727f6b7..64b9320 100644 --- a/sidecar/src/rowing_tracker_sidecar/contract.py +++ b/sidecar/src/rowing_tracker_sidecar/contract.py @@ -3,8 +3,11 @@ from dataclasses import dataclass from typing import Literal, TypedDict +from . import __version__ + KEYPOINT_SCHEMA_VERSION = 2 KEYPOINT_COUNT = 33 +SIDECAR_CONTRACT_VERSION = 1 class Keypoint(TypedDict): @@ -37,6 +40,8 @@ class SourceHealth: fps: float camera_count: int schema_version: int = KEYPOINT_SCHEMA_VERSION + contract_version: int = SIDECAR_CONTRACT_VERSION + sidecar_version: str = __version__ source: str = "synthetic" calibration_id: str | None = None diagnostics: tuple[str, ...] = () @@ -47,6 +52,8 @@ def to_json(self) -> dict[str, object]: "fps": self.fps, "cameras": self.camera_count, "schemaVersion": self.schema_version, + "contractVersion": self.contract_version, + "sidecarVersion": self.sidecar_version, "source": self.source, "calibrationId": self.calibration_id, "diagnostics": list(self.diagnostics), diff --git a/sidecar/src/rowing_tracker_sidecar/server.py b/sidecar/src/rowing_tracker_sidecar/server.py index cf9a7c0..9f9b98f 100644 --- a/sidecar/src/rowing_tracker_sidecar/server.py +++ b/sidecar/src/rowing_tracker_sidecar/server.py @@ -35,39 +35,67 @@ def start(self) -> tuple[int, dict[str, object]]: if health.status != "ready": return ( HTTPStatus.CONFLICT, - { - "status": health.status, - "schemaVersion": health.schema_version, - "diagnostics": list(health.diagnostics), - }, + health.to_json(), ) if health.schema_version != KEYPOINT_SCHEMA_VERSION: + body = health.to_json() + body["status"] = "incompatible-schema" + body["expectedSchemaVersion"] = KEYPOINT_SCHEMA_VERSION return ( HTTPStatus.CONFLICT, - { - "status": "incompatible-schema", - "schemaVersion": health.schema_version, - "expectedSchemaVersion": KEYPOINT_SCHEMA_VERSION, - }, + body, ) - self.session_id = str(uuid.uuid4()) + try: + source_session = self.source.start_session(str(uuid.uuid4())) + except Exception as exc: + LOGGER.exception("sidecar source failed to start") + body = health.to_json() + body["status"] = "error" + diagnostics = list(body.get("diagnostics", ())) + diagnostics.append(f"Failed to start source session: {exc}") + body["diagnostics"] = diagnostics + self.active = False + self.session_id = None + return ( + HTTPStatus.CONFLICT, + body, + ) + self.session_id = source_session.session_id self.active = True LOGGER.info("sidecar session started: %s", self.session_id) return ( HTTPStatus.OK, { "sessionId": self.session_id, - "calibrationId": self.source.calibration_id, + "calibrationId": source_session.calibration_id, }, ) - def stop(self) -> dict[str, object]: + def stop(self) -> tuple[int, dict[str, object]]: with self.lock: stopped_id = self.session_id + if stopped_id is not None: + try: + self.source.stop_session(stopped_id) + except Exception as exc: + LOGGER.exception("sidecar source failed to stop") + return ( + HTTPStatus.CONFLICT, + { + "status": "error", + "sessionId": stopped_id, + "diagnostics": [ + f"Failed to stop source session: {exc}", + ], + }, + ) self.active = False self.session_id = None LOGGER.info("sidecar session stopped: %s", stopped_id) - return {"status": "stopped", "sessionId": stopped_id} + return ( + HTTPStatus.OK, + {"status": "stopped", "sessionId": stopped_id}, + ) def is_active(self) -> bool: with self.lock: @@ -133,8 +161,8 @@ def do_POST(self) -> None: self._send_json(status, body) return if self.path == "/session/stop": - body = self.sidecar_state.stop() - self._send_json(HTTPStatus.OK, body) + status, body = self.sidecar_state.stop() + self._send_json(status, body) return self._send_json(HTTPStatus.NOT_FOUND, {"error": f"Unknown path: {self.path}"}) diff --git a/sidecar/src/rowing_tracker_sidecar/sources.py b/sidecar/src/rowing_tracker_sidecar/sources.py index c3c4e55..964daf0 100644 --- a/sidecar/src/rowing_tracker_sidecar/sources.py +++ b/sidecar/src/rowing_tracker_sidecar/sources.py @@ -1,15 +1,40 @@ from __future__ import annotations +import importlib +import importlib.metadata +import importlib.util import itertools import json import math import os import pathlib +import threading +from dataclasses import dataclass from typing import Protocol from .contract import KEYPOINT_COUNT, KEYPOINT_SCHEMA_VERSION, SidecarFrame, SourceHealth +@dataclass(frozen=True) +class RecordedFrame: + keypoints: list[list[float]] + reprojection_error_mm: float | None = None + + +@dataclass(frozen=True) +class SourceSession: + session_id: str + calibration_id: str | None + + +@dataclass(frozen=True) +class RuntimeFrame: + keypoints: list[list[float]] + timestamp_ms: int | None + reprojection_error_mm: float | None = None + camera_count: int | None = None + + class FrameSource(Protocol): @property def fps(self) -> float: ... @@ -22,9 +47,23 @@ def calibration_id(self) -> str | None: ... def health(self) -> SourceHealth: ... + def start_session(self, session_id: str) -> SourceSession: ... + + def stop_session(self, session_id: str) -> None: ... + def frame(self, frame_index: int, timestamp_ms: int) -> SidecarFrame: ... +class FreeMocapRuntimeProvider(Protocol): + def health(self) -> SourceHealth: ... + + def start_session(self, session_id: str) -> SourceSession: ... + + def stop_session(self, session_id: str) -> None: ... + + def frame(self) -> RuntimeFrame: ... + + def build_frame_source( *, source: str, @@ -32,6 +71,9 @@ def build_frame_source( camera_count: int, calibration_id: str | None, freemocap_data: str | None, + freemocap_recording: str | None = None, + freemocap_calibration_toml: str | None = None, + freemocap_provider: str | None = None, ) -> FrameSource: if source == "synthetic": return SyntheticFrameSource( @@ -40,11 +82,27 @@ def build_frame_source( calibration_id=calibration_id or "synthetic-calibration", ) if source == "freemocap": - return FreeMocapRecordedSource( - data_path=freemocap_data, - fps=fps, - camera_count=camera_count, - calibration_id=calibration_id, + if freemocap_data: + return FreeMocapRecordedSource( + data_path=freemocap_data, + fps=fps, + camera_count=camera_count, + calibration_id=calibration_id, + ) + if freemocap_recording: + return FreeMocapProcessedRecordingSource( + recording_path=freemocap_recording, + calibration_toml_path=freemocap_calibration_toml, + fps=fps, + camera_count=camera_count, + calibration_id=calibration_id, + ) + return FreeMocapRuntimeSource( + provider=_load_freemocap_runtime_provider( + freemocap_provider, + fps=fps, + calibration_id=calibration_id, + ), ) raise ValueError(f"Unsupported sidecar source: {source}") @@ -95,6 +153,15 @@ def health(self) -> SourceHealth: diagnostics=("synthetic source ready",), ) + def start_session(self, session_id: str) -> SourceSession: + return SourceSession( + session_id=session_id, + calibration_id=self.calibration_id, + ) + + def stop_session(self, session_id: str) -> None: + return None + def frame(self, frame_index: int, timestamp_ms: int) -> SidecarFrame: hip_knee_depth = _stroke_depth(frame_index) keypoints = [_untracked_point(index) for index in range(KEYPOINT_COUNT)] @@ -148,7 +215,7 @@ def __init__( self._fps = fps self._camera_count = camera_count self._data_path = pathlib.Path(data_path).expanduser() if data_path else None - self._frames: list[list[list[float]]] = [] + self._frames: list[RecordedFrame] = [] self._diagnostics: tuple[str, ...] = () self._calibration_id = calibration_id self._load() @@ -197,39 +264,25 @@ def health(self) -> SourceHealth: diagnostics=self._diagnostics, ) + def start_session(self, session_id: str) -> SourceSession: + return SourceSession( + session_id=session_id, + calibration_id=self.calibration_id, + ) + + def stop_session(self, session_id: str) -> None: + return None + def frame(self, frame_index: int, timestamp_ms: int) -> SidecarFrame: if not self._frames: raise RuntimeError("FreeMoCap source is not ready") raw = self._frames[frame_index % len(self._frames)] - keypoints = [] - confidences = [] - for index, values in enumerate(raw): - x, y, z, confidence = values - confidences.append(confidence) - keypoints.append( - { - "index": index, - "x": float(x), - "y": float(y), - "z": float(z), - "confidence": float(confidence), - }, - ) - tracked = sum(1 for confidence in confidences if confidence >= 0.5) - mean_confidence = sum(confidences) / len(confidences) - return { - "schema_version": KEYPOINT_SCHEMA_VERSION, - "frame_index": frame_index, - "timestamp_ms": timestamp_ms, - "source": "sidecar-3d", - "keypoints": keypoints, - "quality": { - "tracked_count": tracked, - "mean_confidence": mean_confidence, - "reprojection_error_mm": 0.0, - "camera_count": self.camera_count, - }, - } + return _sidecar_frame_from_keypoints( + raw, + frame_index=frame_index, + timestamp_ms=timestamp_ms, + camera_count=self.camera_count, + ) def _load(self) -> None: if self._data_path is None: @@ -242,6 +295,10 @@ def _load(self) -> None: if path is None or not path.exists(): self._diagnostics = (f"FreeMoCap data path not found: {self._data_path}",) return + reprojection_error_path = _resolve_freemocap_reprojection_path( + requested_path=self._data_path, + resolved_data_path=path, + ) try: if path.suffix.lower() in {".json", ".jsonl"}: frames = _load_json_frames(path) @@ -249,21 +306,370 @@ def _load(self) -> None: frames = _load_npy_frames(path) else: raise ValueError(f"Unsupported FreeMoCap data file: {path}") + if reprojection_error_path is not None: + frames = _attach_reprojection_errors(frames, reprojection_error_path) _validate_recorded_frames(frames) self._frames = frames self._calibration_id = self._calibration_id or f"freemocap-{path.stem}" - self._diagnostics = (f"loaded FreeMoCap data from {path}",) + diagnostics = [f"loaded FreeMoCap data from {path}"] + if reprojection_error_path is not None: + diagnostics.append( + f"loaded FreeMoCap reprojection quality from {reprojection_error_path}", + ) + self._diagnostics = tuple(diagnostics) except Exception as exc: self._frames = [] self._diagnostics = (f"Failed to load FreeMoCap data: {exc}",) +class FreeMocapProcessedRecordingSource(FreeMocapRecordedSource): + def __init__( + self, + *, + recording_path: str, + calibration_toml_path: str | None, + fps: float, + camera_count: int, + calibration_id: str | None, + ) -> None: + self._recording_path = pathlib.Path(recording_path).expanduser() + self._calibration_toml_path = ( + pathlib.Path(calibration_toml_path).expanduser() + if calibration_toml_path + else None + ) + self._fps = fps + self._camera_count = camera_count + self._data_path = _freemocap_output_data_path(self._recording_path) + self._frames: list[RecordedFrame] = [] + self._diagnostics: tuple[str, ...] = ( + f"processing FreeMoCap recording {self._recording_path}", + ) + self._calibration_id = calibration_id or _processed_recording_calibration_id( + self._recording_path, + self._calibration_toml_path, + ) + self._status = "initializing" + self._lock = threading.RLock() + self._worker = threading.Thread( + target=self._process_and_load, + name="freemocap-recording-processor", + daemon=True, + ) + self._worker.start() + + def health(self) -> SourceHealth: + invalid = _invalid_runtime_diagnostics(self.fps, self.camera_count) + if invalid: + return SourceHealth( + status="error", + fps=self.fps, + camera_count=max(0, self.camera_count), + source=self.source_name, + calibration_id=self.calibration_id, + diagnostics=invalid, + ) + with self._lock: + status = self._status + diagnostics = self._diagnostics + frame_count = len(self._frames) + calibration_id = self._calibration_id + if status == "ready" and frame_count > 0: + return SourceHealth( + status="ready", + fps=self.fps, + camera_count=self.camera_count, + source=self.source_name, + calibration_id=calibration_id, + diagnostics=diagnostics, + ) + if status == "initializing": + return SourceHealth( + status="initializing", + fps=self.fps, + camera_count=max(0, self.camera_count), + source=self.source_name, + calibration_id=calibration_id, + diagnostics=diagnostics, + ) + return SourceHealth( + status="error", + fps=self.fps, + camera_count=max(0, self.camera_count if frame_count else 0), + source=self.source_name, + calibration_id=calibration_id, + diagnostics=diagnostics, + ) + + def _process_and_load(self) -> None: + if not self._recording_path.exists(): + self._set_processing_result( + "error", + f"FreeMoCap recording path not found: {self._recording_path}", + ) + return + if not self._recording_path.is_dir(): + self._set_processing_result( + "error", + f"FreeMoCap recording path must be a directory: {self._recording_path}", + ) + return + if self._calibration_toml_path and not self._calibration_toml_path.exists(): + self._set_processing_result( + "error", + f"FreeMoCap calibration TOML not found: {self._calibration_toml_path}", + ) + return + try: + _process_freemocap_recording( + recording_path=self._recording_path, + calibration_toml_path=self._calibration_toml_path, + ) + except Exception as exc: + self._set_processing_result( + "error", + f"Failed to process FreeMoCap recording {self._recording_path}: {exc}", + 'install with python -m pip install -e "sidecar[freemocap]" on Python 3.10-3.12, ' + "or pass --freemocap-data with an already processed FreeMoCap output directory", + ) + return + + loaded = FreeMocapRecordedSource( + data_path=os.fspath(self._data_path), + fps=self._fps, + camera_count=self._camera_count, + calibration_id=self._calibration_id, + ) + with self._lock: + self._frames = loaded._frames + self._calibration_id = loaded.calibration_id + if self._frames: + self._status = "ready" + self._diagnostics = ( + f"processed FreeMoCap recording {self._recording_path}", + *loaded._diagnostics, + ) + else: + self._status = "error" + self._diagnostics = loaded._diagnostics + + def _set_processing_result( + self, + status: str, + *diagnostics: str, + ) -> None: + with self._lock: + self._frames = [] + self._status = status + self._diagnostics = tuple(diagnostics) + + +class FreeMocapRuntimeSource: + source_name = "freemocap" + + def __init__(self, *, provider: FreeMocapRuntimeProvider) -> None: + self._provider = provider + + @property + def fps(self) -> float: + return self.health().fps + + @property + def camera_count(self) -> int: + return self.health().camera_count + + @property + def calibration_id(self) -> str | None: + return self.health().calibration_id + + def health(self) -> SourceHealth: + health = self._provider.health() + return SourceHealth( + status=health.status, + fps=health.fps, + camera_count=health.camera_count, + schema_version=health.schema_version, + contract_version=health.contract_version, + sidecar_version=health.sidecar_version, + source=self.source_name, + calibration_id=health.calibration_id, + diagnostics=health.diagnostics, + ) + + def start_session(self, session_id: str) -> SourceSession: + return self._provider.start_session(session_id) + + def stop_session(self, session_id: str) -> None: + self._provider.stop_session(session_id) + + def frame(self, frame_index: int, timestamp_ms: int) -> SidecarFrame: + raw = self._provider.frame() + if not isinstance(raw.timestamp_ms, int): + raise ValueError("runtime frame timestamp_ms must be an integer") + return _sidecar_frame_from_keypoints( + RecordedFrame( + keypoints=raw.keypoints, + reprojection_error_mm=raw.reprojection_error_mm, + ), + frame_index=frame_index, + timestamp_ms=raw.timestamp_ms, + camera_count=raw.camera_count or self.camera_count, + ) + + +class UnavailableFreeMocapRuntimeProvider: + def __init__( + self, + *, + fps: float, + calibration_id: str | None, + diagnostics: tuple[str, ...] | None = None, + ) -> None: + self._fps = fps + self._calibration_id = calibration_id + self._diagnostics = diagnostics or ( + "FreeMoCap live camera runtime is not configured in this build", + "pass --freemocap-provider module:factory for a live runtime adapter or --freemocap-data for recorded output", + ) + + def health(self) -> SourceHealth: + return SourceHealth( + status="error", + fps=self._fps, + camera_count=0, + source="freemocap", + calibration_id=self._calibration_id, + diagnostics=self._diagnostics, + ) + + def start_session(self, session_id: str) -> SourceSession: + raise RuntimeError(self._diagnostics[0]) + + def stop_session(self, session_id: str) -> None: + return None + + def frame(self) -> RuntimeFrame: + raise RuntimeError(self._diagnostics[0]) + + +class FreeMocapPackageRuntimeProvider(UnavailableFreeMocapRuntimeProvider): + def __init__( + self, + *, + fps: float, + calibration_id: str | None, + ) -> None: + super().__init__( + fps=fps, + calibration_id=calibration_id, + diagnostics=_detect_freemocap_package_runtime_diagnostics(), + ) + + +def _load_freemocap_runtime_provider( + provider_spec: str | None, + *, + fps: float, + calibration_id: str | None, +) -> FreeMocapRuntimeProvider: + if provider_spec is None: + return FreeMocapPackageRuntimeProvider( + fps=fps, + calibration_id=calibration_id, + ) + module_name, separator, factory_name = provider_spec.partition(":") + if not separator or not module_name or not factory_name: + return UnavailableFreeMocapRuntimeProvider( + fps=fps, + calibration_id=calibration_id, + diagnostics=( + "FreeMoCap runtime provider must be formatted as module:factory", + ), + ) + try: + module = importlib.import_module(module_name) + factory = getattr(module, factory_name) + provider = factory() + except Exception as exc: + return UnavailableFreeMocapRuntimeProvider( + fps=fps, + calibration_id=calibration_id, + diagnostics=(f"Failed to load FreeMoCap runtime provider {provider_spec}: {exc}",), + ) + return provider + + +def _detect_freemocap_package_runtime_diagnostics() -> tuple[str, ...]: + package_spec = importlib.util.find_spec("freemocap") + if package_spec is None: + return ( + "FreeMoCap package is not installed in this sidecar environment", + 'install with python -m pip install -e "sidecar[freemocap]" on Python 3.10-3.12, ' + "or pass --freemocap-provider module:factory for an operator-owned live runtime adapter", + "pass --freemocap-recording to process a FreeMoCap recording folder headlessly", + "pass --freemocap-data to stream recorded FreeMoCap output through the ADR-0005 contract", + ) + + try: + version = importlib.metadata.version("freemocap") + except importlib.metadata.PackageNotFoundError: + version = "unknown version" + + return ( + f"FreeMoCap package {version} is installed, but it does not expose a supported headless live camera API", + "upstream FreeMoCap exposes stable batch/headless processing and recorded outputs; " + "live calibrated camera frames require an injected --freemocap-provider adapter", + "pass --freemocap-recording to process a FreeMoCap recording folder headlessly", + "pass --freemocap-data to stream recorded FreeMoCap output through the ADR-0005 contract", + ) + + +def _process_freemocap_recording( + *, + recording_path: pathlib.Path, + calibration_toml_path: pathlib.Path | None, +) -> None: + try: + from freemocap.core_processes.process_motion_capture_videos.process_recording_headless import ( # type: ignore + process_recording_headless, + ) + except Exception as exc: + raise RuntimeError( + "FreeMoCap headless recording processing requires the optional freemocap package", + ) from exc + + process_recording_headless( + recording_path=os.fspath(recording_path), + path_to_camera_calibration_toml=( + os.fspath(calibration_toml_path) if calibration_toml_path else None + ), + run_blender=False, + make_jupyter_notebook=False, + use_tqdm=False, + ) + + +def _freemocap_output_data_path(recording_path: pathlib.Path) -> pathlib.Path: + return recording_path / "output_data" + + +def _processed_recording_calibration_id( + recording_path: pathlib.Path, + calibration_toml_path: pathlib.Path | None, +) -> str: + if calibration_toml_path is not None: + return f"freemocap-{calibration_toml_path.stem}" + return f"freemocap-{recording_path.name}" + + def _resolve_freemocap_path(path: pathlib.Path) -> pathlib.Path | None: if path.is_file(): return path if not path.is_dir(): return path candidates = [ + "skeleton_3d.npy", + "3dData_numFrames_numTrackedPoints_spatialXYZ.npy", "mediapipe_body_3d_xyz.npy", "mediapipe_body_3d_xyz_confidence.npy", "body_3d_xyz.npy", @@ -279,7 +685,24 @@ def _resolve_freemocap_path(path: pathlib.Path) -> pathlib.Path | None: return None -def _load_json_frames(path: pathlib.Path) -> list[list[list[float]]]: +def _resolve_freemocap_reprojection_path( + *, + requested_path: pathlib.Path, + resolved_data_path: pathlib.Path, +) -> pathlib.Path | None: + search_dir = requested_path if requested_path.is_dir() else resolved_data_path.parent + candidates = [ + "3dData_numFrames_numTrackedPoints_reprojectionError.npy", + "3dData_numCams_numFrames_numTrackedPoints_reprojectionError.npy", + ] + for relative in candidates: + candidate = search_dir / relative + if candidate.exists() and candidate != resolved_data_path: + return candidate + return None + + +def _load_json_frames(path: pathlib.Path) -> list[RecordedFrame]: if path.suffix.lower() == ".jsonl": frames = [] for line in path.read_text().splitlines(): @@ -295,27 +718,48 @@ def _load_json_frames(path: pathlib.Path) -> list[list[list[float]]]: return [_extract_frame_values(frame) for frame in body] -def _extract_frame_values(frame: object) -> list[list[float]]: +def _extract_frame_values(frame: object) -> RecordedFrame: if isinstance(frame, dict) and "keypoints" in frame: keypoints = frame["keypoints"] if not isinstance(keypoints, list): raise ValueError("frame keypoints must be a list") - ordered = sorted(keypoints, key=lambda item: int(item["index"])) - return [ - [ - float(item["x"]), - float(item["y"]), - float(item["z"]), - float(item.get("confidence", 1.0)), - ] - for item in ordered + indexed: list[tuple[int, dict[str, object]]] = [] + for position, item in enumerate(keypoints): + if not isinstance(item, dict): + raise ValueError(f"frame keypoint {position} must be an object") + if "index" not in item: + raise ValueError(f"frame keypoint {position} is missing index") + indexed.append((int(item["index"]), item)) + indexes = sorted(index for index, _item in indexed) + if indexes != list(range(KEYPOINT_COUNT)): + raise ValueError( + f"keypoint indexes must contain 0 through {KEYPOINT_COUNT - 1} exactly once", + ) + ordered = [ + item for _index, item in sorted(indexed, key=lambda entry: entry[0]) ] + quality = frame.get("quality") + reprojection_error_mm = None + if isinstance(quality, dict) and quality.get("reprojection_error_mm") is not None: + reprojection_error_mm = float(quality["reprojection_error_mm"]) + return RecordedFrame( + keypoints=[ + [ + float(item["x"]), + float(item["y"]), + float(item["z"]), + float(item.get("confidence", 1.0)), + ] + for item in ordered + ], + reprojection_error_mm=reprojection_error_mm, + ) if isinstance(frame, list): - return [[float(value) for value in point] for point in frame] + return RecordedFrame(keypoints=[[float(value) for value in point] for point in frame]) raise ValueError("frame must be a keypoint frame object or a 33x4 array") -def _load_npy_frames(path: pathlib.Path) -> list[list[list[float]]]: +def _load_npy_frames(path: pathlib.Path) -> list[RecordedFrame]: try: import numpy as np # type: ignore except Exception as exc: @@ -325,26 +769,153 @@ def _load_npy_frames(path: pathlib.Path) -> list[list[list[float]]]: raise ValueError(f"expected NPY array with shape (frames, 33, 4), got {array.shape}") if array.shape[1] != KEYPOINT_COUNT or array.shape[2] < 3: raise ValueError(f"expected NPY array with shape (frames, 33, 4), got {array.shape}") - frames: list[list[list[float]]] = [] + frames: list[RecordedFrame] = [] for frame in array: converted = [] for point in frame: confidence = float(point[3]) if len(point) > 3 else 1.0 converted.append([float(point[0]), float(point[1]), float(point[2]), confidence]) - frames.append(converted) + frames.append(RecordedFrame(keypoints=converted)) return frames -def _validate_recorded_frames(frames: list[list[list[float]]]) -> None: +def _attach_reprojection_errors( + frames: list[RecordedFrame], + path: pathlib.Path, +) -> list[RecordedFrame]: + errors = _load_reprojection_errors(path, frame_count=len(frames)) + return [ + RecordedFrame( + keypoints=frame.keypoints, + reprojection_error_mm=frame.reprojection_error_mm + if frame.reprojection_error_mm is not None + else errors[index], + ) + for index, frame in enumerate(frames) + ] + + +def _load_reprojection_errors( + path: pathlib.Path, + *, + frame_count: int, +) -> list[float | None]: + try: + import numpy as np # type: ignore + except Exception as exc: + raise ValueError("reading FreeMoCap reprojection data requires numpy") from exc + array = np.load(os.fspath(path)) + shape = tuple(getattr(array, "shape", ())) + if not shape: + raise ValueError(f"expected reprojection NPY array with frames, got {shape}") + if shape[0] == frame_count: + frame_values = [array[index] for index in range(frame_count)] + elif len(shape) >= 2 and shape[1] == frame_count: + frame_values = [ + [camera_values[index] for camera_values in array] + for index in range(frame_count) + ] + else: + raise ValueError( + f"expected reprojection NPY to include {frame_count} frames, got {shape}", + ) + return [_mean_finite_values(values) for values in frame_values] + + +def _mean_finite_values(value: object) -> float | None: + values = list(_iter_finite_numbers(value)) + if not values: + return None + return sum(values) / len(values) + + +def _iter_finite_numbers(value: object): + if isinstance(value, (int, float)): + number = float(value) + if math.isfinite(number): + yield number + return + if isinstance(value, (str, bytes)): + return + tolist = getattr(value, "tolist", None) + if callable(tolist): + yield from _iter_finite_numbers(tolist()) + return + try: + iterator = iter(value) # type: ignore[arg-type] + except TypeError: + return + for item in iterator: + yield from _iter_finite_numbers(item) + + +def _validate_recorded_frames(frames: list[RecordedFrame]) -> None: if not frames: raise ValueError("no frames found") - for frame_index, frame in enumerate(frames): - if len(frame) != KEYPOINT_COUNT: - raise ValueError(f"frame {frame_index} has {len(frame)} keypoints; expected 33") - for point_index, point in enumerate(frame): - if len(point) < 4: + for frame_index, recorded_frame in enumerate(frames): + _validate_recorded_frame(recorded_frame, frame_index) + + +def _sidecar_frame_from_keypoints( + recorded_frame: RecordedFrame, + *, + frame_index: int, + timestamp_ms: int, + camera_count: int, +) -> SidecarFrame: + _validate_recorded_frame(recorded_frame, frame_index) + keypoints = [] + confidences = [] + for index, values in enumerate(recorded_frame.keypoints): + x, y, z, confidence = values[:4] + x_value = float(x) + y_value = float(y) + z_value = float(z) + confidence_value = float(confidence) + confidences.append(confidence_value) + keypoints.append( + { + "index": index, + "x": x_value, + "y": y_value, + "z": z_value, + "confidence": confidence_value, + }, + ) + tracked = sum(1 for confidence in confidences if confidence >= 0.5) + mean_confidence = sum(confidences) / len(confidences) + return { + "schema_version": KEYPOINT_SCHEMA_VERSION, + "frame_index": frame_index, + "timestamp_ms": timestamp_ms, + "source": "sidecar-3d", + "keypoints": keypoints, + "quality": { + "tracked_count": tracked, + "mean_confidence": mean_confidence, + "reprojection_error_mm": recorded_frame.reprojection_error_mm + if recorded_frame.reprojection_error_mm is not None + else 0.0, + "camera_count": camera_count, + }, + } + + +def _validate_recorded_frame(recorded_frame: RecordedFrame, frame_index: int) -> None: + frame = recorded_frame.keypoints + if len(frame) != KEYPOINT_COUNT: + raise ValueError( + f"frame {frame_index} has {len(frame)} keypoints; expected 33", + ) + for point_index, point in enumerate(frame): + if len(point) < 4: + raise ValueError( + f"frame {frame_index} keypoint {point_index} has {len(point)} values; expected 4", + ) + for value_index, value in enumerate(point[:4]): + if not math.isfinite(float(value)): raise ValueError( - f"frame {frame_index} keypoint {point_index} has {len(point)} values; expected 4", + f"frame {frame_index} keypoint {point_index} value {value_index} must be finite", ) diff --git a/src/app/api/mocap/sessions/[id]/sidecar/connect/route.ts b/src/app/api/mocap/sessions/[id]/sidecar/connect/route.ts index b707cf3..285a581 100644 --- a/src/app/api/mocap/sessions/[id]/sidecar/connect/route.ts +++ b/src/app/api/mocap/sessions/[id]/sidecar/connect/route.ts @@ -7,6 +7,9 @@ import { KEYPOINT_SCHEMA_V2 } from "@/lib/mocap/poseFrameStream"; import { checkSidecarHealth, SIDECAR_DEFAULT_PORT, + SIDECAR_CONTRACT_VERSION, + SidecarSessionStartError, + sidecarReadinessError, startSidecarSession, } from "@/lib/mocap/sidecarClient"; @@ -52,24 +55,19 @@ export async function POST( try { const health = await checkSidecarHealth(port); - if (health.status !== "ready") { + const readinessError = sidecarReadinessError(health); + if (readinessError) { return NextResponse.json( { - status: health.status, + status: health.status === "ready" ? "incompatible-sidecar" : health.status, port, + error: readinessError, diagnostics: health.diagnostics ?? [], source: health.source ?? null, - }, - { status: 409 }, - ); - } - if (health.schemaVersion !== KEYPOINT_SCHEMA_V2) { - return NextResponse.json( - { - status: "incompatible-schema", - port, schemaVersion: health.schemaVersion, expectedSchemaVersion: KEYPOINT_SCHEMA_V2, + contractVersion: health.contractVersion ?? null, + expectedContractVersion: SIDECAR_CONTRACT_VERSION, }, { status: 409 }, ); @@ -92,6 +90,21 @@ export async function POST( calibrationId: sidecarSession.calibrationId ?? null, }); } catch (err) { + if (err instanceof SidecarSessionStartError) { + const body = isRecord(err.body) ? err.body : {}; + return NextResponse.json( + { + ...body, + status: optionalString(body.status) ?? "start-failed", + port, + error: err.message, + diagnostics: Array.isArray(body.diagnostics) ? body.diagnostics : [], + expectedSchemaVersion: KEYPOINT_SCHEMA_V2, + expectedContractVersion: SIDECAR_CONTRACT_VERSION, + }, + { status: err.status }, + ); + } return NextResponse.json( { status: "unreachable", @@ -102,3 +115,11 @@ export async function POST( ); } } + +function optionalString(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/src/app/mocap/page.tsx b/src/app/mocap/page.tsx index 6f81a0c..e0c960e 100644 --- a/src/app/mocap/page.tsx +++ b/src/app/mocap/page.tsx @@ -36,6 +36,7 @@ import { BYTES_PER_FRAME_V1, BYTES_PER_FRAME_V2, KEYPOINT_SCHEMA_V2, + QUALITY_FLAG, decodeFrame, } from "@/lib/mocap/poseFrameStream"; import { VideoUploader } from "@/lib/mocap/videoUploader"; @@ -60,7 +61,7 @@ import type { import { settings } from "@/lib/settings"; import { checkSidecarHealth, - sidecarReadinessMessage, + sidecarReadinessError, type SidecarHealth, } from "@/lib/mocap/sidecarClient"; import { resolveSidecarPort } from "@/lib/mocap/sidecarPoseSource"; @@ -156,6 +157,7 @@ export default function MocapCapturePage() { const sourceRef = useRef(null); const calibrationSourceRef = useRef(null); const startedAtRef = useRef(0); + const activeCaptureSessionIdRef = useRef(null); const latestPoseFrameRef = useRef(EMPTY_QUALITY); const qualityHistoryRef = useRef([]); const effectiveFpsSamplesRef = useRef([]); @@ -263,16 +265,20 @@ export default function MocapCapturePage() { setPoseStatus("loading"); try { const health = await checkSidecarHealth(port); - if (health.status !== "ready") { - throw new Error(sidecarReadinessMessage(health)); + const readinessError = sidecarReadinessError(health); + if (readinessError) { + throw new Error(readinessError); } setPoseStatus("ready"); setSidecarHealth(health); setSidecarError(null); return health; } catch (err) { + const message = err instanceof Error ? err.message : ""; const error = - err instanceof Error && err.message.startsWith("Sidecar not ready:") + err instanceof Error && + (message.startsWith("Sidecar not ready:") || + message.startsWith("Sidecar incompatible:")) ? err : new Error(`Sidecar not reachable on port ${port}`); setPoseStatus("error"); @@ -370,6 +376,12 @@ export default function MocapCapturePage() { if (decodedFrame) engineRef.current.pushFrame(decodedFrame); } + if ((info.qualityFlags & QUALITY_FLAG.HIGH_REPROJECTION_ERROR) !== 0) { + setSessionQualityFlags((flags) => + appendUniqueFlags(flags, ["high-reprojection-error"]), + ); + } + if (!monitorDegradedFraming) return; if (!readiness.sustainedDegraded) { @@ -408,6 +420,9 @@ export default function MocapCapturePage() { const handleError = useCallback( async (err: unknown, sessionId?: string) => { const message = err instanceof Error ? err.message : String(err); + const deleteAbandonedSession = Boolean( + sessionId && activeCaptureSessionIdRef.current !== sessionId, + ); setState({ kind: "error", message }); try { await sourceRef.current?.stop(); @@ -426,7 +441,7 @@ export default function MocapCapturePage() { } await uploaderRef.current?.drain(); await teardown(); - if (sessionId) { + if (deleteAbandonedSession && sessionId) { // Best-effort delete of the abandoned row fetch(`/api/mocap/sessions/${sessionId}`, { method: "DELETE" }).catch( () => {}, @@ -526,6 +541,7 @@ export default function MocapCapturePage() { const start = useCallback(async () => { // Sidecar path: skip browser camera/calibration, use sidecar-3d perspective if (useSidecar) { + activeCaptureSessionIdRef.current = null; setState({ kind: "starting" }); let sessionId: string | undefined; try { @@ -536,7 +552,7 @@ export default function MocapCapturePage() { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ source: "sidecar", - captureModelVersion: `freemocap-sidecar@schemaV${health.schemaVersion}`, + captureModelVersion: `freemocap-sidecar@${health.sidecarVersion ?? "unknown"}+schemaV${health.schemaVersion}`, capturePerspective: "sidecar-3d", captureFps: health.fps, recordOnly: false, @@ -575,6 +591,7 @@ export default function MocapCapturePage() { sourceRef.current = source; await source.init(); await source.start(); + activeCaptureSessionIdRef.current = created.id; setState({ kind: "capturing", @@ -615,6 +632,7 @@ export default function MocapCapturePage() { } setState({ kind: "starting" }); + activeCaptureSessionIdRef.current = null; let sessionId: string | undefined; try { const stream = @@ -720,6 +738,7 @@ export default function MocapCapturePage() { } recorder.start(VIDEO_TIMESLICE_MS); + activeCaptureSessionIdRef.current = sessionId; sourceRef.current?.start(); startedAtRef.current = Date.now(); setElapsedSec(0); @@ -821,6 +840,7 @@ export default function MocapCapturePage() { throw new Error(`Finalize failed: ${finalized.message}`); } await teardown(); + activeCaptureSessionIdRef.current = null; setCalibration({ kind: "idle" }); // Fetch faults for end-of-session summary @@ -1313,7 +1333,10 @@ export default function MocapCapturePage() { ) : null} {state.kind === "error" ? ( -
+
Error: {state.message}
) : null} diff --git a/src/lib/mocap/browserPoseSource.ts b/src/lib/mocap/browserPoseSource.ts index 6d19eba..c2f7ed6 100644 --- a/src/lib/mocap/browserPoseSource.ts +++ b/src/lib/mocap/browserPoseSource.ts @@ -121,11 +121,14 @@ export class BrowserPoseSource implements PoseCaptureSource { clearInterval(this.flushTimer); this.flushTimer = null; } - await this.drain(); - this.worker?.postMessage({ type: "close" }); - this.worker?.terminate(); - this.worker = null; - this.setStatus("stopped"); + try { + await this.drain(); + } finally { + this.worker?.postMessage({ type: "close" }); + this.worker?.terminate(); + this.worker = null; + this.setStatus("stopped"); + } } private scheduleFrame(): void { diff --git a/src/lib/mocap/freemocapSidecarSource.ts b/src/lib/mocap/freemocapSidecarSource.ts index 01df11a..6f30836 100644 --- a/src/lib/mocap/freemocapSidecarSource.ts +++ b/src/lib/mocap/freemocapSidecarSource.ts @@ -38,6 +38,7 @@ interface SidecarConnectResponse { } const SIDECAR_RELEVANT_TRACKED_KEYPOINTS = 13; +export const SIDECAR_REPROJECTION_ERROR_WARNING_MM = 50; export class FreemocapSidecarSource implements PoseCaptureSource { private closeStream: (() => void) | null = null; @@ -240,14 +241,15 @@ export function validateSidecarKeypointFrame( throw new Error("Sidecar frame must be an object"); } - const schemaVersion = optionalNumber( + const schemaVersion = requiredInteger( pickField(message, "schemaVersion", "schema_version"), + "schemaVersion", ); - if (schemaVersion !== undefined && schemaVersion !== KEYPOINT_SCHEMA_V2) { + if (schemaVersion !== KEYPOINT_SCHEMA_V2) { throw new Error("Sidecar frame schemaVersion must be 2"); } - const source = optionalString(message.source); - if (source !== undefined && source !== "sidecar-3d") { + const source = requiredString(message.source, "source"); + if (source !== "sidecar-3d") { throw new Error("Sidecar frame source must be sidecar-3d"); } @@ -341,13 +343,20 @@ function sidecarKeypointsToQuads(frame: SidecarKeypointFrame): Float32Array { } function sidecarQualityFlags(frame: SidecarKeypointFrame): number { + let flags = 0; if ( frame.quality.meanConfidence < 0.6 || frame.quality.trackedCount < 13 ) { - return QUALITY_FLAG.LOW_CONFIDENCE; + flags |= QUALITY_FLAG.LOW_CONFIDENCE; } - return 0; + if ( + frame.quality.reprojectionErrorMm !== undefined && + frame.quality.reprojectionErrorMm > SIDECAR_REPROJECTION_ERROR_WARNING_MM + ) { + flags |= QUALITY_FLAG.HIGH_REPROJECTION_ERROR; + } + return flags; } function sidecarTrackedCountForPoseQuality(trackedCount: number): number { @@ -409,6 +418,13 @@ function optionalString(value: unknown): string | undefined { return typeof value === "string" && value.length > 0 ? value : undefined; } +function requiredString(value: unknown, field: string): string { + if (typeof value !== "string" || value.length === 0) { + throw new Error(`Sidecar ${field} must be a string`); + } + return value; +} + function pickField( value: Record, camelCase: string, diff --git a/src/lib/mocap/poseFrameStream.ts b/src/lib/mocap/poseFrameStream.ts index 159a6ef..2016ea0 100644 --- a/src/lib/mocap/poseFrameStream.ts +++ b/src/lib/mocap/poseFrameStream.ts @@ -48,6 +48,7 @@ export const QUALITY_FLAG = { OUT_OF_FRAME: 1 << 2, LOW_CONFIDENCE: 1 << 3, CAMERA_MOTION: 1 << 4, + HIGH_REPROJECTION_ERROR: 1 << 5, } as const; export interface PoseStreamHeader { diff --git a/src/lib/mocap/poseStreamUploadBuffer.ts b/src/lib/mocap/poseStreamUploadBuffer.ts index 328f396..6c35734 100644 --- a/src/lib/mocap/poseStreamUploadBuffer.ts +++ b/src/lib/mocap/poseStreamUploadBuffer.ts @@ -8,6 +8,7 @@ export class PoseStreamUploadBuffer { private pendingChunks: Uint8Array[] = []; private pendingBytes = 0; private uploadInflight: Promise = Promise.resolve(); + private uploadError: Error | null = null; constructor(private readonly opts: PoseStreamUploadBufferOptions) {} @@ -15,7 +16,7 @@ export class PoseStreamUploadBuffer { this.pendingChunks.push(bytes); this.pendingBytes += bytes.byteLength; if (this.pendingBytes >= this.opts.flushBytes) { - void this.flush(false); + void this.flush(false).catch(() => undefined); } } @@ -25,7 +26,10 @@ export class PoseStreamUploadBuffer { async flush(final: boolean): Promise { if (this.pendingChunks.length === 0) { - if (final) await this.uploadInflight; + if (final) { + await this.uploadInflight; + if (this.uploadError) throw this.uploadError; + } return; } const chunks = this.pendingChunks; @@ -40,14 +44,15 @@ export class PoseStreamUploadBuffer { off += chunk.byteLength; } - this.uploadInflight = this.uploadInflight.then(() => - this.upload(buf).catch((err) => { - this.opts.onError?.( - err instanceof Error ? err : new Error(String(err)), - ); - }), - ); - if (final) await this.uploadInflight; + this.uploadInflight = this.uploadInflight.then(() => this.upload(buf)).catch((err) => { + const error = err instanceof Error ? err : new Error(String(err)); + this.uploadError ??= error; + this.opts.onError?.(error); + }); + if (final) { + await this.uploadInflight; + if (this.uploadError) throw this.uploadError; + } } private async upload(buf: Uint8Array): Promise { diff --git a/src/lib/mocap/sidecarClient.ts b/src/lib/mocap/sidecarClient.ts index 1401e94..1456bc8 100644 --- a/src/lib/mocap/sidecarClient.ts +++ b/src/lib/mocap/sidecarClient.ts @@ -1,10 +1,15 @@ +import { KEYPOINT_SCHEMA_V2 } from "./poseFrameStream"; + export const SIDECAR_DEFAULT_PORT = 8765; +export const SIDECAR_CONTRACT_VERSION = 1; export interface SidecarHealth { status: "ready" | "initializing" | "error"; fps: number; cameras: number; schemaVersion: number; + contractVersion?: number; + sidecarVersion?: string; source?: string; calibrationId?: string | null; diagnostics?: string[]; @@ -15,6 +20,17 @@ export interface SidecarSessionInfo { calibrationId?: string; } +export class SidecarSessionStartError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: unknown, + ) { + super(message); + this.name = "SidecarSessionStartError"; + } +} + export interface SidecarKeypointFrame { frameIndex: number; timestampMs: number; @@ -40,6 +56,38 @@ export async function checkSidecarHealth(port = SIDECAR_DEFAULT_PORT): Promise typeof entry === "string" && entry.length > 0) : undefined; @@ -50,8 +98,14 @@ export function sidecarReadinessMessage(health: SidecarHealth): string { export async function startSidecarSession(port = SIDECAR_DEFAULT_PORT): Promise { const res = await fetch(`http://localhost:${port}/session/start`, { method: "POST" }); - if (!res.ok) throw new Error(`Sidecar session/start failed: ${res.status}`); const body = (await res.json().catch(() => ({}))) as unknown; + if (!res.ok) { + throw new SidecarSessionStartError( + sidecarSessionFailureMessage("start", res.status, body), + res.status, + body, + ); + } if (!isRecord(body)) return {}; return { sessionId: optionalString(body.sessionId), @@ -85,6 +139,24 @@ function optionalString(value: unknown): string | undefined { return typeof value === "string" && value.length > 0 ? value : undefined; } +function sidecarSessionFailureMessage( + action: "start" | "stop", + status: number, + body: unknown, +): string { + if (!isRecord(body)) return `Sidecar session/${action} failed: ${status}`; + const statusText = optionalString(body.status) ?? optionalString(body.error); + const diagnostics = Array.isArray(body.diagnostics) + ? body.diagnostics.filter( + (entry): entry is string => typeof entry === "string" && entry.length > 0, + ) + : []; + const details = [statusText, ...diagnostics].filter(Boolean).join(": "); + return details + ? `Sidecar session/${action} failed: ${status} (${details})` + : `Sidecar session/${action} failed: ${status}`; +} + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } diff --git a/src/lib/mocap/sidecarPoseSource.ts b/src/lib/mocap/sidecarPoseSource.ts index 5197c5b..4084a2c 100644 --- a/src/lib/mocap/sidecarPoseSource.ts +++ b/src/lib/mocap/sidecarPoseSource.ts @@ -5,7 +5,7 @@ import { stopSidecarSession, type SidecarHealth, type SidecarSessionInfo, - sidecarReadinessMessage, + sidecarReadinessError, } from "./sidecarClient"; import type { PoseCaptureSource, @@ -56,14 +56,18 @@ export class FreemocapSidecarSource implements PoseCaptureSource { this.setStatus("loading"); try { const health = await checkSidecarHealth(this.port); - if (health.status !== "ready") { - throw new Error(sidecarReadinessMessage(health)); + const readinessError = sidecarReadinessError(health); + if (readinessError) { + throw new Error(readinessError); } this.currentHealth = health; this.setStatus("ready"); } catch (err) { + const message = err instanceof Error ? err.message : ""; const error = - err instanceof Error && err.message.startsWith("Sidecar not ready:") + err instanceof Error && + (message.startsWith("Sidecar not ready:") || + message.startsWith("Sidecar incompatible:")) ? err : new Error(`Sidecar not reachable on port ${this.port}`); this.reportError(error); diff --git a/tests/browserPoseSource.test.ts b/tests/browserPoseSource.test.ts index 6511b4b..39a2546 100644 --- a/tests/browserPoseSource.test.ts +++ b/tests/browserPoseSource.test.ts @@ -103,7 +103,7 @@ test("browser pose source reports drain upload errors through the shared contrac await contract.init(); FakeWorker.instances[0].emitMessage(makeFrameMessage(1, [9, 8, 7])); - await contract.drain(); + await assert.rejects(() => contract.drain(), /Pose upload failed: 500/); assert.equal(errors.length, 1); assert.equal(errors[0].message, "Pose upload failed: 500"); @@ -112,6 +112,27 @@ test("browser pose source reports drain upload errors through the shared contrac ); }); +test("browser pose source cleans up worker when stop drain fails", async () => { + await withBrowserPoseSourceGlobals( + async () => { + const source = new BrowserPoseSource({ + sessionId: "session-3", + videoEl: fakeVideoElement(), + flushBytes: 100, + }); + + await source.init(); + FakeWorker.instances[0].emitMessage(makeFrameMessage(1, [3, 2, 1])); + + await assert.rejects(() => source.stop(), /Pose upload failed: 500/); + + assert.equal(FakeWorker.instances[0].terminated, true); + assert.equal(source.status, "stopped"); + }, + { fetchOk: false }, + ); +}); + async function withBrowserPoseSourceGlobals( run: (context: { fetchCalls: { input: RequestInfo | URL; init?: RequestInit }[]; diff --git a/tests/e2e/mocap-capture.spec.ts b/tests/e2e/mocap-capture.spec.ts index f962cb5..ba9378a 100644 --- a/tests/e2e/mocap-capture.spec.ts +++ b/tests/e2e/mocap-capture.spec.ts @@ -10,7 +10,12 @@ * Wire that variant in CI once the auth fixture lands; for now this spec * keeps the contract regression-proof on the UI surface. */ -import { test, expect } from "@playwright/test"; +import { test, expect, type Page } from "@playwright/test"; +import { SIDECAR_REPROJECTION_ERROR_WARNING_MM } from "../../src/lib/mocap/freemocapSidecarSource"; +import { + makeTestSidecarFrame, + type TestSidecarFrame, +} from "../helpers/testSidecarMock"; const sidecarCorsHeaders = { "Access-Control-Allow-Origin": "*", @@ -19,6 +24,12 @@ const sidecarCorsHeaders = { "Access-Control-Allow-Private-Network": "true", }; +type FinalizeBody = { + durationSec: number; + qualityFlags?: string[]; + skipAnalysis?: boolean; +}; + test("capture page loads without prompting for camera", async ({ page }) => { let mediaRequested = false; await page.exposeFunction("__markMediaRequested", () => { @@ -51,14 +62,160 @@ test("capture page loads without prompting for camera", async ({ page }) => { }); test("sidecar capture can start and finalize as analyzable", async ({ page }) => { - let finalizeBody: - | { - durationSec: number; - qualityFlags?: string[]; - skipAnalysis?: boolean; + let finalizeBody: FinalizeBody | null = null; + const highReprojectionFrame = makeTestSidecarFrame(0, 20); + highReprojectionFrame.quality.reprojection_error_mm = + SIDECAR_REPROJECTION_ERROR_WARNING_MM + 1; + + await mockSidecarCapture(page, { + appSessionId: "mock-sidecar", + onFinalize: (body) => { + finalizeBody = body; + }, + }); + + await openReadySidecarCapture(page, { frames: [highReprojectionFrame] }); + await page.getByTestId("mocap-start").click(); + await expect(page.getByTestId("mocap-recording-indicator")).toBeVisible(); + + await page.waitForTimeout(100); + await page.getByTestId("mocap-stop").click(); + + await expect(page.getByTestId("mocap-done")).toContainText( + "12 pose frames", + ); + expect(finalizeBody).not.toBeNull(); + expect(finalizeBody!.durationSec).toBeGreaterThan(0); + expect(finalizeBody!.durationSec).toBeLessThan(60 * 60 * 8); + expect(finalizeBody!.skipAnalysis).toBe(false); + expect(finalizeBody!.qualityFlags ?? []).toContain( + "high-reprojection-error", + ); + expect(finalizeBody!.qualityFlags ?? []).not.toContain("record-only"); +}); + +test("sidecar capture preserves active sessions when finalize fails", async ({ + page, +}) => { + let deleteRequested = false; + + await mockSidecarCapture(page, { + appSessionId: "mock-sidecar-finalize-fails", + finalizeStatus: 500, + finalizeError: "analysis worker crashed", + onDelete: () => { + deleteRequested = true; + }, + }); + + await openReadySidecarCapture(page); + await page.getByTestId("mocap-start").click(); + await expect(page.getByTestId("mocap-recording-indicator")).toBeVisible(); + await page.getByTestId("mocap-stop").click(); + + await expect(page.getByTestId("mocap-error")).toContainText( + "Finalize failed: analysis worker crashed", + ); + expect(deleteRequested).toBe(false); +}); + +test("sidecar capture preserves active sessions when sidecar stop fails", async ({ + page, +}) => { + let deleteRequested = false; + + await mockSidecarCapture(page, { + appSessionId: "mock-sidecar-stop-fails", + stopStatus: 500, + onDelete: () => { + deleteRequested = true; + }, + }); + + await openReadySidecarCapture(page); + await page.getByTestId("mocap-start").click(); + await expect(page.getByTestId("mocap-recording-indicator")).toBeVisible(); + await page.getByTestId("mocap-stop").click(); + + await expect(page.getByTestId("mocap-error")).toContainText( + "Sidecar session/stop failed: 500", + ); + expect(deleteRequested).toBe(false); +}); + +async function openReadySidecarCapture( + page: Page, + opts: { frames?: TestSidecarFrame[] } = {}, +) { + await page.goto("/mocap"); + await page.waitForLoadState("networkidle"); + const sidecarToggle = page.getByTestId("mocap-sidecar-toggle"); + if (await sidecarToggle.isChecked()) { + await sidecarToggle.uncheck(); + } + await sidecarToggle.check(); + await expect(page.getByTestId("mocap-start")).toContainText( + "Start sidecar capture", + ); + await expect(page.getByTestId("mocap-sidecar-status")).toContainText( + "Sidecar ready", + ); + await expect(page.getByTestId("mocap-start")).toBeEnabled(); + await installMockSidecarWebSocket(page, opts.frames); +} + +async function installMockSidecarWebSocket( + page: Page, + frames: TestSidecarFrame[] = [], +) { + await page.evaluate((mockFrames) => { + class MockWebSocket { + onmessage: ((event: MessageEvent) => void) | null = null; + onerror: ((event: Event) => void) | null = null; + onopen: ((event: Event) => void) | null = null; + onclose: ((event: CloseEvent) => void) | null = null; + readyState = 1; + readonly url: string; + + constructor(url: string) { + this.url = url; + setTimeout(() => { + this.onopen?.(new Event("open")); + for (const frame of mockFrames) { + this.onmessage?.( + new MessageEvent("message", { data: JSON.stringify(frame) }), + ); + } + }, 0); + } + + send() {} + + close() { + this.readyState = 3; + this.onclose?.(new CloseEvent("close")); } - | null = null; + } + + Object.defineProperty(window, "WebSocket", { + configurable: true, + value: MockWebSocket, + writable: true, + }); + }, frames); +} +async function mockSidecarCapture( + page: Page, + opts: { + appSessionId: string; + stopStatus?: number; + finalizeStatus?: number; + finalizeError?: string; + onFinalize?: (body: FinalizeBody) => void; + onDelete?: () => void; + }, +) { await page.route("**/health", async (route) => { await route.fulfill({ status: 200, @@ -69,6 +226,8 @@ test("sidecar capture can start and finalize as analyzable", async ({ page }) => fps: 60, cameras: 3, schemaVersion: 2, + contractVersion: 1, + sidecarVersion: "0.1.0", }), }); }); @@ -84,7 +243,15 @@ test("sidecar capture can start and finalize as analyzable", async ({ page }) => }); }); await page.route("**/session/stop", async (route) => { - await route.fulfill({ status: 200, headers: sidecarCorsHeaders, body: "" }); + await route.fulfill({ + status: opts.stopStatus ?? 200, + contentType: "application/json", + headers: sidecarCorsHeaders, + body: + opts.stopStatus && opts.stopStatus >= 400 + ? JSON.stringify({ error: "sidecar stop failed" }) + : "", + }); }); await page.route("**/api/mocap/sessions", async (route) => { if (route.request().method() !== "POST") { @@ -94,11 +261,11 @@ test("sidecar capture can start and finalize as analyzable", async ({ page }) => await route.fulfill({ status: 200, contentType: "application/json", - body: JSON.stringify({ id: "mock-sidecar" }), + body: JSON.stringify({ id: opts.appSessionId }), }); }); await page.route( - "**/api/mocap/sessions/mock-sidecar/sidecar/connect", + `**/api/mocap/sessions/${opts.appSessionId}/sidecar/connect`, async (route) => { expect(route.request().postDataJSON()).toEqual({ port: 8765 }); await route.fulfill({ @@ -115,17 +282,34 @@ test("sidecar capture can start and finalize as analyzable", async ({ page }) => }, ); await page.route( - "**/api/mocap/sessions/mock-sidecar/finalize", + `**/api/mocap/sessions/${opts.appSessionId}/pose-stream`, async (route) => { - finalizeBody = route.request().postDataJSON(); + await route.fulfill({ status: 204, body: "" }); + }, + ); + await page.route( + `**/api/mocap/sessions/${opts.appSessionId}/finalize`, + async (route) => { + const body = route.request().postDataJSON() as FinalizeBody; + opts.onFinalize?.(body); + if (opts.finalizeStatus && opts.finalizeStatus >= 400) { + await route.fulfill({ + status: opts.finalizeStatus, + contentType: "application/json", + body: JSON.stringify({ + error: opts.finalizeError ?? "finalize failed", + }), + }); + return; + } await route.fulfill({ status: 200, contentType: "application/json", body: JSON.stringify({ - id: "mock-sidecar", + id: opts.appSessionId, status: "ready", analysisMode: "pose-segmented", - durationSec: finalizeBody?.durationSec ?? 0, + durationSec: body.durationSec, frameCount: 12, poseStreamBytes: 4096, strokeMetricCount: 2, @@ -134,69 +318,16 @@ test("sidecar capture can start and finalize as analyzable", async ({ page }) => }); }, ); - await page.route("**/api/mocap/sessions/mock-sidecar", async (route) => { + await page.route(`**/api/mocap/sessions/${opts.appSessionId}`, async (route) => { + if (route.request().method() === "DELETE") { + opts.onDelete?.(); + await route.fulfill({ status: 204, body: "" }); + return; + } await route.fulfill({ status: 200, contentType: "application/json", body: JSON.stringify({ session: { postureFaults: [] } }), }); }); - - await page.goto("/mocap"); - await page.waitForLoadState("networkidle"); - const sidecarToggle = page.getByTestId("mocap-sidecar-toggle"); - if (await sidecarToggle.isChecked()) { - await sidecarToggle.uncheck(); - } - await sidecarToggle.check(); - await expect(page.getByTestId("mocap-start")).toContainText( - "Start sidecar capture", - ); - await expect(page.getByTestId("mocap-sidecar-status")).toContainText( - "Sidecar ready", - ); - - await expect(page.getByTestId("mocap-start")).toBeEnabled(); - await page.evaluate(() => { - class MockWebSocket { - onmessage: ((event: MessageEvent) => void) | null = null; - onerror: ((event: Event) => void) | null = null; - onopen: ((event: Event) => void) | null = null; - onclose: ((event: CloseEvent) => void) | null = null; - readyState = 1; - readonly url: string; - - constructor(url: string) { - this.url = url; - setTimeout(() => this.onopen?.(new Event("open")), 0); - } - - send() {} - - close() { - this.readyState = 3; - this.onclose?.(new CloseEvent("close")); - } - } - - Object.defineProperty(window, "WebSocket", { - configurable: true, - value: MockWebSocket, - writable: true, - }); - }); - await page.getByTestId("mocap-start").click(); - await expect(page.getByTestId("mocap-recording-indicator")).toBeVisible(); - - await page.waitForTimeout(100); - await page.getByTestId("mocap-stop").click(); - - await expect(page.getByTestId("mocap-done")).toContainText( - "12 pose frames", - ); - expect(finalizeBody).not.toBeNull(); - expect(finalizeBody!.durationSec).toBeGreaterThan(0); - expect(finalizeBody!.durationSec).toBeLessThan(60 * 60 * 8); - expect(finalizeBody!.skipAnalysis).toBe(false); - expect(finalizeBody!.qualityFlags ?? []).not.toContain("record-only"); -}); +} diff --git a/tests/e2e/sidecar-hardware-smoke.spec.ts b/tests/e2e/sidecar-hardware-smoke.spec.ts new file mode 100644 index 0000000..56ac769 --- /dev/null +++ b/tests/e2e/sidecar-hardware-smoke.spec.ts @@ -0,0 +1,369 @@ +/** + * Opt-in smoke test for a real FreeMoCap sidecar runtime. + * + * This is skipped by default so normal CI remains hardware-free. To run it, + * start Rowing Tracker's dev server and a ready sidecar on localhost, then set: + * + * ROWING_TRACKER_SIDECAR_HARDWARE_SMOKE=1 + * + * The test mocks only Rowing Tracker's auth/database API surface. Sidecar + * health, session start/stop, and WebSocket frame streaming use the real local + * sidecar process. + */ +import { expect, test, type Page, type Route } from "@playwright/test"; +import { + BYTES_PER_FRAME_V2, + HEADER_SIZE, + KEYPOINT_SCHEMA_V2, + encodeHeader, +} from "../../src/lib/mocap/poseFrameStream"; + +const SMOKE_ENABLED = + process.env.ROWING_TRACKER_SIDECAR_HARDWARE_SMOKE === "1"; +const ALLOW_SYNTHETIC = + process.env.ROWING_TRACKER_SIDECAR_HARDWARE_ALLOW_SYNTHETIC === "1"; +const SIDECAR_PORT = Number.parseInt( + process.env.ROWING_TRACKER_SIDECAR_PORT ?? "8765", + 10, +); +const SESSION_ID = "hardware-sidecar-smoke"; + +type SidecarHealth = { + status?: string; + fps?: number; + cameras?: number; + schemaVersion?: number; + contractVersion?: number; + sidecarVersion?: string; + source?: string; + calibrationId?: string | null; + diagnostics?: string[]; +}; + +type FinalizeBody = { + durationSec: number; + qualityFlags?: string[]; + skipAnalysis?: boolean; +}; + +test.describe("hardware-gated FreeMoCap sidecar smoke", () => { + test.skip( + !SMOKE_ENABLED, + "Set ROWING_TRACKER_SIDECAR_HARDWARE_SMOKE=1 with a real ready sidecar to run.", + ); + + test("records through Rowing Tracker and verifies replay-ready sidecar output", async ({ + page, + }) => { + const health = await fetchSidecarHealth(); + assertSmokeHealth(health); + + const uploadedFrameChunks: Uint8Array[] = []; + let finalizeBody: FinalizeBody | null = null; + let finalizedFrameCount = 0; + + await installSidecarPortSetting(page, SIDECAR_PORT); + await mockPersistenceAroundRealSidecar(page, { + health, + uploadedFrameChunks, + onFinalize: (body) => { + finalizeBody = body; + finalizedFrameCount = uploadedFrameCount(uploadedFrameChunks); + }, + }); + + await page.goto("/mocap"); + await page.waitForLoadState("networkidle"); + await page.getByTestId("mocap-sidecar-toggle").check(); + + await expect(page.getByTestId("mocap-sidecar-status")).toContainText( + "Sidecar ready", + ); + await expect(page.getByTestId("mocap-start")).toBeEnabled(); + + await page.getByTestId("mocap-start").click(); + await expect(page.getByTestId("mocap-recording-indicator")).toBeVisible(); + + await expect + .poll(() => uploadedFrameCount(uploadedFrameChunks), { + message: "sidecar frame uploads", + timeout: 20_000, + }) + .toBeGreaterThanOrEqual(1); + + await page.getByTestId("mocap-stop").click(); + await expect(page.getByTestId("mocap-done")).toContainText("pose frames"); + + expect(finalizeBody).not.toBeNull(); + expect(finalizeBody!.skipAnalysis).toBe(false); + expect(finalizedFrameCount).toBeGreaterThanOrEqual(1); + + await page.getByTestId("mocap-replay-link").click(); + await expect(page).toHaveURL(/\/mocap\/sessions\/hardware-sidecar-smoke$/); + await expect(page.getByTestId("mocap-replay-sidecar")).toBeVisible(); + await expect(page.getByText("pose-segmented")).toBeVisible(); + }); +}); + +async function installSidecarPortSetting(page: Page, port: number) { + if (port === 8765) return; + await page.addInitScript((sidecarPort) => { + window.localStorage.setItem( + "rowing_app_settings", + JSON.stringify({ + version: "sidecar-hardware-smoke", + updatedAt: new Date().toISOString(), + mocapSettings: { sidecarPort }, + }), + ); + }, port); +} + +async function mockPersistenceAroundRealSidecar( + page: Page, + opts: { + health: SidecarHealth; + uploadedFrameChunks: Uint8Array[]; + onFinalize: (body: FinalizeBody) => void; + }, +) { + await page.route("**/api/mocap/sessions", async (route) => { + if (route.request().method() !== "POST") { + await route.fallback(); + return; + } + await fulfillJson(route, { id: SESSION_ID }); + }); + + await page.route( + `**/api/mocap/sessions/${SESSION_ID}/sidecar/connect`, + async (route) => { + expect(route.request().postDataJSON()).toEqual({ port: SIDECAR_PORT }); + const startRes = await fetch( + `http://localhost:${SIDECAR_PORT}/session/start`, + { method: "POST" }, + ); + const started = (await startRes.json().catch(() => ({}))) as Record< + string, + unknown + >; + if (!startRes.ok) { + await fulfillJson(route, started, startRes.status); + return; + } + await fulfillJson(route, { + status: "connected", + fps: opts.health.fps, + cameras: opts.health.cameras, + schemaVersion: opts.health.schemaVersion, + contractVersion: opts.health.contractVersion, + sidecarVersion: opts.health.sidecarVersion, + source: opts.health.source, + port: SIDECAR_PORT, + sidecarSessionId: + typeof started.sessionId === "string" ? started.sessionId : null, + calibrationId: + typeof started.calibrationId === "string" + ? started.calibrationId + : opts.health.calibrationId, + }); + }, + ); + + await page.route( + `**/api/mocap/sessions/${SESSION_ID}/pose-stream`, + async (route) => { + if (route.request().method() === "POST") { + const body = route.request().postDataBuffer(); + expect(body).not.toBeNull(); + opts.uploadedFrameChunks.push(new Uint8Array(body!)); + await route.fulfill({ status: 200, body: "" }); + return; + } + await fulfillPoseRange(route, buildPoseStreamBlob(opts)); + }, + ); + + await page.route( + `**/api/mocap/sessions/${SESSION_ID}/finalize`, + async (route) => { + const body = route.request().postDataJSON() as FinalizeBody; + opts.onFinalize(body); + const frameCount = uploadedFrameCount(opts.uploadedFrameChunks); + await fulfillJson(route, { + id: SESSION_ID, + status: "ready", + analysisMode: "pose-segmented", + durationSec: body.durationSec, + frameCount, + poseStreamBytes: HEADER_SIZE + frameCount * BYTES_PER_FRAME_V2, + strokeMetricCount: 1, + faultCount: 0, + }); + }, + ); + + await page.route( + `**/api/mocap/sessions/${SESSION_ID}/assignment-candidates`, + async (route) => { + await fulfillJson(route, { + linkable: false, + reason: "hardware-smoke", + message: "Hardware smoke sessions are not linked to CSV rows.", + candidates: [], + }); + }, + ); + + await page.route( + `**/api/mocap/sessions/${SESSION_ID}/sidecar/stop`, + async (route) => { + const stopRes = await fetch(`http://localhost:${SIDECAR_PORT}/session/stop`, { + method: "POST", + }); + const body = (await stopRes.json().catch(() => ({}))) as Record< + string, + unknown + >; + await fulfillJson(route, body, stopRes.status); + }, + ); + + await page.route(`**/api/mocap/sessions/${SESSION_ID}`, async (route) => { + if (route.request().method() === "DELETE") { + await route.fulfill({ status: 204, body: "" }); + return; + } + await fulfillJson(route, { session: smokeSession(opts) }); + }); +} + +async function fetchSidecarHealth(): Promise { + const res = await fetch(`http://localhost:${SIDECAR_PORT}/health`); + if (!res.ok) { + throw new Error(`Sidecar health check failed: ${res.status}`); + } + return (await res.json()) as SidecarHealth; +} + +function assertSmokeHealth(health: SidecarHealth) { + expect(health.status).toBe("ready"); + expect(health.schemaVersion).toBe(KEYPOINT_SCHEMA_V2); + expect(health.contractVersion).toBe(1); + expect(health.cameras).toBeGreaterThan(0); + expect(health.fps).toBeGreaterThan(0); + if (!ALLOW_SYNTHETIC) { + expect(health.source).toBe("freemocap"); + expect(health.calibrationId).toBeTruthy(); + } +} + +function buildPoseStreamBlob(opts: { + health: SidecarHealth; + uploadedFrameChunks: Uint8Array[]; +}): Uint8Array { + const frameBytes = concatBytes(opts.uploadedFrameChunks); + const frameCount = Math.floor(frameBytes.byteLength / BYTES_PER_FRAME_V2); + const header = encodeHeader({ + fps: opts.health.fps ?? 30, + keypointSchemaVersion: KEYPOINT_SCHEMA_V2, + frameCount, + coordinateSpace: "world-mm-3d", + cameraCount: opts.health.cameras ?? 1, + }); + return concatBytes([header, frameBytes]); +} + +function uploadedFrameCount(chunks: Uint8Array[]): number { + return Math.floor( + chunks.reduce((total, chunk) => total + chunk.byteLength, 0) / + BYTES_PER_FRAME_V2, + ); +} + +function concatBytes(chunks: Uint8Array[]): Uint8Array { + const total = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0); + const out = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.byteLength; + } + return out; +} + +async function fulfillPoseRange(route: Route, blob: Uint8Array) { + const range = route.request().headers().range; + const match = range?.match(/^bytes=(\d+)-(\d+)$/); + const start = match ? Number.parseInt(match[1], 10) : 0; + const requestedEnd = match ? Number.parseInt(match[2], 10) : blob.byteLength - 1; + const end = Math.min(requestedEnd, blob.byteLength - 1); + const slice = blob.slice(start, end + 1); + await route.fulfill({ + status: range ? 206 : 200, + contentType: "application/octet-stream", + headers: { + "Accept-Ranges": "bytes", + "Content-Range": `bytes ${start}-${end}/${blob.byteLength}`, + }, + body: Buffer.from(slice), + }); +} + +async function fulfillJson( + route: Route, + body: unknown, + status = 200, +) { + await route.fulfill({ + status, + contentType: "application/json", + body: JSON.stringify(body), + }); +} + +function smokeSession(opts: { + health: SidecarHealth; + uploadedFrameChunks: Uint8Array[]; +}) { + return { + id: SESSION_ID, + status: "ready", + capturePerspective: "sidecar-3d", + captureFps: opts.health.fps ?? 30, + durationSec: Math.max(1, uploadedFrameCount(opts.uploadedFrameChunks) / 30), + qualityScore: 0.9, + qualityFlags: [], + createdAt: new Date().toISOString(), + rowingSessionId: null, + rowingSession: null, + strokePostureMetrics: [ + { + id: "hardware-smoke-metric-0", + strokeIndex: 0, + segmentationSource: "pose-segmented", + phaseBoundariesJson: { + catchFrameIndex: 0, + driveStartFrameIndex: 0, + finishFrameIndex: 0, + recoveryStartFrameIndex: 0, + nextCatchFrameIndex: 0, + confidence: 0.8, + }, + metricsJson: { + backAngleAtCatchDeg: 30, + backAngleAtFinishDeg: 55, + laybackAngleDeg: 25, + recoveryDriveRatio: 2, + sidecar3D: { + lateralShoulderSymmetryMm: 0, + leftKneeTrackDeviationMm: 0, + rightKneeTrackDeviationMm: 0, + nearShinAngleDeg: 90, + }, + }, + }, + ], + postureFaults: [], + }; +} diff --git a/tests/freemocapSidecarSource.test.ts b/tests/freemocapSidecarSource.test.ts index 2155b6a..c8b7f8e 100644 --- a/tests/freemocapSidecarSource.test.ts +++ b/tests/freemocapSidecarSource.test.ts @@ -1,7 +1,12 @@ import { test } from "node:test"; import assert from "node:assert/strict"; +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { once } from "node:events"; +import net from "node:net"; +import path from "node:path"; import { FreemocapSidecarSource, + SIDECAR_REPROJECTION_ERROR_WARNING_MM, encodeSidecarPoseFrame, validateSidecarKeypointFrame, } from "../src/lib/mocap/freemocapSidecarSource"; @@ -9,6 +14,7 @@ import type { PoseCaptureSource } from "../src/lib/mocap/poseCaptureSource"; import { BYTES_PER_FRAME_V2, KEYPOINT_SCHEMA_V2, + QUALITY_FLAG, decodeFrame, } from "../src/lib/mocap/poseFrameStream"; @@ -85,6 +91,73 @@ test("sidecar source starts capture, records calibration, and encodes v2 frames" }); }); +test("sidecar source captures from a real synthetic sidecar process", async () => { + const port = await getFreePort(); + const sidecar = startPythonSidecar(port); + await waitForSidecarHealth(port); + const originalFetch = globalThis.fetch; + const poseUploads: Uint8Array[] = []; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/sidecar/connect")) { + assert.deepEqual(JSON.parse(init?.body as string), { port }); + const startRes = await originalFetch(`http://localhost:${port}/session/start`, { + method: "POST", + }); + assert.equal(startRes.status, 200); + const started = (await startRes.json()) as { + sessionId?: string; + calibrationId?: string; + }; + return jsonResponse( + { + status: "connected", + fps: 30, + cameras: 3, + schemaVersion: 2, + port, + sidecarSessionId: started.sessionId, + calibrationId: started.calibrationId, + }, + 200, + ); + } + if (url.endsWith("/pose-stream")) { + const body = init?.body; + assert.ok(body instanceof Blob); + poseUploads.push(new Uint8Array(await body.arrayBuffer())); + return new Response(null, { status: 200 }); + } + return originalFetch(input, init); + }) as typeof fetch; + + try { + const source = new FreemocapSidecarSource({ + sessionId: "session-real-sidecar", + port, + flushBytes: BYTES_PER_FRAME_V2, + }); + + await source.init(); + await source.start(); + await waitFor(() => source.framesCaptured >= 2 && poseUploads.length >= 1); + await source.stop(); + + assert.match(source.sidecarSessionId ?? "", /^[0-9a-f-]{36}$/); + assert.equal(source.calibrationId, "synthetic-calibration"); + assert.ok(source.framesCaptured >= 2); + assert.ok(poseUploads.length >= 1); + assert.equal( + poseUploads.reduce((total, chunk) => total + chunk.byteLength, 0) % + BYTES_PER_FRAME_V2, + 0, + ); + } finally { + globalThis.fetch = originalFetch; + await stopProcess(sidecar); + } +}); + test("sidecar source flushes buffered v2 bytes by threshold and drains on stop", async () => { await withSidecarGlobals(async ({ fetchCalls }) => { const source = new FreemocapSidecarSource({ @@ -129,6 +202,39 @@ test("sidecar source flushes buffered v2 bytes by threshold and drains on stop", }); }); +test("sidecar source preserves high reprojection quality as frame flags", async () => { + await withSidecarGlobals(async () => { + const capturedFlags: number[] = []; + const capturedBytes: Uint8Array[] = []; + const source = new FreemocapSidecarSource({ + sessionId: "session-reprojection", + flushBytes: BYTES_PER_FRAME_V2 * 4, + onFrame: (frame) => { + capturedFlags.push(frame.qualityFlags); + capturedBytes.push(frame.poseFrameBytes); + }, + }); + + await source.init(); + await source.start(); + const highReprojectionFrame = makeSidecarFrame(0); + highReprojectionFrame.quality.reprojection_error_mm = + SIDECAR_REPROJECTION_ERROR_WARNING_MM + 1; + FakeWebSocket.instances[0].emitMessage(highReprojectionFrame); + await source.stop(); + + assert.equal( + (capturedFlags[0] & QUALITY_FLAG.HIGH_REPROJECTION_ERROR) !== 0, + true, + ); + const decoded = decodeFrame(capturedBytes[0], 0, KEYPOINT_SCHEMA_V2); + assert.equal( + (decoded.qualityFlags & QUALITY_FLAG.HIGH_REPROJECTION_ERROR) !== 0, + true, + ); + }); +}); + test("sidecar source reports malformed frames without uploading them", async () => { await withSidecarGlobals(async ({ fetchCalls }) => { const errors: Error[] = []; @@ -221,6 +327,31 @@ test("sidecar source reports stop errors after pending uploads are drained", asy ); }); +test("sidecar source rejects final drain upload failures", async () => { + await withSidecarGlobals( + async () => { + const errors: Error[] = []; + const statuses: string[] = []; + const source = new FreemocapSidecarSource({ + sessionId: "session-7", + flushBytes: BYTES_PER_FRAME_V2 * 4, + onStatus: (status) => statuses.push(status), + onError: (err) => errors.push(err), + }); + + await source.init(); + await source.start(); + FakeWebSocket.instances[0].emitMessage(makeSidecarFrame(0)); + + await assert.rejects(() => source.stop(), /Pose upload failed: 500/); + + assert.equal(errors.at(-1)?.message, "Pose upload failed: 500"); + assert.equal(statuses.includes("stopped"), false); + }, + { poseUploadStatus: 500 }, + ); +}); + test("sidecar frame validator accepts ADR-0005 snake_case and rejects duplicates", () => { const frame = validateSidecarKeypointFrame(makeSidecarFrame(7)); assert.equal(frame.frameIndex, 7); @@ -239,11 +370,31 @@ test("sidecar frame validator accepts ADR-0005 snake_case and rejects duplicates ); }); +test("sidecar frame validator requires schema and source fields", () => { + const missingSchema = makeSidecarFrame(0) as Partial>; + delete missingSchema.schema_version; + assert.throws( + () => validateSidecarKeypointFrame(missingSchema), + /schemaVersion must be a finite number/, + ); + + const missingSource = makeSidecarFrame(0) as Partial>; + delete missingSource.source; + assert.throws( + () => validateSidecarKeypointFrame(missingSource), + /source must be a string/, + ); +}); + async function withSidecarGlobals( run: (context: { fetchCalls: { input: RequestInfo | URL; init?: RequestInit }[]; }) => Promise, - options: { connectStatus?: number; stopStatus?: number } = {}, + options: { + connectStatus?: number; + stopStatus?: number; + poseUploadStatus?: number; + } = {}, ): Promise { const originalFetch = globalThis.fetch; const originalWebSocket = globalThis.WebSocket; @@ -266,7 +417,7 @@ async function withSidecarGlobals( ); } if (url.endsWith("/pose-stream")) { - return new Response(null, { status: 200 }); + return new Response(null, { status: options.poseUploadStatus ?? 200 }); } if (url === "http://localhost:8765/session/stop") { return new Response(null, { status: options.stopStatus ?? 200 }); @@ -333,3 +484,79 @@ function jsonResponse(body: unknown, status: number): Response { function tick(): Promise { return new Promise((resolve) => setTimeout(resolve, 0)); } + +async function getFreePort(): Promise { + const server = net.createServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert.ok(address && typeof address === "object"); + const port = address.port; + server.close(); + await once(server, "close"); + return port; +} + +function startPythonSidecar(port: number): ChildProcessWithoutNullStreams { + const sidecarSrc = path.join(process.cwd(), "sidecar", "src"); + const env = { + ...process.env, + PYTHONPATH: process.env.PYTHONPATH + ? `${sidecarSrc}${path.delimiter}${process.env.PYTHONPATH}` + : sidecarSrc, + }; + return spawn( + "python3", + [ + "-m", + "rowing_tracker_sidecar", + "--host", + "127.0.0.1", + "--port", + String(port), + "--source", + "synthetic", + ], + { env }, + ); +} + +async function waitForSidecarHealth(port: number): Promise { + const deadline = Date.now() + 5000; + let lastError: unknown; + while (Date.now() < deadline) { + try { + const res = await fetch(`http://localhost:${port}/health`); + if (res.ok) return; + } catch (err) { + lastError = err; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(`sidecar did not become healthy: ${String(lastError)}`); +} + +async function waitFor( + predicate: () => boolean, + timeoutMs = 5000, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 25)); + } + throw new Error("timed out waiting for sidecar frames"); +} + +async function stopProcess(proc: ChildProcessWithoutNullStreams): Promise { + if (proc.exitCode !== null || proc.signalCode !== null) return; + proc.kill("SIGTERM"); + await Promise.race([ + once(proc, "exit"), + new Promise((resolve) => setTimeout(resolve, 1000)), + ]); + if (proc.exitCode === null && proc.signalCode === null) { + proc.kill("SIGKILL"); + await once(proc, "exit"); + } +} diff --git a/tests/sidecarCliContract.test.ts b/tests/sidecarCliContract.test.ts index 17081bd..4ebdc28 100644 --- a/tests/sidecarCliContract.test.ts +++ b/tests/sidecarCliContract.test.ts @@ -1,11 +1,18 @@ import assert from "node:assert/strict"; -import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; -import { mkdtemp, writeFile } from "node:fs/promises"; +import { + execFile, + spawn, + type ChildProcessWithoutNullStreams, +} from "node:child_process"; +import { mkdir, mkdtemp, readFile, stat, writeFile } from "node:fs/promises"; import net from "node:net"; import { once } from "node:events"; import os from "node:os"; import path from "node:path"; import { test } from "node:test"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); test("rowing-tracker-sidecar serves the ADR-0005 contract with synthetic frames", async () => { const port = await getFreePort(); @@ -15,6 +22,8 @@ test("rowing-tracker-sidecar serves the ADR-0005 contract with synthetic frames" const health = await waitForHealth(port); assert.equal(health.status, "ready"); assert.equal(health.schemaVersion, 2); + assert.equal(health.contractVersion, 1); + assert.equal(health.sidecarVersion, "0.1.0"); assert.equal(health.cameras, 3); assert.equal(health.fps, 30); assert.equal(health.source, "synthetic"); @@ -53,12 +62,18 @@ test("rowing-tracker-sidecar streams recorded FreeMoCap-style frames", async () dataPath, JSON.stringify({ frames: [ - Array.from({ length: 33 }, (_, index) => [ - index + 0.5, - index + 100.5, - index + 200.5, - index < 13 ? 0.91 : 0.25, - ]), + { + keypoints: Array.from({ length: 33 }, (_, index) => ({ + index, + x: index + 0.5, + y: index + 100.5, + z: index + 200.5, + confidence: index < 13 ? 0.91 : 0.25, + })), + quality: { + reprojection_error_mm: 4.5, + }, + }, ], }), ); @@ -96,6 +111,100 @@ test("rowing-tracker-sidecar streams recorded FreeMoCap-style frames", async () assert.equal(frame.keypoints[12].confidence, 0.91); assert.equal(frame.quality.tracked_count, 13); assert.equal(frame.quality.camera_count, 4); + assert.equal(frame.quality.reprojection_error_mm, 4.5); + + await fetch(`http://localhost:${port}/session/stop`, { method: "POST" }); + } finally { + await stopProcess(sidecar); + } +}); + +test("rowing-tracker-sidecar maps FreeMoCap directory reprojection quality", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "sidecar-freemocap-dir-")); + await writeFile( + path.join(root, "3dData_numFrames_numTrackedPoints_spatialXYZ.npy"), + "", + ); + await writeFile( + path.join(root, "3dData_numFrames_numTrackedPoints_reprojectionError.npy"), + "", + ); + + const fakeNumpyRoot = await mkdtemp(path.join(os.tmpdir(), "sidecar-numpy-")); + await writeFile( + path.join(fakeNumpyRoot, "numpy.py"), + ` +import math +import os + +class FakeArray: + def __init__(self, data): + self.data = data + self.shape = self._shape(data) + self.ndim = len(self.shape) + + def _shape(self, data): + if isinstance(data, list): + if not data: + return (0,) + return (len(data),) + self._shape(data[0]) + return () + + def __iter__(self): + return iter(self.data) + + def __getitem__(self, index): + return self.data[index] + +def _frame(offset): + return [ + [float(index + offset), float(index + 100 + offset), float(index + 200 + offset)] + for index in range(33) + ] + +def load(path): + basename = os.path.basename(path) + if basename == "3dData_numFrames_numTrackedPoints_spatialXYZ.npy": + return FakeArray([_frame(0), _frame(1000)]) + if basename == "3dData_numFrames_numTrackedPoints_reprojectionError.npy": + return FakeArray([[1.0, 2.0, 3.0], [4.0, math.nan, 6.0]]) + raise FileNotFoundError(path) +`, + ); + + const port = await getFreePort(); + const sidecar = startSidecar( + port, + [ + "--source", + "freemocap", + "--freemocap-data", + root, + "--camera-count", + "3", + ], + { pythonPath: [fakeNumpyRoot] }, + ); + + try { + const health = await waitForHealth(port); + assert.equal(health.status, "ready"); + assert.equal(health.source, "freemocap"); + assert.equal(health.calibrationId, "freemocap-3dData_numFrames_numTrackedPoints_spatialXYZ"); + assert.match( + String((health.diagnostics as unknown[])[1]), + /loaded FreeMoCap reprojection quality/, + ); + + const startRes = await fetch(`http://localhost:${port}/session/start`, { + method: "POST", + }); + assert.equal(startRes.status, 200); + + const frame = await readOneSidecarFrame(port); + assert.equal(frame.keypoints[12].x, 12); + assert.equal(frame.keypoints[12].confidence, 1); + assert.equal(frame.quality.reprojection_error_mm, 2); await fetch(`http://localhost:${port}/session/stop`, { method: "POST" }); } finally { @@ -103,6 +212,704 @@ test("rowing-tracker-sidecar streams recorded FreeMoCap-style frames", async () } }); +test("rowing-tracker-sidecar processes a FreeMoCap recording folder before streaming", async () => { + const recordingRoot = await mkdtemp(path.join(os.tmpdir(), "sidecar-freemocap-recording-")); + const calibrationPath = path.join(recordingRoot, "rig-calibration.toml"); + await writeFile(calibrationPath, "fake calibration"); + + const fakePackageRoot = await mkdtemp(path.join(os.tmpdir(), "sidecar-freemocap-headless-")); + const processorDir = path.join( + fakePackageRoot, + "freemocap", + "core_processes", + "process_motion_capture_videos", + ); + await mkdir(processorDir, { recursive: true }); + await writeFile(path.join(fakePackageRoot, "freemocap", "__init__.py"), ""); + await writeFile(path.join(fakePackageRoot, "freemocap", "core_processes", "__init__.py"), ""); + await writeFile(path.join(processorDir, "__init__.py"), ""); + await writeFile( + path.join(processorDir, "process_recording_headless.py"), + ` +from pathlib import Path + +def process_recording_headless( + recording_path, + path_to_camera_calibration_toml=None, + path_to_blender_executable=None, + recording_processing_parameter_model=None, + recording_info_model=None, + run_blender=True, + make_jupyter_notebook=True, + use_tqdm=True, +): + output = Path(recording_path) / "output_data" + output.mkdir(parents=True, exist_ok=True) + (output / "skeleton_3d.npy").write_text("fake npy") + (output / "processor_args.txt").write_text( + f"{path_to_camera_calibration_toml}|{run_blender}|{make_jupyter_notebook}|{use_tqdm}" + ) +`, + ); + await writeFile( + path.join(fakePackageRoot, "numpy.py"), + ` +import os + +class FakeArray: + def __init__(self, data): + self.data = data + self.shape = self._shape(data) + self.ndim = len(self.shape) + + def _shape(self, data): + if isinstance(data, list): + if not data: + return (0,) + return (len(data),) + self._shape(data[0]) + return () + + def __iter__(self): + return iter(self.data) + +def load(path): + if os.path.basename(path) != "skeleton_3d.npy": + raise FileNotFoundError(path) + return FakeArray([ + [ + [float(index), float(index + 100), float(index + 200), 0.91] + for index in range(33) + ] + ]) +`, + ); + + const port = await getFreePort(); + const sidecar = startSidecar( + port, + [ + "--source", + "freemocap", + "--freemocap-recording", + recordingRoot, + "--freemocap-calibration-toml", + calibrationPath, + "--camera-count", + "3", + ], + { pythonPath: [fakePackageRoot] }, + ); + + try { + const health = await waitForHealth(port); + assert.equal(health.status, "ready"); + assert.equal(health.source, "freemocap"); + assert.equal(health.calibrationId, "freemocap-rig-calibration"); + assert.match( + String((health.diagnostics as unknown[])[0]), + /processed FreeMoCap recording/, + ); + + const processorArgs = await readFile( + path.join(recordingRoot, "output_data", "processor_args.txt"), + "utf8", + ); + assert.equal(processorArgs, `${calibrationPath}|False|False|False`); + + const startRes = await fetch(`http://localhost:${port}/session/start`, { + method: "POST", + }); + assert.equal(startRes.status, 200); + + const frame = await readOneSidecarFrame(port); + assert.equal(frame.schema_version, 2); + assert.equal(frame.source, "sidecar-3d"); + assert.equal(frame.keypoints[12].x, 12); + assert.equal(frame.keypoints[12].y, 112); + assert.equal(frame.keypoints[12].z, 212); + assert.equal(frame.keypoints[12].confidence, 0.91); + assert.equal(frame.quality.camera_count, 3); + + await fetch(`http://localhost:${port}/session/stop`, { method: "POST" }); + } finally { + await stopProcess(sidecar); + } +}); + +test("rowing-tracker-sidecar reports initializing while FreeMoCap recording processing runs", async () => { + const recordingRoot = await mkdtemp( + path.join(os.tmpdir(), "sidecar-freemocap-processing-"), + ); + const fakePackageRoot = await mkdtemp( + path.join(os.tmpdir(), "sidecar-freemocap-processing-pkg-"), + ); + const processorDir = path.join( + fakePackageRoot, + "freemocap", + "core_processes", + "process_motion_capture_videos", + ); + await mkdir(processorDir, { recursive: true }); + await writeFile(path.join(fakePackageRoot, "freemocap", "__init__.py"), ""); + await writeFile( + path.join(fakePackageRoot, "freemocap", "core_processes", "__init__.py"), + "", + ); + await writeFile(path.join(processorDir, "__init__.py"), ""); + await writeFile( + path.join(processorDir, "process_recording_headless.py"), + ` +import time +from pathlib import Path + +def process_recording_headless( + recording_path, + path_to_camera_calibration_toml=None, + path_to_blender_executable=None, + recording_processing_parameter_model=None, + recording_info_model=None, + run_blender=True, + make_jupyter_notebook=True, + use_tqdm=True, +): + root = Path(recording_path) + (root / "processor-started.txt").write_text("started") + release = root / "processor-release.txt" + while not release.exists(): + time.sleep(0.02) + output = root / "output_data" + output.mkdir(parents=True, exist_ok=True) + (output / "skeleton_3d.npy").write_text("fake npy") +`, + ); + await writeFile( + path.join(fakePackageRoot, "numpy.py"), + ` +import os + +class FakeArray: + def __init__(self, data): + self.data = data + self.shape = (len(data), len(data[0]), len(data[0][0])) + self.ndim = 3 + + def __iter__(self): + return iter(self.data) + +def load(path): + if os.path.basename(path) != "skeleton_3d.npy": + raise FileNotFoundError(path) + return FakeArray([ + [ + [float(index), float(index + 100), float(index + 200), 0.91] + for index in range(33) + ] + ]) +`, + ); + + const port = await getFreePort(); + const sidecar = startSidecar( + port, + [ + "--source", + "freemocap", + "--freemocap-recording", + recordingRoot, + "--camera-count", + "3", + ], + { pythonPath: [fakePackageRoot] }, + ); + + try { + await waitForFile(path.join(recordingRoot, "processor-started.txt"), 1_000); + const initializing = await waitForHealthStatus(port, "initializing", 1_000); + assert.equal(initializing.source, "freemocap"); + assert.equal(initializing.fps, 30); + assert.equal(initializing.cameras, 3); + assert.equal(initializing.schemaVersion, 2); + assert.equal(initializing.contractVersion, 1); + assert.equal(initializing.calibrationId, `freemocap-${path.basename(recordingRoot)}`); + assert.match( + String((initializing.diagnostics as unknown[])[0]), + /processing FreeMoCap recording/, + ); + + const initializingStartRes = await fetch( + `http://localhost:${port}/session/start`, + { method: "POST" }, + ); + assert.equal(initializingStartRes.status, 409); + const initializingStartBody = + (await initializingStartRes.json()) as Record; + assert.equal(initializingStartBody.status, "initializing"); + assert.equal(initializingStartBody.calibrationId, initializing.calibrationId); + assert.deepEqual( + initializingStartBody.diagnostics, + initializing.diagnostics, + ); + + await writeFile(path.join(recordingRoot, "processor-release.txt"), "go"); + const ready = await waitForHealthStatus(port, "ready", 5_000); + assert.equal(ready.calibrationId, `freemocap-${path.basename(recordingRoot)}`); + const readyStartRes = await fetch(`http://localhost:${port}/session/start`, { + method: "POST", + }); + assert.equal(readyStartRes.status, 200); + const frame = await readOneSidecarFrame(port); + assert.equal(frame.source, "sidecar-3d"); + assert.equal(frame.keypoints.length, 33); + await fetch(`http://localhost:${port}/session/stop`, { method: "POST" }); + } finally { + await stopProcess(sidecar); + } +}); + +test("rowing-tracker-sidecar reports FreeMoCap recording processing failures after initializing", async () => { + const recordingRoot = await mkdtemp( + path.join(os.tmpdir(), "sidecar-freemocap-processing-fails-"), + ); + const fakePackageRoot = await mkdtemp( + path.join(os.tmpdir(), "sidecar-freemocap-processing-fails-pkg-"), + ); + const processorDir = path.join( + fakePackageRoot, + "freemocap", + "core_processes", + "process_motion_capture_videos", + ); + await mkdir(processorDir, { recursive: true }); + await writeFile(path.join(fakePackageRoot, "freemocap", "__init__.py"), ""); + await writeFile( + path.join(fakePackageRoot, "freemocap", "core_processes", "__init__.py"), + "", + ); + await writeFile(path.join(processorDir, "__init__.py"), ""); + await writeFile( + path.join(processorDir, "process_recording_headless.py"), + ` +import time +from pathlib import Path + +def process_recording_headless( + recording_path, + path_to_camera_calibration_toml=None, + path_to_blender_executable=None, + recording_processing_parameter_model=None, + recording_info_model=None, + run_blender=True, + make_jupyter_notebook=True, + use_tqdm=True, +): + root = Path(recording_path) + (root / "processor-started.txt").write_text("started") + release = root / "processor-release.txt" + while not release.exists(): + time.sleep(0.02) + raise RuntimeError("calibration solve failed") +`, + ); + + const port = await getFreePort(); + const sidecar = startSidecar( + port, + [ + "--source", + "freemocap", + "--freemocap-recording", + recordingRoot, + "--camera-count", + "3", + ], + { pythonPath: [fakePackageRoot] }, + ); + + try { + await waitForFile(path.join(recordingRoot, "processor-started.txt"), 1_000); + const initializing = await waitForHealthStatus(port, "initializing", 1_000); + assert.equal(initializing.source, "freemocap"); + + await writeFile(path.join(recordingRoot, "processor-release.txt"), "go"); + const failed = await waitForHealthStatus(port, "error", 5_000); + assert.equal(failed.source, "freemocap"); + assert.match( + String((failed.diagnostics as unknown[])[0]), + /Failed to process FreeMoCap recording/, + ); + assert.match( + String((failed.diagnostics as unknown[])[0]), + /calibration solve failed/, + ); + + const startRes = await fetch(`http://localhost:${port}/session/start`, { + method: "POST", + }); + assert.equal(startRes.status, 409); + const startBody = (await startRes.json()) as Record; + assert.equal(startBody.status, "error"); + assert.deepEqual(startBody.diagnostics, failed.diagnostics); + } finally { + await stopProcess(sidecar); + } +}); + +test("rowing-tracker-sidecar rejects recorded FreeMoCap frames with incomplete keypoint indexes", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "sidecar-freemocap-")); + const dataPath = path.join(root, "frames.json"); + await writeFile( + dataPath, + JSON.stringify({ + frames: [ + { + keypoints: Array.from({ length: 33 }, (_, index) => ({ + index: Math.min(index, 31), + x: index + 0.5, + y: index + 100.5, + z: index + 200.5, + confidence: 0.91, + })), + }, + ], + }), + ); + + const port = await getFreePort(); + const sidecar = startSidecar(port, [ + "--source", + "freemocap", + "--freemocap-data", + dataPath, + ]); + + try { + const health = await waitForHealth(port); + assert.equal(health.status, "error"); + assert.equal(health.source, "freemocap"); + assert.match( + String((health.diagnostics as unknown[])[0]), + /keypoint indexes must contain 0 through 32 exactly once/, + ); + + const startRes = await fetch(`http://localhost:${port}/session/start`, { + method: "POST", + }); + assert.equal(startRes.status, 409); + } finally { + await stopProcess(sidecar); + } +}); + +test("rowing-tracker-sidecar streams injected FreeMoCap runtime provider frames", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "sidecar-runtime-")); + await writeFile( + path.join(root, "fake_runtime_provider.py"), + ` +from rowing_tracker_sidecar.contract import SourceHealth +from rowing_tracker_sidecar.sources import RuntimeFrame, SourceSession + +class FakeRuntimeProvider: + def __init__(self): + self.active = False + self.session_id = None + + def health(self): + return SourceHealth( + status="ready", + fps=24, + camera_count=5, + source="freemocap", + calibration_id="runtime-calibration", + diagnostics=("fake runtime ready",), + ) + + def start_session(self, session_id): + self.active = True + self.session_id = f"runtime-{session_id}" + return SourceSession( + session_id=self.session_id, + calibration_id="runtime-calibration", + ) + + def stop_session(self, session_id): + if session_id != self.session_id: + raise RuntimeError(f"unexpected session {session_id}") + self.active = False + + def frame(self): + if not self.active: + raise RuntimeError("runtime provider is not active") + return RuntimeFrame( + keypoints=[ + [index + 0.25, index + 100.25, index + 200.25, 0.92] + for index in range(33) + ], + timestamp_ms=1710000000123, + reprojection_error_mm=2.75, + camera_count=5, + ) + +def create_provider(): + return FakeRuntimeProvider() +`, + ); + + const port = await getFreePort(); + const sidecar = startSidecar( + port, + [ + "--source", + "freemocap", + "--freemocap-provider", + "fake_runtime_provider:create_provider", + ], + { pythonPath: [root] }, + ); + + try { + const health = await waitForHealth(port); + assert.equal(health.status, "ready"); + assert.equal(health.source, "freemocap"); + assert.equal(health.fps, 24); + assert.equal(health.cameras, 5); + assert.equal(health.calibrationId, "runtime-calibration"); + + const startRes = await fetch(`http://localhost:${port}/session/start`, { + method: "POST", + }); + assert.equal(startRes.status, 200); + const started = (await startRes.json()) as Record; + assert.match(String(started.sessionId), /^runtime-[0-9a-f-]+$/); + assert.equal(started.calibrationId, "runtime-calibration"); + + const frame = await readOneSidecarFrame(port); + assert.equal(frame.schema_version, 2); + assert.equal(frame.timestamp_ms, 1710000000123); + assert.equal(frame.source, "sidecar-3d"); + assert.equal(frame.keypoints.length, 33); + assert.equal(frame.keypoints[12].x, 12.25); + assert.equal(frame.keypoints[12].y, 112.25); + assert.equal(frame.keypoints[12].z, 212.25); + assert.equal(frame.keypoints[12].confidence, 0.92); + assert.equal(frame.quality.tracked_count, 33); + assert.equal(frame.quality.camera_count, 5); + assert.equal(frame.quality.reprojection_error_mm, 2.75); + + const stopRes = await fetch(`http://localhost:${port}/session/stop`, { + method: "POST", + }); + assert.equal(stopRes.status, 200); + } finally { + await stopProcess(sidecar); + } +}); + +test("freemocap runtime source rejects non-finite live provider frames", async () => { + const script = String.raw` +import math + +from rowing_tracker_sidecar.contract import SourceHealth +from rowing_tracker_sidecar.sources import FreeMocapRuntimeSource, RuntimeFrame, SourceSession + +class BadRuntimeProvider: + def health(self): + return SourceHealth( + status="ready", + fps=24, + camera_count=3, + source="freemocap", + calibration_id="runtime-calibration", + ) + + def start_session(self, session_id): + return SourceSession( + session_id=session_id, + calibration_id="runtime-calibration", + ) + + def stop_session(self, session_id): + return None + + def frame(self): + return RuntimeFrame( + keypoints=[ + [math.nan if index == 12 else index, index + 100, index + 200, 0.9] + for index in range(33) + ], + timestamp_ms=1710000000123, + reprojection_error_mm=1.5, + camera_count=3, + ) + +source = FreeMocapRuntimeSource(provider=BadRuntimeProvider()) + +try: + source.frame(0, 1710000000999) +except ValueError as exc: + assert "must be finite" in str(exc) +else: + raise AssertionError("expected non-finite runtime frame rejection") +`; + await execFileAsync("python3", ["-c", script], { + env: sidecarPythonEnv(), + timeout: 30_000, + }); +}); + +test("sidecar state delegates source session lifecycle", async () => { + const script = String.raw` +from rowing_tracker_sidecar.contract import SourceHealth +from rowing_tracker_sidecar.server import SidecarState +from rowing_tracker_sidecar.sources import SourceSession + +class FakeSource: + fps = 42 + camera_count = 2 + calibration_id = "health-calibration" + + def __init__(self): + self.started_with = None + self.stopped_with = None + + def health(self): + return SourceHealth( + status="ready", + fps=self.fps, + camera_count=self.camera_count, + source="freemocap", + calibration_id=self.calibration_id, + ) + + def start_session(self, session_id): + self.started_with = session_id + return SourceSession( + session_id=f"runtime-{session_id}", + calibration_id="runtime-calibration", + ) + + def stop_session(self, session_id): + self.stopped_with = session_id + + def frame(self, frame_index, timestamp_ms): + raise AssertionError("frame should not be read by lifecycle test") + +source = FakeSource() +state = SidecarState(source) +status, body = state.start() + +assert status == 200 +assert source.started_with is not None +assert body["sessionId"] == f"runtime-{source.started_with}" +assert body["calibrationId"] == "runtime-calibration" +assert state.session_id == body["sessionId"] +assert state.is_active() + +stop_status, stop_body = state.stop() +assert stop_status == 200 +assert source.stopped_with == body["sessionId"] +assert stop_body == {"status": "stopped", "sessionId": body["sessionId"]} +assert state.session_id is None +assert not state.is_active() +`; + await execFileAsync("python3", ["-c", script], { + env: sidecarPythonEnv(), + timeout: 30_000, + }); +}); + +test("sidecar state refuses source session start failures without arming", async () => { + const script = String.raw` +from rowing_tracker_sidecar.contract import SourceHealth +from rowing_tracker_sidecar.server import SidecarState + +class FailingSource: + fps = 42 + camera_count = 2 + calibration_id = "health-calibration" + + def health(self): + return SourceHealth( + status="ready", + fps=self.fps, + camera_count=self.camera_count, + source="freemocap", + calibration_id=self.calibration_id, + ) + + def start_session(self, session_id): + raise RuntimeError("camera runtime refused start") + + def stop_session(self, session_id): + raise AssertionError("stop should not be called") + + def frame(self, frame_index, timestamp_ms): + raise AssertionError("frame should not be read") + +state = SidecarState(FailingSource()) +status, body = state.start() + +assert status == 409 +assert body["status"] == "error" +assert "camera runtime refused start" in body["diagnostics"][-1] +assert state.session_id is None +assert not state.is_active() +`; + await execFileAsync("python3", ["-c", script], { + env: sidecarPythonEnv(), + timeout: 30_000, + }); +}); + +test("sidecar state reports source session stop failures without clearing active session", async () => { + const script = String.raw` +from rowing_tracker_sidecar.contract import SourceHealth +from rowing_tracker_sidecar.server import SidecarState +from rowing_tracker_sidecar.sources import SourceSession + +class FailingStopSource: + fps = 42 + camera_count = 2 + calibration_id = "runtime-calibration" + + def health(self): + return SourceHealth( + status="ready", + fps=self.fps, + camera_count=self.camera_count, + source="freemocap", + calibration_id=self.calibration_id, + ) + + def start_session(self, session_id): + return SourceSession( + session_id=f"runtime-{session_id}", + calibration_id=self.calibration_id, + ) + + def stop_session(self, session_id): + raise RuntimeError(f"camera runtime refused stop for {session_id}") + + def frame(self, frame_index, timestamp_ms): + raise AssertionError("frame should not be read") + +state = SidecarState(FailingStopSource()) +start_status, start_body = state.start() +stop_status, stop_body = state.stop() + +assert start_status == 200 +assert stop_status == 409 +assert stop_body["status"] == "error" +assert stop_body["sessionId"] == start_body["sessionId"] +assert "camera runtime refused stop" in stop_body["diagnostics"][-1] +assert state.session_id == start_body["sessionId"] +assert state.is_active() +`; + await execFileAsync("python3", ["-c", script], { + env: sidecarPythonEnv(), + timeout: 30_000, + }); +}); + test("rowing-tracker-sidecar refuses unconfigured FreeMoCap capture", async () => { const port = await getFreePort(); const sidecar = startSidecar(port, ["--source", "freemocap"]); @@ -114,15 +921,68 @@ test("rowing-tracker-sidecar refuses unconfigured FreeMoCap capture", async () = assert.deepEqual(health.cameras, 0); assert.match( String((health.diagnostics as unknown[])[0]), - /FreeMoCap live camera runtime is not configured/, + new RegExp( + "FreeMoCap package (is not installed|.+ is installed, " + + "but it does not expose a supported headless live camera API)", + ), ); const startRes = await fetch(`http://localhost:${port}/session/start`, { method: "POST", }); assert.equal(startRes.status, 409); - const startBody = (await startRes.json()) as { status: string }; + const startBody = (await startRes.json()) as Record; assert.equal(startBody.status, "error"); + assert.equal(startBody.source, "freemocap"); + assert.equal(startBody.fps, 30); + assert.equal(startBody.cameras, 0); + assert.equal(startBody.schemaVersion, 2); + assert.equal(startBody.contractVersion, 1); + assert.equal(startBody.sidecarVersion, "0.1.0"); + assert.deepEqual(startBody.diagnostics, health.diagnostics); + } finally { + await stopProcess(sidecar); + } +}); + +test("rowing-tracker-sidecar diagnoses installed FreeMoCap package without a supported live API", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "sidecar-freemocap-pkg-")); + await mkdir(path.join(root, "freemocap"), { recursive: true }); + await writeFile(path.join(root, "freemocap", "__init__.py"), ""); + await mkdir(path.join(root, "freemocap-1.8.2.dist-info"), { + recursive: true, + }); + await writeFile( + path.join(root, "freemocap-1.8.2.dist-info", "METADATA"), + "Metadata-Version: 2.1\nName: freemocap\nVersion: 1.8.2\n", + ); + + const port = await getFreePort(); + const sidecar = startSidecar(port, ["--source", "freemocap"], { + pythonPath: [root], + }); + + try { + const health = await waitForHealth(port); + assert.equal(health.status, "error"); + assert.equal(health.source, "freemocap"); + assert.equal(health.cameras, 0); + assert.match( + String((health.diagnostics as unknown[])[0]), + new RegExp( + "FreeMoCap package 1\\.8\\.2 is installed, " + + "but it does not expose a supported headless live camera API", + ), + ); + assert.match( + String((health.diagnostics as unknown[])[1]), + /live calibrated camera frames require an injected --freemocap-provider adapter/, + ); + + const startRes = await fetch(`http://localhost:${port}/session/start`, { + method: "POST", + }); + assert.equal(startRes.status, 409); } finally { await stopProcess(sidecar); } @@ -150,6 +1010,15 @@ test("rowing-tracker-sidecar refuses capture when no cameras are available", asy method: "POST", }); assert.equal(startRes.status, 409); + const startBody = (await startRes.json()) as Record; + assert.equal(startBody.status, "error"); + assert.equal(startBody.source, "synthetic"); + assert.equal(startBody.fps, 30); + assert.equal(startBody.cameras, 0); + assert.equal(startBody.schemaVersion, 2); + assert.equal(startBody.contractVersion, 1); + assert.equal(startBody.sidecarVersion, "0.1.0"); + assert.deepEqual(startBody.diagnostics, health.diagnostics); } finally { await stopProcess(sidecar); } @@ -170,14 +1039,8 @@ async function getFreePort(): Promise { function startSidecar( port: number, extraArgs: string[], + options: { pythonPath?: string[] } = {}, ): ChildProcessWithoutNullStreams { - const sidecarSrc = path.join(process.cwd(), "sidecar", "src"); - const env = { - ...process.env, - PYTHONPATH: process.env.PYTHONPATH - ? `${sidecarSrc}${path.delimiter}${process.env.PYTHONPATH}` - : sidecarSrc, - }; return spawn( "python3", [ @@ -189,10 +1052,23 @@ function startSidecar( String(port), ...extraArgs, ], - { env }, + { env: sidecarPythonEnv(options.pythonPath) }, ); } +function sidecarPythonEnv(extraPythonPath: string[] = []): NodeJS.ProcessEnv { + const sidecarSrc = path.join(process.cwd(), "sidecar", "src"); + const pythonPath = [ + sidecarSrc, + ...extraPythonPath, + process.env.PYTHONPATH, + ].filter((entry): entry is string => Boolean(entry)); + return { + ...process.env, + PYTHONPATH: pythonPath.join(path.delimiter), + }; +} + async function waitForHealth( port: number, ): Promise> { @@ -210,8 +1086,51 @@ async function waitForHealth( throw new Error(`sidecar did not become healthy: ${String(lastError)}`); } +async function waitForFile(filePath: string, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + let lastError: unknown; + while (Date.now() < deadline) { + try { + await stat(filePath); + return; + } catch (err) { + lastError = err; + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`Timed out waiting for ${filePath}: ${String(lastError)}`); +} + +async function waitForHealthStatus( + port: number, + status: string, + timeoutMs: number, +): Promise> { + const deadline = Date.now() + timeoutMs; + let lastHealth: Record | null = null; + let lastError: unknown; + while (Date.now() < deadline) { + try { + const res = await fetch(`http://localhost:${port}/health`); + if (res.ok) { + lastHealth = (await res.json()) as Record; + if (lastHealth.status === status) return lastHealth; + } + } catch (err) { + lastError = err; + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error( + `sidecar did not report health status ${status}: ${JSON.stringify( + lastHealth, + )}; ${String(lastError)}`, + ); +} + async function readOneSidecarFrame(port: number): Promise<{ schema_version: number; + timestamp_ms: number; source: string; keypoints: Array<{ x: number; @@ -219,7 +1138,11 @@ async function readOneSidecarFrame(port: number): Promise<{ z: number; confidence: number; }>; - quality: { tracked_count: number; camera_count: number }; + quality: { + tracked_count: number; + camera_count: number; + reprojection_error_mm: number; + }; }> { return new Promise((resolve, reject) => { const ws = new WebSocket(`ws://localhost:${port}/pose-stream`); diff --git a/tests/sidecarPackageInstall.test.ts b/tests/sidecarPackageInstall.test.ts index 0ea4488..22a4837 100644 --- a/tests/sidecarPackageInstall.test.ts +++ b/tests/sidecarPackageInstall.test.ts @@ -1,6 +1,12 @@ import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; +import { + execFile, + spawn, + type ChildProcessWithoutNullStreams, +} from "node:child_process"; +import { once } from "node:events"; import { mkdtemp } from "node:fs/promises"; +import net from "node:net"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; @@ -18,7 +24,7 @@ test("sidecar package installs locally and exposes rowing-tracker-sidecar", asyn await execFileAsync(python, ["-m", "pip", "install", "-e", "sidecar"], { cwd: process.cwd(), - timeout: 60_000, + timeout: 180_000, }); const { stdout } = await execFileAsync(command, ["--help"], { timeout: 30_000, @@ -27,4 +33,68 @@ test("sidecar package installs locally and exposes rowing-tracker-sidecar", asyn assert.match(stdout, /Run the Rowing Tracker local mocap sidecar/); assert.match(stdout, /--source/); assert.match(stdout, /--freemocap-data/); + assert.match(stdout, /--freemocap-recording/); + assert.match(stdout, /--freemocap-calibration-toml/); + assert.match(stdout, /--freemocap-provider/); + + const port = await getFreePort(); + const sidecar = spawn(command, [ + "--host", + "127.0.0.1", + "--port", + String(port), + "--source", + "synthetic", + ]); + try { + const health = await waitForHealth(port); + assert.equal(health.status, "ready"); + assert.equal(health.schemaVersion, 2); + assert.equal(health.contractVersion, 1); + assert.equal(health.sidecarVersion, "0.1.0"); + } finally { + await stopProcess(sidecar); + } }); + +async function getFreePort(): Promise { + const server = net.createServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert.ok(address && typeof address === "object"); + const port = address.port; + server.close(); + await once(server, "close"); + return port; +} + +async function waitForHealth( + port: number, +): Promise> { + const deadline = Date.now() + 5000; + let lastError: unknown; + while (Date.now() < deadline) { + try { + const res = await fetch(`http://localhost:${port}/health`); + if (res.ok) return (await res.json()) as Record; + } catch (err) { + lastError = err; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(`installed sidecar did not start: ${String(lastError)}`); +} + +async function stopProcess(proc: ChildProcessWithoutNullStreams): Promise { + if (proc.exitCode !== null || proc.signalCode !== null) return; + proc.kill("SIGTERM"); + await Promise.race([ + once(proc, "exit"), + new Promise((resolve) => setTimeout(resolve, 1000)), + ]); + if (proc.exitCode === null && proc.signalCode === null) { + proc.kill("SIGKILL"); + await once(proc, "exit"); + } +} diff --git a/tests/sidecarPoseSource.test.ts b/tests/sidecarPoseSource.test.ts index 9af32be..b0b44c9 100644 --- a/tests/sidecarPoseSource.test.ts +++ b/tests/sidecarPoseSource.test.ts @@ -116,6 +116,123 @@ test("sidecar source reports non-ready health through status and error callbacks ); }); +test("sidecar source rejects ready health with no cameras", async () => { + await withSidecarGlobals( + async () => { + const source = new FreemocapSidecarSource(); + + await assert.rejects( + () => source.init(), + /Sidecar not ready: ready \(No cameras available\)/, + ); + }, + { cameras: 0 }, + ); +}); + +test("sidecar source rejects ready health with zero fps", async () => { + await withSidecarGlobals( + async () => { + const source = new FreemocapSidecarSource(); + + await assert.rejects( + () => source.init(), + /Sidecar not ready: ready \(FPS must be greater than zero\)/, + ); + }, + { fps: 0 }, + ); +}); + +test("sidecar source rejects freemocap readiness without calibration traceability", async () => { + await withSidecarGlobals( + async () => { + const source = new FreemocapSidecarSource(); + + await assert.rejects( + () => source.init(), + /Sidecar not ready: ready \(Missing sidecar calibration id\)/, + ); + }, + { source: "freemocap", calibrationId: null }, + ); +}); + +test("sidecar source rejects incompatible schema during readiness", async () => { + await withSidecarGlobals( + async () => { + const statuses: Array<[PoseCaptureSourceStatus, string | undefined]> = []; + const errors: Error[] = []; + const source = new FreemocapSidecarSource({ + onStatus: (status, detail) => statuses.push([status, detail]), + onError: (err) => errors.push(err), + }); + + await assert.rejects( + () => source.init(), + /Sidecar incompatible: schema version 1 \(expected 2\)/, + ); + + assert.deepEqual(statuses, [ + ["loading", undefined], + ["error", "Sidecar incompatible: schema version 1 (expected 2)"], + ]); + assert.equal(errors[0].message, "Sidecar incompatible: schema version 1 (expected 2)"); + }, + { schemaVersion: 1 }, + ); +}); + +test("sidecar source rejects incompatible contract during readiness", async () => { + await withSidecarGlobals( + async () => { + const source = new FreemocapSidecarSource(); + + await assert.rejects( + () => source.init(), + /Sidecar incompatible: contract version 99 \(expected 1\)/, + ); + }, + { contractVersion: 99 }, + ); +}); + +test("sidecar source rejects missing contract during readiness", async () => { + await withSidecarGlobals( + async () => { + const source = new FreemocapSidecarSource(); + + await assert.rejects( + () => source.init(), + /Sidecar incompatible: contract version missing \(expected 1\)/, + ); + }, + { contractVersion: null }, + ); +}); + +test("sidecar source reports direct start refusal diagnostics", async () => { + await withSidecarGlobals( + async () => { + const errors: Error[] = []; + const source = new FreemocapSidecarSource({ + onError: (err) => errors.push(err), + }); + + await source.init(); + await assert.rejects( + () => source.start(), + /Sidecar session\/start failed: 409 \(error: No cameras available\)/, + ); + assert.equal( + errors[0].message, + "Sidecar session/start failed: 409 (error: No cameras available)", + ); + }, + { startResponse: "notReady" }, + ); +}); + test("sidecar source stops sidecar after start response omits metadata", async () => { await withSidecarGlobals( async ({ fetchCalls }) => { @@ -157,7 +274,13 @@ async function withSidecarGlobals( options: { healthResponse?: "ready" | "initializing" | "unreachable"; healthDiagnostics?: string[]; - startResponse?: "ready" | "malformed"; + fps?: number; + cameras?: number; + schemaVersion?: number; + contractVersion?: number | null; + source?: string; + calibrationId?: string | null; + startResponse?: "ready" | "malformed" | "notReady"; } = {}, ): Promise { const originalFetch = globalThis.fetch; @@ -170,13 +293,23 @@ async function withSidecarGlobals( if (options.healthResponse === "unreachable") { throw new TypeError("fetch failed"); } - return Response.json({ + const healthBody: Record = { status: options.healthResponse ?? "ready", - fps: 60, - cameras: 3, - schemaVersion: 2, + fps: options.fps ?? 60, + cameras: options.cameras ?? 3, + schemaVersion: options.schemaVersion ?? 2, + sidecarVersion: "0.1.0", + source: options.source ?? "synthetic", + calibrationId: + options.calibrationId === undefined + ? "synthetic-calibration" + : options.calibrationId, diagnostics: options.healthDiagnostics, - }); + }; + if (options.contractVersion !== null) { + healthBody.contractVersion = options.contractVersion ?? 1; + } + return Response.json(healthBody); } if (url.includes("/session/start")) { if (options.startResponse === "malformed") { @@ -185,6 +318,21 @@ async function withSidecarGlobals( headers: { "content-type": "application/json" }, }); } + if (options.startResponse === "notReady") { + return Response.json( + { + status: "error", + fps: 30, + cameras: 0, + schemaVersion: 2, + contractVersion: 1, + sidecarVersion: "0.1.0", + source: "synthetic", + diagnostics: ["No cameras available"], + }, + { status: 409 }, + ); + } return Response.json({ sessionId: "sidecar-session", calibrationId: "calibration-1",