From a32b5e797c69cb59e4262e5fd4770804e34be1c9 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Sat, 31 Oct 2026 15:24:11 -0400 Subject: [PATCH 1/2] examples/egocap: serve SENSING plugin cameras from CaptureServer Add a SENSING_COLOR camera type served from the SENSING plugin's existing command line. CaptureServer launches the plugin's camera_plugin_sensing executable, tails each selected sensor's H.264 output file, decodes it on the CPU and sends JPEGs on the existing image path. The same code serves real hardware and the plugin's mock backend, and the server process uses no CUDA. - protocol: CameraType.SENSING_COLOR = 2; OAK_D_COLOR and the wire format are unchanged. The camera cap is now four so both eyes of a stereo head camera can be shown. - Station config (--sensing-config): one rig plus named cameras selected by module name and sensor_index. Discovery uses the plugin's --list-sensors; expected_backend is a station assertion checked against --mock. - One owned producer process per session (private runtime directory, bounded log tail, terminate/kill/reap, parent-death signal) and one adapter per stream, with honest degraded status and stall detection. - Reader: one persistent parser and decoder per sensor; every access unit is decoded in order and the newest decoded frame is JPEG-encoded at the station quality. Consumed bytes are reclaimed with a checked fallocate and the undecoded backlog is bounded. - station_config.py accepts SENSING_COLOR video_sources. - av (PyAV) is a declared dependency, imported only when a SENSING stream attaches. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../backend/egocap_web/capture_preview.py | 4 +- .../backend/egocap_web/station_config.py | 21 +- examples/egocap/capture_server/README.md | 13 + .../capture_server/acquisition/__init__.py | 3 + .../capture_server/acquisition/acquisition.py | 122 +- .../capture_server/acquisition/sensing.py | 795 ++++++++++++ .../acquisition/sensing_h264.py | 400 ++++++ .../acquisition/sensing_jpeg.py | 42 + .../capture_server/capture_server/client.py | 2 +- .../capture_server/capture_server/launch.py | 99 +- .../capture_server/capture_server/protocol.py | 19 +- .../capture_server/capture_server/server.py | 157 ++- examples/egocap/frontend/README.md | 2 +- .../frontend/src/captureProtocol.test.ts | 18 + .../egocap/frontend/src/captureProtocol.ts | 2 +- examples/egocap/pyproject.toml | 2 + .../tests/backend/test_station_config.py | 40 +- .../capture_server/bench_sensing_decode.py | 84 ++ .../tests/capture_server/h264_fixtures.py | 76 ++ .../test_acquisition_adapters.py | 57 +- .../tests/capture_server/test_protocol.py | 60 +- .../capture_server/test_sensing_adapter.py | 1123 +++++++++++++++++ .../tests/capture_server/test_sensing_h264.py | 318 +++++ .../egocap/tests/support/capture_server.py | 7 +- examples/egocap/uv.lock | 19 + 25 files changed, 3382 insertions(+), 103 deletions(-) create mode 100644 examples/egocap/capture_server/capture_server/acquisition/sensing.py create mode 100644 examples/egocap/capture_server/capture_server/acquisition/sensing_h264.py create mode 100644 examples/egocap/capture_server/capture_server/acquisition/sensing_jpeg.py create mode 100644 examples/egocap/tests/capture_server/bench_sensing_decode.py create mode 100644 examples/egocap/tests/capture_server/h264_fixtures.py create mode 100644 examples/egocap/tests/capture_server/test_sensing_adapter.py create mode 100644 examples/egocap/tests/capture_server/test_sensing_h264.py diff --git a/examples/egocap/backend/egocap_web/capture_preview.py b/examples/egocap/backend/egocap_web/capture_preview.py index d680334ff7..90e05ea0f9 100644 --- a/examples/egocap/backend/egocap_web/capture_preview.py +++ b/examples/egocap/backend/egocap_web/capture_preview.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -"""Latest-sample preview cache: one tracking slot and at most three camera +"""Latest-sample preview cache: one tracking slot and at most four camera slots, each holding its immutable callback value and backend-local receipt time. Replacing a slot drops the superseded value -- there is no history, replay queue or recording buffer here. @@ -18,7 +18,7 @@ # capture. MAX_TRACKING_PAYLOAD_BYTES = 64 * 1024 MAX_CAMERA_JPEG_BYTES = 8 * 1024 * 1024 -MAX_CAMERAS = 3 +MAX_CAMERAS = 4 @dataclass(frozen=True) diff --git a/examples/egocap/backend/egocap_web/station_config.py b/examples/egocap/backend/egocap_web/station_config.py index 53f873185b..1d9d9353b7 100644 --- a/examples/egocap/backend/egocap_web/station_config.py +++ b/examples/egocap/backend/egocap_web/station_config.py @@ -34,9 +34,16 @@ STATION_CONFIG_ENV = "EGOCAP_STATION_CONFIG" _DEFAULT_FLAT_STATION_ID = "station_01" -#: Matches capture_server.protocol's own BeginRequest.cameras limit -- checked here too so an -#: over-limit YAML is a configuration error, not a later capture-startup failure. -_MAX_CAMERAS = 3 +#: capture_server.protocol's own BeginRequest.cameras limit, reused rather than duplicated so this +#: loader and the server can never drift apart -- checked here too so an over-limit YAML is a +#: configuration error, not a later capture-startup failure. +_MAX_CAMERAS = p._MAX_CAMERAS + +#: Maps each supported YAML ``video_sources[].type`` spelling to its wire CameraType. +_SOURCE_TYPES: dict[str, p.CameraType] = { + "oak_camera": p.CameraType.OAK_D_COLOR, + "sensing_camera": p.CameraType.SENSING_COLOR, +} class StationConfigError(ValueError): @@ -122,11 +129,13 @@ def _resolve_cameras(station_id: str, entry: dict) -> tuple[p.CameraSelection, . raise StationConfigError(f"station {station_id!r}: duplicate video source name {name!r}") seen_names.add(name) source_type = source.get("type") - if source_type != "oak_camera": + camera_type = _SOURCE_TYPES.get(source_type) if isinstance(source_type, str) else None + if camera_type is None: + supported = ", ".join(sorted(repr(t) for t in _SOURCE_TYPES)) raise StationConfigError( - f"station {station_id!r}: video_sources[{i}].type {source_type!r} is not 'oak_camera'" + f"station {station_id!r}: video_sources[{i}].type {source_type!r} is not one of {supported}" ) - cameras.append(p.CameraSelection(name=name, type=p.CameraType.OAK_D_COLOR)) + cameras.append(p.CameraSelection(name=name, type=camera_type)) return tuple(cameras) diff --git a/examples/egocap/capture_server/README.md b/examples/egocap/capture_server/README.md index bfaebbdde3..6c85e3a8f1 100644 --- a/examples/egocap/capture_server/README.md +++ b/examples/egocap/capture_server/README.md @@ -82,6 +82,19 @@ the link at USB 2; some host ports or cables cannot boot an OAK-D at USB 3. The runs on-device NV12 to MJPEG; each frame's own intrinsics are reported in the camera status when the device provides them. +**SENSING cameras.** Cameras served by IsaacCapture's SENSING plugin are configured with one +YAML file (`sensing_rig:` plus `cameras:` entries of `type: SENSING_COLOR`) and +`--sensing-config FILE`; the file format is documented on `load_sensing_station` in +`capture_server/acquisition/sensing.py`. The server launches the plugin's executable itself, reads each +sensor's H.264 output file, decodes it on the CPU (every unit in order) and sends the newest decoded frame as a JPEG +(`av`, i.e. PyAV, is a declared dependency; nothing is loaded unless a SENSING stream is used). +`--mock` with a SENSING station requires the plugin's MOCK build, whose encoder must write real H.264; +an older mock encoder wrote placeholder bytes that are not video, which the decoder rejects, so such a +build shows a degraded camera instead of pictures. With `--sensing-config`, `cameras` accepts only the +SENSING names it lists (`--camera` is real-acquisition only), so the OAK mock cameras cannot be mixed in. +Measure per-frame cost on the target with `tests/capture_server/bench_sensing_decode.py` +before choosing `delivery_fps`. + **Failure policy.** A camera that produces no frame for max(5 s, five frame periods) counts as failed. A required camera failure ends the session; an optional one reports a degraded camera. There is no automatic reconnection: retry by starting a new session. diff --git a/examples/egocap/capture_server/capture_server/acquisition/__init__.py b/examples/egocap/capture_server/capture_server/acquisition/__init__.py index 11761359e0..baa1c4de60 100644 --- a/examples/egocap/capture_server/capture_server/acquisition/__init__.py +++ b/examples/egocap/capture_server/capture_server/acquisition/__init__.py @@ -12,12 +12,15 @@ from .depthai import DepthAIAdapter from .errors import CameraSourceError, SourceError, SourceExited from .isaac_capture import IsaacCaptureAdapter, TrackingSelection +from .sensing import SensingAdapter, SensingRig __all__ = [ "Acquisition", "CameraSourceError", "DepthAIAdapter", "IsaacCaptureAdapter", + "SensingAdapter", + "SensingRig", "SourceError", "SourceExited", "TrackingSelection", diff --git a/examples/egocap/capture_server/capture_server/acquisition/acquisition.py b/examples/egocap/capture_server/capture_server/acquisition/acquisition.py index 8bdb1dbbe0..0373b1a623 100644 --- a/examples/egocap/capture_server/capture_server/acquisition/acquisition.py +++ b/examples/egocap/capture_server/capture_server/acquisition/acquisition.py @@ -2,20 +2,17 @@ # # SPDX-License-Identifier: Apache-2.0 -"""``Acquisition``: the one concrete object ``_Session`` holds for its -acquisition source -- composing one optional :class:`IsaacCaptureAdapter` -(``None`` for a camera-only request, which must never construct an IC -session) with zero or more :class:`DepthAIAdapter`, one per configured -camera stream. - -This is not a generic protocol/interface: it is one -fixed, concretely named composition that always holds exactly these two -concrete adapter types, mode-agnostic by construction since both adapters -already are. ``_Session`` calls it directly -- -``enter``/``next_tracking_sample``/``next_camera_jpeg``/``status``/``close``. -``next_tracking_sample``/``next_camera_jpeg`` take no -``t``/``tick`` argument: each adapter paces and times itself -- see their -own docstrings. +"""``Acquisition``: the one concrete object ``_Session`` holds for its acquisition source. + +It composes one optional :class:`IsaacCaptureAdapter` (``None`` for a camera-only request, which must never +construct an IC session), zero or more camera adapters (:class:`DepthAIAdapter` for OAK-D, +:class:`SensingAdapter` for SENSING), and the optional :class:`SensingRig` -- the one native producer process +the SENSING adapters read from. The rig is owned here and only borrowed by its adapters, so it is started +before them and reaped after they have settled. + +This is NOT a generic source interface: it is one fixed, concretely named composition. ``_Session`` calls it +directly -- ``enter``/``next_tracking_sample``/``next_camera_jpeg``/``status``/``close`` -- and each adapter +paces and times itself (see their own docstrings). """ from __future__ import annotations @@ -28,14 +25,30 @@ from .depthai import DepthAIAdapter from .errors import CameraSourceError from .isaac_capture import IsaacCaptureAdapter +from .sensing import SensingAdapter, SensingRig __all__ = ["Acquisition"] +def _observe_close(task: "asyncio.Future") -> "asyncio.Future": + """Register an observer on a close task as soon as it exists, so a late failure is never left unretrieved. + + Observation only: it does not own, cancel or await the task. + """ + task.add_done_callback(lambda f: f.cancelled() or f.exception()) + return task + + +# The server's whole source-release budget for one session; adapters and the SENSING producer share it. +_CLOSE_BUDGET_S = 5.0 + + @dataclass class Acquisition: tracking: IsaacCaptureAdapter | None - cameras: tuple[DepthAIAdapter, ...] + cameras: tuple[DepthAIAdapter | SensingAdapter, ...] + # The one native SENSING producer shared by the SENSING adapters in ``cameras``; owned here, borrowed by them. + sensing_rig: SensingRig | None = None async def enter(self) -> None: """A REQUIRED source's startup failure is fatal and propagates @@ -51,6 +64,10 @@ async def enter(self) -> None: for that camera.""" if self.tracking is not None: await self.tracking.enter() + if self.sensing_rig is not None: + # Never raises: a start failure is recorded on the rig and each SENSING adapter reports it under + # its own required/optional policy below, so optional streams degrade while OAK/tracking go on. + await self.sensing_rig.start() for camera in self.cameras: try: await camera.enter() @@ -84,28 +101,59 @@ async def status(self, updated_at_ns: int) -> p.StatusSnapshot: ) async def close(self) -> bool: - """Closes every adapter CONCURRENTLY, each on its own dedicated - worker thread (native thread affinity is per-adapter already -- - see ThreadRunner -- so running them concurrently never puts two - threads on the same native resource). Up to four independent - adapters (one IC + up to three cameras) each bound to their own - release timeout would, awaited sequentially, sum to up to four - times that bound -- comfortably exceeding the outer release-wait - budget server.py's own finalization assumes for ONE source - release. Running them concurrently keeps the total wall-clock - cost to whichever single adapter is slowest, not their sum. - Every adapter's close is still attempted regardless of whether - another raised or timed out -- one release failure must never - skip releasing the others, and a failure to confirm release - within bound is reported (``ok=False``), never silently assumed - successful.""" - closers = ([self.tracking.close()] if self.tracking is not None else []) + [ - camera.close() for camera in self.cameras + """Releases every adapter and the SENSING producer, concurrently, each on its own bound. + + Every adapter is closed on its own dedicated worker thread (native thread affinity is per-adapter -- see + ThreadRunner). The tracking and OAK adapters each keep their own bound. The SENSING stream adapters and + the producer they borrow form ONE chain (:meth:`_close_rig_chain`): the adapters settle first, then the + producer is terminated and reaped, so a worker still reading never races its teardown. The chain runs + concurrently with the other closes and is sized to fit the same ``_CLOSE_BUDGET_S``, so a slow OAK or + tracking release is not cut short by the producer's shutdown, nor the reverse. + Waiting is bounded by ``_CLOSE_BUDGET_S``; a timeout does not cancel any close job -- it keeps running and + is only reported as unconfirmed (False). Every close is attempted; one failure never skips another. A + False means a worker MAY still be running: the wait bounds how long we waited, it does not prove native + settlement. A cancelled caller does not prove release either: every job keeps running, observed, so the + producer is still reaped. + """ + borrowing = [c for c in self.cameras if getattr(c, "borrows_rig", False)] + independent = ([self.tracking] if self.tracking is not None else []) + [ + c for c in self.cameras if not getattr(c, "borrows_rig", False) ] - if not closers: - return True - results = await asyncio.gather(*closers, return_exceptions=True) - first_exc = next((r for r in results if isinstance(r, BaseException)), None) - if first_exc is not None: + tasks = [_observe_close(asyncio.ensure_future(a.close())) for a in independent] + if borrowing or self.sensing_rig is not None: + tasks.append(_observe_close(asyncio.ensure_future(self._close_rig_chain(borrowing)))) + pending: set[asyncio.Future] = set() + if tasks: + _done, pending = await asyncio.wait(tasks, timeout=_CLOSE_BUDGET_S) + results: list[bool | BaseException] = [] + for task in tasks: + if task in pending or task.cancelled(): + results.append(False) # unconfirmed: may still be running + elif (exc := task.exception()) is not None: + results.append(exc) + else: + results.append(task.result()) + if (first_exc := next((r for r in results if isinstance(r, BaseException)), None)) is not None: raise first_exc return all(results) + + async def _close_rig_chain(self, adapters: list) -> bool: + """Close the adapters that borrow the producer, then terminate and reap it. + + The producer is reaped whether or not an adapter failed to close. True only if every adapter and the + producer confirmed release; the first failure is raised after both have been attempted. + """ + results = await asyncio.gather(*(a.close() for a in adapters), return_exceptions=True) + rig_result: bool | BaseException = True + if self.sensing_rig is not None: + try: + rig_result = await self.sensing_rig.close() + except Exception as exc: # noqa: BLE001 -- reported below, after the adapters' own results + rig_result = exc + exceptions = [r for r in (*results, rig_result) if isinstance(r, BaseException)] + if exceptions: + # A cancellation takes priority over any other exception collected here: dropping it in favor of + # an unrelated close failure would silently defeat the caller's own cancellation. + cancelled = next((e for e in exceptions if isinstance(e, asyncio.CancelledError)), None) + raise cancelled if cancelled is not None else exceptions[0] + return all(results) and rig_result is True diff --git a/examples/egocap/capture_server/capture_server/acquisition/sensing.py b/examples/egocap/capture_server/capture_server/acquisition/sensing.py new file mode 100644 index 0000000000..58b9eaabf0 --- /dev/null +++ b/examples/egocap/capture_server/capture_server/acquisition/sensing.py @@ -0,0 +1,795 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""SENSING camera source: station configuration, producer ownership and the per-stream adapter. + +One native ``camera_plugin_sensing`` producer per rig writes every selected sensor's H.264 stream to its own +file; each stream's adapter tails one file, decodes it in order and JPEG-encodes the newest decoded frame for the +existing CaptureServer image path. Nothing loads the decoder until a stream attaches, so OAK-only and tracking-only +runs never need it. + +The producer's existing interface is used unchanged: each selected sensor's H.264 output file +(``--add-stream=sensor=N,output=PATH``). The adapter tails that file, decodes every unit in order on the CPU, +and JPEG-encodes the newest decoded frame. No CUDA is touched in this process. + +Native configuration is server-owned: a client's Begin selects logical camera names only, never paths, +masks or arguments. +""" + +from __future__ import annotations + +import asyncio +import ctypes +import functools +import os +import re +import shutil +import subprocess +import sys +import tempfile +import threading +import time +from dataclasses import dataclass +from typing import Any, Callable + +from capture_server import protocol as p + +from ._thread_runner import ThreadRunner +from .clock import Clock, MonotonicClock +from .errors import CameraSourceError, SourceError +from .sensing_h264 import SensingStreamError, SensingStreamReader +from .sensing_jpeg import encode_rgb_jpeg + +__all__ = [ + "SensingAdapter", + "SensingDescription", + "SensingRig", + "SensingRigConfig", + "SensingSensor", + "SensingStationCamera", + "inspect_rig", + "parse_sensor_listing", + "load_sensing_station", + "resolve_sensors", +] + +_BACKENDS = ("JETSON", "MOCK") +# Producer startup (camera bring-up) and first-frame budget per stream. +_STARTUP_TIMEOUT_S = 30.0 +_INSPECT_TIMEOUT_S = 10.0 +# A started stream that produces no decoded frame for this long (or five sensor frame periods, if longer) has +# failed, the same rule DepthAIAdapter applies to an OAK camera. +_NO_FRAME_TIMEOUT_S = 5.0 +# Shutdown of the SENSING chain fits inside the server's single 5 s source-release budget: the stream adapters +# settle first (_ADAPTER_RELEASE_S), then the producer gets _RIG_CLOSE_BUDGET_S, of which its own steps use +# _RIG_STEPS_S (half of that is the SIGTERM grace before SIGKILL). The OAK and tracking adapters are not part of +# this chain; they close concurrently on their own bounds. The producer's real teardown time on hardware is not +# measured yet; these are bounds, not measurements. +_ADAPTER_RELEASE_S = 1.5 +_RIG_CLOSE_BUDGET_S = 3.5 +_RIG_STEPS_S = 3.0 +_LOG_TAIL_BYTES = 4096 +# Linux's prctl(2) option that asks the kernel to deliver a signal to this process when its parent dies, so an +# orphaned producer (this server killed hard -- OOM, SIGKILL, a native crash) is reaped by the kernel instead of +# running unsupervised and filling its tmpfs runtime directory forever. No effect on an ordinary close(): that +# path already terminates the producer itself. Linux-only; harmless no-op everywhere else (see _set_pdeathsig). +_PR_SET_PDEATHSIG = 1 + + +@dataclass(frozen=True) +class SensingRigConfig: + """Server-owned settings of the one native producer: which binary, backend, platform and link masks.""" + + executable: str + expected_backend: str + platform_config: str + config_name: str + link_masks: tuple[str, ...] + gpu_id: int = 0 + nito_dir: str | None = None + # Encoder settings passed to the producer only when set; otherwise its own defaults apply. The decoder handles + # whatever GOP the producer uses, so none of these is a decoding requirement. + gop: int | None = None + bitrate_bps: int | None = None + # Whether the producer's NV12 uses the full [0,255] luma range; the decoder must convert with the same. + full_range: bool = False + # Where per-session stream files live; None prefers /dev/shm. tmpfs is memory, not extra capacity: the files + # stay bounded only because consumed bytes are reclaimed as they are read (see sensing_h264.py). + runtime_dir: str | None = None + + def __post_init__(self) -> None: + """Reject an unknown backend, a missing executable/platform/name, or empty link masks.""" + if self.expected_backend not in _BACKENDS: + raise ValueError(f"SensingRigConfig.expected_backend must be one of {_BACKENDS}") + if not (self.executable and self.platform_config and self.config_name and self.link_masks): + raise ValueError("SensingRigConfig needs executable, platform_config, config_name and link_masks") + + def base_args(self) -> list[str]: + """The producer's rig arguments as an argument list (never a shell command).""" + args = [ + self.executable, + f"--platform-config={self.platform_config}", + f"--config-name={self.config_name}", + f"--link-masks={' '.join(self.link_masks)}", + f"--gpu-id={self.gpu_id}", + ] + if self.nito_dir: + args.append(f"--nito-dir={self.nito_dir}") + if self.gop is not None: + args.append(f"--gop={self.gop}") + if self.bitrate_bps is not None: + args += [f"--bitrate={self.bitrate_bps}", f"--peak-bitrate={self.bitrate_bps * 3 // 2}"] + if self.full_range: + args.append("--full-range") + return args + + +@dataclass(frozen=True) +class SensingStationCamera: + """Server-owned settings of one SENSING sensor, keyed by the logical name clients request.""" + + module_name: str + delivery_fps: float = 30.0 + jpeg_quality: int = 80 + required: bool = True + # Which sensor of the module, counted in pipeline-id order, when a module has several sensors (a stereo + # camera's two eyes share one module name). Required exactly when the module name is not unique. + sensor_index: int | None = None + + def __post_init__(self) -> None: + """Reject an empty module name, a bad sensor index, a non-positive delivery rate or a bad JPEG quality.""" + if not self.module_name: + raise ValueError("SensingStationCamera.module_name must be nonempty") + if self.sensor_index is not None and (type(self.sensor_index) is not int or self.sensor_index < 0): + raise ValueError( + f"SensingStationCamera.sensor_index must be a non-negative integer, got {self.sensor_index!r}" + ) + if not 0 < self.delivery_fps < float("inf"): + raise ValueError(f"SensingStationCamera.delivery_fps must be positive, got {self.delivery_fps}") + if not 1 <= self.jpeg_quality <= 100: + raise ValueError(f"SensingStationCamera.jpeg_quality must be in 1..100, got {self.jpeg_quality}") + + +@dataclass(frozen=True) +class SensingSensor: + """One sensor as the native producer resolved it: module name, pipeline id, native geometry.""" + + module_name: str + pipeline_id: int + width: int + height: int + fps: float + + +@dataclass(frozen=True) +class SensingDescription: + """What the producer reports for the configured platform: each sensor's module, pipeline id, size and rate. + + The producer's existing ``--list-sensors`` does not say which backend it was built with, so the backend is + the station's declared ``expected_backend`` (checked against ``--mock`` at configuration time), not + something read back from the executable. + """ + + sensors: tuple[SensingSensor, ...] + + +_RIG_KEYS = { + "executable", + "expected_backend", + "platform_config", + "config_name", + "link_masks", + "gpu_id", + "nito_dir", + "gop", + "bitrate_bps", + "full_range", + "runtime_dir", +} +_CAMERA_KEYS = {"type", "module_name", "delivery_fps", "jpeg_quality", "required", "sensor_index"} + + +# Scalar types accepted for each key. bool is checked separately because it is a subclass of int in Python. +_RIG_TYPES: dict[str, type] = { + "executable": str, + "expected_backend": str, + "platform_config": str, + "config_name": str, + "gpu_id": int, + "nito_dir": str, + "gop": int, + "bitrate_bps": int, + "full_range": bool, + "runtime_dir": str, +} +_CAMERA_TYPES: dict[str, type | tuple[type, ...]] = { + "type": str, + "module_name": str, + "delivery_fps": (int, float), + "jpeg_quality": int, + "required": bool, + "sensor_index": int, +} + + +def _check_types(where: str, mapping: dict, types: dict) -> None: + """Reject a value whose YAML type is not the one the key needs (a bool is never accepted as a number).""" + for key, value in mapping.items(): + wanted = types[key] + accepted = wanted if isinstance(wanted, tuple) else (wanted,) + if (isinstance(value, bool) and bool not in accepted) or not isinstance(value, accepted): + names = "/".join(t.__name__ for t in accepted) + raise ValueError(f"{where}.{key} must be {names}, got {type(value).__name__} {value!r}") + + +def load_sensing_station(path: str) -> tuple[SensingRigConfig, dict[str, SensingStationCamera]]: + """Load the station's SENSING rig and cameras from a YAML file; the one configuration entry point. + + Expected shape: ``sensing_rig:`` (see :class:`SensingRigConfig`) and ``cameras:`` mapping logical names + to ``type: SENSING_COLOR`` entries (see :class:`SensingStationCamera`). Unknown keys, wrongly typed values, + a non-SENSING type, a missing section or a file that is not valid YAML all raise ``ValueError``; nothing is + defaulted silently. Link masks must be quoted strings (``'0x0000'``): an unquoted ``0x0000`` is a YAML + integer and would reach the producer as a decimal number. + """ + import yaml + + try: + with open(path, encoding="utf-8") as f: + data = yaml.safe_load(f) + except yaml.YAMLError as e: + raise ValueError(f"{path}: not valid YAML: {e}") from e + if not isinstance(data, dict) or set(data) - {"sensing_rig", "cameras"} or not data.get("sensing_rig"): + raise ValueError(f"{path}: expected only 'sensing_rig' and 'cameras' sections, with a sensing_rig") + rig = data["sensing_rig"] + if not isinstance(rig, dict) or set(rig) - _RIG_KEYS: + raise ValueError(f"{path}: sensing_rig keys must be among {sorted(_RIG_KEYS)}") + masks = rig.get("link_masks", []) + if not isinstance(masks, list) or not all(isinstance(m, str) for m in masks): + raise ValueError(f"{path}: sensing_rig.link_masks must be a list of quoted strings such as '0x0000'") + scalars = {k: v for k, v in rig.items() if k != "link_masks"} + try: + _check_types(f"{path}: sensing_rig", scalars, _RIG_TYPES) + rig_config = SensingRigConfig(**{**scalars, "link_masks": tuple(masks)}) + except TypeError as e: + raise ValueError(f"{path}: invalid sensing_rig: {e}") from e + raw_cameras = data.get("cameras") + if not isinstance(raw_cameras, dict): + raise ValueError(f"{path}: cameras must be a mapping of logical name to camera settings") + cameras: dict[str, SensingStationCamera] = {} + for name, entry in raw_cameras.items(): + if not isinstance(entry, dict) or set(entry) - _CAMERA_KEYS or entry.get("type") != "SENSING_COLOR": + raise ValueError( + f"{path}: camera {name!r} must be type SENSING_COLOR with keys among {sorted(_CAMERA_KEYS)}" + ) + _check_types(f"{path}: cameras.{name}", entry, _CAMERA_TYPES) + try: + cameras[str(name)] = SensingStationCamera(**{k: v for k, v in entry.items() if k != "type"}) + except TypeError as e: + raise ValueError(f"{path}: invalid camera {name!r}: {e}") from e + if not cameras: + raise ValueError(f"{path}: no SENSING cameras configured") + return rig_config, cameras + + +# One line of the producer's ``--list-sensors``: " sensor= x @ fps". +_SENSOR_LINE = re.compile(r"^\s*sensor=(\d+)\s+(\S+)\s+(\d+)x(\d+)\s+@\s+([0-9.]+)\s+fps\s*$") + + +def parse_sensor_listing(text: str) -> SensingDescription: + """Parse the producer's ``--list-sensors`` output; raises ``ValueError`` on anything unexpected. + + The first line (``CONFIG: N sensor(s)``) states how many sensor lines must follow, so a truncated or + garbled listing is rejected instead of silently describing fewer sensors. + """ + lines = [line for line in text.splitlines() if line.strip()] + header = re.match(r"^\S+: (\d+) sensor\(s\)$", lines[0].strip()) if lines else None + if header is None: + raise ValueError(f"malformed sensor listing: unexpected first line {lines[0] if lines else ''!r}") + sensors = [] + for line in lines[1:]: + match = _SENSOR_LINE.match(line) + if match is None: + raise ValueError(f"malformed sensor listing: unexpected line {line!r}") + pipeline_id, module, width, height, fps = match.groups() + if int(width) <= 0 or int(height) <= 0 or not 0 < float(fps) < float("inf"): + raise ValueError(f"malformed sensor listing: non-positive geometry or rate in {line!r}") + sensors.append(SensingSensor(module, int(pipeline_id), int(width), int(height), float(fps))) + if len(sensors) != int(header.group(1)): + raise ValueError(f"malformed sensor listing: header says {header.group(1)} sensors, found {len(sensors)}") + return SensingDescription(tuple(sensors)) + + +def _producer_env() -> dict[str, str]: + """The environment for every run of the producer binary, listing and capture alike. + + The reduced-size test knob must never reach either, or the listed geometry and the captured geometry could + differ from the station's. + """ + return {k: v for k, v in os.environ.items() if k != "SENSING_MOCK_SCALE_DIV"} + + +@functools.cache +def _bind_prctl() -> Callable[[int, int, int, int, int], int] | None: + """The C library's ``prctl`` with explicit argument types, bound once; ``None`` where it does not exist + (anywhere but Linux). Called in the parent before each spawn (see ``_start_sync``), so the ``preexec_fn`` + below only ever calls an already-resolved C function pointer -- see ``_set_pdeathsig``. + """ + try: + prctl = ctypes.CDLL(None, use_errno=True).prctl + except (OSError, AttributeError): + return None + prctl.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong] + return prctl + + +def _set_pdeathsig() -> None: + """``subprocess.Popen``'s ``preexec_fn``: ask the kernel to SIGTERM the producer if this server dies first. + + Runs in the forked child between ``fork()`` and ``exec()``. It calls only a ctypes function pointer already + bound in the parent, with primitive integer arguments, so it does no dlopen or ctypes construction there. + The kernel delivers the signal when the *spawning thread* (the rig's executor worker), not the process, + exits, and exec clears the setting for a setuid or file-capability binary. A failed or unsupported call (not + Linux, or the kernel refuses it) is silently skipped: PR_SET_PDEATHSIG only narrows the orphaned-producer + window from "unbounded" to "none", it is not the only thing cleaning up the producer on an ordinary close(). + """ + if sys.platform != "linux": + return + prctl = _bind_prctl() + if prctl is not None: + prctl(_PR_SET_PDEATHSIG, 15, 0, 0, 0) # 15 == SIGTERM; avoids importing signal for one constant + + +def inspect_rig( + config: SensingRigConfig, + *, + run: Callable[..., Any] = subprocess.run, + timeout_s: float = _INSPECT_TIMEOUT_S, +) -> SensingDescription: + """Ask the native executable which sensors the configured platform resolves to; blocking and bounded. + + Uses the producer's existing ``--list-sensors`` (hardware-free; it never starts capture). Run it off the + event loop. + """ + try: + done = run( + config.base_args() + ["--list-sensors"], + capture_output=True, + text=True, + timeout=timeout_s, + check=False, + env=_producer_env(), + ) + except (OSError, subprocess.TimeoutExpired) as e: + raise SourceError(f"could not inspect {config.executable}: {e}") from e + if done.returncode != 0: + raise SourceError( + f"{config.executable} --list-sensors failed ({done.returncode}): {done.stderr.strip()[-500:]}" + ) + try: + return parse_sensor_listing(done.stdout) + except ValueError as e: + raise SourceError(str(e)) from e + + +def resolve_sensors( + description: SensingDescription, stations: dict[str, SensingStationCamera] +) -> dict[str, SensingSensor]: + """Map each logical camera name to its resolved sensor by module name. + + A module with several sensors needs ``sensor_index``. Rejects a missing module, an ambiguous or out-of-range + selection, and two logical names selecting the same physical sensor. + """ + resolved: dict[str, SensingSensor] = {} + for name, station in stations.items(): + # Pipeline-id order makes sensor_index stable: the link mask renumbers pipelines, not their order. + matches = sorted( + (s for s in description.sensors if s.module_name == station.module_name), key=lambda s: s.pipeline_id + ) + if not matches: + raise SourceError(f"camera {name!r}: module {station.module_name!r} is not in the platform config") + if station.sensor_index is None: + if len(matches) > 1: + raise SourceError( + f"camera {name!r}: module {station.module_name!r} has {len(matches)} sensors; " + "select one with sensor_index" + ) + resolved[name] = matches[0] + elif station.sensor_index >= len(matches): + raise SourceError( + f"camera {name!r}: module {station.module_name!r} has {len(matches)} sensor(s), " + f"no sensor_index {station.sensor_index}" + ) + else: + resolved[name] = matches[station.sensor_index] + pipelines = [s.pipeline_id for s in resolved.values()] + dup = sorted({i for i in pipelines if pipelines.count(i) > 1}) + if dup: + raise SourceError(f"physical sensor(s) selected by more than one logical camera: pipeline {dup}") + return resolved + + +class SensingRig: + """Owns one producer process, its private runtime directory and per-stream H.264 output file paths. + + ``start`` and ``close`` run on the rig's own dedicated worker, one job at a time, so a spawn still in + flight when Stop or cancellation arrives is always finished and then reaped by the queued close; it can + never escape cleanup. The producer is launched with one ``--add-stream=...,output=FILE`` per selected + sensor, and with PR_SET_PDEATHSIG (Linux; a silent no-op elsewhere) so a hard death of this server + process -- OOM, SIGKILL, a native crash -- still gets the producer reaped by the kernel, rather than + left running unsupervised with nothing left to bound its tmpfs output. Diagnostics are drained + continuously into a byte-bounded tail so a full pipe can never stall capture. A start failure is + recorded in ``start_error`` for each adapter to report under its own required/optional policy rather + than ending the attempt here. + """ + + def __init__( + self, + config: SensingRigConfig, + pipeline_ids: tuple[int, ...], + *, + popen: Callable[..., Any] = subprocess.Popen, + runtime_root: str | None = None, + ) -> None: + """Records the config and selected pipeline ids; nothing is started or created until :meth:`start`.""" + self._config = config + self._pipeline_ids = pipeline_ids + self._popen = popen + self._runtime_root = runtime_root + self._dir: str | None = None + self._proc: Any = None + self._drain: threading.Thread | None = None + self._tail = b"" + self._abort_reason = "" + self._runner = ThreadRunner(release_timeout_s=_RIG_CLOSE_BUDGET_S) + self.start_error: str | None = None + # Sticky: once any close failed to confirm release, no later call reports success. + self.cleanup_ok = True + + def stream_path(self, pipeline_id: int) -> str: + """The H.264 output file for one selected sensor; valid after a successful :meth:`start`.""" + assert self._dir is not None, "rig not started" + return os.path.join(self._dir, f"sensor{pipeline_id}.h264") + + async def start(self) -> None: + """Launch the producer on the rig's worker; a failure is recorded in ``start_error``, not raised.""" + try: + await self._runner.run(self._start_sync) + except SourceError as e: + self.start_error = str(e) + + def _start_sync(self) -> None: + # Local filesystem failures are start failures too (an optional stream then degrades instead of + # aborting healthy OAK/tracking). The directory, once created, stays owned for close() to remove. + """Create the private runtime directory and spawn the producer on the rig worker. + + Raises ``SourceError`` on failure. + """ + try: + root = self._runtime_root or self._config.runtime_dir + if root is None and os.path.isdir("/dev/shm") and os.access("/dev/shm", os.W_OK): + root = "/dev/shm" + self._dir = tempfile.mkdtemp(prefix="sensing-", dir=root) + os.chmod(self._dir, 0o700) + except OSError as e: + raise SourceError(f"could not prepare the SENSING runtime directory: {e}") from e + args = self._config.base_args() + [ + f"--add-stream=sensor={i},output={self.stream_path(i)}" for i in self._pipeline_ids + ] + env = _producer_env() + _bind_prctl() # resolve in the parent, so the post-fork preexec_fn does no dlopen + try: + self._proc = self._popen( + args, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + preexec_fn=_set_pdeathsig, + ) + except OSError as e: + self._remove_dir() + raise SourceError(f"could not start {self._config.executable}: {e}") from e + self._drain = threading.Thread(target=self._drain_output, name="sensing-rig-log", daemon=True) + self._drain.start() + + def _drain_output(self) -> None: + """Log-drain thread body. + + Keeps the producer's most recent output so its pipe can never fill and stall it. + """ + stream = self._proc.stdout + try: + while chunk := stream.read1(4096): + # Byte-bounded (a newline-free flood cannot grow it); one atomic assignment, so readers + # always see a consistent snapshot. + self._tail = (self._tail + chunk)[-_LOG_TAIL_BYTES:] + except (OSError, ValueError): + pass # the pipe was closed under us during shutdown + finally: + stream.close() + + def exit_code(self) -> int | None: + """The producer's exit code once it has exited on its own, else None.""" + return None if self._proc is None else self._proc.poll() + + def diagnostics(self) -> str: + """The producer's most recent output, decoded leniently, for error messages.""" + tail = self._tail.decode("utf-8", errors="replace").strip().replace("\n", " | ") + return f"stopped by the server: {self._abort_reason} | {tail}" if self._abort_reason else tail + + def abort(self, reason: str) -> None: + """Stop the producer because its output can no longer be bounded; ``close`` still reaps it.""" + self._abort_reason = self._abort_reason or reason + if self._proc is not None and self._proc.poll() is None: + self._proc.terminate() + + async def close(self) -> bool: + """Terminate and reap the producer, join the log drain, remove the runtime directory. + + Runs on the rig's worker after any pending start, within one bounded budget. Returns True only when + every step is confirmed, and False on every later call once any attempt failed or timed out. + """ + try: + done = await self._runner.close(self._close_sync) + except Exception: + self.cleanup_ok = False + raise + if not done: + self.cleanup_ok = False + return done and self.cleanup_ok + + def _close_sync(self) -> None: + """Terminate, reap and clean up within ``_RIG_STEPS_S`` on the rig worker. + + Sets ``cleanup_ok`` False if anything is unconfirmed. + """ + deadline = time.monotonic() + _RIG_STEPS_S + ok = True + proc = self._proc + if proc is not None: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=_RIG_STEPS_S / 2) + except subprocess.TimeoutExpired: + proc.kill() + try: + proc.wait(timeout=max(deadline - time.monotonic(), 0.05)) + except subprocess.TimeoutExpired: + ok = False + if ok and self._drain is not None: + self._drain.join(timeout=max(deadline - time.monotonic(), 0.05)) + ok = not self._drain.is_alive() + if ok: + ok = self._remove_dir() + if not ok: + self.cleanup_ok = False + + def _remove_dir(self) -> bool: + """Remove this rig's own runtime directory only; False if it could not be removed.""" + if self._dir is None: + return True + try: + shutil.rmtree(self._dir) + except FileNotFoundError: + pass + except OSError: + return False + self._dir = None + return True + + +class SensingAdapter: + """One SENSING stream: tail its H.264 file, deliver the newest frame as JPEG at the delivery rate. + + The same instance shape the server expects of every camera adapter. All blocking file, decode and + JPEG work runs on this adapter's dedicated worker; the reader is created, used and closed there. + """ + + # Acquisition closes adapters that borrow the shared producer before it reaps the producer. + borrows_rig = True + + def __init__( + self, + *, + rig: SensingRig, + sensor: SensingSensor, + camera: p.CameraStreamConfig, + full_range: bool = False, + clock: Clock | None = None, + startup_timeout_s: float = _STARTUP_TIMEOUT_S, + ) -> None: + """Bind one sensor of the shared rig to one accepted camera stream. + + The reader is created later, in :meth:`enter`. + """ + self._rig = rig + self._sensor = sensor + self._camera = camera + self._interval = 1.0 / camera.fps + self._clock = clock or MonotonicClock() + self._startup_timeout_s = startup_timeout_s + self._runner = ThreadRunner(release_timeout_s=_ADAPTER_RELEASE_S) + self._full_range = full_range + # Built in enter(): the stream file path exists only once Acquisition has started the rig. + self._reader: SensingStreamReader | None = None + self._deadline: float | None = None + self._dead = False + self._flushed = False + # Set synchronously by close(), before its job queues behind the worker, so startup, reads and the final + # flush stop within a fraction of a second. + self._stop = threading.Event() + # Host time of the last decoded frame (or of the first confirmed one); drives stall detection. + self._last_frame_s: float | None = None + self._degraded = True + self._last_error = "camera not yet started" + # Sticky: a close that failed to confirm release is never forgotten, whatever happens to the stream. + self.cleanup_failed = False + + @property + def stream_id(self) -> int: + """The server-assigned stream id this adapter delivers.""" + return self._camera.stream_id + + @property + def required(self) -> bool: + """Whether this stream's failure ends the whole session.""" + return self._camera.required + + def _fail(self, message: str) -> CameraSourceError: + """Mark the stream dead and degraded, and return the ``CameraSourceError`` for the caller to raise.""" + self._dead, self._degraded, self._last_error = True, True, message + return CameraSourceError(message, camera_name=self._camera.name) + + def _enter_sync(self) -> None: + """Worker-side startup: wait for the stream file and a first complete, decodable frame. + + Raises a camera error if either does not arrive. + """ + if self._rig.start_error is not None: + # A shared-producer failure degrades exactly the streams that depend on it, each under its own + # required/optional policy in Acquisition.enter. + raise self._fail(f"SENSING producer did not start: {self._rig.start_error}") + deadline = time.monotonic() + self._startup_timeout_s + self._reader = reader = SensingStreamReader( + self._rig.stream_path(self._sensor.pipeline_id), + width=self._sensor.width, + height=self._sensor.height, + full_range=self._full_range, + stop=self._stop, + ) + # The file existing is not readiness: attach, then wait for a complete, decodable frame. + while True: + if self._stop.is_set(): + raise self._fail("stopped during startup") + if (code := self._rig.exit_code()) is not None: + raise self._fail(f"SENSING producer exited with {code}: {self._rig.diagnostics()}") + left = deadline - time.monotonic() + if left <= 0: + raise self._fail(f"timed out waiting for the SENSING stream: {self._rig.diagnostics()}") + try: + reader.connect(min(left, 2.0)) + break + except OSError: + self._stop.wait(0.1) # the producer has not created the file yet + except SensingStreamError as e: + if self._stop.is_set(): + raise self._fail("stopped during startup") from e + raise self._fail(f"SENSING stream rejected: {e}") from e + try: + while reader.read_latest(min(max(deadline - time.monotonic(), 0.0), 0.5)) is None: + if (code := self._rig.exit_code()) is not None: + raise self._fail(f"SENSING producer exited with {code}: {self._rig.diagnostics()}") + if time.monotonic() >= deadline: + raise self._fail("no frame from the SENSING stream before the startup deadline") + except SensingStreamError as e: + if self._stop.is_set(): + raise self._fail("stopped during startup") from e + raise self._fail(f"SENSING stream failed at startup: {e}") from e + self._last_frame_s = self._clock.monotonic() + + async def enter(self) -> None: + """Attach and confirm a usable frame; the deadline is set first so an optional failure stays pollable.""" + self._deadline = self._clock.monotonic() + self._interval + await self._runner.run(self._enter_sync) + self._degraded = False + self._last_error = "" + + def _poll_sync(self) -> tuple[int, bytes] | None: + """Worker-side delivery: JPEG-encode the newest decoded frame, or None when nothing new has arrived.""" + if self._reader is None: + return None + if self._dead: + # The shared producer keeps writing this sensor's file; keep handing the bytes back (no decoding). + try: + self._reader.reclaim_all() + except SensingStreamError as e: + # Unreclaimable spool: stop the shared producer (every stream then reports its exit) rather + # than let its output fill memory. + self._rig.abort(str(e)) + return None + try: + if (code := self._rig.exit_code()) is not None: + # The producer ended: flush the decoder once for its final frame, then report the exit. + if not self._flushed: + self._flushed = True + if (last := self._reader.finish()) is not None: + return self._to_jpeg(last) + raise self._fail(f"SENSING producer exited with {code}: {self._rig.diagnostics()}") + got = self._reader.read_latest(0.0) + if got is None: + self._check_stall() + return None + return self._to_jpeg(got) + except (SensingStreamError, SourceError, ValueError, OSError) as e: + raise self._fail(str(e) or type(e).__name__) from e + + def _check_stall(self) -> None: + """Fail the stream if the producer is alive but no decoded frame has arrived for too long. + + The clock starts at the first confirmed frame, so startup keeps its own deadline; a producer that stops + writing without exiting (link loss, an encoder or plugin hang) is otherwise indistinguishable from a + quiet camera, and a required camera would never end the session. + """ + if self._last_frame_s is None: + return + silent = self._clock.monotonic() - self._last_frame_s + if silent > max(_NO_FRAME_TIMEOUT_S, 5.0 / self._sensor.fps): + raise SensingStreamError( + f"no frame from the SENSING stream for {silent:.1f}s: {self._rig.diagnostics()}" + ) + + def _to_jpeg(self, got: tuple[Any, int]) -> tuple[int, bytes]: + """Encode a decoded RGB frame at the station quality and mark the stream live: ``(host_ns, jpeg)``.""" + rgb, selected_ns = got + jpeg = encode_rgb_jpeg(rgb, self._camera.jpeg_quality) + self._last_frame_s = self._clock.monotonic() + self._degraded, self._last_error = False, "" + return selected_ns, jpeg + + async def next_camera_jpeg(self) -> tuple[int, bytes] | None: + """Sleep to the next delivery tick, then deliver the newest frame as ``(host_ns, jpeg)`` or None. + + None means nothing new this tick (or the stream already failed; later ticks stay paced and empty + rather than looping on exceptions). The timestamp is host monotonic time at frame selection. + """ + assert self._deadline is not None, "next_camera_jpeg() called before enter()" + await asyncio.sleep(max(0.0, self._deadline - self._clock.monotonic())) + now = self._clock.monotonic() + while self._deadline <= now: + self._deadline += self._interval + return await self._runner.run(self._poll_sync) + + def camera_status(self) -> p.CameraStatus: + """This stream's honest current status; never CONNECTED before a frame was confirmed.""" + return p.CameraStatus( + name=self._camera.name, + stream_id=self._camera.stream_id, + status=p.DeviceState.DEGRADED if self._degraded else p.DeviceState.CONNECTED, + reason=p.StatusReason.NO_CURRENT_DATA if self._degraded else p.StatusReason.NONE, + error=self._last_error if self._degraded else "", + intrinsics=None, + ) + + async def close(self) -> bool: + """Close the reader on its worker; False (sticky) if release could not be confirmed. + + The stop signal is raised here, synchronously, before the close job queues behind whatever the worker is + doing, so a startup or read in progress ends promptly instead of delaying the release. + """ + self._stop.set() + + def _close() -> None: + """Worker-side release of the reader; a failure propagates to the awaiting caller below.""" + if self._reader is not None: + self._reader.close() + + try: + confirmed = await self._runner.close(_close) + except Exception: + self.cleanup_failed = True + raise + if not confirmed: + self.cleanup_failed = True + return confirmed and not self.cleanup_failed diff --git a/examples/egocap/capture_server/capture_server/acquisition/sensing_h264.py b/examples/egocap/capture_server/capture_server/acquisition/sensing_h264.py new file mode 100644 index 0000000000..91a88dc8e7 --- /dev/null +++ b/examples/egocap/capture_server/capture_server/acquisition/sensing_h264.py @@ -0,0 +1,400 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Reader for the SENSING plugin's existing H.264 file output, decoded to RGB on the CPU. + +The producer (``camera_plugin_sensing``) writes each sensor's H.264 Annex-B elementary stream to the ``output=`` +path of its ``--add-stream``. This module tails that file with one persistent parser and one persistent software +decoder per sensor: every complete access unit is decoded IN ORDER (ordinary H.264 P-frames reference earlier +frames, so units cannot be skipped before decoding), and only the newest DECODED frame is kept for delivery. The +same code serves the real Jetson encoder and the native mock; neither is special-cased. + +Everything is blocking and synchronous: run it on one dedicated worker (see ``ThreadRunner``). The parser and +decoder are PyAV's, created lazily so ordinary OAK-only runs do not load them until a SENSING stream attaches. +""" + +from __future__ import annotations + +import collections +import ctypes +import errno +import functools +import os +import re +import threading +import time +from typing import Any, Callable + +import numpy as np + +__all__ = ["SensingStreamError", "SensingStreamReader"] + +# Bytes consumed from the file before the already-read prefix is handed back to the filesystem. The producer +# writes at the encoder bitrate for the whole session, so without this the file grows without bound. +_PUNCH_CHUNK_BYTES = 8 * 1024 * 1024 +_READ_BYTES = 4 * 1024 * 1024 +# Work done per poll is capped by bytes (a producer appending faster than we parse cannot hold the worker) and +# by the poll's own deadline; the rest of a backlog is simply read on the next poll. +_MAX_POLL_BYTES = 32 * 1024 * 1024 +_MIN_WORK_S = 0.2 +# Bytes fed to the parser without it completing a single access unit before the stream is declared bad: a +# malformed stream must not make the parser retain input without bound. +_MAX_UNIT_BYTES = 32 * 1024 * 1024 +# Complete but not yet decoded stream bytes the reader may hold. A decoder that falls this far behind resumes at +# the newest restart point (an IDR unit carrying its own SPS/PPS; nothing before it is needed), or ends the stream. +_MAX_PENDING_BYTES = 64 * 1024 * 1024 +# A failed stream's file may stay unreclaimed (a failing fallocate) only this far before the producer stops. +_MAX_UNRECLAIMED_BYTES = 64 * 1024 * 1024 +# Time allowed to read, decode and flush whatever the producer left behind when it ended. +_FINISH_BUDGET_S = 30.0 +_POLL_INTERVAL_S = 0.005 +_FALLOC_FL_KEEP_SIZE = 0x01 +_FALLOC_FL_PUNCH_HOLE = 0x02 + + +# NAL units read straight from the Annex-B bytes (a start code, then a header byte whose low five bits are the +# type): an IDR slice (5), a sequence parameter set (7) and a picture parameter set (8), for any nal_ref_idc. +_IDR_NAL = re.compile(rb"\x00\x00\x01[\x25\x45\x65]") +_SPS_NAL = re.compile(rb"\x00\x00\x01[\x27\x47\x67]") +_PPS_NAL = re.compile(rb"\x00\x00\x01[\x28\x48\x68]") + + +def _is_restart_point(packet: Any) -> bool: + """Whether decoding can safely begin at this access unit with nothing decoded before it. + + That needs an IDR slice (no earlier frame referenced) AND in-band SPS and PPS in the same unit: an IDR alone + says nothing about whether the decoder has the parameter sets it refers to. Read from the bytes themselves, + never from a parser flag. + """ + data = bytes(packet) + return all(nal.search(data) is not None for nal in (_IDR_NAL, _SPS_NAL, _PPS_NAL)) + + +class SensingStreamError(Exception): + """The producer's stream is unreadable, undecodable, or disagrees with the expected geometry.""" + + +@functools.cache +def _bind_fallocate() -> Callable[[int, int, int, int], int]: + """The C library's ``fallocate`` with explicit argument types, bound once; ``-1`` plus ``errno`` on failure.""" + fallocate = ctypes.CDLL(None, use_errno=True).fallocate + fallocate.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_longlong, ctypes.c_longlong] + return fallocate + + +class SensingStreamReader: + """One sensor's H.264 file: attach, then repeatedly take the newest decoded frame as an RGB array. + + A parse, decode or geometry failure is terminal: later reads raise and a new attempt builds a new reader. + The returned array is owned by the reader and valid until the next read. + """ + + def __init__( + self, + path: str, + *, + width: int, + height: int, + full_range: bool = False, + fallocate: Callable[[int, int, int, int], int] | None = None, + stop: threading.Event | None = None, + ) -> None: + """Record the expected geometry and colour range; the file is opened later, in :meth:`connect`. + + ``stop`` is set (from any thread) when the stream is being shut down: every wait and loop here checks it + at least every fraction of a second and ends with :class:`SensingStreamError`, so Stop never queues + behind a long startup, read or final flush. + """ + self._path = path + self._width, self._height = width, height + self._full_range = full_range + self._fd: int | None = None + self._offset = 0 + self._punched = 0 + self._closed = False + self._parser: Any = None + self._decoder: Any = None + self._failed = False + # Complete access units (PyAV packets) waiting to be decoded, in stream order, and their total size. + self._pending: collections.deque[Any] = collections.deque() + self._pending_bytes = 0 + self._newest: Any = None # newest decoded frame not yet delivered + self._finished = False + self._fed_since_packet = 0 + # Injectable so tests can make the native call fail; production binds the C library's once. + self._fallocate = fallocate + self._stop = stop + + def _check_stopped(self) -> None: + """End the current operation, with the reason, if the stream is being shut down.""" + if self._stop is not None and self._stop.is_set(): + raise SensingStreamError("stopped") + + def _wait(self, seconds: float) -> None: + """Sleep for ``seconds``, or less if the stream is shut down meanwhile.""" + if self._stop is not None: + self._stop.wait(seconds) + else: + time.sleep(seconds) + + def connect(self, timeout_s: float) -> None: + """Wait up to ``timeout_s`` for the producer to create the file, then open it. + + Raises ``FileNotFoundError`` (an ``OSError``) if it never appears, and :class:`SensingStreamError` + if it exists but cannot be opened for reading and space reclamation: the stream file is a bounded + spool, so a file we cannot reclaim from is an error, never a quietly unbounded one. + """ + deadline = time.monotonic() + timeout_s + while True: + self._check_stopped() + try: + # Read-write: handing consumed bytes back (hole punching) needs a writable descriptor. + self._fd = os.open(self._path, os.O_RDWR) + break + except FileNotFoundError: + if time.monotonic() >= deadline: + raise + self._wait(_POLL_INTERVAL_S * 4) + except OSError as e: + raise SensingStreamError(f"cannot open {self._path} for reading and reclamation: {e}") from e + if self._fallocate is None: + self._fallocate = _bind_fallocate() + import av # noqa: PLC0415 -- lazy: only a SENSING stream needs the decoder + + self._parser = av.CodecContext.create("h264", "r") + self._decoder = av.CodecContext.create("h264", "r") + # One thread: frame threading delays output by several frames, which a live preview should not pay. + self._decoder.thread_count = 1 + + def _drain_file(self, deadline: float) -> None: + """Read newly written bytes within this poll's work bounds and queue the complete access units.""" + assert self._fd is not None + # A snapshot of the size, not "until EOF": a producer that keeps appending cannot extend this poll. + end = min(os.fstat(self._fd).st_size, self._offset + _MAX_POLL_BYTES) + while self._offset < end: + chunk = os.pread(self._fd, min(_READ_BYTES, end - self._offset), self._offset) + if not chunk: + break + self._offset += len(chunk) + self._fed_since_packet += len(chunk) + # The parser holds a unit back until it sees the next one begin, so a half-written frame is never + # returned; a temporary end of file on a growing file just means waiting for more. + self._queue(self._parser.parse(chunk)) + if self._fed_since_packet > _MAX_UNIT_BYTES: + raise SensingStreamError(f"no complete access unit in {self._fed_since_packet} bytes of stream") + if time.monotonic() >= deadline: + break # always one chunk of progress, then yield to the caller's deadline + self._reclaim() + + def _queue(self, packets: list[Any]) -> None: + """Append parsed access units for decoding, and bound what is held when the decoder is behind.""" + for packet in packets: + self._pending.append(packet) + self._pending_bytes += packet.size + self._fed_since_packet = 0 + if self._pending_bytes <= _MAX_PENDING_BYTES: + return + # Overloaded: resume at the newest restart point, dropping everything older undecoded. This is the only + # place units are skipped before decoding, and it is safe only where nothing earlier is needed: an IDR + # unit that also carries its SPS and PPS. + newest_key = max((i for i, pk in enumerate(self._pending) if _is_restart_point(pk)), default=None) + if newest_key is None: + raise SensingStreamError( + f"decoder cannot keep up: {self._pending_bytes} bytes undecoded and no restart point to resume at" + ) + for _ in range(newest_key): + self._pending_bytes -= self._pending.popleft().size + if self._pending_bytes > _MAX_PENDING_BYTES: + raise SensingStreamError( + f"decoder cannot keep up: {self._pending_bytes} bytes undecoded even from the newest restart point" + ) + + def _decode_pending(self, deadline: float) -> None: + """Decode queued units in order until none are left or ``deadline`` passes (one unit of progress at least). + + A decode that returns nothing is the decoder buffering, not an error; every returned frame is checked + and the newest one is kept. + """ + while self._pending: + self._check_stopped() + packet = self._pending.popleft() + self._pending_bytes -= packet.size + for frame in self._decoder.decode(packet): + self._newest = self._checked(frame) + if time.monotonic() >= deadline: + break + + def _checked(self, frame: Any) -> Any: + """The decoded frame, if it has the geometry the station and the producer agreed on.""" + if (frame.width, frame.height) != (self._width, self._height): + raise SensingStreamError( + f"decoded {frame.width}x{frame.height}, expected {self._width}x{self._height}" + ) + return frame + + def _deliver(self) -> tuple[np.ndarray, int] | None: + """Convert the newest undelivered decoded frame to RGB, or None if there is none.""" + frame, self._newest = self._newest, None + if frame is None: + return None + selected_ns = time.monotonic_ns() + # The producer converts with BT.601; the range follows its --full-range flag. Stated explicitly so + # colours never depend on a decoder's guess for an unspecified stream. + rgb = frame.reformat( + format="rgb24", src_colorspace="ITU601", src_color_range="JPEG" if self._full_range else "MPEG" + ).to_ndarray() + return rgb, selected_ns + + def _reclaim(self) -> None: + """Hand the consumed prefix back to the filesystem once enough has accumulated; failure is an error. + + Space reclamation is what keeps a long session bounded, so an unsupported or failing call ends the + stream with the reason instead of letting the file grow silently; the reclaimed offset advances only + after the call succeeds. + """ + pending = self._offset - self._punched + if pending < _PUNCH_CHUNK_BYTES: + return + assert self._fd is not None and self._fallocate is not None + ctypes.set_errno(0) + if self._fallocate(self._fd, _FALLOC_FL_PUNCH_HOLE | _FALLOC_FL_KEEP_SIZE, self._punched, pending) != 0: + code = ctypes.get_errno() or errno.EIO + raise SensingStreamError(f"cannot reclaim consumed stream bytes: {os.strerror(code)} (errno {code})") + self._punched = self._offset + + def read_latest(self, timeout_s: float) -> tuple[np.ndarray, int] | None: + """Wait up to ``timeout_s`` for a decoded frame, then return ``(rgb, selected_ns)`` of the newest one. + + Every complete unit is decoded in order; frames decoded but not yet delivered are dropped in favour of + the newest. ``selected_ns`` is host monotonic time when that frame is selected (after decoding; the + protocol's HOST_POLL semantics). Returns None when no frame was produced in time; never fabricates one. + Any failure makes the reader terminal and is raised as :class:`SensingStreamError`. + """ + if self._failed: + raise SensingStreamError("reader failed earlier; a new connection is required") + if self._finished: + raise SensingStreamError("stream already finished; the decoder has been flushed") + if self._fd is None: + raise SensingStreamError("reader is not connected") + deadline = time.monotonic() + max(timeout_s, 0.0) + try: + while True: + self._check_stopped() + # Reading may use a little longer than a zero-timeout poll so it can still make progress. + work_deadline = max(deadline, time.monotonic() + _MIN_WORK_S) + self._drain_file(work_deadline) + self._decode_pending(work_deadline) + if self._newest is not None: + return self._deliver() + if time.monotonic() >= deadline: + return None + self._wait(_POLL_INTERVAL_S) + except SensingStreamError: + self._fail() + raise + except Exception as e: # noqa: BLE001 -- parser/decoder/OS errors all end this stream the same way + self._fail() + raise SensingStreamError(f"{type(e).__name__}: {e}") from e + + def _check_finish_budget(self, deadline: float) -> None: + """Fail the stream, with the reason, if finishing was cancelled or has used up its time budget.""" + self._check_stopped() + if time.monotonic() >= deadline: + raise SensingStreamError( + f"could not finish decoding the remaining stream within {_FINISH_BUDGET_S:g} s" + ) + + def finish(self) -> tuple[np.ndarray, int] | None: + """Flush the parser and decoder once and return the final frame; call only after the producer has ended. + + A temporary end of file on a growing file is not the end of the stream, so this is never called for it. + Reads whatever the producer left in the file first, then drains the parser and the decoder's delayed + output. Returns None if no frame remains. Terminal: the reader cannot be read again. + """ + if self._failed or self._fd is None or self._finished: + return None + self._finished = True + try: + far = time.monotonic() + _FINISH_BUDGET_S + # Everything the producer left behind is read and decoded, in order, before the flush: flushing with + # units still pending would drop or reorder them. The budget bounds the whole sequence. + while self._offset < os.fstat(self._fd).st_size or self._pending: + self._check_finish_budget(far) + self._drain_file(far) + self._decode_pending(far) + self._queue(self._parser.parse(None)) # flush the parser's held-back last unit + while self._pending: + self._check_finish_budget(far) + self._decode_pending(far) + for frame in self._decoder.decode(None): # flush the decoder's delayed output + self._newest = self._checked(frame) + return self._deliver() + except SensingStreamError: + self._fail() + raise + except Exception as e: # noqa: BLE001 -- parser/decoder/OS errors all end this stream the same way + self._fail() + raise SensingStreamError(f"{type(e).__name__}: {e}") from e + + def _fail(self) -> None: + """Make the reader terminal. The descriptor stays open so :meth:`reclaim_all` can keep reclaiming.""" + self._failed = True + self._pending.clear() + self._pending_bytes = 0 + + def reclaim_all(self) -> None: + """For a stream that has failed: keep handing the file's bytes back to the filesystem, without decoding. + + The shared producer keeps appending to every sensor's file for the whole session, so a failed stream's + file would otherwise grow in memory (tmpfs) until Stop. A file the producer created after this stream + failed (before it ever attached) is opened here. A single failing call is simply tried again on the next + tick, but the spool is bounded: once more than ``_MAX_UNRECLAIMED_BYTES`` cannot be handed back, this + raises :class:`SensingStreamError` so the caller can stop the producer rather than let it fill memory. + """ + self._failed = True + if self._fd is None and not self._closed: + try: + self._fd = os.open(self._path, os.O_RDWR) + except FileNotFoundError: + return # the producer has not created the file yet; nothing is accumulating + except OSError: + self._check_unreclaimed(self._path_size(), "cannot open it") + return + if self._fd is None: + return + if self._fallocate is None: + self._fallocate = _bind_fallocate() + size = os.fstat(self._fd).st_size + if size <= self._punched: + return + ctypes.set_errno(0) + mode = _FALLOC_FL_PUNCH_HOLE | _FALLOC_FL_KEEP_SIZE + try: + if self._fallocate(self._fd, mode, self._punched, size - self._punched) == 0: + self._punched = size + return + reason = os.strerror(ctypes.get_errno()) or "fallocate failed" + except OSError as e: + reason = str(e) + self._check_unreclaimed(size - self._punched, reason) + + def _path_size(self) -> int: + try: + return os.stat(self._path).st_size + except OSError: + return 0 + + def _check_unreclaimed(self, unreclaimed: int, reason: str) -> None: + if unreclaimed > _MAX_UNRECLAIMED_BYTES: + raise SensingStreamError( + f"{self._path} holds {unreclaimed >> 20} MiB that cannot be handed back ({reason}); " + "the spool is unbounded" + ) + + def close(self) -> None: + """Close the file; nothing else is held (the producer owns the file's lifetime and directory).""" + self._closed = True + fd, self._fd = self._fd, None + if fd is not None: + os.close(fd) diff --git a/examples/egocap/capture_server/capture_server/acquisition/sensing_jpeg.py b/examples/egocap/capture_server/capture_server/acquisition/sensing_jpeg.py new file mode 100644 index 0000000000..dc9d9c2549 --- /dev/null +++ b/examples/egocap/capture_server/capture_server/acquisition/sensing_jpeg.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""JPEG encoding for SENSING frames: owned RGB host pixels in, one complete JPEG out. + +The wire protocol carries only independently decodable JPEG images with a hard 2 MiB payload ceiling. +A frame that does not fit is a configuration/source error -- never silently resized, re-quantized or +replaced, so the diagnostic names the dimensions and quality that produced it. +""" + +from __future__ import annotations + +import io + +import numpy as np +from PIL import Image + +from .errors import SourceError + +__all__ = ["MAX_JPEG_BYTES", "encode_rgb_jpeg"] + +MAX_JPEG_BYTES = 2 * 1024 * 1024 + + +def encode_rgb_jpeg(rgb: np.ndarray, quality: int, *, max_bytes: int = MAX_JPEG_BYTES) -> bytes: + """Encode an ``(H, W, 3)`` uint8 RGB frame as a complete JPEG at ``quality``; returns owned bytes.""" + if rgb.ndim != 3 or rgb.shape[2] != 3 or rgb.dtype != np.uint8: + raise ValueError(f"expected (H, W, 3) uint8 RGB pixels, got {rgb.shape} {rgb.dtype}") + height, width = rgb.shape[:2] + out = io.BytesIO() + # Encoding finishes before this returns, so the caller may reuse its buffer immediately afterwards; + # nothing here keeps a reference to it. + Image.fromarray(rgb, "RGB").save(out, format="JPEG", quality=quality) + jpeg = out.getvalue() + if not (jpeg.startswith(b"\xff\xd8") and jpeg.endswith(b"\xff\xd9")): + raise SourceError(f"encoder produced an incomplete JPEG for {width}x{height} at quality {quality}") + if len(jpeg) > max_bytes: + raise SourceError( + f"{width}x{height} JPEG at quality {quality} is {len(jpeg)} B, over the {max_bytes} B payload ceiling" + ) + return jpeg diff --git a/examples/egocap/capture_server/capture_server/client.py b/examples/egocap/capture_server/capture_server/client.py index 9776eb38ce..267852433c 100644 --- a/examples/egocap/capture_server/capture_server/client.py +++ b/examples/egocap/capture_server/capture_server/client.py @@ -305,7 +305,7 @@ async def _invoke(callback: Callback[T] | None, value: T) -> None: @dataclass class _StreamRegistry: """Caches SessionInit's stream descriptors for decoding binary - messages -- a tracking stream (ID 0) and up to three camera streams.""" + messages -- a tracking stream (ID 0) and up to four camera streams.""" tracking: p.TrackingStream | None = None cameras: dict[int, p.CameraStream] = dataclass_field(default_factory=dict) diff --git a/examples/egocap/capture_server/capture_server/launch.py b/examples/egocap/capture_server/capture_server/launch.py index 626ebe06be..3878e175a4 100644 --- a/examples/egocap/capture_server/capture_server/launch.py +++ b/examples/egocap/capture_server/capture_server/launch.py @@ -30,7 +30,15 @@ from typing import TYPE_CHECKING, Any, Callable from . import protocol as p -from .acquisition import Acquisition, DepthAIAdapter, IsaacCaptureAdapter, TrackingSelection +from .acquisition import ( + Acquisition, + DepthAIAdapter, + IsaacCaptureAdapter, + SensingAdapter, + SensingRig, + TrackingSelection, +) +from .acquisition.sensing import SensingStationCamera, resolve_sensors from .server import ServerConfig, StationCamera if TYPE_CHECKING: @@ -86,15 +94,47 @@ def _tracking_adapter( ) +def _sensing_composition( + config: ServerConfig | None, cameras: tuple[p.CameraStreamConfig, ...] +) -> tuple[SensingRig | None, dict[int, SensingAdapter]]: + """The shared SENSING rig and one adapter per SENSING-station camera, keyed by stream id. + + Identical for mock and real: a SENSING camera always runs the native producer (its MOCK build on + desktop) writing H.264 files this process tails. ``(None, {})`` when no requested camera is a SENSING + station camera, so such sessions never construct a rig or load the decoder. + """ + stations = (config.cameras if config else None) or {} + wanted = {c.name: stations[c.name] for c in cameras if isinstance(stations.get(c.name), SensingStationCamera)} + if not wanted: + return None, {} + if config is None or config.sensing_rig is None or config.sensing_description is None: + # Reached only if this factory was built from a different config object than the server's, so the + # server's inspection result never arrived: say so instead of rejecting with an empty message. + raise ValueError("SENSING cameras are requested but the factory has no rig or sensor description") + sensors = resolve_sensors(config.sensing_description, wanted) + rig = SensingRig(config.sensing_rig, tuple(sensors[name].pipeline_id for name in wanted)) + adapters = { + c.stream_id: SensingAdapter( + rig=rig, sensor=sensors[c.name], camera=c, full_range=config.sensing_rig.full_range + ) + for c in cameras + if c.name in wanted + } + return rig, adapters + + def build_mock_source_factory( *, scenario: str, seed: int, ic_faults: set[str] | frozenset[str] = frozenset(), camera_faults: set[str] | frozenset[str] = frozenset(), + config: ServerConfig | None = None, ) -> SourceFactory: """The default ``CaptureServer`` source factory, wired to the two mock - SDK files. ``ic_faults``/``camera_faults`` are captured here, once, + SDK files for tracking and OAK cameras (SENSING cameras always run the + real plugin executable, its mock backend on a desktop). ``ic_faults``/``camera_faults`` are + captured here, once, from CLI/test configuration -- "[e]xisting documented local CLI controls may delegate to the two mocks without becoming protocol fields" -- and baked into each new mock session/pipeline this @@ -112,8 +152,11 @@ def source_factory(begin: p.BeginRequest, cameras: tuple[p.CameraStreamConfig, . ), ) + rig, sensing_adapters = _sensing_composition(config, cameras) camera_adapters = tuple( - DepthAIAdapter( + sensing_adapters[camera.stream_id] + if camera.stream_id in sensing_adapters + else DepthAIAdapter( sdk=mock_depthai, pipeline_factory=partial( mock_depthai.Pipeline, @@ -125,7 +168,7 @@ def source_factory(begin: p.BeginRequest, cameras: tuple[p.CameraStreamConfig, . ) for camera in cameras ) - return Acquisition(tracking=tracking, cameras=camera_adapters) + return Acquisition(tracking=tracking, cameras=camera_adapters, sensing_rig=rig) # Lets CaptureServer reject a launch-built factory that disagrees with ServerConfig.mock; # untagged custom factories are not checked. @@ -135,11 +178,15 @@ def source_factory(begin: p.BeginRequest, cameras: tuple[p.CameraStreamConfig, . def _real_camera_adapters( config: ServerConfig, cameras: tuple[p.CameraStreamConfig, ...] -) -> tuple[DepthAIAdapter, ...]: - """One real OAK-D adapter per requested camera, each bound to its station-configured physical device.""" +) -> tuple[tuple[DepthAIAdapter | SensingAdapter, ...], SensingRig | None]: + """One adapter per requested camera: real OAK-D by station device ID, or SENSING by module name. + + Also returns the SENSING rig when any camera uses it. The OAK SDK is imported only if an OAK camera + is requested. + """ if not cameras: - return () - # Every camera needs a station device: report that configuration error before importing the optional SDK. + return (), None + # Every camera needs a station entry: report that configuration error before importing optional SDKs. stations = {} for camera in cameras: station = (config.cameras or {}).get(camera.name) @@ -148,17 +195,26 @@ def _real_camera_adapters( f"camera {camera.name!r} has no station device; configure it with --camera NAME=DEVICE_ID" ) stations[camera.name] = station - import depthai as dai - - return tuple( - DepthAIAdapter( - sdk=dai, - pipeline_factory=partial( - _open_pipeline, dai, stations[camera.name].device_id, stations[camera.name].max_usb_speed - ), - camera=camera, - ) - for camera in cameras + rig, sensing_adapters = _sensing_composition(config, cameras) + oak = [c for c in cameras if c.stream_id not in sensing_adapters] + dai = None + if oak: + import depthai as dai + + return ( + tuple( + sensing_adapters[camera.stream_id] + if camera.stream_id in sensing_adapters + else DepthAIAdapter( + sdk=dai, + pipeline_factory=partial( + _open_pipeline, dai, stations[camera.name].device_id, stations[camera.name].max_usb_speed + ), + camera=camera, + ) + for camera in cameras + ), + rig, ) @@ -184,7 +240,8 @@ def source_factory(begin: p.BeginRequest, cameras: tuple[p.CameraStreamConfig, . sdk = load_isaac_capture() if begin.tracking.sources else None tracking = _tracking_adapter(begin, sdk=sdk, session_factory=sdk.TeleopSession) if sdk else None - return Acquisition(tracking=tracking, cameras=_real_camera_adapters(config, cameras)) + camera_adapters, rig = _real_camera_adapters(config, cameras) + return Acquisition(tracking=tracking, cameras=camera_adapters, sensing_rig=rig) source_factory.mock = False # type: ignore[attr-defined] return source_factory @@ -368,5 +425,5 @@ def build_source_factory( if scenario not in SCENARIOS: raise ValueError(f"unknown mock scenario {scenario!r}; choose one of {', '.join(SCENARIOS)}") return build_mock_source_factory( - scenario=scenario, seed=seed, ic_faults=set(faults), camera_faults=set(faults) + scenario=scenario, seed=seed, ic_faults=set(faults), camera_faults=set(faults), config=config ) diff --git a/examples/egocap/capture_server/capture_server/protocol.py b/examples/egocap/capture_server/capture_server/protocol.py index 04ef35bd86..d78f9f84b5 100644 --- a/examples/egocap/capture_server/capture_server/protocol.py +++ b/examples/egocap/capture_server/capture_server/protocol.py @@ -135,6 +135,7 @@ class WireDType(IntEnum): class CameraType(IntEnum): OAK_D_COLOR = 1 + SENSING_COLOR = 2 class ImageCodec(IntEnum): @@ -701,7 +702,7 @@ def tracking_stream_descriptor( # --------------------------------------------------------------------------- -# Camera stream descriptor (stream IDs 1-3). +# Camera stream descriptor (stream IDs 1-4). # --------------------------------------------------------------------------- @@ -745,7 +746,7 @@ class CameraStreamConfig: @dataclass(frozen=True) class CameraStream: - """One camera descriptor (stream_id 1-3) of SessionInit's ``streams`` + """One camera descriptor (stream_id 1-4) of SessionInit's ``streams`` array.""" stream_id: int @@ -798,8 +799,8 @@ def from_json(cls, data: Any) -> CameraStream: "a camera stream descriptor", ) stream_id = data.get("stream_id") - if type(stream_id) is not int or not (1 <= stream_id <= 3): - raise ProtocolError("camera stream descriptor stream_id must be an integer 1-3") + if type(stream_id) is not int or not (1 <= stream_id <= 4): + raise ProtocolError("camera stream descriptor stream_id must be an integer 1-4") name = data.get("name") if not isinstance(name, str) or not name: raise ProtocolError("camera stream.name must be a nonempty string") @@ -855,7 +856,7 @@ def camera_stream_descriptor(camera: CameraStreamConfig) -> CameraStream: _MAX_SESSION_ID_LEN = 128 _MAX_RATE_HZ = 120.0 -_MAX_CAMERAS = 3 +_MAX_CAMERAS = 4 def _require_keys(data: Any, allowed: set[str], context: str) -> dict: @@ -1482,8 +1483,8 @@ def from_json(cls, data: Any) -> CameraStatus: if not isinstance(name, str) or not name: raise ProtocolError("camera status name must be a nonempty string") stream_id = data.get("stream_id") - if type(stream_id) is not int or not (1 <= stream_id <= 3): - raise ProtocolError("camera status stream_id must be an integer 1-3") + if type(stream_id) is not int or not (1 <= stream_id <= 4): + raise ProtocolError("camera status stream_id must be an integer 1-4") error = data.get("error") if not isinstance(error, str): raise ProtocolError("camera status error must be a string") @@ -1738,8 +1739,8 @@ def to_json(self) -> dict[str, Any]: def from_json(cls, data: Any) -> StreamCounters: _require_keys(data, {"stream_id", "produced", "sent", "dropped"}, "stream counters") stream_id = data.get("stream_id") - if type(stream_id) is not int or not (0 <= stream_id <= 3): - raise ProtocolError("stream counters stream_id must be an integer 0-3") + if type(stream_id) is not int or not (0 <= stream_id <= 4): + raise ProtocolError("stream counters stream_id must be an integer 0-4") values = {} for key in ("produced", "sent", "dropped"): value = data.get(key) diff --git a/examples/egocap/capture_server/capture_server/server.py b/examples/egocap/capture_server/capture_server/server.py index 6a46e8c895..0ff53feb58 100644 --- a/examples/egocap/capture_server/capture_server/server.py +++ b/examples/egocap/capture_server/capture_server/server.py @@ -53,6 +53,15 @@ from . import protocol as p from .acquisition import Acquisition, CameraSourceError, SourceExited +from .acquisition.acquisition import _CLOSE_BUDGET_S as _SOURCE_RELEASE_TIMEOUT_S +from .acquisition.sensing import ( + SensingDescription, + SensingRigConfig, + SensingStationCamera, + inspect_rig, + load_sensing_station, + resolve_sensors, +) __all__ = ["CaptureServer", "ServerConfig", "main"] @@ -71,7 +80,6 @@ _CONTROL_QUEUE_CAPACITY = 64 _SEND_DEADLINE_S = 2.0 _CLOSE_TIMEOUT_S = 2.0 -_SOURCE_RELEASE_TIMEOUT_S = 5.0 _COMPLETED_TOMBSTONE_CAPACITY = 16 @@ -153,21 +161,46 @@ class ServerConfig: # registry yet); set to reject an unknown camera name as INVALID_CONFIG # instead of inventing settings for it. known_cameras: tuple[str, ...] | None = None - # Real acquisition: logical camera name -> its station settings. When set, only these names are - # accepted and each stream uses its entry; None keeps the mock's fixed 1280x720@30 defaults. - cameras: dict[str, StationCamera] | None = None + # Logical camera name -> its station settings (OAK-D or SENSING). When set, only these names are + # accepted and each stream uses its entry, including in mock mode: a mock station with SENSING cameras + # therefore cannot also select the OAK mock cameras. None keeps the OAK mock's fixed 1280x720@30 defaults. + cameras: dict[str, StationCamera | SensingStationCamera] | None = None # Operator-facing note appended to the startup banner, saying where the served data comes from. None # prints the mode's default (the mock's SYNTHETIC-data warning, nothing for live); replay sets it so # recorded data is never announced as synthetic. data_note: str | None = None + # The one SENSING producer (at most one rig per session); required iff ``cameras`` has SENSING entries. + sensing_rig: SensingRigConfig | None = None + # Resolved once by the server before the first SENSING Begin and then reused (the platform is fixed). + sensing_description: SensingDescription | None = None def __post_init__(self) -> None: - """Reject one physical camera assigned to more than one logical name.""" + """Reject a physical camera assigned to more than one logical name, within each identity domain. + + OAK devices are identified by MXID and SENSING sensors by (module name, sensor index); the two never + collide. Final physical-pipeline duplicates are caught when the sensors are resolved. A SENSING camera + needs a rig whose declared backend matches ``mock``. + """ if self.cameras is not None: - ids = [c.device_id for c in self.cameras.values()] - duplicates = sorted({i for i in ids if ids.count(i) > 1}) - if duplicates: - raise ValueError(f"physical camera(s) assigned to more than one logical name: {duplicates}") + oak = [c.device_id for c in self.cameras.values() if isinstance(c, StationCamera)] + sensing = [ + (c.module_name, c.sensor_index) + for c in self.cameras.values() + if isinstance(c, SensingStationCamera) + ] + for ids in (oak, sensing): + duplicates = sorted({i for i in ids if ids.count(i) > 1}, key=str) + if duplicates: + raise ValueError(f"physical camera(s) assigned to more than one logical name: {duplicates}") + if sensing and self.sensing_rig is None: + raise ValueError("SENSING cameras are configured but no sensing_rig is") + if self.sensing_rig is not None: + required = "MOCK" if self.mock else "JETSON" + if self.sensing_rig.expected_backend != required: + raise ValueError( + f"sensing_rig.expected_backend must be {required} when mock={self.mock}, " + f"got {self.sensing_rig.expected_backend}" + ) @dataclass @@ -1131,6 +1164,7 @@ def __init__( # ws.send() directly; everything (rejections, replays, session # lifecycle, status, frames) goes through this. self._outbox: _Outbox | None = None + self._sensing_inspection: asyncio.Future[SensingDescription] | None = None self._active_session: _Session | None = None # Scoped to the CURRENT owner connection, reset in _handle_connection # -- a session ID completed under a since-disconnected owner must @@ -1148,6 +1182,19 @@ def request_stop(self) -> None: def _camera_stream_config(self, stream_id: int, camera: Any) -> p.CameraStreamConfig: """The server-owned settings for one requested camera: its station entry, else the mock defaults.""" station = (self.config.cameras or {}).get(camera.name) + if isinstance(station, SensingStationCamera): + assert self.config.sensing_description is not None # resolved by _prepare_sensing before this + sensor = resolve_sensors(self.config.sensing_description, {camera.name: station})[camera.name] + return p.CameraStreamConfig( + stream_id=stream_id, + name=camera.name, + type=camera.type, + required=station.required, + width=sensor.width, + height=sensor.height, + fps=station.delivery_fps, + jpeg_quality=station.jpeg_quality, + ) return p.CameraStreamConfig( stream_id=stream_id, name=camera.name, @@ -1159,6 +1206,61 @@ def _camera_stream_config(self, stream_id: int, camera: Any) -> p.CameraStreamCo jpeg_quality=station.jpeg_quality if station else 80, ) + def _begin_gate_error(self) -> str | None: + """Why a Begin cannot be admitted right now (shutdown, or this connection's writer is gone), else None.""" + if self._stop_event.is_set(): + return "server is shutting down" + if self._outbox is None or self._outbox.closed: + return "this connection is no longer usable" + return None + + async def _prepare_sensing(self, begin: p.BeginRequest) -> str | None: + """Validate requested camera types against the station and resolve SENSING sensors; error text or None. + + The native inspection is blocking, so it runs off the event loop and its result is cached for the + server's lifetime (a failed inspection is not cached). It happens before the admission checks so + nothing awaits between admitting a Begin and recording it as the active session. + """ + stations = self.config.cameras or {} + wanted: dict[str, SensingStationCamera] = {} + for camera in begin.cameras: + station = stations.get(camera.name) + if station is None: + if camera.type == p.CameraType.SENSING_COLOR: + # Never let the permissive fallback build a mock OAK labelled as SENSING. + return f"camera {camera.name!r} requests SENSING_COLOR but has no SENSING station entry" + continue + expected = ( + p.CameraType.SENSING_COLOR + if isinstance(station, SensingStationCamera) + else p.CameraType.OAK_D_COLOR + ) + if camera.type != expected: + return f"camera {camera.name!r} is a {expected.name} station camera, not {camera.type.name}" + if isinstance(station, SensingStationCamera): + wanted[camera.name] = station + # A Begin that is already BUSY (or a replay) builds nothing, so it never triggers an inspection. + if not wanted or self._active_session is not None: + return None + assert self.config.sensing_rig is not None # ServerConfig requires it for SENSING cameras + try: + if self.config.sensing_description is None: + # One shared in-flight inspection: concurrent first Begins await the same blocking call. + if self._sensing_inspection is None: + self._sensing_inspection = asyncio.ensure_future( + asyncio.to_thread(inspect_rig, self.config.sensing_rig) + ) + task = self._sensing_inspection + try: + self.config.sensing_description = await asyncio.shield(task) + finally: + if task.done() and self._sensing_inspection is task: + self._sensing_inspection = None + resolve_sensors(self.config.sensing_description, wanted) + except Exception as exc: # noqa: BLE001 -- reported to the client as a configuration rejection + return f"SENSING camera selection failed: {exc}" + return None + async def run(self) -> None: self._ws_server = await websockets.serve( self._handle_connection, @@ -1374,13 +1476,8 @@ async def _handle_begin(self, ws: ServerConnection, data: dict[str, Any]) -> Non # session onto a dead writer would create a fresh quiesce # acknowledgment that writer will never resolve, hanging that # session's own _finalize forever. - if self._stop_event.is_set() or self._outbox is None or self._outbox.closed: - self._reject( - session_id, - p.MessageType.BEGIN, - p.RejectionReason.BUSY, - "server is shutting down" if self._stop_event.is_set() else "this connection is no longer usable", - ) + if (gate_error := self._begin_gate_error()) is not None: + self._reject(session_id, p.MessageType.BEGIN, p.RejectionReason.BUSY, gate_error) return try: begin = p.parse_begin(data) @@ -1420,6 +1517,16 @@ async def _handle_begin(self, ws: ServerConnection, data: dict[str, Any]) -> Non ) return + sensing_error = await self._prepare_sensing(begin) + if sensing_error is not None: + self._reject(begin.session_id, p.MessageType.BEGIN, p.RejectionReason.INVALID_CONFIG, sensing_error) + return + # The inspection may have awaited for seconds: shutdown or a dead writer can have happened meanwhile, + # so the gate is re-checked before admission rather than trusting the check made before the await. + if (gate_error := self._begin_gate_error()) is not None: + self._reject(begin.session_id, p.MessageType.BEGIN, p.RejectionReason.BUSY, gate_error) + return + if self._active_session is not None: active = self._active_session # Owner identity, not just session_id, gates replay: a session @@ -1645,6 +1752,13 @@ def _build_arg_parser() -> argparse.ArgumentParser: metavar="NAME=DEVICE_ID", help="real only: station camera, logical NAME -> stable physical DEVICE_ID (OAK-D MXID); repeatable", ) + parser.add_argument( + "--sensing-config", + default=None, + metavar="FILE", + help="YAML with the SENSING rig and its cameras (type SENSING_COLOR); works with --mock, " + "where the rig must be a MOCK build, and without it, where it must be a JETSON build", + ) parser.add_argument("--camera-width", type=int, default=1280, help="real only") parser.add_argument("--camera-height", type=int, default=720, help="real only") parser.add_argument("--camera-fps", type=float, default=30.0, help="real only") @@ -1782,12 +1896,21 @@ async def _run_cli(args: argparse.Namespace) -> int: elif args.gmsl_ipc is not None: config, source_factory = _gmsl_setup(args) else: + cameras: dict[str, Any] | None = _station_cameras(args) + sensing_rig = None + if args.sensing_config: + sensing_rig, sensing_cameras = load_sensing_station(args.sensing_config) + clash = sorted(set(cameras or {}) & set(sensing_cameras)) + if clash: + raise ValueError(f"camera name(s) configured both as --camera and SENSING: {clash}") + cameras = {**(cameras or {}), **sensing_cameras} config = ServerConfig( mock=args.mock, host=args.host, port=args.port, connect_delay=args.connect_delay, - cameras=_station_cameras(args), + cameras=cameras, + sensing_rig=sensing_rig, ) # The one place the implementation is chosen, before any session is accepted. source_factory = build_source_factory( diff --git a/examples/egocap/frontend/README.md b/examples/egocap/frontend/README.md index 6d3d502d86..d9314ef698 100644 --- a/examples/egocap/frontend/README.md +++ b/examples/egocap/frontend/README.md @@ -137,7 +137,7 @@ Fixed defaults the backend starts with, set in `backend/egocap_web/config.py`'s | Station label | `Station 01` | the station YAML's label, see below | | Profile label | `Headset and hands` | fixed | | Sources | head, controllers, hands | fixed | -| Cameras | none requested | the station YAML's `video_sources`, see below | +| Cameras | none requested | the station YAML's `video_sources`, see below (`type` `1` = `OAK_D_COLOR`, `2` = `SENSING_COLOR`) | | Plugins | none | `EGOCAP_CAPTURE_PLUGINS`, a comma-separated list of plugin names (case-insensitive); only `manus` exists today, so `manus` requests glove hand tracking. An unknown name stops the backend at startup. | | Tracking sources | head, controllers, hands | `EGOCAP_CAPTURE_SOURCES`, a comma-separated list of `head`, `controllers`, `hands`, or `none` for a cameras-only session that needs no headset and no IsaacCapture (it cannot be combined with `EGOCAP_CAPTURE_PLUGINS`). | | Preview sample rates | 10 Hz cameras, 10 Hz tracking | `EGOCAP_CAMERA_SAMPLE_HZ`, `EGOCAP_TRACKING_SAMPLE_HZ`: how often the page is sent camera frames / tracking samples (positive Hz). Raise to the capture rate (e.g. `60`) to see the native feed; every viewer pays for each sample. | diff --git a/examples/egocap/frontend/src/captureProtocol.test.ts b/examples/egocap/frontend/src/captureProtocol.test.ts index 19fff01897..74cb7130f7 100644 --- a/examples/egocap/frontend/src/captureProtocol.test.ts +++ b/examples/egocap/frontend/src/captureProtocol.test.ts @@ -131,6 +131,24 @@ describe("parseSnapshot", () => { expect(() => parseSnapshot(body)).toThrow(CapturePreviewParseError); }); + it("accepts a camera stream_id of 4 and rejects 5", () => { + const parse = (streamId: number) => + parseSnapshot( + validSnapshot({ + capture: { + connecting: false, + started: startedWith([trackingStreamJson(), cameraStreamJson({ stream_id: streamId })]), + status: null, + stopped: null, + error_message: null, + binary_generation: null, + }, + }), + ); + expect(() => parse(4)).not.toThrow(); + expect(() => parse(5)).toThrow(CapturePreviewParseError); + }); + it("rejects a duplicate field name in a tracking descriptor", () => { const stream = trackingStreamJson(); (stream.layout as { fields: unknown[] }).fields.push({ name: "head.present", dtype: 1, shape: [], offset: 5 }); diff --git a/examples/egocap/frontend/src/captureProtocol.ts b/examples/egocap/frontend/src/captureProtocol.ts index 9519d2a36b..1ed237a76b 100644 --- a/examples/egocap/frontend/src/captureProtocol.ts +++ b/examples/egocap/frontend/src/captureProtocol.ts @@ -328,7 +328,7 @@ function parseStreamDescriptor(value: unknown, field: string): StreamDescriptor }; } // kind === StreamKind.IMAGE (the only other value expectEnum(StreamKind, ...) allows) - const streamId = expectBoundedPositiveInt(value.stream_id, `${field}.stream_id`, 3); + const streamId = expectBoundedPositiveInt(value.stream_id, `${field}.stream_id`, 4); return { kind: "camera", stream_id: streamId, diff --git a/examples/egocap/pyproject.toml b/examples/egocap/pyproject.toml index 8789a81fc6..672964fb25 100644 --- a/examples/egocap/pyproject.toml +++ b/examples/egocap/pyproject.toml @@ -32,6 +32,8 @@ dependencies = [ # decode/encode MP4 files on disk. # --- capture_server --- "websockets>=17.2", + "av>=14", # PyAV: decodes the SENSING producer's H.264 output (capture_server's + # acquisition/sensing_h264.py); loaded lazily, only for SENSING streams. # --- backend (egocap_web) --- "fastapi>=0.115", "uvicorn[standard]>=0.30", diff --git a/examples/egocap/tests/backend/test_station_config.py b/examples/egocap/tests/backend/test_station_config.py index d5df52a5d0..69f60d99a1 100644 --- a/examples/egocap/tests/backend/test_station_config.py +++ b/examples/egocap/tests/backend/test_station_config.py @@ -226,12 +226,48 @@ def test_a_duplicate_video_source_name_is_a_station_config_error(tmp_path): def test_too_many_video_sources_is_a_station_config_error(tmp_path): - entries = "\n".join(f"- type: oak_camera\n name: cam{i}" for i in range(4)) + entries = "\n".join(f"- type: oak_camera\n name: cam{i}" for i in range(5)) path = _write(tmp_path, f"video_sources:\n{entries}\n") - with pytest.raises(StationConfigError, match="at most 3"): + with pytest.raises(StationConfigError, match="at most 4"): load_station_config(path) +def test_sensing_camera_video_source_is_accepted(tmp_path): + path = _write( + tmp_path, + "video_sources:\n- type: sensing_camera\n name: head_left\n", + ) + resolved = load_station_config(path) + assert resolved.cameras == (p.CameraSelection(name="head_left", type=p.CameraType.SENSING_COLOR),) + + +def test_four_sensing_cameras_are_accepted_and_forwarded(tmp_path): + entries = "\n".join(f"- type: sensing_camera\n name: cam{i}" for i in range(4)) + path = _write(tmp_path, f"video_sources:\n{entries}\n") + resolved = load_station_config(path) + assert len(resolved.cameras) == 4 + assert all(c["type"] == p.CameraType.SENSING_COLOR for c in resolved.cameras) + assert [c["name"] for c in resolved.cameras] == [f"cam{i}" for i in range(4)] + + +def test_oak_and_sensing_cameras_can_be_mixed_in_one_station(tmp_path): + path = _write( + tmp_path, + "video_sources:\n" + "- type: oak_camera\n name: ego\n" + "- type: sensing_camera\n name: head_left\n" + "- type: sensing_camera\n name: head_right\n" + "- type: sensing_camera\n name: wrist_left\n", + ) + resolved = load_station_config(path) + assert resolved.cameras == ( + p.CameraSelection(name="ego", type=p.CameraType.OAK_D_COLOR), + p.CameraSelection(name="head_left", type=p.CameraType.SENSING_COLOR), + p.CameraSelection(name="head_right", type=p.CameraType.SENSING_COLOR), + p.CameraSelection(name="wrist_left", type=p.CameraType.SENSING_COLOR), + ) + + def test_station_config_path_from_env(monkeypatch, tmp_path): monkeypatch.delenv(STATION_CONFIG_ENV, raising=False) assert station_config_path_from_env() is None diff --git a/examples/egocap/tests/capture_server/bench_sensing_decode.py b/examples/egocap/tests/capture_server/bench_sensing_decode.py new file mode 100644 index 0000000000..aa6638d701 --- /dev/null +++ b/examples/egocap/tests/capture_server/bench_sensing_decode.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Per-frame cost of the SENSING CPU path at native size: H.264 decode, YUV->RGB, JPEG encode. + +Run on the target (e.g. the Orin) to learn what delivery rate one stream can sustain per core: + + uv run python tests/capture_server/bench_sensing_decode.py [IMAGE.png] [--quality 80] [--frames 30] + +With no image it uses synthetic noise (a worst case for both the H.264 unit size and JPEG size); pass a real +scene capture for realistic numbers. The units here are intra-only fixtures; an inter-coded stream decodes +cheaper per frame. Each iteration decodes exactly one unit, the cost the reader pays per frame. +""" + +from __future__ import annotations + +import argparse +import statistics +import time +from pathlib import Path + +import numpy as np +from PIL import Image + +from capture_server.acquisition.sensing_h264 import SensingStreamReader # noqa: E402 +from capture_server.acquisition.sensing_jpeg import encode_rgb_jpeg # noqa: E402 + +WIDTH, HEIGHT = 2560, 1984 + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("image", nargs="?") + parser.add_argument("--quality", type=int, default=80) + parser.add_argument("--frames", type=int, default=30) + args = parser.parse_args() + + if args.image: + rgb = np.asarray(Image.open(args.image).convert("RGB").resize((WIDTH, HEIGHT))) + else: + rgb = np.random.default_rng(0).integers(0, 256, (HEIGHT, WIDTH, 3), dtype=np.uint8) + # encode_units takes solid colours; encode the real picture the same way through PyAV directly. + from fractions import Fraction + + import av + + enc = av.CodecContext.create("libx264", "w") + enc.width, enc.height, enc.pix_fmt = WIDTH, HEIGHT, "yuv420p" + enc.framerate, enc.time_base, enc.gop_size = 30, Fraction(1, 30), 1 + enc.options = {"preset": "ultrafast", "tune": "zerolatency", "crf": "20", "bframes": "0"} + units = [] + for i in range(3): + frame = av.VideoFrame.from_ndarray(rgb, format="rgb24").reformat( + format="yuv420p", dst_colorspace="ITU601", dst_color_range="MPEG" + ) + frame.pts = i + units += [bytes(p) for p in enc.encode(frame)] + units += [bytes(p) for p in enc.encode(None)] + print(f"intra-only unit: {len(units[0]) / 1e6:.2f} MB; JPEG q{args.quality} cap 2.00 MB") + + path = Path("/tmp/bench_sensing.h264") + decode_ms, jpeg_ms, sizes = [], [], [] + for _ in range(args.frames): + path.write_bytes(b"".join(units[:2])) # the first unit is complete; the parser holds the second back + reader = SensingStreamReader(str(path), width=WIDTH, height=HEIGHT) + reader.connect(1.0) + t = time.perf_counter() + out, _ = reader.read_latest(5.0) + decode_ms.append((time.perf_counter() - t) * 1000) + reader.close() + t = time.perf_counter() + sizes.append(len(encode_rgb_jpeg(out, args.quality, max_bytes=1 << 30))) # measure, do not enforce + jpeg_ms.append((time.perf_counter() - t) * 1000) + d, j = statistics.median(decode_ms), statistics.median(jpeg_ms) + size = statistics.median(sizes) + verdict = "OVER the 2 MiB wire ceiling" if size > 2 * 1024 * 1024 else "within the 2 MiB wire ceiling" + print(f"decode+yuv->rgb: {d:.1f} ms jpeg: {j:.1f} ms JPEG size: {size / 1e6:.2f} MB ({verdict})") + total = d + j + print(f"per frame {total:.1f} ms => ~{1000 / total:.1f} fps per stream per core; two streams need 2x that") + + +if __name__ == "__main__": + main() diff --git a/examples/egocap/tests/capture_server/h264_fixtures.py b/examples/egocap/tests/capture_server/h264_fixtures.py new file mode 100644 index 0000000000..53d289a91d --- /dev/null +++ b/examples/egocap/tests/capture_server/h264_fixtures.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Real H.264 for SENSING reader tests, encoded with libx264 (fixtures, not native output). + +Encoded with PyAV/libx264 (IDR frames carry in-band SPS/PPS; no B-frames), so the reader and adapter +are exercised against genuinely decodable Annex-B data rather than hand-made bytes. +""" + +from __future__ import annotations + +import os +import threading +from fractions import Fraction + +import numpy as np + + +def encode_units(width: int, height: int, colors: list[tuple[int, int, int]], *, gop: int = 1) -> list[bytes]: + """One Annex-B access unit per solid-colour frame; ``gop=1`` makes each independently decodable, a larger GOP + makes later units P-frames that reference earlier ones.""" + import av + + enc = av.CodecContext.create("libx264", "w") + enc.width, enc.height, enc.pix_fmt = width, height, "yuv420p" + enc.framerate, enc.time_base, enc.gop_size = 30, Fraction(1, 30), gop + enc.options = {"preset": "ultrafast", "tune": "zerolatency", "crf": "18", "bframes": "0"} + packets = [] + for i, (r, g, b) in enumerate(colors): + # BT.601 limited-range YUV, the producer's own conversion, so decoded RGB should match the input. + rgb = np.empty((height, width, 3), np.uint8) + rgb[...] = (r, g, b) + frame = av.VideoFrame.from_ndarray(rgb, format="rgb24").reformat( + format="yuv420p", dst_colorspace="ITU601", dst_color_range="MPEG" + ) + frame.pts = i + packets += enc.encode(frame) + packets += enc.encode(None) + return [bytes(p) for p in packets] + + +class FileProducer: + """Appends units to ``path`` like the native ``output=`` writer: buffered, so a unit's tail lags.""" + + def __init__(self, path: str, units: list[bytes], *, interval_s: float = 0.02, loop: bool = False) -> None: + self.path, self.units, self.interval_s, self.loop = path, units, interval_s, loop + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + def start(self) -> "FileProducer": + self._thread.start() + return self + + def _run(self) -> None: + with open(self.path, "wb") as f: + while not self._stop.is_set(): + for unit in self.units: + f.write(unit) + f.flush() + if self._stop.wait(self.interval_s): + return + if not self.loop: + # Like the encoder tail: a final unit only becomes complete when another begins. + f.write(self.units[-1]) + f.flush() + self._stop.wait() + return + + def stop(self) -> None: + self._stop.set() + self._thread.join(timeout=2) + + +def allocated_bytes(path: str) -> int: + return os.stat(path).st_blocks * 512 diff --git a/examples/egocap/tests/capture_server/test_acquisition_adapters.py b/examples/egocap/tests/capture_server/test_acquisition_adapters.py index aaad25f998..ad6e3a6958 100644 --- a/examples/egocap/tests/capture_server/test_acquisition_adapters.py +++ b/examples/egocap/tests/capture_server/test_acquisition_adapters.py @@ -6,16 +6,22 @@ acquisition adapters directly: an object whose own `__enter__()` raised must never see a matching `__exit__()` call, while an object that *did* enter must still be released. See isaac_capture.py's `close()` and -depthai.py's `close()` for the guard these tests cover.""" +depthai.py's `close()` for the guard these tests cover. + +Also exercises Acquisition._close_rig_chain's exception-priority rule: a +cancellation collected alongside another close failure must still be the +one raised, never silently dropped in favor of the other failure.""" from __future__ import annotations +import asyncio from functools import partial from unittest.mock import patch import pytest from capture_server import protocol as p +from capture_server.acquisition.acquisition import Acquisition from capture_server.acquisition.depthai import DepthAIAdapter from capture_server.acquisition.errors import CameraSourceError, SourceError from capture_server.acquisition.isaac_capture import IsaacCaptureAdapter, TrackingSelection @@ -80,3 +86,52 @@ async def test_depthai_skips_pipeline_exit_when_pipeline_enter_raised(): await adapter.close() pipeline_exit.assert_not_called() + + +class _FakeRigAdapter: + """A minimal adapter standing in for a SENSING stream in `_close_rig_chain`.""" + + borrows_rig = True + + def __init__(self, close_raises: BaseException | None = None) -> None: + self._close_raises = close_raises + + async def close(self) -> bool: + if self._close_raises is not None: + raise self._close_raises + return True + + +class _FakeRig: + """A minimal stand-in for `SensingRig`, closed after its adapters in `_close_rig_chain`.""" + + def __init__(self, close_raises: BaseException | None = None) -> None: + self._close_raises = close_raises + + async def close(self) -> bool: + if self._close_raises is not None: + raise self._close_raises + return True + + +async def test_close_rig_chain_raises_cancellation_over_a_concurrent_close_failure(): + """One adapter's close() is cancelled while another's raises an ordinary error. The cancellation must win even + though the ordinary failure is first in iteration order: silently raising it instead would defeat the + caller's own cancellation.""" + acquisition = Acquisition(tracking=None, cameras=(), sensing_rig=_FakeRig()) + failed = _FakeRigAdapter(close_raises=RuntimeError("ordinary close failure")) + cancelled = _FakeRigAdapter(close_raises=asyncio.CancelledError()) + + with pytest.raises(asyncio.CancelledError): + await acquisition._close_rig_chain([failed, cancelled]) + + +async def test_close_rig_chain_raises_cancellation_from_the_rig_itself(): + """The rig's own close() raising CancelledError must take priority over an adapter's ordinary failure too.""" + acquisition = Acquisition( + tracking=None, cameras=(), sensing_rig=_FakeRig(close_raises=asyncio.CancelledError()) + ) + failed = _FakeRigAdapter(close_raises=RuntimeError("ordinary close failure")) + + with pytest.raises(asyncio.CancelledError): + await acquisition._close_rig_chain([failed]) diff --git a/examples/egocap/tests/capture_server/test_protocol.py b/examples/egocap/tests/capture_server/test_protocol.py index b721e2cd35..5acf50783a 100644 --- a/examples/egocap/tests/capture_server/test_protocol.py +++ b/examples/egocap/tests/capture_server/test_protocol.py @@ -218,6 +218,58 @@ def test_camera_stream_descriptor_matches_doc_shape(): assert p.CameraStream.from_json(descriptor) == stream +def test_sensing_camera_descriptor_round_trips_with_jpeg_codec_and_oak_value_unchanged(): + camera = p.CameraStreamConfig( + stream_id=1, + name="camera_left", + type=p.CameraType.SENSING_COLOR, + required=True, + width=2560, + height=1984, + fps=30.0, + jpeg_quality=80, + ) + descriptor = p.camera_stream_descriptor(camera).to_json() + assert descriptor["type"] == 2 and descriptor["codec"] == int(p.ImageCodec.JPEG) + assert (descriptor["width"], descriptor["height"]) == (2560, 1984) + assert p.CameraStream.from_json(descriptor).type is p.CameraType.SENSING_COLOR + assert int(p.CameraType.OAK_D_COLOR) == 1 # the existing value is untouched + # An old peer that only knows OAK_D_COLOR cannot parse the new value; that is stated, not negotiated away. + with pytest.raises(p.ProtocolError): + p.CameraStream.from_json({**descriptor, "type": 3}) + + +def test_four_camera_streams_are_accepted_through_descriptor_status_and_counters(): + camera = p.CameraStreamConfig( + stream_id=4, + name="right_wrist_camera", + type=p.CameraType.SENSING_COLOR, + required=False, + width=1920, + height=1536, + fps=15.0, + jpeg_quality=80, + ) + stream = p.camera_stream_descriptor(camera) + assert p.CameraStream.from_json(stream.to_json()) == stream + counters = p.StreamCounters(stream_id=4, produced=1, sent=1, dropped=0) + assert p.StreamCounters.from_json(counters.to_json()) == counters + with pytest.raises(p.ProtocolError, match="1-4"): + p.CameraStream.from_json({**stream.to_json(), "stream_id": 5}) + status = p.CameraStatus( + name="right_wrist_camera", + stream_id=4, + status=p.DeviceState.CONNECTED, + reason=p.StatusReason.NONE, + error="", + ) + assert p.CameraStatus.from_json(status.to_json()) == status + with pytest.raises(p.ProtocolError, match="1-4"): + p.CameraStatus.from_json({**status.to_json(), "stream_id": 5}) + four = tuple(p.CameraRequest(name=f"camera_{i}", type=p.CameraType.SENSING_COLOR) for i in range(4)) + assert len(p.BeginRequest(tracking=p.TrackingConfig(sources=()), cameras=four).cameras) == 4 + + def test_camera_stream_descriptor_with_intrinsics(): camera = p.CameraStreamConfig( stream_id=2, @@ -357,8 +409,8 @@ def test_begin_request_rejects_duplicate_plugin_kinds(): def test_begin_request_rejects_too_many_cameras(): - cameras = tuple(p.CameraRequest(name=f"camera_{i}", type=p.CameraType.OAK_D_COLOR) for i in range(4)) - with pytest.raises(ValueError, match="at most 3"): + cameras = tuple(p.CameraRequest(name=f"camera_{i}", type=p.CameraType.OAK_D_COLOR) for i in range(5)) + with pytest.raises(ValueError, match="at most 4"): p.BeginRequest(tracking=p.TrackingConfig(sources=()), cameras=cameras) @@ -573,8 +625,8 @@ def test_parse_begin_rejects_cameras_not_a_list(): def test_parse_begin_rejects_too_many_cameras(): - cameras = [{"name": f"camera_{i}", "type": 1} for i in range(4)] - with pytest.raises(p.ProtocolError, match="at most 3"): + cameras = [{"name": f"camera_{i}", "type": 1} for i in range(5)] + with pytest.raises(p.ProtocolError, match="at most 4"): p.parse_begin(_valid_begin_dict(cameras=cameras)) diff --git a/examples/egocap/tests/capture_server/test_sensing_adapter.py b/examples/egocap/tests/capture_server/test_sensing_adapter.py new file mode 100644 index 0000000000..e5e65278b5 --- /dev/null +++ b/examples/egocap/tests/capture_server/test_sensing_adapter.py @@ -0,0 +1,1123 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""SENSING configuration, inspection, rig ownership, adapter lifecycle and Acquisition composition. + +CPU-only: a fake producer writing real intra-only H.264 files stands in for the native binary, +so these do not replace the desktop MOCK-binary or Orin integration runs. +""" + +from __future__ import annotations + +import asyncio +import io +import os +import subprocess +import threading +import time +import types + +import pytest +from PIL import Image + +from capture_server import protocol as p +from capture_server.acquisition import Acquisition, CameraSourceError +from capture_server.acquisition import sensing as sn +from capture_server.acquisition import sensing_h264 as h264 +from capture_server.acquisition.errors import SourceError +from capture_server.server import CaptureServer, ServerConfig, StationCamera + +from .h264_fixtures import FileProducer, allocated_bytes, encode_units + +W, H = 64, 32 + +RIG = dict( + executable="/opt/x/camera_plugin_sensing", + expected_backend="MOCK", + platform_config="/opt/x/p.json", + config_name="CFG", + link_masks=("0x0000", "0x1100"), +) +DESCRIPTION = sn.SensingDescription((sn.SensingSensor("L", 1, W, H, 30.0), sn.SensingSensor("R", 0, W, H, 30.0))) + + +def _listing(*lines: str, header: str = "CFG: 2 sensor(s)") -> str: + """The native plugin's ``--list-sensors`` text: a header, then one line per sensor.""" + return "\n".join([header, *lines]) + "\n" + + +GOOD_LISTING = _listing( + f" sensor=1 L {W}x{H} @ 30 fps", + f" sensor=0 R {W}x{H} @ 30 fps", +) + + +# --- configuration -------------------- + + +@pytest.mark.parametrize( + "over", [dict(expected_backend="CPU"), dict(executable=""), dict(link_masks=()), dict(config_name="")] +) +def test_rig_config_rejects_bad_values(over): + with pytest.raises(ValueError): + sn.SensingRigConfig(**{**RIG, **over}) + + +@pytest.mark.parametrize("over", [dict(module_name=""), dict(delivery_fps=0), dict(jpeg_quality=101)]) +def test_station_camera_rejects_bad_values(over): + with pytest.raises(ValueError): + sn.SensingStationCamera(**{"module_name": "L", **over}) + + +def test_server_config_dedups_within_each_identity_domain_only(): + rig = sn.SensingRigConfig(**RIG) + cams = { + "a": StationCamera("mxid-1"), + "b": sn.SensingStationCamera("mxid-1"), # same string, different identity domain: fine + } + ServerConfig(mock=True, cameras=cams, sensing_rig=rig) + with pytest.raises(ValueError, match="more than one logical name"): + ServerConfig( + mock=True, + cameras={"a": sn.SensingStationCamera("L"), "b": sn.SensingStationCamera("L")}, + sensing_rig=rig, + ) + + +def test_server_config_allows_both_eyes_of_one_module_but_not_the_same_eye_twice(): + rig = sn.SensingRigConfig(**RIG) + eyes = { + "head_left": sn.SensingStationCamera("HEAD", sensor_index=0), + "head_right": sn.SensingStationCamera("HEAD", sensor_index=1), + } + ServerConfig(mock=True, cameras=eyes, sensing_rig=rig) + twice = { + "a": sn.SensingStationCamera("HEAD", sensor_index=0), + "b": sn.SensingStationCamera("HEAD", sensor_index=0), + } + with pytest.raises(ValueError, match="more than one logical name"): + ServerConfig(mock=True, cameras=twice, sensing_rig=rig) + # One name selecting the module as a whole and another selecting one eye is also a conflict to resolve later. + mixed = {"a": sn.SensingStationCamera("HEAD"), "b": sn.SensingStationCamera("HEAD", sensor_index=1)} + ServerConfig(mock=True, cameras=mixed, sensing_rig=rig) # config-level distinct; resolve_sensors rejects it + + +def test_server_config_requires_rig_and_matching_backend(): + with pytest.raises(ValueError, match="no sensing_rig"): + ServerConfig(mock=True, cameras={"a": sn.SensingStationCamera("L")}) + with pytest.raises(ValueError, match="must be JETSON"): + ServerConfig(mock=False, sensing_rig=sn.SensingRigConfig(**RIG)) + with pytest.raises(ValueError, match="must be MOCK"): + ServerConfig(mock=True, sensing_rig=sn.SensingRigConfig(**{**RIG, "expected_backend": "JETSON"})) + + +def test_every_documented_rig_key_loads_from_yaml(tmp_path): + f = tmp_path / "s.yaml" + f.write_text( + "sensing_rig:\n executable: /x/e\n expected_backend: MOCK\n platform_config: /x/p.json\n" + " config_name: C\n link_masks: ['0x0000']\n gpu_id: 1\n nito_dir: /n\n" + " bitrate_bps: 50000000\n full_range: true\n runtime_dir: /r\n" + "cameras:\n left: {type: SENSING_COLOR, module_name: L}\n" + ) + rig, _ = sn.load_sensing_station(str(f)) + assert (rig.gpu_id, rig.nito_dir, rig.bitrate_bps, rig.full_range, rig.runtime_dir) == ( + 1, + "/n", + 50_000_000, + True, + "/r", + ) + args = rig.base_args() + assert "--full-range" in args and "--bitrate=50000000" in args and "--peak-bitrate=75000000" in args + + +def test_configured_encoder_settings_reach_the_launch_arguments(): + args = sn.SensingRigConfig(**RIG, gop=30, bitrate_bps=20_000_000).base_args() + assert "--gop=30" in args and "--bitrate=20000000" in args and "--peak-bitrate=30000000" in args + + +def test_load_station_yaml(tmp_path): + f = tmp_path / "s.yaml" + f.write_text( + "sensing_rig:\n executable: /x/e\n expected_backend: MOCK\n platform_config: /x/p.json\n" + " config_name: C\n link_masks: ['0x0000', '0x1100']\n" + "cameras:\n left: {type: SENSING_COLOR, module_name: L, delivery_fps: 15}\n" + ) + rig, cams = sn.load_sensing_station(str(f)) + assert rig.link_masks == ("0x0000", "0x1100") and cams["left"].delivery_fps == 15.0 + + +@pytest.mark.parametrize( + "text", + [ + "cameras: {}\n", + "sensing_rig: {executable: e}\ncameras: {}\n", + "sensing_rig: {executable: e, bogus: 1}\ncameras: {l: {type: SENSING_COLOR, module_name: L}}\n", + ], +) +def test_load_station_yaml_rejects_bad_files(tmp_path, text): + f = tmp_path / "s.yaml" + f.write_text(text) + with pytest.raises(ValueError): + sn.load_sensing_station(str(f)) + + +def test_load_station_yaml_rejects_non_sensing_camera(tmp_path): + f = tmp_path / "s.yaml" + f.write_text( + "sensing_rig: {executable: e, expected_backend: MOCK, platform_config: p, config_name: c," + " link_masks: ['0x0']}\n" + "cameras: {l: {type: OAK_D_COLOR, module_name: L}}\n" + ) + with pytest.raises(ValueError, match="SENSING_COLOR"): + sn.load_sensing_station(str(f)) + + +# --- inspection and resolution -------------------- + + +def _fake_run(stdout="", returncode=0, stderr="", exc=None): + def run(args, **kw): + assert args[-1] == "--list-sensors" and kw["timeout"] > 0 and "shell" not in kw + if exc: + raise exc + return subprocess.CompletedProcess(args, returncode, stdout, stderr) + + return run + + +def test_inspect_rig_happy_path_uses_argument_list_and_the_existing_listing(): + d = sn.inspect_rig(sn.SensingRigConfig(**RIG), run=_fake_run(GOOD_LISTING)) + assert [(s.module_name, s.pipeline_id, s.width, s.height) for s in d.sensors] == [ + ("L", 1, W, H), + ("R", 0, W, H), + ] + + +@pytest.mark.parametrize( + "run,msg", + [ + (_fake_run(returncode=2, stderr="bad masks"), "bad masks"), + (_fake_run("not a listing"), "malformed"), + (_fake_run(_listing(f" sensor=0 L {W}x{H} @ 30 fps")), "header says 2 sensors, found 1"), + (_fake_run(_listing(" sensor=0 L garbage", " sensor=1 R 1x1 @ 1 fps")), "unexpected line"), + ( + _fake_run(_listing(f" sensor=0 L 0x{H} @ 30 fps", f" sensor=1 R {W}x{H} @ 30 fps")), + "non-positive", + ), + (_fake_run(exc=subprocess.TimeoutExpired("x", 1)), "could not inspect"), + (_fake_run(exc=FileNotFoundError("nope")), "could not inspect"), + ], +) +def test_inspect_rig_failures(run, msg): + with pytest.raises(SourceError, match=msg): + sn.inspect_rig(sn.SensingRigConfig(**RIG), run=run) + + +@pytest.mark.parametrize( + "bad", ["", "\n\n", "CFG: x sensor(s)\n", "CFG: 1 sensor(s)\n sensor=-1 L 1x1 @ 30 fps\n"] +) +def test_parse_sensor_listing_rejects_instead_of_guessing(bad): + with pytest.raises(ValueError, match="malformed"): + sn.parse_sensor_listing(bad) + + +def test_resolve_sensors_uses_module_names_not_positions(): + got = sn.resolve_sensors( + DESCRIPTION, {"left": sn.SensingStationCamera("L"), "right": sn.SensingStationCamera("R")} + ) + assert got["left"].pipeline_id == 1 and got["right"].pipeline_id == 0 + + +def test_sensor_index_selects_one_eye_of_a_stereo_module_in_pipeline_order(): + stereo = sn.SensingDescription( + ( + sn.SensingSensor("HEAD", 1, W, H, 30.0), + sn.SensingSensor("HEAD", 0, W, H, 30.0), + sn.SensingSensor("WRIST", 2, W, H, 30.0), + ), + ) + got = sn.resolve_sensors( + stereo, + { + "head_left_eye": sn.SensingStationCamera("HEAD", sensor_index=0), + "head_right_eye": sn.SensingStationCamera("HEAD", sensor_index=1), + "wrist": sn.SensingStationCamera("WRIST"), + }, + ) + assert [got[n].pipeline_id for n in ("head_left_eye", "head_right_eye", "wrist")] == [0, 1, 2] + with pytest.raises(SourceError, match="select one with sensor_index"): + sn.resolve_sensors(stereo, {"head": sn.SensingStationCamera("HEAD")}) + with pytest.raises(SourceError, match="no sensor_index 2"): + sn.resolve_sensors(stereo, {"head": sn.SensingStationCamera("HEAD", sensor_index=2)}) + with pytest.raises(SourceError, match="more than one logical"): + sn.resolve_sensors( + stereo, + { + "a": sn.SensingStationCamera("HEAD", sensor_index=0), + "b": sn.SensingStationCamera("HEAD", sensor_index=0), + }, + ) + with pytest.raises(ValueError): + sn.SensingStationCamera("HEAD", sensor_index=-1) + + +def test_resolve_sensors_rejects_missing_ambiguous_and_duplicate(): + with pytest.raises(SourceError, match="not in the platform"): + sn.resolve_sensors(DESCRIPTION, {"x": sn.SensingStationCamera("Z")}) + amb = sn.SensingDescription((sn.SensingSensor("L", 0, W, H, 30.0), sn.SensingSensor("L", 1, W, H, 30.0))) + with pytest.raises(SourceError, match="select one with sensor_index"): + sn.resolve_sensors(amb, {"x": sn.SensingStationCamera("L")}) + with pytest.raises(SourceError, match="more than one logical"): + sn.resolve_sensors(DESCRIPTION, {"a": sn.SensingStationCamera("L"), "b": sn.SensingStationCamera("L")}) + + +# --- rig ownership -------------------- + + +class FakeProc: + def __init__(self, *, ignore_terminate=False, ignore_kill=False) -> None: + self.code: int | None = None + self.stdout = io.BytesIO(b"line one\nline two\n") + self.ignore_terminate, self.ignore_kill = ignore_terminate, ignore_kill + self.calls: list[str] = [] + + def poll(self): + return self.code + + def terminate(self): + self.calls.append("terminate") + if not self.ignore_terminate: + self.code = -15 + + def kill(self): + self.calls.append("kill") + if not self.ignore_kill: + self.code = -9 + + def wait(self, timeout=None): + if self.code is None: + raise subprocess.TimeoutExpired("p", timeout) + return self.code + + +def _rig(tmp_path, proc, popen=None): + captured = {} + + def default_popen(args, **kw): + captured["args"], captured["kw"] = args, kw + return proc + + rig = sn.SensingRig( + sn.SensingRigConfig(**RIG), (1, 0), popen=popen or default_popen, runtime_root=str(tmp_path) + ) + return rig, captured + + +@pytest.mark.asyncio +async def test_rig_start_builds_file_output_argument_list_and_cleans_env(tmp_path, monkeypatch): + monkeypatch.setenv("SENSING_MOCK_SCALE_DIV", "4") + rig, cap = _rig(tmp_path, FakeProc()) + await rig.start() + args = cap["args"] + assert isinstance(args, list) and args[0] == RIG["executable"] and rig.start_error is None + assert "--link-masks=0x0000 0x1100" in args + # Encoder settings are passed only when configured; the decoder does not require any particular GOP. + assert not any(a.startswith(("--gop", "--bitrate", "--peak-bitrate")) for a in args) + streams = [a for a in args if a.startswith("--add-stream=")] + assert len(streams) == 2 and all(",output=" in a and "ipc=" not in a for a in streams) + assert streams[0].startswith("--add-stream=sensor=1,") and rig.stream_path(1) in streams[0] + assert "SENSING_MOCK_SCALE_DIV" not in cap["kw"]["env"] and "shell" not in cap["kw"] + run_dir = os.path.dirname(rig.stream_path(1)) + assert os.stat(run_dir).st_mode & 0o777 == 0o700 + await asyncio.sleep(0.05) # let the log drain read the fake output + assert "line two" in rig.diagnostics() + assert await rig.close() and rig.cleanup_ok + assert not os.path.exists(run_dir) + + +@pytest.mark.asyncio +async def test_rig_close_escalates_to_kill_and_removes_only_its_directory(tmp_path): + other = tmp_path / "keep" + other.mkdir() + proc = FakeProc(ignore_terminate=True) + rig, _ = _rig(tmp_path, proc) + await rig.start() + run_dir = os.path.dirname(rig.stream_path(1)) + assert await rig.close() and proc.calls == ["terminate", "kill"] and other.exists() + assert not os.path.exists(run_dir) + + +@pytest.mark.asyncio +async def test_rig_unconfirmed_exit_stays_false_on_every_later_call(tmp_path, monkeypatch): + monkeypatch.setattr(sn, "_RIG_STEPS_S", 0.1) + rig, _ = _rig(tmp_path, FakeProc(ignore_terminate=True, ignore_kill=True)) + await rig.start() + assert await rig.close() is False and rig.cleanup_ok is False + assert await rig.close() is False # never reported as released just because a wait ended + + +@pytest.mark.asyncio +async def test_rig_close_stays_inside_one_total_budget(tmp_path, monkeypatch): + monkeypatch.setattr(sn, "_RIG_STEPS_S", 0.3) + rig, _ = _rig(tmp_path, FakeProc(ignore_terminate=True, ignore_kill=True)) + await rig.start() + started = time.monotonic() + assert await rig.close() is False + assert time.monotonic() - started < 1.5 + + +@pytest.mark.asyncio +async def test_rig_start_failure_is_recorded_not_raised_and_leaves_no_directory(tmp_path): + def boom(*a, **k): + raise FileNotFoundError("no binary") + + rig, _ = _rig(tmp_path, None, popen=boom) + await rig.start() + assert "could not start" in rig.start_error and list(tmp_path.iterdir()) == [] + assert await rig.close() is True + + +@pytest.mark.asyncio +async def test_slow_spawn_still_in_flight_at_stop_is_reaped_by_the_queued_close(tmp_path): + release = threading.Event() + proc = FakeProc() + + def slow_popen(args, **kw): + release.wait(5) + return proc + + rig, _ = _rig(tmp_path, None, popen=slow_popen) + start = asyncio.ensure_future(rig.start()) + await asyncio.sleep(0.05) + start.cancel() # Stop/cancellation while the spawn is still running on the worker + threading.Timer(0.2, release.set).start() + assert await rig.close() is True + assert proc.calls == ["terminate"] and list(tmp_path.iterdir()) == [] + + +@pytest.mark.asyncio +async def test_log_tail_is_byte_bounded_decoded_leniently_and_closed(tmp_path): + proc = FakeProc() + proc.stdout = io.BytesIO(b"x" * 100_000 + b"\xff\xfe tail-marker") + rig, _ = _rig(tmp_path, proc) + await rig.start() + await asyncio.sleep(0.1) + text = rig.diagnostics() + assert len(text) <= sn._LOG_TAIL_BYTES and text.endswith("tail-marker") + assert proc.stdout.closed + await rig.close() + + +# --- adapter lifecycle ------------------------------------------------------------------------------------------- + + +class FakeRig: + start_error: str | None = None + + def __init__(self, path: str) -> None: + self.path, self.code = path, None + self.aborted: list[str] = [] + + def abort(self, reason: str) -> None: + self.aborted.append(reason) + + def stream_path(self, _pid: int) -> str: + return self.path + + def exit_code(self): + return self.code + + def diagnostics(self) -> str: + return "producer log" + + +def _camera(required=True, fps=200.0) -> p.CameraStreamConfig: + return p.CameraStreamConfig( + stream_id=1, + name="cam", + type=p.CameraType.SENSING_COLOR, + required=required, + width=W, + height=H, + fps=fps, + jpeg_quality=80, + ) + + +def _adapter(tmp_path, **kw): + rig = FakeRig(str(tmp_path / "sensor1.h264")) + adapter = sn.SensingAdapter( + rig=rig, + sensor=sn.SensingSensor("M", 1, W, H, 30.0), + camera=kw.pop("camera", _camera()), + startup_timeout_s=kw.pop("startup_timeout_s", 5.0), + ) + return adapter, rig + + +RED, GREEN, BLUE = (220, 20, 20), (20, 220, 20), (20, 20, 220) + + +@pytest.mark.asyncio +async def test_adapter_delivers_decodable_jpeg_and_closes_cleanly(tmp_path): + adapter, rig = _adapter(tmp_path) + assert adapter.camera_status().status == p.DeviceState.DEGRADED # never CONNECTED before a frame + producer = FileProducer(rig.path, encode_units(W, H, [RED, GREEN, BLUE]), loop=True).start() + try: + await adapter.enter() + assert adapter.camera_status().status == p.DeviceState.CONNECTED + got = None + for _ in range(100): + got = await adapter.next_camera_jpeg() + if got: + break + ts, jpeg = got + img = Image.open(io.BytesIO(jpeg)) + assert img.size == (W, H) and ts > 0 + assert await adapter.close() is True and not adapter.cleanup_failed + finally: + producer.stop() + + +@pytest.mark.asyncio +async def test_producer_exit_during_startup_is_a_camera_error(tmp_path): + adapter, rig = _adapter(tmp_path, startup_timeout_s=3.0) + rig.code = 3 + with pytest.raises(CameraSourceError, match="exited with 3: producer log"): + await adapter.enter() + assert adapter.camera_status().status == p.DeviceState.DEGRADED + assert await adapter.close() is True + + +@pytest.mark.asyncio +async def test_startup_deadline_without_a_frame(tmp_path): + adapter, rig = _adapter(tmp_path, startup_timeout_s=0.5) + open(rig.path, "wb").close() # the file exists but no unit is ever written + with pytest.raises(CameraSourceError, match="no frame"): + await adapter.enter() + await adapter.close() + + +@pytest.mark.asyncio +async def test_undecodable_producer_output_is_a_camera_error_not_a_synthetic_frame(tmp_path): + adapter, rig = _adapter(tmp_path, startup_timeout_s=2.0) + # Annex-B shaped but not video: the placeholder bytes an older mock encoder wrote. + with open(rig.path, "wb") as f: + f.write((b"\x00\x00\x00\x01\x65MOCK" + b"U" * 300) * 6) + with pytest.raises(CameraSourceError, match="failed at startup|no frame"): + await adapter.enter() + assert adapter.camera_status().status == p.DeviceState.DEGRADED + await adapter.close() + + +@pytest.mark.asyncio +async def test_mid_session_producer_exit_then_paced_empty_polls(tmp_path): + adapter, rig = _adapter(tmp_path, camera=_camera(required=False, fps=50.0)) + producer = FileProducer(rig.path, encode_units(W, H, [RED, GREEN]), loop=True).start() + try: + await adapter.enter() + rig.code = 1 + with pytest.raises(CameraSourceError, match="exited with 1"): + for _ in range(20): + await adapter.next_camera_jpeg() + assert adapter.camera_status().status == p.DeviceState.DEGRADED + started = time.monotonic() + for _ in range(3): # after the failure: no exception loop, still one tick per interval + assert await adapter.next_camera_jpeg() is None + assert time.monotonic() - started >= 0.03 + await adapter.close() + finally: + producer.stop() + + +@pytest.mark.asyncio +async def test_failed_close_is_sticky_even_after_stream_degradation(tmp_path): + adapter, rig = _adapter(tmp_path) + producer = FileProducer(rig.path, encode_units(W, H, [RED, GREEN]), loop=True).start() + try: + await adapter.enter() + + def leaky_close(): + raise OSError("could not release") + + adapter._reader.close = leaky_close + with pytest.raises(OSError, match="could not release"): + await adapter.close() + assert adapter.cleanup_failed + finally: + producer.stop() + + +# --- Acquisition ownership -------------------- + + +class SpyAdapter: + stream_id, required = 1, False + + def __init__( + self, + order, + name, + fail=False, + gate: threading.Event | None = None, + enter_error=None, + required=False, + borrows_rig=True, + ): + self.order, self.name, self.fail, self.gate = order, name, fail, gate + self.enter_error, self.required, self.borrows_rig = enter_error, required, borrows_rig + + async def enter(self): + if self.enter_error: + raise CameraSourceError(self.enter_error, camera_name=self.name) + + async def close(self): + self.order.append(f"{self.name}:close-start") + if self.gate is not None: + await asyncio.get_running_loop().run_in_executor(None, self.gate.wait, 5) + self.order.append(f"{self.name}:close-done") + if self.fail: + raise RuntimeError("release failed") + return True + + def camera_status(self): + raise AssertionError + + +class SpyRig: + def __init__(self, order, ok=True, start_error=None): + self.order, self.ok, self.started, self.start_error = order, ok, False, start_error + + async def start(self): + self.started = True + + async def close(self): + self.order.append("rig:close") + return self.ok + + +@pytest.mark.asyncio +async def test_acquisition_starts_rig_then_closes_adapters_before_rig(): + order: list[str] = [] + rig = SpyRig(order) + acq = Acquisition(tracking=None, cameras=(SpyAdapter(order, "a"),), sensing_rig=rig) + await acq.enter() + assert rig.started + assert await acq.close() is True and order == ["a:close-start", "a:close-done", "rig:close"] + + +@pytest.mark.asyncio +async def test_rig_is_not_terminated_while_an_adapter_close_is_still_held(): + order: list[str] = [] + gate = threading.Event() + acq = Acquisition(tracking=None, cameras=(SpyAdapter(order, "a", gate=gate),), sensing_rig=SpyRig(order)) + closing = asyncio.ensure_future(acq.close()) + await asyncio.sleep(0.2) + assert order == ["a:close-start"] # producer teardown has NOT begun while the adapter is still closing + gate.set() + assert await closing is True + assert order == ["a:close-start", "a:close-done", "rig:close"] + + +@pytest.mark.asyncio +async def test_acquisition_reaps_rig_even_if_an_adapter_close_raises_and_reports_unconfirmed_rig(): + order: list[str] = [] + acq = Acquisition(tracking=None, cameras=(SpyAdapter(order, "a", fail=True),), sensing_rig=SpyRig(order)) + with pytest.raises(RuntimeError, match="release failed"): + await acq.close() + assert order[-1] == "rig:close" + acq2 = Acquisition(tracking=None, cameras=(SpyAdapter([], "a"),), sensing_rig=SpyRig([], ok=False)) + assert await acq2.close() is False + + +@pytest.mark.asyncio +async def test_shared_producer_start_failure_degrades_optional_and_fails_required_streams(tmp_path): + class DeadRig(FakeRig): + start_error = "could not start /x: no binary" + + def make(required): + return sn.SensingAdapter( + rig=DeadRig("/nope"), + sensor=sn.SensingSensor("M", 1, W, H, 30.0), + camera=_camera(required=required), + ) + + healthy_oak = SpyAdapter([], "oak") + optional = make(False) + await Acquisition(tracking=None, cameras=(healthy_oak, optional), sensing_rig=None).enter() # tolerated + assert optional.camera_status().status == p.DeviceState.DEGRADED + assert "did not start" in optional.camera_status().error + with pytest.raises(CameraSourceError, match="did not start"): + await Acquisition(tracking=None, cameras=(healthy_oak, make(True)), sensing_rig=None).enter() + for adapter in (optional,): + assert await adapter.close() is True + + +# --- server type validation and launch selection -------------------- + + +def _server(mock=True, cameras=None): + cfg = ServerConfig(mock=mock, cameras=cameras, sensing_rig=sn.SensingRigConfig(**RIG) if cameras else None) + return CaptureServer(cfg, source_factory=lambda *_: None) + + +def _begin(*cams): + return p.BeginRequest( + session_id="s1", + tracking=p.TrackingConfig(sources=(), rate_hz=30.0), + cameras=tuple(p.CameraRequest(name=n, type=t) for n, t in cams), + ) + + +@pytest.mark.asyncio +async def test_prepare_sensing_rejects_type_mismatch_before_inspection(monkeypatch): + monkeypatch.setattr("capture_server.server.inspect_rig", lambda *_: pytest.fail("must not inspect")) + server = _server(cameras={"left": sn.SensingStationCamera("L")}) + err = await server._prepare_sensing(_begin(("left", p.CameraType.OAK_D_COLOR))) + assert "SENSING_COLOR station camera" in err + + +@pytest.mark.asyncio +async def test_prepare_sensing_inspects_once_and_caches(monkeypatch): + calls = [] + + def fake_inspect(cfg): + calls.append(cfg) + return DESCRIPTION + + monkeypatch.setattr("capture_server.server.inspect_rig", fake_inspect) + server = _server(cameras={"left": sn.SensingStationCamera("L")}) + begin = _begin(("left", p.CameraType.SENSING_COLOR)) + assert await server._prepare_sensing(begin) is None and await server._prepare_sensing(begin) is None + assert len(calls) == 1 + cam = server._camera_stream_config(1, begin.cameras[0]) + assert (cam.width, cam.height, cam.fps) == (W, H, 30.0) and cam.type == p.CameraType.SENSING_COLOR + + +@pytest.mark.asyncio +async def test_prepare_sensing_failure_is_reported_and_not_cached(monkeypatch): + def boom(_cfg): + raise SourceError("backend mismatch") + + monkeypatch.setattr("capture_server.server.inspect_rig", boom) + server = _server(cameras={"left": sn.SensingStationCamera("L")}) + err = await server._prepare_sensing(_begin(("left", p.CameraType.SENSING_COLOR))) + assert "backend mismatch" in err and server.config.sensing_description is None + + +@pytest.mark.asyncio +async def test_oak_only_begin_needs_no_sensing(monkeypatch): + monkeypatch.setattr("capture_server.server.inspect_rig", lambda *_: pytest.fail("must not inspect")) + server = _server(cameras={"w": StationCamera("mxid")}) + assert await server._prepare_sensing(_begin(("w", p.CameraType.OAK_D_COLOR))) is None + + +def test_launch_builds_no_rig_for_oak_only_sessions(): + from capture_server.launch import _sensing_composition + + cfg = ServerConfig(mock=True) + assert _sensing_composition(cfg, (_camera(),)) == (None, {}) + + +def test_launch_mixes_sensing_and_mock_oak_in_one_acquisition(): + from capture_server.launch import build_source_factory + + cfg = ServerConfig( + mock=True, + cameras={"left": sn.SensingStationCamera("L")}, + sensing_rig=sn.SensingRigConfig(**RIG), + sensing_description=DESCRIPTION, + ) + factory = build_source_factory(cfg) + begin = _begin(("left", p.CameraType.SENSING_COLOR), ("wrist", p.CameraType.OAK_D_COLOR)) + cams = ( + p.CameraStreamConfig(1, "left", p.CameraType.SENSING_COLOR, True, W, H, 30.0, 80), + p.CameraStreamConfig(2, "wrist", p.CameraType.OAK_D_COLOR, True, 1280, 720, 30.0, 80), + ) + acq = factory(begin, cams) + assert isinstance(acq.cameras[0], sn.SensingAdapter) and not isinstance(acq.cameras[1], sn.SensingAdapter) + assert acq.sensing_rig is not None + + +# --- Begin admission through the real server ----------- + + +async def _begin_rejection(url: str, camera_type: p.CameraType, name: str = "left") -> p.RequestRejected: + """Send one real Begin with a single camera over a raw socket; return the RequestRejected it draws.""" + import json + + import websockets + + begin = p.BeginRequest( + tracking=p.TrackingConfig(sources=()), cameras=(p.CameraRequest(name=name, type=camera_type),) + ) + async with websockets.connect(url) as ws: + await ws.send( + json.dumps( + { + "type": int(p.MessageType.BEGIN), + "version": p.PROTOCOL_VERSION, + "session_id": "sensing-begin-1", + **p.effective_config(begin).to_json(), + } + ) + ) + for _ in range(20): + raw = await asyncio.wait_for(ws.recv(), timeout=3) + if isinstance(raw, bytes): + continue + event = p.parse_server_message(json.loads(raw)) + if isinstance(event, p.RequestRejected): + return event + raise AssertionError("no RequestRejected arrived") + + +@pytest.mark.asyncio +async def test_sensing_request_without_a_station_entry_is_rejected_not_built_as_a_mock_oak(unused_tcp_port): + from tests.support.capture_server import start_capture_server, stop_capture_server + + running = await start_capture_server(unused_tcp_port) # permissive legacy config: cameras=None + try: + rejected = await _begin_rejection(f"ws://127.0.0.1:{unused_tcp_port}", p.CameraType.SENSING_COLOR) + assert rejected.reason == p.RejectionReason.INVALID_CONFIG + assert "no SENSING station entry" in rejected.error_message + assert running.server._active_session is None + finally: + await stop_capture_server(running) + + +@pytest.mark.asyncio +async def test_shutdown_during_a_paused_inspection_never_admits_a_session(unused_tcp_port, monkeypatch): + from tests.support.capture_server import start_capture_server, stop_capture_server + + started, release = threading.Event(), threading.Event() + + def slow_inspect(_cfg): + started.set() + release.wait(5) + return DESCRIPTION + + monkeypatch.setattr("capture_server.server.inspect_rig", slow_inspect) + cfg = ServerConfig( + mock=True, cameras={"left": sn.SensingStationCamera("L")}, sensing_rig=sn.SensingRigConfig(**RIG) + ) + running = await start_capture_server(unused_tcp_port, config=cfg) + try: + pending = asyncio.ensure_future( + _begin_rejection(f"ws://127.0.0.1:{unused_tcp_port}", p.CameraType.SENSING_COLOR) + ) + await asyncio.get_running_loop().run_in_executor(None, started.wait, 5) + running.server.request_stop() # shutdown begins while the Begin is parked in inspection + release.set() + rejected = await pending + assert rejected.reason == p.RejectionReason.BUSY and "shutting down" in rejected.error_message + assert running.server._active_session is None + finally: + release.set() + await stop_capture_server(running) + + +@pytest.mark.asyncio +async def test_concurrent_first_begins_share_one_inspection(monkeypatch): + calls: list[int] = [] + release = threading.Event() + + def slow_inspect(_cfg): + calls.append(1) + release.wait(5) + return DESCRIPTION + + monkeypatch.setattr("capture_server.server.inspect_rig", slow_inspect) + server = _server(cameras={"left": sn.SensingStationCamera("L")}) + begin = _begin(("left", p.CameraType.SENSING_COLOR)) + both = asyncio.gather(server._prepare_sensing(begin), server._prepare_sensing(begin)) + await asyncio.sleep(0.1) + release.set() + assert await both == [None, None] and len(calls) == 1 + + +@pytest.mark.asyncio +async def test_a_begin_that_is_already_busy_triggers_no_inspection(monkeypatch): + monkeypatch.setattr("capture_server.server.inspect_rig", lambda *_: pytest.fail("must not inspect")) + server = _server(cameras={"left": sn.SensingStationCamera("L")}) + server._active_session = object() # a session is already running + assert await server._prepare_sensing(_begin(("left", p.CameraType.SENSING_COLOR))) is None + + +# --- shared shutdown budget and local start failures ------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_slow_oak_release_is_not_cut_short_by_the_producer_shutdown(monkeypatch): + from capture_server.acquisition import acquisition as acq_mod + + monkeypatch.setattr(acq_mod, "_CLOSE_BUDGET_S", 0.8) + order: list[str] = [] + gate = threading.Event() + threading.Timer(0.45, gate.set).start() # a release taking 0.45 s of a 0.8 s budget + oak = SpyAdapter(order, "oak", gate=gate, borrows_rig=False) + acq = Acquisition(tracking=None, cameras=(oak, SpyAdapter(order, "sensing")), sensing_rig=SpyRig(order)) + started = time.monotonic() + assert await acq.close() is True # the old shared budget cut OAK at budget minus the producer's reserve + assert time.monotonic() - started < 0.8 + assert order.count("oak:close-done") == 1 and order[-1] in ("rig:close", "oak:close-done") + + +@pytest.mark.asyncio +async def test_a_held_oak_release_does_not_delay_the_producer_reap_and_stays_unconfirmed(monkeypatch): + from capture_server.acquisition import acquisition as acq_mod + + monkeypatch.setattr(acq_mod, "_CLOSE_BUDGET_S", 0.4) + order: list[str] = [] + gate = threading.Event() + oak = SpyAdapter(order, "oak", gate=gate, borrows_rig=False) + acq = Acquisition(tracking=None, cameras=(oak, SpyAdapter(order, "sensing")), sensing_rig=SpyRig(order)) + started = time.monotonic() + assert await acq.close() is False # the OAK release is still pending: unconfirmed, admission stays blocked + assert time.monotonic() - started < 0.9 + assert order.index("rig:close") < order.index("oak:close-done") if "oak:close-done" in order else True + assert "rig:close" in order and "sensing:close-done" in order # the producer chain was not held up by OAK + gate.set() # the retained OAK close finishes later and is observed, not dropped + await asyncio.sleep(0.1) + assert order.count("oak:close-done") == 1 + + +@pytest.mark.asyncio +async def test_runtime_directory_failures_are_start_failures_and_leave_nothing_behind(tmp_path, monkeypatch): + rig = sn.SensingRig( + sn.SensingRigConfig(**RIG), (1,), popen=lambda *a, **k: FakeProc(), runtime_root=str(tmp_path / "missing") + ) + await rig.start() + assert "runtime directory" in rig.start_error and await rig.close() is True + + monkeypatch.setattr(os, "chmod", lambda *a, **k: (_ for _ in ()).throw(PermissionError("denied"))) + rig2 = sn.SensingRig( + sn.SensingRigConfig(**RIG), (1,), popen=lambda *a, **k: FakeProc(), runtime_root=str(tmp_path) + ) + await rig2.start() + assert "denied" in rig2.start_error and list(tmp_path.iterdir()) != [] # created, so retained for cleanup + assert await rig2.close() is True and list(tmp_path.iterdir()) == [] + + +@pytest.mark.asyncio +async def test_cancelled_close_still_reaps_the_producer_and_observes_late_failures(): + import gc + + unobserved: list[dict] = [] + asyncio.get_running_loop().set_exception_handler(lambda _loop, ctx: unobserved.append(ctx)) + order: list[str] = [] + gate = threading.Event() + acq = Acquisition( + tracking=None, + cameras=(SpyAdapter(order, "slow", gate=gate), SpyAdapter(order, "late-fail", fail=True, gate=gate)), + sensing_rig=SpyRig(order), + ) + closing = asyncio.ensure_future(acq.close()) + await asyncio.sleep(0.1) + closing.cancel() # the caller gives up while both adapter closes are still held + with pytest.raises(asyncio.CancelledError): + await closing + gate.set() # the held closes finish later; one of them fails with nobody awaiting it + for _ in range(100): + if "rig:close" in order and "late-fail:close-done" in order: + break + await asyncio.sleep(0.02) + assert "rig:close" in order # the producer was still reaped + del acq, closing + gc.collect() + await asyncio.sleep(0.05) + assert unobserved == [] # no "exception was never retrieved" + + +# --- strict station file, shared producer environment, factory misuse, cancellation precedence ------------------- + + +def _yaml(tmp_path, rig_extra="", camera="{type: SENSING_COLOR, module_name: L}", masks="['0x0000']"): + f = tmp_path / "s.yaml" + f.write_text( + "sensing_rig: {executable: e, expected_backend: MOCK, platform_config: p, config_name: c," + f" link_masks: {masks}{rig_extra}}}\ncameras: {{l: {camera}}}\n" + ) + return str(f) + + +@pytest.mark.parametrize( + "kw,msg", + [ + (dict(masks="[0x0000, 0x1100]"), "quoted strings"), # unquoted masks are YAML integers + (dict(rig_extra=", full_range: yes-please"), "full_range must be bool"), + (dict(rig_extra=", gpu_id: true"), "gpu_id must be int"), + (dict(camera="{type: SENSING_COLOR, module_name: L, jpeg_quality: 80.5}"), "jpeg_quality must be int"), + (dict(camera="{type: SENSING_COLOR, module_name: L, required: 1}"), "required must be bool"), + ( + dict(camera="{type: SENSING_COLOR, module_name: L, delivery_fps: true}"), + "delivery_fps must be int/float", + ), + (dict(camera="{type: SENSING_COLOR, module_name: 7}"), "module_name must be str"), + ], +) +def test_station_yaml_scalars_are_type_checked(tmp_path, kw, msg): + with pytest.raises(ValueError, match=msg): + sn.load_sensing_station(_yaml(tmp_path, **kw)) + + +def test_a_file_that_is_not_yaml_is_a_value_error_not_a_traceback(tmp_path): + f = tmp_path / "s.yaml" + f.write_text("sensing_rig: [unclosed\n") + with pytest.raises(ValueError, match="not valid YAML"): + sn.load_sensing_station(str(f)) + + +def test_the_listing_and_the_capture_run_share_one_sanitized_environment(monkeypatch): + monkeypatch.setenv("SENSING_MOCK_SCALE_DIV", "4") + seen = {} + + def run(args, **kw): + seen["env"] = kw["env"] + return subprocess.CompletedProcess(args, 0, GOOD_LISTING, "") + + sn.inspect_rig(sn.SensingRigConfig(**RIG), run=run) + assert "SENSING_MOCK_SCALE_DIV" not in seen["env"] and seen["env"] == sn._producer_env() + + +def test_a_factory_without_the_servers_sensor_description_says_so(): + from capture_server.launch import _sensing_composition + + cfg = ServerConfig( + mock=True, cameras={"left": sn.SensingStationCamera("L")}, sensing_rig=sn.SensingRigConfig(**RIG) + ) # no sensing_description: what a copied config would look like + camera = p.CameraStreamConfig(1, "left", p.CameraType.SENSING_COLOR, True, W, H, 30.0, 80) + with pytest.raises(ValueError, match="no rig or sensor description"): + _sensing_composition(cfg, (camera,)) + + +@pytest.mark.asyncio +async def test_a_producer_that_goes_quiet_but_stays_alive_fails_the_stream(tmp_path, monkeypatch): + monkeypatch.setattr(sn, "_NO_FRAME_TIMEOUT_S", 0.3) + adapter, rig = _adapter(tmp_path, camera=_camera(required=True, fps=100.0)) + producer = FileProducer(rig.path, encode_units(W, H, [RED, GREEN, BLUE]), loop=False).start() + try: + await adapter.enter() + assert adapter.camera_status().status == p.DeviceState.CONNECTED + with pytest.raises(CameraSourceError, match="no frame from the SENSING stream for"): + for _ in range(200): # rig.exit_code() stays None: the producer is alive, just silent + await adapter.next_camera_jpeg() + assert adapter.camera_status().status == p.DeviceState.DEGRADED + assert await adapter.close() is True + finally: + producer.stop() + + +def test_reclaim_all_hands_back_a_failed_streams_file_without_decoding(tmp_path): + path = tmp_path / "s.h264" + path.write_bytes(os.urandom(4 * 1024 * 1024)) # real, allocated bytes the producer keeps appending to + reader = h264.SensingStreamReader(str(path), width=W, height=H) + reader.connect(1.0) + before = allocated_bytes(str(path)) + reader.reclaim_all() + assert allocated_bytes(str(path)) < before / 2 and os.stat(path).st_size == 4 * 1024 * 1024 + with pytest.raises(h264.SensingStreamError, match="failed earlier"): + reader.read_latest(0.0) # reclaiming never revives the stream + with open(path, "ab") as f: + f.write(os.urandom(2 * 1024 * 1024)) + reader.reclaim_all() # more output arrived: reclaimed again on the next call + assert allocated_bytes(str(path)) < 1024 * 1024 + reader.close() + + +def test_reclaim_all_tolerates_a_failing_native_call_only_up_to_the_spool_bound(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_MAX_UNRECLAIMED_BYTES", 2 * 1024 * 1024) + path = tmp_path / "s.h264" + path.write_bytes(os.urandom(1024 * 1024)) + reader = h264.SensingStreamReader(str(path), width=W, height=H, fallocate=lambda *a: -1) + reader.connect(1.0) + reader.reclaim_all() + assert reader._punched == 0 # nothing marked reclaimed; tried again next tick + with open(path, "ab") as f: + f.write(os.urandom(2 * 1024 * 1024)) + with pytest.raises(h264.SensingStreamError, match="cannot be handed back"): + reader.reclaim_all() # 3 MiB the producer keeps growing and we cannot return: the spool is unbounded + reader.close() + + +def test_reclaim_all_reclaims_a_file_created_after_the_stream_failed_before_attaching(tmp_path): + path = tmp_path / "late.h264" + reader = h264.SensingStreamReader(str(path), width=W, height=H) + reader.reclaim_all() # no file yet: nothing accumulating, nothing raised + path.write_bytes(os.urandom(4 * 1024 * 1024)) # the producer creates it later + reader.reclaim_all() + assert allocated_bytes(str(path)) < 1024 * 1024 + reader.close() + + +def test_reclaim_all_bounds_a_late_file_it_cannot_open(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_MAX_UNRECLAIMED_BYTES", 1024) + path = tmp_path / "late.h264" + path.write_bytes(os.urandom(64 * 1024)) + path.chmod(0o400) # readable only: O_RDWR open fails, so nothing can be handed back + if os.access(path, os.W_OK): + pytest.skip("running as a user that can write read-only files") + reader = h264.SensingStreamReader(str(path), width=W, height=H) + with pytest.raises(h264.SensingStreamError, match="cannot be handed back"): + reader.reclaim_all() + reader.close() + + +@pytest.mark.asyncio +async def test_an_unreclaimable_dead_stream_stops_the_shared_producer(tmp_path): + adapter, rig = _adapter(tmp_path, camera=_camera(required=False, fps=200.0)) + + def unbounded(): + raise h264.SensingStreamError("spool is unbounded") + + adapter._reader = types.SimpleNamespace(reclaim_all=unbounded, close=lambda: None) + adapter._dead = True + adapter._deadline = time.monotonic() + assert await adapter.next_camera_jpeg() is None + assert rig.aborted == ["spool is unbounded"] + await adapter.close() + + +@pytest.mark.asyncio +async def test_a_dead_stream_keeps_reclaiming_on_every_paced_poll(tmp_path): + calls = [] + adapter, _ = _adapter(tmp_path, camera=_camera(required=False, fps=200.0)) + adapter._reader = types.SimpleNamespace(reclaim_all=lambda: calls.append(1), close=lambda: None) + adapter._dead = True + adapter._deadline = time.monotonic() + for _ in range(3): + assert await adapter.next_camera_jpeg() is None + assert len(calls) == 3 + await adapter.close() + + +@pytest.mark.asyncio +async def test_stop_during_bring_up_ends_the_startup_and_the_release_is_confirmed_promptly(tmp_path): + adapter, rig = _adapter(tmp_path, startup_timeout_s=30.0) # the stream file never appears + entering = asyncio.ensure_future(adapter.enter()) + await asyncio.sleep(0.3) + started = time.monotonic() + assert await adapter.close() is True # without the stop signal this queues behind the 30 s startup + assert time.monotonic() - started < 1.0 + with pytest.raises(CameraSourceError, match="stopped during startup"): + await entering + + +def test_a_stopped_reader_ends_every_blocking_operation_with_the_reason(tmp_path): + stop = threading.Event() + path = tmp_path / "s.h264" + path.write_bytes(b"".join(encode_units(W, H, [RED, GREEN, BLUE]) * 50)) + reader = h264.SensingStreamReader(str(path), width=W, height=H, stop=stop) + reader.connect(1.0) + stop.set() + with pytest.raises(h264.SensingStreamError, match="stopped"): + reader.read_latest(5.0) + with pytest.raises(h264.SensingStreamError, match="stopped"): + h264.SensingStreamReader(str(tmp_path / "never"), width=W, height=H, stop=stop).connect(5.0) + finisher = h264.SensingStreamReader(str(path), width=W, height=H, stop=threading.Event()) + finisher.connect(1.0) + finisher._stop.set() + with pytest.raises(h264.SensingStreamError, match="stopped"): + finisher.finish() diff --git a/examples/egocap/tests/capture_server/test_sensing_h264.py b/examples/egocap/tests/capture_server/test_sensing_h264.py new file mode 100644 index 0000000000..5f8d0d88eb --- /dev/null +++ b/examples/egocap/tests/capture_server/test_sensing_h264.py @@ -0,0 +1,318 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""SensingStreamReader: tails the producer's H.264 file, decodes every unit in order, delivers the newest frame. + +CPU-only, against real libx264-encoded units (intra-only and inter-coded GOPs); the native producer's own +output is covered separately. +""" + +from __future__ import annotations + +import ctypes +import errno +import os +import threading +import time + +import numpy as np +import pytest + +from capture_server.acquisition import sensing_h264 as h264 +from capture_server.acquisition.errors import SourceError +from capture_server.acquisition.sensing_jpeg import encode_rgb_jpeg + +from .h264_fixtures import allocated_bytes, encode_units + +W, H = 64, 32 +RED, GREEN, BLUE = (220, 20, 20), (20, 220, 20), (20, 20, 220) + + +def _close(rgb: np.ndarray, expected: tuple[int, int, int], tol: int = 14) -> bool: + return bool(np.abs(rgb[H // 2, W // 2].astype(int) - np.array(expected)).max() <= tol) + + +def _reader(path, **kw) -> h264.SensingStreamReader: + reader = h264.SensingStreamReader(str(path), width=kw.pop("width", W), height=kw.pop("height", H), **kw) + reader.connect(1.0) + return reader + + +def test_decodes_correct_colours_and_geometry(tmp_path): + units = encode_units(W, H, [RED, GREEN]) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) + units[-1]) + reader = _reader(path) + rgb, selected_ns = reader.read_latest(1.0) + assert rgb.shape == (H, W, 3) and selected_ns > 0 and _close(rgb, GREEN) + reader.close() + + +def test_newest_complete_unit_wins_and_older_ones_are_skipped(tmp_path): + units = encode_units(W, H, [RED, GREEN, BLUE]) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) # the last unit is not yet known complete: GREEN is the newest + reader = _reader(path) + rgb, _ = reader.read_latest(1.0) + assert _close(rgb, GREEN) # RED was decoded first but only the newest decoded frame is delivered + assert reader.read_latest(0.05) is None # nothing newer; never invents a frame + reader.close() + + +def test_a_half_written_unit_is_not_delivered_until_complete(tmp_path): + units = encode_units(W, H, [RED, GREEN]) + path = tmp_path / "s.h264" + reader_path = str(path) + with open(path, "wb") as f: + f.write(units[0][: len(units[0]) // 2]) + f.flush() + reader = h264.SensingStreamReader(reader_path, width=W, height=H) + reader.connect(1.0) + assert reader.read_latest(0.05) is None + f.write(units[0][len(units[0]) // 2 :] + units[1]) + f.flush() + rgb, _ = reader.read_latest(1.0) + assert _close(rgb, RED) + reader.close() + + +def test_missing_file_times_out_with_oserror(tmp_path): + reader = h264.SensingStreamReader(str(tmp_path / "never"), width=W, height=H) + with pytest.raises(FileNotFoundError): + reader.connect(0.05) + + +def test_geometry_mismatch_is_terminal(tmp_path): + units = encode_units(W, H, [RED, GREEN]) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) + units[-1]) + reader = _reader(path, width=W * 2) + with pytest.raises(h264.SensingStreamError, match="expected 128x32"): + reader.read_latest(1.0) + with pytest.raises(h264.SensingStreamError, match="failed earlier"): + reader.read_latest(0.0) + + +def test_undecodable_bytes_are_a_terminal_stream_error_not_a_frame(tmp_path): + path = tmp_path / "s.h264" + # Annex-B shaped but not video, like the native mock encoder's units. + path.write_bytes((b"\x00\x00\x00\x01\x67MOCK" + b"U" * 200 + b"\x00\x00\x00\x01\x65MOCK" + b"U" * 200) * 3) + reader = _reader(path) + with pytest.raises(h264.SensingStreamError): + for _ in range(5): + reader.read_latest(0.2) + with pytest.raises(h264.SensingStreamError, match="failed earlier"): + reader.read_latest(0.0) + + +def test_read_before_connect_is_an_error(tmp_path): + reader = h264.SensingStreamReader(str(tmp_path / "x"), width=W, height=H) + with pytest.raises(h264.SensingStreamError, match="not connected"): + reader.read_latest(0.0) + + +def test_consumed_prefix_is_returned_to_the_filesystem_while_reading_continues(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_PUNCH_CHUNK_BYTES", 4096) + units = encode_units(W, H, [RED, GREEN, BLUE]) * 600 # each unit is an independent IDR + path = tmp_path / "s.h264" + with open(path, "wb") as f: + reader = None + for i, unit in enumerate(units): + f.write(unit) + f.flush() + if reader is None: + reader = h264.SensingStreamReader(str(path), width=W, height=H) + reader.connect(1.0) + if i > 2: + assert reader.read_latest(1.0) is not None + size = os.stat(path).st_size + assert allocated_bytes(str(path)) < size # holes punched behind the read offset (best effort) + reader.close() + + +def test_native_size_unit_decodes_independently_and_fits_the_payload_ceiling(tmp_path): + width, height = 2560, 1984 + units = encode_units(width, height, [RED, GREEN, BLUE]) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) + reader = h264.SensingStreamReader(str(path), width=width, height=height) + reader.connect(1.0) + started = time.monotonic() + rgb, _ = reader.read_latest(5.0) # RED then GREEN decoded in order; GREEN delivered + assert rgb.shape == (height, width, 3) + assert bool(np.abs(rgb[900, 1200].astype(int) - np.array(GREEN)).max() <= 14) + assert len(encode_rgb_jpeg(rgb, 80)) < 2 * 1024 * 1024 + assert time.monotonic() - started < 5.0 + reader.close() + + +def test_oversized_jpeg_names_dimensions_and_quality(): + noise = np.random.default_rng(0).integers(0, 256, (64, 64, 3), dtype=np.uint8) + with pytest.raises(SourceError, match=r"64x64 JPEG at quality 95.*ceiling"): + encode_rgb_jpeg(noise, 95, max_bytes=100) + + +def test_jpeg_rejects_wrong_shape(): + with pytest.raises(ValueError): + encode_rgb_jpeg(np.zeros((4, 4, 4), np.uint8), 80) + + +def _failing_fallocate(code: int): + """A stand-in for the C call that fails the way the real one does: returns -1 and sets errno.""" + + def fallocate(_fd, _mode, _offset, _length): + ctypes.set_errno(code) + return -1 + + return fallocate + + +def test_a_failing_space_reclamation_ends_the_stream_with_the_reason_and_never_marks_bytes_reclaimed( + tmp_path, monkeypatch +): + monkeypatch.setattr(h264, "_PUNCH_CHUNK_BYTES", 256) + units = encode_units(W, H, [RED, GREEN, BLUE]) * 40 + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) + reader = h264.SensingStreamReader(str(path), width=W, height=H, fallocate=_failing_fallocate(errno.EOPNOTSUPP)) + reader.connect(1.0) + with pytest.raises(h264.SensingStreamError, match=r"cannot reclaim.*errno 95"): + reader.read_latest(1.0) + assert reader._punched == 0 # the reclaimed offset advances only after the call succeeds + with pytest.raises(h264.SensingStreamError, match="failed earlier"): + reader.read_latest(0.0) + + +def test_a_file_we_cannot_reclaim_from_is_an_error_not_a_quiet_read_only_fallback(tmp_path): + path = tmp_path / "s.h264" + path.write_bytes(b"x") + path.chmod(0o444) + if os.access(path, os.W_OK): + pytest.skip("running with privileges that ignore file modes") + reader = h264.SensingStreamReader(str(path), width=W, height=H) + with pytest.raises(h264.SensingStreamError, match="reclamation"): + reader.connect(0.5) + + +def test_one_poll_reads_at_most_its_byte_cap_even_with_a_larger_backlog(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_MAX_POLL_BYTES", 2048) + monkeypatch.setattr(h264, "_READ_BYTES", 512) + units = encode_units(W, H, [RED, GREEN, BLUE]) * 200 + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) + reader = _reader(path) + assert reader.read_latest(1.0) is not None + assert reader._offset <= 2048 and os.stat(path).st_size > 20 * 2048 # the backlog is read over later polls + reader.close() + + +def test_a_producer_that_keeps_appending_cannot_extend_a_poll_past_its_deadline(tmp_path): + units = encode_units(W, H, [RED, GREEN]) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 3) + stop = threading.Event() + + def append() -> None: + with open(path, "ab") as f: + while not stop.is_set(): + f.write(units[0]) + f.flush() + + writer = threading.Thread(target=append, daemon=True) + reader = _reader(path) + writer.start() + started = time.monotonic() + assert reader.read_latest(0.3) is not None + assert time.monotonic() - started < 2.0 + stop.set() + writer.join(timeout=2) + reader.close() + + +def test_input_that_never_completes_a_unit_is_rejected_instead_of_retained_without_bound(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_MAX_UNIT_BYTES", 8192) + path = tmp_path / "s.h264" + path.write_bytes(b"\x01" * 64 * 1024) # no start codes: the parser never completes a unit + reader = _reader(path) + with pytest.raises(h264.SensingStreamError, match="no complete access unit in"): + reader.read_latest(1.0) + + +def test_inter_coded_units_are_decoded_in_order_and_the_newest_decoded_frame_is_delivered(tmp_path): + # gop=30: after the first IDR the units are P-frames that cannot be decoded without their predecessors. + units = encode_units(W, H, [RED] * 3 + [GREEN] * 3 + [BLUE] * 3, gop=30) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) + reader = _reader(path) + rgb, _ = reader.read_latest(2.0) + assert _close(rgb, BLUE) # the last complete unit; the final one is still held back by the parser + reader.close() + + +def test_finish_flushes_the_unit_the_parser_was_holding_back_and_only_once(tmp_path): + units = encode_units(W, H, [RED, GREEN, BLUE]) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) + reader = _reader(path) + rgb, _ = reader.read_latest(1.0) + assert _close(rgb, GREEN) and reader.read_latest(0.05) is None # BLUE is incomplete while the file may grow + last = reader.finish() # the producer ended: now the end of the file is the end of the stream + assert last is not None and _close(last[0], BLUE) + assert reader.finish() is None + reader.close() + + +def test_a_backlog_beyond_the_pending_bound_resumes_at_the_newest_idr(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_MAX_PENDING_BYTES", 400) + units = encode_units(W, H, [RED, GREEN, BLUE] * 17) # 51 independent IDR units + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) + reader = _reader(path) + rgb, _ = reader.read_latest(2.0) + assert _close(rgb, GREEN) # unit 49, the newest complete one; everything older was skipped undecoded + reader.close() + + +def test_an_inter_coded_backlog_that_cannot_be_caught_up_ends_the_stream_with_the_reason(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_MAX_PENDING_BYTES", 100) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(encode_units(W, H, [RED] * 40, gop=1000))) # one IDR, then only P-frames + reader = _reader(path) + with pytest.raises(h264.SensingStreamError, match="decoder cannot keep up"): + reader.read_latest(2.0) + + +def test_a_restart_point_needs_an_idr_slice_and_in_band_parameter_sets(): + idr, p_frame = encode_units(W, H, [RED, RED], gop=30)[:2] + assert h264._is_restart_point(idr) and not h264._is_restart_point(p_frame) + slice_at = idr.index(b"\x00\x00\x01\x65") + assert not h264._is_restart_point(idr[slice_at:]) # the IDR slice alone: no SPS/PPS to restart from + + +def test_finish_straight_away_decodes_every_pending_unit_in_order_before_flushing(tmp_path): + path = tmp_path / "s.h264" + path.write_bytes(b"".join(encode_units(W, H, [RED] * 3 + [GREEN] * 3 + [BLUE] * 3, gop=30))) + reader = _reader(path) + last = reader.finish() # no read_latest first: the units are still queued/held back, all must be decoded + assert last is not None and _close(last[0], BLUE) + reader.close() + + +def test_finish_is_bounded_by_its_budget_and_fails_with_the_reason(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_FINISH_BUDGET_S", 0.0) + monkeypatch.setattr(h264, "_READ_BYTES", 512) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(encode_units(W, H, [RED, GREEN, BLUE]) * 300)) + reader = _reader(path) + with pytest.raises(h264.SensingStreamError, match="could not finish decoding"): + reader.finish() + + +def test_reading_after_finish_is_an_error_not_a_decode_on_a_flushed_decoder(tmp_path): + path = tmp_path / "s.h264" + path.write_bytes(b"".join(encode_units(W, H, [RED, GREEN]))) + reader = _reader(path) + reader.finish() + with pytest.raises(h264.SensingStreamError, match="already finished"): + reader.read_latest(0.0) diff --git a/examples/egocap/tests/support/capture_server.py b/examples/egocap/tests/support/capture_server.py index ec92979aa2..e524a3579a 100644 --- a/examples/egocap/tests/support/capture_server.py +++ b/examples/egocap/tests/support/capture_server.py @@ -64,7 +64,12 @@ async def start_capture_server( effective_config, source_factory=source_factory or build_mock_source_factory( - scenario="demo", seed=1, ic_faults=ic_faults or set(), camera_faults=camera_faults or set() + scenario="demo", + seed=1, + ic_faults=ic_faults or set(), + camera_faults=camera_faults or set(), + # The SENSING composition shares this config with the server (its cached sensor description). + config=effective_config, ), ) run_task = asyncio.ensure_future(server.run()) diff --git a/examples/egocap/uv.lock b/examples/egocap/uv.lock index b4a29b14fa..92723f87c0 100644 --- a/examples/egocap/uv.lock +++ b/examples/egocap/uv.lock @@ -42,6 +42,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "av" +version = "19.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/01/76b6bf563fe907597cf58f10d10168e93fe588b6db9f2658db87e87efa7e/av-19.0.2.tar.gz", hash = "sha256:bebbc52598886480f4d861c575dc2b020785d6c04e1308680ddcc545e1b94c66", size = 4277157, upload-time = "2026-10-21T03:45:30.799Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/2f/f51f37820741134516e841053d5ecebf6350fbd451adcbff9ead96aa1244/av-19.0.2-cp312-abi3-macosx_11_0_x86_64.whl", hash = "sha256:f5f7bcf987567cd3f9b01f1c375c59bf59d8a5b98c9c5c154aeda7a11e0c5dce", size = 22627374, upload-time = "2026-10-21T03:44:28.296Z" }, + { url = "https://files.pythonhosted.org/packages/7c/42/067d0bcd2c07103f18cc23b510a8289dd80330e984c49d51ce0bafcee017/av-19.0.2-cp312-abi3-macosx_14_0_arm64.whl", hash = "sha256:b0dff6dfcc6ff9944fbbff15bc4ee2d90a2c1695d493bdea96e86f48823e9feb", size = 18556422, upload-time = "2026-10-21T03:44:31.267Z" }, + { url = "https://files.pythonhosted.org/packages/de/0c/448f89f76cf5e4da35dd6294a1e8155b75daa8839f2f2fa353775855d152/av-19.0.2-cp312-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:04c8e412642c652cb2b1997a9cb2114546ee7d894600c631f4be8ceb26a0d0d2", size = 32683287, upload-time = "2026-10-21T03:44:34.588Z" }, + { url = "https://files.pythonhosted.org/packages/e1/7f/9f1237feb894afa472ef834aa1fbfe4d6398f28dc7e86a1b418650204c36/av-19.0.2-cp312-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c1da7aad0fa6c13b4a5c1e7985be0d13f8526a96284573acaf3d55ead6d3fe8", size = 34990159, upload-time = "2026-10-21T03:44:37.637Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b5/1bde0e54e525d88ad8f5becae657f8142543fd3be9d10b72141f041ca868/av-19.0.2-cp312-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9d4f9f78c0ce1f20deb72626fdb3c723757f61a8cd6fbb7dd44d9401884a91fd", size = 44422922, upload-time = "2026-10-21T03:44:41.847Z" }, + { url = "https://files.pythonhosted.org/packages/08/21/65997929fa4dd7c20f64e742aa87205a8d65485718a2ab51633756078e83/av-19.0.2-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:dfe7af5aa8243afd5d4778f85ad1a7c948e37fad272869da1f8f93f8d57fa7ad", size = 33754699, upload-time = "2026-10-21T03:44:45.772Z" }, + { url = "https://files.pythonhosted.org/packages/30/44/4de8c0ff5a5e25ceb0841efc0a4a7530a22d61800bc0cc19a1d03aa04358/av-19.0.2-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:44d3faae747cf04a33f39db8ae983ff3d93adc864ee663bc186b25b686803c3e", size = 36014845, upload-time = "2026-10-21T03:44:49.64Z" }, + { url = "https://files.pythonhosted.org/packages/32/5a/743a83666ded197df8b9957e2800bef9a39d52b23be2ca211bbd14b5fb22/av-19.0.2-cp312-abi3-win_amd64.whl", hash = "sha256:3ce5a1d7b916d13c16ac58f1756f0fee4af2d53f02bc1403e18550b50d375a22", size = 28150949, upload-time = "2026-10-21T03:44:53.434Z" }, + { url = "https://files.pythonhosted.org/packages/42/fe/e79d786628d9c0428f72214dbe4b90ee0c625e04d552a604d5235ff35515/av-19.0.2-cp312-abi3-win_arm64.whl", hash = "sha256:79a277b077e3768a0ea9f881ca0b3bc96d4135a8d3a7a579938adb58718d0dfd", size = 20708023, upload-time = "2026-10-21T03:44:56.435Z" }, +] + [[package]] name = "certifi" version = "2026.7.22" @@ -201,6 +218,7 @@ name = "egocap" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "av" }, { name = "fastapi" }, { name = "loguru" }, { name = "numpy" }, @@ -232,6 +250,7 @@ oak = [ [package.metadata] requires-dist = [ + { name = "av", specifier = ">=14" }, { name = "cupy-cuda12x", marker = "extra == 'gmsl'", specifier = ">=13,<14" }, { name = "depthai", marker = "extra == 'oak'", specifier = "==3.6.1" }, { name = "fastapi", specifier = ">=0.115" }, From 35f18ee283f6b54eb2a82f0be2453390e7d1fd77 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Wed, 4 Nov 2026 09:10:21 -0800 Subject: [PATCH 2/2] examples/egocap: decode SENSING H.264 on the Jetson NVDEC hardware decoder Software decode of a 2560x1984 stream at 60 fps is far slower than real time on one Orin core, so the reader fell behind and the preview lagged by seconds. Add a hardware path that hands access units to a gst-launch nvv4l2decoder subprocess and reads back I420 frames. - sensing_nvdec: availability probe, pipeline and the decoder process. A one-buffer leaky queue before the converter keeps the decoder at full speed when Python is slow to read frames. - sensing_h264: decoder= auto|hardware|software on the reader. auto uses NVDEC when reachable and logs which decoder it chose; hardware fails rather than downgrading. Parsing, the overload policy, the newest-frame rule and the RGB conversion are shared by both paths. - sensing: sensing_rig.decoder (default auto), passed through launch. - sensing_jpeg: encode with OpenCV, which is faster and releases the GIL. - Lower _MAX_PENDING_BYTES to 8 MiB so a stalled decoder jumps to a restart point after under two seconds instead of about thirteen. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- examples/egocap/capture_server/README.md | 7 +- .../capture_server/acquisition/sensing.py | 16 +- .../acquisition/sensing_h264.py | 173 +++++- .../acquisition/sensing_jpeg.py | 21 +- .../acquisition/sensing_nvdec.py | 366 ++++++++++++ .../capture_server/capture_server/launch.py | 6 +- .../capture_server/test_sensing_adapter.py | 1 + .../tests/capture_server/test_sensing_h264.py | 27 + .../capture_server/test_sensing_nvdec.py | 538 ++++++++++++++++++ 9 files changed, 1132 insertions(+), 23 deletions(-) create mode 100644 examples/egocap/capture_server/capture_server/acquisition/sensing_nvdec.py create mode 100644 examples/egocap/tests/capture_server/test_sensing_nvdec.py diff --git a/examples/egocap/capture_server/README.md b/examples/egocap/capture_server/README.md index 6c85e3a8f1..32ce484a31 100644 --- a/examples/egocap/capture_server/README.md +++ b/examples/egocap/capture_server/README.md @@ -86,8 +86,13 @@ status when the device provides them. YAML file (`sensing_rig:` plus `cameras:` entries of `type: SENSING_COLOR`) and `--sensing-config FILE`; the file format is documented on `load_sensing_station` in `capture_server/acquisition/sensing.py`. The server launches the plugin's executable itself, reads each -sensor's H.264 output file, decodes it on the CPU (every unit in order) and sends the newest decoded frame as a JPEG +sensor's H.264 output file, decodes it (every unit in order) and sends the newest decoded frame as a JPEG (`av`, i.e. PyAV, is a declared dependency; nothing is loaded unless a SENSING stream is used). +The decoder is `sensing_rig.decoder`: `auto` (default) uses the Jetson NVDEC hardware decoder through GStreamer's +`nvv4l2decoder` when `/dev/v4l2-nvdec`, `gst-launch-1.0` and that element exist, and otherwise PyAV on the CPU; +`hardware` fails the stream if NVDEC is unavailable; `software` forces the CPU. Each stream logs which decoder it +uses (`isaaccapture.capture_server.sensing_h264`). A 2560x1984 stream at 60 fps is beyond one CPU core of software +decoding (about 30 fps on an Orin core), which shows as a picture that falls further behind; use hardware decode there. `--mock` with a SENSING station requires the plugin's MOCK build, whose encoder must write real H.264; an older mock encoder wrote placeholder bytes that are not video, which the decoder rejects, so such a build shows a degraded camera instead of pictures. With `--sensing-config`, `cameras` accepts only the diff --git a/examples/egocap/capture_server/capture_server/acquisition/sensing.py b/examples/egocap/capture_server/capture_server/acquisition/sensing.py index 58b9eaabf0..5a6bbb0bb8 100644 --- a/examples/egocap/capture_server/capture_server/acquisition/sensing.py +++ b/examples/egocap/capture_server/capture_server/acquisition/sensing.py @@ -38,7 +38,7 @@ from ._thread_runner import ThreadRunner from .clock import Clock, MonotonicClock from .errors import CameraSourceError, SourceError -from .sensing_h264 import SensingStreamError, SensingStreamReader +from .sensing_h264 import DECODERS, SensingStreamError, SensingStreamReader from .sensing_jpeg import encode_rgb_jpeg __all__ = [ @@ -97,11 +97,16 @@ class SensingRigConfig: # Where per-session stream files live; None prefers /dev/shm. tmpfs is memory, not extra capacity: the files # stay bounded only because consumed bytes are reclaimed as they are read (see sensing_h264.py). runtime_dir: str | None = None + # How the H.264 is decoded: "auto" uses the Jetson's NVDEC hardware decoder when it is reachable and the CPU + # otherwise (the choice is logged), "hardware" requires NVDEC, "software" forces the CPU. See sensing_h264.py. + decoder: str = "auto" def __post_init__(self) -> None: - """Reject an unknown backend, a missing executable/platform/name, or empty link masks.""" + """Reject an unknown backend or decoder, a missing executable/platform/name, or empty link masks.""" if self.expected_backend not in _BACKENDS: raise ValueError(f"SensingRigConfig.expected_backend must be one of {_BACKENDS}") + if self.decoder not in DECODERS: + raise ValueError(f"SensingRigConfig.decoder must be one of {DECODERS}, got {self.decoder!r}") if not (self.executable and self.platform_config and self.config_name and self.link_masks): raise ValueError("SensingRigConfig needs executable, platform_config, config_name and link_masks") @@ -186,6 +191,7 @@ class SensingDescription: "bitrate_bps", "full_range", "runtime_dir", + "decoder", } _CAMERA_KEYS = {"type", "module_name", "delivery_fps", "jpeg_quality", "required", "sensor_index"} @@ -202,6 +208,7 @@ class SensingDescription: "bitrate_bps": int, "full_range": bool, "runtime_dir": str, + "decoder": str, } _CAMERA_TYPES: dict[str, type | tuple[type, ...]] = { "type": str, @@ -597,13 +604,17 @@ def __init__( sensor: SensingSensor, camera: p.CameraStreamConfig, full_range: bool = False, + decoder: str = "software", clock: Clock | None = None, startup_timeout_s: float = _STARTUP_TIMEOUT_S, ) -> None: """Bind one sensor of the shared rig to one accepted camera stream. + ``decoder`` is the reader's decoder choice (see ``SensingStreamReader``); the server passes the rig's + setting, and direct construction defaults to the deterministic CPU decoder. The reader is created later, in :meth:`enter`. """ + self._decoder = decoder self._rig = rig self._sensor = sensor self._camera = camera @@ -658,6 +669,7 @@ def _enter_sync(self) -> None: height=self._sensor.height, full_range=self._full_range, stop=self._stop, + decoder=self._decoder, ) # The file existing is not readiness: attach, then wait for a complete, decodable frame. while True: diff --git a/examples/egocap/capture_server/capture_server/acquisition/sensing_h264.py b/examples/egocap/capture_server/capture_server/acquisition/sensing_h264.py index 91a88dc8e7..f1539e7133 100644 --- a/examples/egocap/capture_server/capture_server/acquisition/sensing_h264.py +++ b/examples/egocap/capture_server/capture_server/acquisition/sensing_h264.py @@ -10,6 +10,10 @@ frames, so units cannot be skipped before decoding), and only the newest DECODED frame is kept for delivery. The same code serves the real Jetson encoder and the native mock; neither is special-cased. +Decoding runs on the CPU (PyAV) or, where the Jetson's NVDEC engine is reachable, on the hardware decoder in +``sensing_nvdec`` (``decoder="auto"`` picks hardware when available and says which one it chose). Parsing, the +overload policy, the newest-frame rule and the YUV->RGB conversion are the same for both. + Everything is blocking and synchronous: run it on one dedicated worker (see ``ThreadRunner``). The parser and decoder are PyAV's, created lazily so ordinary OAK-only runs do not load them until a SENSING stream attaches. """ @@ -20,6 +24,7 @@ import ctypes import errno import functools +import logging import os import re import threading @@ -28,7 +33,16 @@ import numpy as np -__all__ = ["SensingStreamError", "SensingStreamReader"] +from .sensing_nvdec import HardwareDecodeError, HardwareDecoder, hardware_decoder_unavailable_reason + +__all__ = ["DECODERS", "SensingStreamError", "SensingStreamReader"] + +_log = logging.getLogger("isaaccapture.capture_server.sensing_h264") + +DECODERS = ("auto", "hardware", "software") +# Access-unit bytes the hardware decoder may have queued ahead of it. Beyond this the reader keeps units in its +# own pending queue, where the overload policy (resume at the newest restart point) can act on them. +_HW_MAX_INPUT_BYTES = 4 * 1024 * 1024 # Bytes consumed from the file before the already-read prefix is handed back to the filesystem. The producer # writes at the encoder bitrate for the whole session, so without this the file grows without bound. @@ -43,7 +57,9 @@ _MAX_UNIT_BYTES = 32 * 1024 * 1024 # Complete but not yet decoded stream bytes the reader may hold. A decoder that falls this far behind resumes at # the newest restart point (an IDR unit carrying its own SPS/PPS; nothing before it is needed), or ends the stream. -_MAX_PENDING_BYTES = 64 * 1024 * 1024 +# Sized as latency, not memory: at the encoder's default bitrate this is under two seconds of video, so a stalled +# decoder shows a picture that is at most that stale before it jumps to a restart point. +_MAX_PENDING_BYTES = 8 * 1024 * 1024 # A failed stream's file may stay unreclaimed (a failing fallocate) only this far before the producer stops. _MAX_UNRECLAIMED_BYTES = 64 * 1024 * 1024 # Time allowed to read, decode and flush whatever the producer left behind when it ended. @@ -99,13 +115,25 @@ def __init__( full_range: bool = False, fallocate: Callable[[int, int, int, int], int] | None = None, stop: threading.Event | None = None, + decoder: str = "software", + hardware_command: list[str] | None = None, ) -> None: """Record the expected geometry and colour range; the file is opened later, in :meth:`connect`. ``stop`` is set (from any thread) when the stream is being shut down: every wait and loop here checks it at least every fraction of a second and ends with :class:`SensingStreamError`, so Stop never queues behind a long startup, read or final flush. + + ``decoder`` is ``"software"`` (PyAV on the CPU), ``"hardware"`` (Jetson NVDEC; an error if it is not + available) or ``"auto"`` (hardware when available, otherwise software). ``hardware_command`` replaces the + hardware decoder's process for tests. """ + if decoder not in DECODERS: + raise ValueError(f"decoder must be one of {DECODERS}, got {decoder!r}") + self._decoder_choice = decoder + self._hardware_command = hardware_command + self._hw: HardwareDecoder | None = None + self._use_hardware = False self._path = path self._width, self._height = width, height self._full_range = full_range @@ -163,9 +191,30 @@ def connect(self, timeout_s: float) -> None: import av # noqa: PLC0415 -- lazy: only a SENSING stream needs the decoder self._parser = av.CodecContext.create("h264", "r") - self._decoder = av.CodecContext.create("h264", "r") - # One thread: frame threading delays output by several frames, which a live preview should not pay. - self._decoder.thread_count = 1 + self._use_hardware = self._choose_hardware() + if not self._use_hardware: + self._decoder = av.CodecContext.create("h264", "r") + # One thread: frame threading delays output by several frames, which a live preview should not pay. + self._decoder.thread_count = 1 + + def _choose_hardware(self) -> bool: + """Decide between hardware and software decoding for this stream, and log which one is in use. + + ``"hardware"`` that cannot be honoured is an error, never a quiet downgrade; ``"auto"`` falls back and + says why, so a slow preview is traceable to the decoder in use. + """ + if self._decoder_choice == "software": + _log.info("SENSING stream %s: decoding on the CPU (PyAV software), as configured", self._path) + return False + # An injected command stands in for the whole GStreamer stack, so the machine's own capability is moot. + reason = None if self._hardware_command is not None else hardware_decoder_unavailable_reason() + if reason is None: + _log.info("SENSING stream %s: decoding on the Jetson NVDEC hardware decoder", self._path) + return True + if self._decoder_choice == "hardware": + raise SensingStreamError(f"hardware decoding was requested but is unavailable: {reason}") + _log.warning("SENSING stream %s: falling back to CPU (PyAV software) decoding: %s", self._path, reason) + return False def _drain_file(self, deadline: float) -> None: """Read newly written bytes within this poll's work bounds and queue the complete access units.""" @@ -214,8 +263,12 @@ def _decode_pending(self, deadline: float) -> None: """Decode queued units in order until none are left or ``deadline`` passes (one unit of progress at least). A decode that returns nothing is the decoder buffering, not an error; every returned frame is checked - and the newest one is kept. + and the newest one is kept. With the hardware decoder, "decoding" means handing units to its process, + which holds back at most a few MiB before this leaves the rest queued for the overload policy. """ + if self._use_hardware: + self._feed_hardware(deadline) + return while self._pending: self._check_stopped() packet = self._pending.popleft() @@ -225,6 +278,46 @@ def _decode_pending(self, deadline: float) -> None: if time.monotonic() >= deadline: break + def _start_hardware(self) -> HardwareDecoder: + """Start the hardware decoder once the parser knows the stream's size, and check it against the station's. + + The geometry is read from the parsed stream, not from decoded frames, because the hardware decoder's raw + output carries no header: a mismatch would otherwise be read as garbage pixels. + """ + if (self._parser.width, self._parser.height) != (self._width, self._height): + raise SensingStreamError( + f"decoded {self._parser.width}x{self._parser.height}, expected {self._width}x{self._height}" + ) + hw = HardwareDecoder(self._width, self._height, command=self._hardware_command) + hw.start() + self._hw = hw + return hw + + def _feed_hardware(self, deadline: float) -> None: + """Hand queued access units to the hardware decoder, in order, until it is saturated or ``deadline``. + + Writes never block: when the decoder's input is full, units stay in ``_pending`` and are tried again on + the next poll. A failed or exited decoder surfaces here as :class:`SensingStreamError`. + """ + if not self._pending and self._hw is None: + return + hw = self._hw or self._start_hardware() + try: + hw.pump() + while self._pending and hw.pending_input_bytes < _HW_MAX_INPUT_BYTES: + self._check_stopped() + packet = self._pending.popleft() + self._pending_bytes -= packet.size + hw.write(bytes(packet)) + if time.monotonic() >= deadline: + break + hw.check() + if self._pending: + # Saturated: give the decoder a moment instead of letting a caller's loop spin on this method. + self._wait(_POLL_INTERVAL_S) + except HardwareDecodeError as e: + raise SensingStreamError(str(e)) from e + def _checked(self, frame: Any) -> Any: """The decoded frame, if it has the geometry the station and the producer agreed on.""" if (frame.width, frame.height) != (self._width, self._height): @@ -233,9 +326,18 @@ def _checked(self, frame: Any) -> Any: ) return frame + def _has_frame(self) -> bool: + """Whether a decoded frame is waiting to be delivered.""" + if self._use_hardware: + return self._hw is not None and self._hw.has_frame + return self._newest is not None + def _deliver(self) -> tuple[np.ndarray, int] | None: """Convert the newest undelivered decoded frame to RGB, or None if there is none.""" - frame, self._newest = self._newest, None + if self._use_hardware: + frame = self._take_hardware_frame() + else: + frame, self._newest = self._newest, None if frame is None: return None selected_ns = time.monotonic_ns() @@ -246,6 +348,24 @@ def _deliver(self) -> tuple[np.ndarray, int] | None: ).to_ndarray() return rgb, selected_ns + def _take_hardware_frame(self) -> Any: + """The hardware decoder's newest frame as a PyAV ``yuv420p`` frame, or None. + + Building the PyAV frame copies the pixels, so the decoder's buffer is released at once and its output + thread can go on; the conversion to RGB then goes through the same call as the software path. + """ + import av # noqa: PLC0415 -- already loaded by connect(); named here for the call below + + if self._hw is None: + return None + yuv = self._hw.take_frame() + if yuv is None: + return None + try: + return av.VideoFrame.from_ndarray(yuv, format="yuv420p") + finally: + self._hw.release_frame() + def _reclaim(self) -> None: """Hand the consumed prefix back to the filesystem once enough has accumulated; failure is an error. @@ -285,7 +405,7 @@ def read_latest(self, timeout_s: float) -> tuple[np.ndarray, int] | None: work_deadline = max(deadline, time.monotonic() + _MIN_WORK_S) self._drain_file(work_deadline) self._decode_pending(work_deadline) - if self._newest is not None: + if self._has_frame(): return self._deliver() if time.monotonic() >= deadline: return None @@ -327,8 +447,7 @@ def finish(self) -> tuple[np.ndarray, int] | None: while self._pending: self._check_finish_budget(far) self._decode_pending(far) - for frame in self._decoder.decode(None): # flush the decoder's delayed output - self._newest = self._checked(frame) + self._flush_decoder(far) # the decoder's delayed output return self._deliver() except SensingStreamError: self._fail() @@ -337,11 +456,42 @@ def finish(self) -> tuple[np.ndarray, int] | None: self._fail() raise SensingStreamError(f"{type(e).__name__}: {e}") from e + def _flush_decoder(self, far: float) -> None: + """Drain the decoder's delayed output into the newest-frame slot, within the finish budget ``far``. + + The hardware decoder is flushed by closing its input: it then emits what it still holds and exits, and + the output thread ends. A decoder that never saw a unit has nothing to flush. + """ + if not self._use_hardware: + for frame in self._decoder.decode(None): + self._newest = self._checked(frame) + return + hw = self._hw + if hw is None: + return + try: + # Input still queued must be accepted before the end of stream can be signalled. + while not hw.close_input(): + self._check_finish_budget(far) + self._wait(_POLL_INTERVAL_S) + while not hw.wait_finished(_POLL_INTERVAL_S): + self._check_finish_budget(far) + hw.check() + except HardwareDecodeError as e: + raise SensingStreamError(str(e)) from e + def _fail(self) -> None: """Make the reader terminal. The descriptor stays open so :meth:`reclaim_all` can keep reclaiming.""" self._failed = True self._pending.clear() self._pending_bytes = 0 + self._stop_hardware() + + def _stop_hardware(self) -> None: + """End the hardware decoder's process, if one is running; it is never needed again once the reader ends.""" + hw, self._hw = self._hw, None + if hw is not None: + hw.close() def reclaim_all(self) -> None: """For a stream that has failed: keep handing the file's bytes back to the filesystem, without decoding. @@ -393,8 +543,9 @@ def _check_unreclaimed(self, unreclaimed: int, reason: str) -> None: ) def close(self) -> None: - """Close the file; nothing else is held (the producer owns the file's lifetime and directory).""" + """Close the file and the hardware decoder process; the producer owns the file itself.""" self._closed = True + self._stop_hardware() fd, self._fd = self._fd, None if fd is not None: os.close(fd) diff --git a/examples/egocap/capture_server/capture_server/acquisition/sensing_jpeg.py b/examples/egocap/capture_server/capture_server/acquisition/sensing_jpeg.py index dc9d9c2549..11a3c2a6b9 100644 --- a/examples/egocap/capture_server/capture_server/acquisition/sensing_jpeg.py +++ b/examples/egocap/capture_server/capture_server/acquisition/sensing_jpeg.py @@ -7,14 +7,14 @@ The wire protocol carries only independently decodable JPEG images with a hard 2 MiB payload ceiling. A frame that does not fit is a configuration/source error -- never silently resized, re-quantized or replaced, so the diagnostic names the dimensions and quality that produced it. + +OpenCV encodes here rather than Pillow: at 5 MP it is about 40% faster, and it releases the GIL for the whole +encode, which keeps the other streams' decoder and file-reading threads running while one frame is compressed. """ from __future__ import annotations -import io - import numpy as np -from PIL import Image from .errors import SourceError @@ -27,12 +27,17 @@ def encode_rgb_jpeg(rgb: np.ndarray, quality: int, *, max_bytes: int = MAX_JPEG_ """Encode an ``(H, W, 3)`` uint8 RGB frame as a complete JPEG at ``quality``; returns owned bytes.""" if rgb.ndim != 3 or rgb.shape[2] != 3 or rgb.dtype != np.uint8: raise ValueError(f"expected (H, W, 3) uint8 RGB pixels, got {rgb.shape} {rgb.dtype}") + import cv2 # noqa: PLC0415 -- lazy: runs without a SENSING stream never load OpenCV + height, width = rgb.shape[:2] - out = io.BytesIO() - # Encoding finishes before this returns, so the caller may reuse its buffer immediately afterwards; - # nothing here keeps a reference to it. - Image.fromarray(rgb, "RGB").save(out, format="JPEG", quality=quality) - jpeg = out.getvalue() + # OpenCV's encoder takes BGR; the conversion makes a new array, so nothing here keeps a reference to the + # caller's buffer and the caller may reuse it as soon as this returns. + encoded, buffer = cv2.imencode( + ".jpg", cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR), [cv2.IMWRITE_JPEG_QUALITY, quality] + ) + if not encoded: + raise SourceError(f"encoder failed for {width}x{height} at quality {quality}") + jpeg = buffer.tobytes() if not (jpeg.startswith(b"\xff\xd8") and jpeg.endswith(b"\xff\xd9")): raise SourceError(f"encoder produced an incomplete JPEG for {width}x{height} at quality {quality}") if len(jpeg) > max_bytes: diff --git a/examples/egocap/capture_server/capture_server/acquisition/sensing_nvdec.py b/examples/egocap/capture_server/capture_server/acquisition/sensing_nvdec.py new file mode 100644 index 0000000000..216d618cf5 --- /dev/null +++ b/examples/egocap/capture_server/capture_server/acquisition/sensing_nvdec.py @@ -0,0 +1,366 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Hardware H.264 decoding for the SENSING reader: Jetson NVDEC driven through a GStreamer subprocess. + +A 5 MP stream at 60 fps is far beyond one CPU core of software decoding, so the reader hands the access units to +``nvv4l2decoder`` instead and takes planar YUV frames back. It is a subprocess rather than a library binding +because the Jetson decoder is only reachable through NVIDIA's GStreamer/V4L2 stack, which neither the PyAV wheel +(its ``h264_v4l2m2m`` cannot open the NVDEC node) nor the system ffmpeg wrapper serve well: the ffmpeg wrapper +holds about 0.6 s of frames internally, while ``nvv4l2decoder disable-dpb=true`` delivers each frame in tens of +milliseconds. ``disable-dpb`` is correct only for streams without frame reordering; the producer encodes with no +B-frames by construction (``setNumBFrames(0)`` in the Jetson encoder, mirrored by the mock encoder). + +Everything here is thread-based and bounded: three frame buffers, a capped input queue, and a close that returns +within a second. The reader owns when frames are taken, so "newest decoded frame wins" holds exactly as it does +for the software path. +""" + +from __future__ import annotations + +import collections +import fcntl +import functools +import logging +import os +import shutil +import subprocess +import threading +from collections.abc import Sequence + +import numpy as np + +__all__ = ["HardwareDecodeError", "HardwareDecoder", "hardware_decoder_unavailable_reason", "pipeline_command"] + +_log = logging.getLogger("isaaccapture.capture_server.sensing_nvdec") + +_DEVICE_NODE = "/dev/v4l2-nvdec" +_GST_LAUNCH = "gst-launch-1.0" +_GST_INSPECT = "gst-inspect-1.0" +_DECODER_ELEMENT = "nvv4l2decoder" +_PROBE_TIMEOUT_S = 10.0 +# Bytes the decoder process may have queued on its input before the caller is asked to wait. The pipe itself +# holds up to _STDIN_PIPE_BYTES; the rest waits in this process, where the reader's own overload policy sees it. +_STDIN_PIPE_BYTES = 1 << 20 # the largest size an unprivileged process may request (/proc/sys/fs/pipe-max-size) +_WRITE_CHUNK_BYTES = 1 << 20 +# Shutdown bounds: the whole SENSING chain must release inside the server's 5 s budget (see sensing.py). +_CLOSE_GRACE_S = 0.4 +_JOIN_S = 0.3 +_STDERR_TAIL_BYTES = 4096 +# One frame being filled, one waiting to be taken, one held by the consumer while it converts it. +_FRAME_BUFFERS = 3 +_F_SETPIPE_SZ = 1031 # Linux fcntl command; not exposed by the fcntl module on every Python + + +class HardwareDecodeError(Exception): + """The hardware decoder process failed, exited early, or produced output that does not fit the geometry.""" + + +@functools.cache +def hardware_decoder_unavailable_reason() -> str | None: + """Why NVDEC decoding cannot be used on this machine, or ``None`` when it can; cached for the process. + + Checks the decoder device node, the GStreamer tools, and that the ``nvv4l2decoder`` element is installed, + so "auto" falls back to software on a desktop or a Jetson without the NVIDIA GStreamer plugins. + """ + # The node exists only on Jetson platforms with an NVDEC engine. + if not os.path.exists(_DEVICE_NODE): + return f"{_DEVICE_NODE} is missing (this is not a Jetson with NVDEC)" + for tool in (_GST_LAUNCH, _GST_INSPECT): + if shutil.which(tool) is None: + return f"{tool} is not installed" + try: + probe = subprocess.run( # noqa: S603 -- fixed argument list, no shell + [_GST_INSPECT, "--exists", _DECODER_ELEMENT], + capture_output=True, + timeout=_PROBE_TIMEOUT_S, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as e: + return f"could not query GStreamer for {_DECODER_ELEMENT}: {e}" + if probe.returncode != 0: + return f"the GStreamer element {_DECODER_ELEMENT} is not installed" + return None + + +def pipeline_command(*, low_delay: bool = True) -> list[str]: + """The ``gst-launch-1.0`` argument list: Annex-B on stdin, tightly packed I420 frames on stdout. + + ``nvvidconv`` only converts NV12 to planar I420 (no scaling and no colour-matrix change), so the reader's own + explicit BT.601 conversion still decides the colours. ``low_delay`` turns off the decoder's reorder buffer: + required for a live preview, valid only for streams without B-frames. + + The one-buffer leaky queue ahead of the converter is what keeps the decoder at full speed. Reading a 7.6 MB + frame through a pipe takes many GIL acquisitions, and when other Python threads (JPEG encoding) hold the GIL + the reads fall behind; without the queue that backpressure stops the decoder and the input backs up + (measured: 11 fps decode against 124 fps with the queue). With it, frames the reader is too slow to take are + dropped inside GStreamer before the costly copy, so the reader always sees the newest one. + """ + decoder = [_DECODER_ELEMENT, "enable-max-performance=1"] + if low_delay: + decoder.append("disable-dpb=true") + return [ + _GST_LAUNCH, + "-q", + "fdsrc", + "fd=0", + "blocksize=4194304", + "!", + "h264parse", + "!", + *decoder, + "!", + "queue", + "max-size-buffers=1", + "max-size-bytes=0", + "max-size-time=0", + "leaky=downstream", + "!", + "nvvidconv", + "!", + "video/x-raw,format=I420", + "!", + "fdsink", + "fd=1", + "sync=false", + ] + + +class HardwareDecoder: + """One decoder process: feed Annex-B access units in, take the newest decoded I420 frame out. + + Frames are ``(height * 3 // 2, width)`` uint8 arrays (planar Y, U, V), the layout PyAV's ``yuv420p`` frames + are built from. Not thread-safe for its own callers: ``start``/``write``/``pump``/``close_input``/``close`` + belong to the reader's single worker; only the internal output and stderr threads run beside it. + """ + + def __init__(self, width: int, height: int, *, command: Sequence[str] | None = None) -> None: + """Record the geometry; the process starts in :meth:`start`. + + ``command`` replaces the GStreamer pipeline (tests inject a stand-in that follows the same pipe contract). + """ + if width <= 0 or height <= 0 or width % 2 or height % 2: + raise ValueError(f"I420 needs positive even dimensions, got {width}x{height}") + self._width, self._height = width, height + self._frame_bytes = width * height * 3 // 2 + self._command = list(command) if command is not None else pipeline_command() + self._proc: subprocess.Popen[bytes] | None = None + self._buffers = [np.empty((height * 3 // 2, width), dtype=np.uint8) for _ in range(_FRAME_BUFFERS)] + self._lock = threading.Condition() + self._ready: int | None = None # newest complete frame not yet taken + self._held: int | None = None # frame the consumer is converting + self._frames_seen = 0 + self._reader_done = threading.Event() + self._reader_error: Exception | None = None + self._input: collections.deque[bytes] = collections.deque() + self._input_offset = 0 # bytes of _input[0] already written + self._input_bytes = 0 + self._input_closed = False + self._closed = False + self._stderr_tail = bytearray() + self._threads: list[threading.Thread] = [] + + @property + def frames_received(self) -> int: + """Frames that reached this process, including ones replaced by a newer frame before being taken. + + The decoder may decode more: its leaky queue drops frames this process is too slow to read. + """ + with self._lock: + return self._frames_seen + + @property + def pending_input_bytes(self) -> int: + """Bytes handed to :meth:`write` that the decoder process has not yet accepted.""" + return self._input_bytes + + def start(self) -> None: + """Launch the decoder process and its output and diagnostics threads.""" + self._proc = proc = subprocess.Popen( # noqa: S603 -- fixed argument list, no shell + self._command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + close_fds=True, + ) + assert proc.stdin is not None and proc.stdout is not None and proc.stderr is not None + # Writes must never block the reader's worker: a stalled decoder would otherwise outlast Stop. A larger + # pipe lets a whole IDR access unit go in one call; failing to resize it only costs more small writes. + os.set_blocking(proc.stdin.fileno(), False) + try: + fcntl.fcntl(proc.stdin.fileno(), _F_SETPIPE_SZ, _STDIN_PIPE_BYTES) + except OSError: + _log.debug("could not enlarge the decoder input pipe", exc_info=True) + self._threads = [ + threading.Thread(target=self._read_frames, name="nvdec-frames", daemon=True), + threading.Thread(target=self._drain_stderr, name="nvdec-stderr", daemon=True), + ] + for thread in self._threads: + thread.start() + + def write(self, data: bytes) -> None: + """Queue one access unit for the decoder and push as much input as the pipe accepts right now.""" + if self._input_closed or self._closed: + raise HardwareDecodeError("the decoder input is already closed") + self._input.append(data) + self._input_bytes += len(data) + self.pump() + + def pump(self) -> None: + """Push queued input into the decoder's pipe without blocking; what does not fit stays queued.""" + proc = self._proc + if proc is None or proc.stdin is None or self._input_closed: + return + fd = proc.stdin.fileno() + while self._input: + head = memoryview(self._input[0])[self._input_offset :] + try: + written = os.write(fd, head[:_WRITE_CHUNK_BYTES]) + except BlockingIOError: + return # the decoder has not consumed what it already has + except BrokenPipeError as e: + raise HardwareDecodeError( + f"the decoder closed its input unexpectedly; {self._diagnostics()}" + ) from e + self._input_bytes -= written + self._input_offset += written + if self._input_offset >= len(self._input[0]): + self._input.popleft() + self._input_offset = 0 + + def close_input(self) -> bool: + """Signal end of stream once all queued input is written; ``False`` while some is still queued. + + Closing the input makes the decoder flush its remaining frames and exit, which ends :meth:`wait_finished`. + """ + self.pump() + if self._input: + return False + proc = self._proc + if proc is not None and proc.stdin is not None and not self._input_closed: + self._input_closed = True + proc.stdin.close() + return True + + def take_frame(self) -> np.ndarray | None: + """The newest complete frame, or ``None``; the array stays valid until :meth:`release_frame`. + + Taking marks the frame as held so the output thread cannot overwrite it while it is converted; every + successful take must be followed by :meth:`release_frame`. + """ + with self._lock: + if self._ready is None: + return None + if self._held is not None: + raise RuntimeError("release_frame() must be called before taking another frame") + self._held, self._ready = self._ready, None + return self._buffers[self._held] + + def release_frame(self) -> None: + """Let the output thread reuse the frame returned by the last :meth:`take_frame`.""" + with self._lock: + self._held = None + + @property + def has_frame(self) -> bool: + """Whether a complete frame is waiting to be taken.""" + with self._lock: + return self._ready is not None + + def wait_finished(self, timeout_s: float) -> bool: + """Wait for the decoder to end its output after :meth:`close_input`; ``False`` if it did not in time.""" + return self._reader_done.wait(timeout_s) + + def check(self) -> None: + """Raise :class:`HardwareDecodeError` if the decoder failed or exited before its input was closed.""" + if self._reader_error is not None: + raise HardwareDecodeError(f"{self._reader_error}; {self._diagnostics()}") from self._reader_error + # The output ending while input is still open means the process died; a closed input ends it on purpose. + if self._reader_done.is_set() and not self._input_closed: + code = self._proc.poll() if self._proc is not None else None + raise HardwareDecodeError(f"the decoder exited unexpectedly (code {code}); {self._diagnostics()}") + + def close(self) -> None: + """Stop the process and its threads within a fraction of a second; safe to call more than once.""" + if self._closed: + return + self._closed = True + proc = self._proc + if proc is None: + return + # Closing the input lets a healthy decoder exit by itself; the signals cover a stalled one. Both waits are + # short because Stop has to finish inside the server's release budget. + try: + if proc.stdin is not None and not proc.stdin.closed: + proc.stdin.close() + except OSError: + pass + if proc.poll() is None: + proc.terminate() + try: + proc.wait(_CLOSE_GRACE_S) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + for thread in self._threads: + thread.join(_JOIN_S) + for stream in (proc.stdout, proc.stderr): + if stream is not None: + stream.close() + + def _claim_free_buffer(self) -> int: + """The index of a buffer neither waiting to be taken nor held; one always exists with three buffers.""" + with self._lock: + return next(i for i in range(_FRAME_BUFFERS) if i not in (self._ready, self._held)) + + def _read_frames(self) -> None: + """Output thread: read whole frames from the process, keeping only the newest one available.""" + proc = self._proc + assert proc is not None and proc.stdout is not None + try: + while True: + index = self._claim_free_buffer() + flat = memoryview(self._buffers[index]).cast("B") + got = 0 + # A pipe read may return less than a frame; the frame is complete only when every byte is here. + while got < self._frame_bytes: + count = proc.stdout.readinto(flat[got:]) + if not count: + break + got += count + if got == 0: + return # clean end of stream + if got < self._frame_bytes: + raise HardwareDecodeError( + f"decoder output ended mid-frame ({got} of {self._frame_bytes} bytes; " + f"expected {self._width}x{self._height} I420)" + ) + with self._lock: + # A frame the consumer never took is replaced: the preview wants the newest, not every frame. + self._ready = index + self._frames_seen += 1 + self._lock.notify_all() + except Exception as e: # noqa: BLE001 -- any read failure ends this decoder; check() reports it + self._reader_error = e + finally: + self._reader_done.set() + with self._lock: + self._lock.notify_all() + + def _drain_stderr(self) -> None: + """Diagnostics thread: keep the last few KiB of the process's error output for failure messages.""" + proc = self._proc + assert proc is not None and proc.stderr is not None + try: + while chunk := proc.stderr.read(1024): + self._stderr_tail += chunk + del self._stderr_tail[:-_STDERR_TAIL_BYTES] + except (OSError, ValueError): + pass # the stream was closed under us by close(); nothing more to collect + + def _diagnostics(self) -> str: + """The decoder's most recent error output, for the message of a failure.""" + text = bytes(self._stderr_tail).decode("utf-8", errors="replace").strip() + return f"decoder output: {text[-500:]!r}" if text else "the decoder printed no error output" diff --git a/examples/egocap/capture_server/capture_server/launch.py b/examples/egocap/capture_server/capture_server/launch.py index 3878e175a4..95b300e2f7 100644 --- a/examples/egocap/capture_server/capture_server/launch.py +++ b/examples/egocap/capture_server/capture_server/launch.py @@ -115,7 +115,11 @@ def _sensing_composition( rig = SensingRig(config.sensing_rig, tuple(sensors[name].pipeline_id for name in wanted)) adapters = { c.stream_id: SensingAdapter( - rig=rig, sensor=sensors[c.name], camera=c, full_range=config.sensing_rig.full_range + rig=rig, + sensor=sensors[c.name], + camera=c, + full_range=config.sensing_rig.full_range, + decoder=config.sensing_rig.decoder, ) for c in cameras if c.name in wanted diff --git a/examples/egocap/tests/capture_server/test_sensing_adapter.py b/examples/egocap/tests/capture_server/test_sensing_adapter.py index e5e65278b5..c7ee7672dd 100644 --- a/examples/egocap/tests/capture_server/test_sensing_adapter.py +++ b/examples/egocap/tests/capture_server/test_sensing_adapter.py @@ -38,6 +38,7 @@ platform_config="/opt/x/p.json", config_name="CFG", link_masks=("0x0000", "0x1100"), + decoder="software", # tiny fake streams: the CPU decoder keeps these tests machine-independent ) DESCRIPTION = sn.SensingDescription((sn.SensingSensor("L", 1, W, H, 30.0), sn.SensingSensor("R", 0, W, H, 30.0))) diff --git a/examples/egocap/tests/capture_server/test_sensing_h264.py b/examples/egocap/tests/capture_server/test_sensing_h264.py index 5f8d0d88eb..60e13a5ac2 100644 --- a/examples/egocap/tests/capture_server/test_sensing_h264.py +++ b/examples/egocap/tests/capture_server/test_sensing_h264.py @@ -158,6 +158,33 @@ def test_jpeg_rejects_wrong_shape(): encode_rgb_jpeg(np.zeros((4, 4, 4), np.uint8), 80) +def test_jpeg_keeps_rgb_channel_order(): + rgb = np.zeros((16, 16, 3), np.uint8) + rgb[...] = RED + import cv2 + + decoded = cv2.cvtColor( + cv2.imdecode(np.frombuffer(encode_rgb_jpeg(rgb, 90), np.uint8), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB + ) + assert bool(np.abs(decoded[8, 8].astype(int) - np.array(RED)).max() <= 8) + + +def test_jpeg_reports_an_encoder_failure(monkeypatch): + import cv2 + + monkeypatch.setattr(cv2, "imencode", lambda *a, **k: (False, None)) + with pytest.raises(SourceError, match=r"encoder failed for 4x4 at quality 80"): + encode_rgb_jpeg(np.zeros((4, 4, 3), np.uint8), 80) + + +def test_jpeg_rejects_an_incomplete_stream(monkeypatch): + import cv2 + + monkeypatch.setattr(cv2, "imencode", lambda *a, **k: (True, np.frombuffer(b"\xff\xd8 truncated", np.uint8))) + with pytest.raises(SourceError, match="incomplete JPEG"): + encode_rgb_jpeg(np.zeros((4, 4, 3), np.uint8), 80) + + def _failing_fallocate(code: int): """A stand-in for the C call that fails the way the real one does: returns -1 and sets errno.""" diff --git a/examples/egocap/tests/capture_server/test_sensing_nvdec.py b/examples/egocap/tests/capture_server/test_sensing_nvdec.py new file mode 100644 index 0000000000..ddf957a1f8 --- /dev/null +++ b/examples/egocap/tests/capture_server/test_sensing_nvdec.py @@ -0,0 +1,538 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Hardware H.264 decoding for the SENSING reader: the decoder process wrapper and the reader's use of it. + +The Jetson decoder is a GStreamer subprocess; these tests replace it with stand-ins that follow the same pipe +contract (Annex-B on stdin, tightly packed I420 frames on stdout), so every code path runs on any machine. A +separate check at the end exercises the real pipeline when this machine has NVDEC. +""" + +from __future__ import annotations + +import logging +import subprocess +import sys +import textwrap +import time + +import numpy as np +import pytest + +from capture_server.acquisition import sensing_h264 as h264 +from capture_server.acquisition import sensing_nvdec as nvdec + +from .h264_fixtures import encode_units + +W, H = 64, 32 +FRAME = W * H * 3 // 2 +RED, GREEN, BLUE = (220, 20, 20), (20, 220, 20), (20, 20, 220) + +# A stand-in that decodes the Annex-B on stdin with PyAV and writes each frame as raw I420: the real decoder's +# contract, so the reader sees correct pixels end to end. +PYAV_DECODER = textwrap.dedent( + """ + import sys + import av + + parser = av.CodecContext.create("h264", "r") + decoder = av.CodecContext.create("h264", "r") + out = sys.stdout.buffer + + def emit(packets): + for packet in packets: + for frame in decoder.decode(packet): + out.write(frame.to_ndarray(format="yuv420p").tobytes()) + out.flush() + + while chunk := sys.stdin.buffer.read1(65536): + emit(parser.parse(chunk)) + emit(parser.parse(None)) + emit([None]) + """ +) + + +def _command(script: str) -> list[str]: + return [sys.executable, "-c", script] + + +def _counting_decoder(unit_bytes: int = 4) -> list[str]: + """A stand-in that writes one frame, filled with its sequence number, per ``unit_bytes`` read.""" + return _command( + textwrap.dedent( + f""" + import sys + n = 0 + while True: + data = sys.stdin.buffer.read({unit_bytes}) + if len(data) < {unit_bytes}: + break + sys.stdout.buffer.write(bytes([n % 251]) * {FRAME}) + sys.stdout.buffer.flush() + n += 1 + """ + ) + ) + + +def _wait_for(predicate, timeout_s: float = 10.0) -> None: + deadline = time.monotonic() + timeout_s + while not predicate(): + assert time.monotonic() < deadline, "timed out waiting for the decoder" + time.sleep(0.01) + + +@pytest.fixture +def decoders(): + """Decoders created by a test, closed afterwards so a failing test never leaks a process.""" + made = [] + + def make(command, width: int = W, height: int = H) -> nvdec.HardwareDecoder: + decoder = nvdec.HardwareDecoder(width, height, command=command) + made.append(decoder) + return decoder + + yield make + for decoder in made: + decoder.close() + + +# --- availability and the pipeline --- + + +@pytest.fixture +def fresh_probe(): + nvdec.hardware_decoder_unavailable_reason.cache_clear() + yield + nvdec.hardware_decoder_unavailable_reason.cache_clear() + + +def _fake_environment(monkeypatch, *, node=True, tools=("gst-launch-1.0", "gst-inspect-1.0"), probe=0): + monkeypatch.setattr(nvdec.os.path, "exists", lambda p: node if p == nvdec._DEVICE_NODE else True) + monkeypatch.setattr(nvdec.shutil, "which", lambda t: f"/usr/bin/{t}" if t in tools else None) + if isinstance(probe, Exception): + + def run(*a, **k): + raise probe + + else: + + def run(*a, **k): + return subprocess.CompletedProcess(a[0], probe) + + monkeypatch.setattr(nvdec.subprocess, "run", run) + + +def test_available_when_node_tools_and_element_exist(monkeypatch, fresh_probe): + _fake_environment(monkeypatch) + assert nvdec.hardware_decoder_unavailable_reason() is None + + +def test_unavailable_without_the_decoder_node(monkeypatch, fresh_probe): + _fake_environment(monkeypatch, node=False) + assert "not a Jetson" in nvdec.hardware_decoder_unavailable_reason() + + +def test_unavailable_without_gstreamer_tools(monkeypatch, fresh_probe): + _fake_environment(monkeypatch, tools=("gst-launch-1.0",)) + assert "gst-inspect-1.0 is not installed" in nvdec.hardware_decoder_unavailable_reason() + + +def test_unavailable_when_the_element_is_missing(monkeypatch, fresh_probe): + _fake_environment(monkeypatch, probe=1) + assert "nvv4l2decoder is not installed" in nvdec.hardware_decoder_unavailable_reason() + + +def test_unavailable_when_the_probe_cannot_run(monkeypatch, fresh_probe): + _fake_environment(monkeypatch, probe=subprocess.TimeoutExpired("gst-inspect-1.0", 10)) + assert "could not query GStreamer" in nvdec.hardware_decoder_unavailable_reason() + + +def test_pipeline_is_low_delay_with_a_leaky_queue_by_default(): + command = nvdec.pipeline_command() + assert "disable-dpb=true" in command and "leaky=downstream" in command + assert command.index("queue") < command.index("nvvidconv") # frames are dropped before the costly copy + + +def test_pipeline_keeps_the_reorder_buffer_when_asked(): + assert "disable-dpb=true" not in nvdec.pipeline_command(low_delay=False) + + +# --- the decoder process --- + + +@pytest.mark.parametrize("width, height", [(0, 32), (64, 0), (63, 32), (64, 31)]) +def test_geometry_must_be_positive_and_even(width, height): + with pytest.raises(ValueError, match="even dimensions"): + nvdec.HardwareDecoder(width, height) + + +def test_frames_flow_and_the_newest_wins(decoders): + decoder = decoders(_counting_decoder()) + decoder.start() + for _ in range(5): + decoder.write(b"\x00\x00\x00\x00") + _wait_for(lambda: decoder.frames_received == 5) + frame = decoder.take_frame() + assert frame.shape == (H * 3 // 2, W) and frame.dtype == np.uint8 + assert (frame == 4).all() # frames 0..3 were replaced before being taken + assert decoder.take_frame() is None and not decoder.has_frame + decoder.release_frame() + + +def test_a_taken_frame_is_not_overwritten_while_held(decoders): + decoder = decoders(_counting_decoder()) + decoder.start() + decoder.write(b"\x00" * 4) + _wait_for(lambda: decoder.has_frame) + held = decoder.take_frame() + for _ in range(6): + decoder.write(b"\x00" * 4) + _wait_for(lambda: decoder.frames_received == 7) + assert (held == 0).all() # the output thread filled other buffers + decoder.release_frame() + assert (decoder.take_frame() == 6).all() + decoder.release_frame() + + +def test_taking_twice_without_releasing_is_a_programming_error(decoders): + decoder = decoders(_counting_decoder()) + decoder.start() + decoder.write(b"\x00" * 8) + _wait_for(lambda: decoder.frames_received == 2) + decoder.take_frame() + decoder.write(b"\x00" * 4) + _wait_for(lambda: decoder.frames_received == 3) + with pytest.raises(RuntimeError, match="release_frame"): + decoder.take_frame() + + +def test_closing_the_input_flushes_and_ends_the_output(decoders): + decoder = decoders(_counting_decoder()) + decoder.start() + decoder.write(b"\x00" * 12) + assert decoder.close_input() is True + assert decoder.wait_finished(10.0) + decoder.check() # a closed input makes the exit expected, not an error + assert decoder.frames_received == 3 + + +def test_writes_after_the_input_is_closed_are_refused(decoders): + decoder = decoders(_counting_decoder()) + decoder.start() + decoder.close_input() + with pytest.raises(nvdec.HardwareDecodeError, match="already closed"): + decoder.write(b"x") + + +def test_close_input_waits_while_input_is_still_queued(decoders): + decoder = decoders(_command("import time\ntime.sleep(60)")) # never reads its input + decoder.start() + decoder.write(b"\x00" * (8 << 20)) + assert decoder.pending_input_bytes > 0 + assert decoder.close_input() is False + + +def test_a_stalled_decoder_never_blocks_the_writer(decoders): + decoder = decoders(_command("import time\ntime.sleep(60)")) + decoder.start() + started = time.monotonic() + for _ in range(8): + decoder.write(b"\x00" * (1 << 20)) + assert time.monotonic() - started < 2.0 + assert decoder.pending_input_bytes >= 6 << 20 # the pipe took about one MiB; the rest is queued here + + +def test_an_early_exit_is_reported_with_its_error_output(decoders): + decoder = decoders(_command("import sys\nsys.stderr.write('no such device')\nsys.exit(3)")) + decoder.start() + _wait_for(lambda: decoder._reader_done.is_set()) + time.sleep(0.2) # let the diagnostics thread collect stderr + with pytest.raises(nvdec.HardwareDecodeError, match=r"exited unexpectedly \(code 3\).*no such device"): + decoder.check() + + +def test_writing_to_a_dead_decoder_is_reported(decoders): + decoder = decoders(_command("import sys\nsys.exit(1)")) + decoder.start() + _wait_for(lambda: decoder._reader_done.is_set()) + with pytest.raises(nvdec.HardwareDecodeError, match="closed its input"): + for _ in range(50): + decoder.write(b"\x00" * 4096) + time.sleep(0.01) + + +def test_output_that_ends_mid_frame_is_an_error(decoders): + decoder = decoders(_command(f"import sys\nsys.stdout.buffer.write(b'x' * {FRAME // 2})\nsys.stdout.flush()")) + decoder.start() + _wait_for(lambda: decoder._reader_done.is_set()) + with pytest.raises(nvdec.HardwareDecodeError, match="mid-frame"): + decoder.check() + + +def test_close_stops_a_stalled_process_quickly_and_is_repeatable(decoders): + decoder = decoders(_command("import time\ntime.sleep(60)")) + decoder.start() + proc = decoder._proc + started = time.monotonic() + decoder.close() + assert time.monotonic() - started < 1.5 and proc.poll() is not None + decoder.close() # idempotent + + +def test_closing_a_decoder_that_never_started_is_harmless(): + nvdec.HardwareDecoder(W, H).close() + + +def test_pumping_before_the_process_starts_does_nothing(): + decoder = nvdec.HardwareDecoder(W, H) + decoder.pump() + assert decoder.pending_input_bytes == 0 + + +def test_close_kills_a_process_that_ignores_termination(decoders): + ignoring = "import signal, time\nsignal.signal(signal.SIGTERM, signal.SIG_IGN)\ntime.sleep(60)" + decoder = decoders(_command(ignoring)) + decoder.start() + proc = decoder._proc + time.sleep(0.5) # the child must have installed its handler before the terminate arrives + started = time.monotonic() + decoder.close() + assert time.monotonic() - started < 2.0 and proc.poll() is not None + + +def test_a_pipe_that_cannot_be_enlarged_is_tolerated(decoders, monkeypatch): + def refuse(*args): + raise OSError("not permitted") + + monkeypatch.setattr(nvdec.fcntl, "fcntl", refuse) + decoder = decoders(_counting_decoder()) + decoder.start() + decoder.write(b"\x00" * 4) + _wait_for(lambda: decoder.has_frame) + + +def test_close_input_twice_is_harmless(decoders): + decoder = decoders(_counting_decoder()) + decoder.start() + assert decoder.close_input() is True + assert decoder.close_input() is True + + +# --- the reader on the hardware path --- + + +def _reader(path, **kw) -> h264.SensingStreamReader: + kw.setdefault("decoder", "hardware") + kw.setdefault("hardware_command", _command(PYAV_DECODER)) + reader = h264.SensingStreamReader(str(path), width=kw.pop("width", W), height=kw.pop("height", H), **kw) + reader.connect(1.0) + return reader + + +def _close(rgb: np.ndarray, expected, tol: int = 14) -> bool: + return bool(np.abs(rgb[H // 2, W // 2].astype(int) - np.array(expected)).max() <= tol) + + +def _read_until(reader, timeout_s: float = 20.0): + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + out = reader.read_latest(0.2) + if out is not None: + return out + raise AssertionError("no frame decoded in time") + + +def test_hardware_reader_delivers_correct_colours(tmp_path, caplog): + units = encode_units(W, H, [RED, GREEN, BLUE], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 4) + with caplog.at_level(logging.INFO, logger="isaaccapture.capture_server.sensing_h264"): + reader = _reader(path) + assert "NVDEC hardware decoder" in caplog.text + rgb, selected_ns = _read_until(reader) + assert rgb.shape == (H, W, 3) and selected_ns > 0 + assert any(_close(rgb, c) for c in (RED, GREEN, BLUE)) + reader.close() + + +def test_hardware_reader_finish_returns_the_last_frame(tmp_path): + units = encode_units(W, H, [RED, GREEN, BLUE], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) + reader = _reader(path) + reader.read_latest(0.05) + rgb, _ = reader.finish() + assert _close(rgb, BLUE) # the final unit is only known complete once the stream ends + assert reader.finish() is None # terminal + reader.close() + + +def test_hardware_reader_full_range_flag_changes_the_conversion(tmp_path): + units = encode_units(W, H, [GREEN], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 3) + limited = _reader(path) + full = _reader(path, full_range=True) + a, _ = _read_until(limited) + b, _ = _read_until(full) + assert not np.array_equal(a, b) + limited.close() + full.close() + + +def test_hardware_reader_rejects_a_stream_of_the_wrong_size(tmp_path): + units = encode_units(W, H, [RED, GREEN], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 2) + reader = _reader(path, width=W * 2, height=H * 2) + with pytest.raises(h264.SensingStreamError, match=r"decoded 64x32, expected 128x64"): + _read_until(reader) + reader.close() + + +def test_hardware_reader_reports_a_dead_decoder(tmp_path): + units = encode_units(W, H, [RED, GREEN], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 2) + reader = _reader(path, hardware_command=_command("import sys\nsys.stderr.write('boom')\nsys.exit(2)")) + with pytest.raises(h264.SensingStreamError, match="decoder"): + for _ in range(100): + reader.read_latest(0.1) + with pytest.raises(h264.SensingStreamError, match="failed earlier"): + reader.read_latest(0.0) + reader.close() + + +def test_hardware_reader_close_ends_the_process(tmp_path): + units = encode_units(W, H, [RED, GREEN], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 2) + reader = _reader(path) + _read_until(reader) + proc = reader._hw._proc + reader.close() + assert proc.poll() is not None and reader._hw is None + + +def test_input_beyond_the_decoders_capacity_stays_in_the_readers_queue(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_HW_MAX_INPUT_BYTES", 1) + units = encode_units(W, H, [RED, GREEN], gop=1) + path = tmp_path / "s.h264" + # More than the decoder's pipe can hold, so the input really is full and later units must wait here. + path.write_bytes(b"".join(units) * (4 * nvdec._STDIN_PIPE_BYTES // sum(map(len, units)))) + reader = _reader(path, hardware_command=_command("import time\ntime.sleep(60)")) + assert reader.read_latest(0.3) is None + assert reader._pending # not handed over while the decoder's input is full + reader.close() + + +# --- choosing the decoder --- + + +def test_unknown_decoder_name_is_rejected(tmp_path): + with pytest.raises(ValueError, match="decoder must be one of"): + h264.SensingStreamReader(str(tmp_path / "s"), width=W, height=H, decoder="gpu") + + +def test_hardware_requested_but_unavailable_is_an_error(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "hardware_decoder_unavailable_reason", lambda: "no NVDEC here") + (tmp_path / "s.h264").write_bytes(b"") + reader = h264.SensingStreamReader(str(tmp_path / "s.h264"), width=W, height=H, decoder="hardware") + with pytest.raises(h264.SensingStreamError, match="requested but is unavailable: no NVDEC here"): + reader.connect(1.0) + reader.close() + + +def test_auto_falls_back_to_software_and_says_why(tmp_path, monkeypatch, caplog): + monkeypatch.setattr(h264, "hardware_decoder_unavailable_reason", lambda: "no NVDEC here") + units = encode_units(W, H, [RED, GREEN], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 2) + with caplog.at_level(logging.INFO, logger="isaaccapture.capture_server.sensing_h264"): + reader = h264.SensingStreamReader(str(path), width=W, height=H, decoder="auto") + reader.connect(1.0) + assert "falling back to CPU" in caplog.text and "no NVDEC here" in caplog.text + rgb, _ = reader.read_latest(1.0) + assert rgb.shape == (H, W, 3) and reader._hw is None + reader.close() + + +def test_auto_uses_hardware_when_available(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "hardware_decoder_unavailable_reason", lambda: None) + units = encode_units(W, H, [RED, GREEN], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 4) + reader = _reader(path, decoder="auto") + _read_until(reader) + assert reader._hw is not None + reader.close() + + +def test_software_choice_is_logged(tmp_path, caplog): + path = tmp_path / "s.h264" + path.write_bytes(b"") + with caplog.at_level(logging.INFO, logger="isaaccapture.capture_server.sensing_h264"): + reader = h264.SensingStreamReader(str(path), width=W, height=H, decoder="software") + reader.connect(1.0) + assert "CPU (PyAV software), as configured" in caplog.text + reader.close() + + +# --- the rig configuration --- + + +def test_rig_decoder_defaults_to_auto_and_rejects_unknown_values(): + from capture_server.acquisition import sensing as sn + + base = dict( + executable="/x", expected_backend="MOCK", platform_config="/p", config_name="C", link_masks=("0x0000",) + ) + assert sn.SensingRigConfig(**base).decoder == "auto" + with pytest.raises(ValueError, match="decoder must be one of"): + sn.SensingRigConfig(**base, decoder="gpu") + + +def test_station_file_accepts_a_decoder_key_and_rejects_a_bad_one(tmp_path): + from capture_server.acquisition import sensing as sn + + def write(decoder: str) -> str: + path = tmp_path / "station.yaml" + path.write_text( + "sensing_rig:\n" + " executable: /x\n expected_backend: MOCK\n platform_config: /p\n config_name: C\n" + f" link_masks: ['0x0000']\n decoder: {decoder}\n" + "cameras:\n left:\n type: SENSING_COLOR\n module_name: M\n" + ) + return str(path) + + rig, _ = sn.load_sensing_station(write("software")) + assert rig.decoder == "software" + with pytest.raises(ValueError, match="decoder"): + sn.load_sensing_station(write("gpu")) + with pytest.raises(ValueError, match="decoder"): + sn.load_sensing_station(write("1")) + + +# --- the real pipeline, when this machine has NVDEC --- + + +@pytest.mark.skipif( + nvdec.hardware_decoder_unavailable_reason() is not None, reason="no Jetson NVDEC on this machine" +) +def test_real_nvdec_pipeline_decodes_correct_colours(tmp_path): + # Hardware decoders have a minimum surface size; a 640x480 stream is well above it. + width, height = 640, 480 + units = encode_units(width, height, [RED, GREEN, BLUE], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 8) + reader = h264.SensingStreamReader(str(path), width=width, height=height, decoder="hardware") + reader.connect(1.0) + try: + rgb, _ = _read_until(reader, timeout_s=30.0) + finally: + reader.close() + centre = rgb[height // 2, width // 2].astype(int) + assert min(np.abs(centre - np.array(c)).max() for c in (RED, GREEN, BLUE)) <= 16