diff --git a/examples/egocap/backend/egocap_web/capture_preview.py b/examples/egocap/backend/egocap_web/capture_preview.py index d680334ff7..90e05ea0f9 100644 --- a/examples/egocap/backend/egocap_web/capture_preview.py +++ b/examples/egocap/backend/egocap_web/capture_preview.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -"""Latest-sample preview cache: one tracking slot and at most three camera +"""Latest-sample preview cache: one tracking slot and at most four camera slots, each holding its immutable callback value and backend-local receipt time. Replacing a slot drops the superseded value -- there is no history, replay queue or recording buffer here. @@ -18,7 +18,7 @@ # capture. MAX_TRACKING_PAYLOAD_BYTES = 64 * 1024 MAX_CAMERA_JPEG_BYTES = 8 * 1024 * 1024 -MAX_CAMERAS = 3 +MAX_CAMERAS = 4 @dataclass(frozen=True) diff --git a/examples/egocap/backend/egocap_web/station_config.py b/examples/egocap/backend/egocap_web/station_config.py index 53f873185b..1d9d9353b7 100644 --- a/examples/egocap/backend/egocap_web/station_config.py +++ b/examples/egocap/backend/egocap_web/station_config.py @@ -34,9 +34,16 @@ STATION_CONFIG_ENV = "EGOCAP_STATION_CONFIG" _DEFAULT_FLAT_STATION_ID = "station_01" -#: Matches capture_server.protocol's own BeginRequest.cameras limit -- checked here too so an -#: over-limit YAML is a configuration error, not a later capture-startup failure. -_MAX_CAMERAS = 3 +#: capture_server.protocol's own BeginRequest.cameras limit, reused rather than duplicated so this +#: loader and the server can never drift apart -- checked here too so an over-limit YAML is a +#: configuration error, not a later capture-startup failure. +_MAX_CAMERAS = p._MAX_CAMERAS + +#: Maps each supported YAML ``video_sources[].type`` spelling to its wire CameraType. +_SOURCE_TYPES: dict[str, p.CameraType] = { + "oak_camera": p.CameraType.OAK_D_COLOR, + "sensing_camera": p.CameraType.SENSING_COLOR, +} class StationConfigError(ValueError): @@ -122,11 +129,13 @@ def _resolve_cameras(station_id: str, entry: dict) -> tuple[p.CameraSelection, . raise StationConfigError(f"station {station_id!r}: duplicate video source name {name!r}") seen_names.add(name) source_type = source.get("type") - if source_type != "oak_camera": + camera_type = _SOURCE_TYPES.get(source_type) if isinstance(source_type, str) else None + if camera_type is None: + supported = ", ".join(sorted(repr(t) for t in _SOURCE_TYPES)) raise StationConfigError( - f"station {station_id!r}: video_sources[{i}].type {source_type!r} is not 'oak_camera'" + f"station {station_id!r}: video_sources[{i}].type {source_type!r} is not one of {supported}" ) - cameras.append(p.CameraSelection(name=name, type=p.CameraType.OAK_D_COLOR)) + cameras.append(p.CameraSelection(name=name, type=camera_type)) return tuple(cameras) diff --git a/examples/egocap/capture_server/README.md b/examples/egocap/capture_server/README.md index bfaebbdde3..32ce484a31 100644 --- a/examples/egocap/capture_server/README.md +++ b/examples/egocap/capture_server/README.md @@ -82,6 +82,24 @@ the link at USB 2; some host ports or cables cannot boot an OAK-D at USB 3. The runs on-device NV12 to MJPEG; each frame's own intrinsics are reported in the camera status when the device provides them. +**SENSING cameras.** Cameras served by IsaacCapture's SENSING plugin are configured with one +YAML file (`sensing_rig:` plus `cameras:` entries of `type: SENSING_COLOR`) and +`--sensing-config FILE`; the file format is documented on `load_sensing_station` in +`capture_server/acquisition/sensing.py`. The server launches the plugin's executable itself, reads each +sensor's H.264 output file, decodes it (every unit in order) and sends the newest decoded frame as a JPEG +(`av`, i.e. PyAV, is a declared dependency; nothing is loaded unless a SENSING stream is used). +The decoder is `sensing_rig.decoder`: `auto` (default) uses the Jetson NVDEC hardware decoder through GStreamer's +`nvv4l2decoder` when `/dev/v4l2-nvdec`, `gst-launch-1.0` and that element exist, and otherwise PyAV on the CPU; +`hardware` fails the stream if NVDEC is unavailable; `software` forces the CPU. Each stream logs which decoder it +uses (`isaaccapture.capture_server.sensing_h264`). A 2560x1984 stream at 60 fps is beyond one CPU core of software +decoding (about 30 fps on an Orin core), which shows as a picture that falls further behind; use hardware decode there. +`--mock` with a SENSING station requires the plugin's MOCK build, whose encoder must write real H.264; +an older mock encoder wrote placeholder bytes that are not video, which the decoder rejects, so such a +build shows a degraded camera instead of pictures. With `--sensing-config`, `cameras` accepts only the +SENSING names it lists (`--camera` is real-acquisition only), so the OAK mock cameras cannot be mixed in. +Measure per-frame cost on the target with `tests/capture_server/bench_sensing_decode.py` +before choosing `delivery_fps`. + **Failure policy.** A camera that produces no frame for max(5 s, five frame periods) counts as failed. A required camera failure ends the session; an optional one reports a degraded camera. There is no automatic reconnection: retry by starting a new session. diff --git a/examples/egocap/capture_server/capture_server/acquisition/__init__.py b/examples/egocap/capture_server/capture_server/acquisition/__init__.py index 11761359e0..baa1c4de60 100644 --- a/examples/egocap/capture_server/capture_server/acquisition/__init__.py +++ b/examples/egocap/capture_server/capture_server/acquisition/__init__.py @@ -12,12 +12,15 @@ from .depthai import DepthAIAdapter from .errors import CameraSourceError, SourceError, SourceExited from .isaac_capture import IsaacCaptureAdapter, TrackingSelection +from .sensing import SensingAdapter, SensingRig __all__ = [ "Acquisition", "CameraSourceError", "DepthAIAdapter", "IsaacCaptureAdapter", + "SensingAdapter", + "SensingRig", "SourceError", "SourceExited", "TrackingSelection", diff --git a/examples/egocap/capture_server/capture_server/acquisition/acquisition.py b/examples/egocap/capture_server/capture_server/acquisition/acquisition.py index 8bdb1dbbe0..0373b1a623 100644 --- a/examples/egocap/capture_server/capture_server/acquisition/acquisition.py +++ b/examples/egocap/capture_server/capture_server/acquisition/acquisition.py @@ -2,20 +2,17 @@ # # SPDX-License-Identifier: Apache-2.0 -"""``Acquisition``: the one concrete object ``_Session`` holds for its -acquisition source -- composing one optional :class:`IsaacCaptureAdapter` -(``None`` for a camera-only request, which must never construct an IC -session) with zero or more :class:`DepthAIAdapter`, one per configured -camera stream. - -This is not a generic protocol/interface: it is one -fixed, concretely named composition that always holds exactly these two -concrete adapter types, mode-agnostic by construction since both adapters -already are. ``_Session`` calls it directly -- -``enter``/``next_tracking_sample``/``next_camera_jpeg``/``status``/``close``. -``next_tracking_sample``/``next_camera_jpeg`` take no -``t``/``tick`` argument: each adapter paces and times itself -- see their -own docstrings. +"""``Acquisition``: the one concrete object ``_Session`` holds for its acquisition source. + +It composes one optional :class:`IsaacCaptureAdapter` (``None`` for a camera-only request, which must never +construct an IC session), zero or more camera adapters (:class:`DepthAIAdapter` for OAK-D, +:class:`SensingAdapter` for SENSING), and the optional :class:`SensingRig` -- the one native producer process +the SENSING adapters read from. The rig is owned here and only borrowed by its adapters, so it is started +before them and reaped after they have settled. + +This is NOT a generic source interface: it is one fixed, concretely named composition. ``_Session`` calls it +directly -- ``enter``/``next_tracking_sample``/``next_camera_jpeg``/``status``/``close`` -- and each adapter +paces and times itself (see their own docstrings). """ from __future__ import annotations @@ -28,14 +25,30 @@ from .depthai import DepthAIAdapter from .errors import CameraSourceError from .isaac_capture import IsaacCaptureAdapter +from .sensing import SensingAdapter, SensingRig __all__ = ["Acquisition"] +def _observe_close(task: "asyncio.Future") -> "asyncio.Future": + """Register an observer on a close task as soon as it exists, so a late failure is never left unretrieved. + + Observation only: it does not own, cancel or await the task. + """ + task.add_done_callback(lambda f: f.cancelled() or f.exception()) + return task + + +# The server's whole source-release budget for one session; adapters and the SENSING producer share it. +_CLOSE_BUDGET_S = 5.0 + + @dataclass class Acquisition: tracking: IsaacCaptureAdapter | None - cameras: tuple[DepthAIAdapter, ...] + cameras: tuple[DepthAIAdapter | SensingAdapter, ...] + # The one native SENSING producer shared by the SENSING adapters in ``cameras``; owned here, borrowed by them. + sensing_rig: SensingRig | None = None async def enter(self) -> None: """A REQUIRED source's startup failure is fatal and propagates @@ -51,6 +64,10 @@ async def enter(self) -> None: for that camera.""" if self.tracking is not None: await self.tracking.enter() + if self.sensing_rig is not None: + # Never raises: a start failure is recorded on the rig and each SENSING adapter reports it under + # its own required/optional policy below, so optional streams degrade while OAK/tracking go on. + await self.sensing_rig.start() for camera in self.cameras: try: await camera.enter() @@ -84,28 +101,59 @@ async def status(self, updated_at_ns: int) -> p.StatusSnapshot: ) async def close(self) -> bool: - """Closes every adapter CONCURRENTLY, each on its own dedicated - worker thread (native thread affinity is per-adapter already -- - see ThreadRunner -- so running them concurrently never puts two - threads on the same native resource). Up to four independent - adapters (one IC + up to three cameras) each bound to their own - release timeout would, awaited sequentially, sum to up to four - times that bound -- comfortably exceeding the outer release-wait - budget server.py's own finalization assumes for ONE source - release. Running them concurrently keeps the total wall-clock - cost to whichever single adapter is slowest, not their sum. - Every adapter's close is still attempted regardless of whether - another raised or timed out -- one release failure must never - skip releasing the others, and a failure to confirm release - within bound is reported (``ok=False``), never silently assumed - successful.""" - closers = ([self.tracking.close()] if self.tracking is not None else []) + [ - camera.close() for camera in self.cameras + """Releases every adapter and the SENSING producer, concurrently, each on its own bound. + + Every adapter is closed on its own dedicated worker thread (native thread affinity is per-adapter -- see + ThreadRunner). The tracking and OAK adapters each keep their own bound. The SENSING stream adapters and + the producer they borrow form ONE chain (:meth:`_close_rig_chain`): the adapters settle first, then the + producer is terminated and reaped, so a worker still reading never races its teardown. The chain runs + concurrently with the other closes and is sized to fit the same ``_CLOSE_BUDGET_S``, so a slow OAK or + tracking release is not cut short by the producer's shutdown, nor the reverse. + Waiting is bounded by ``_CLOSE_BUDGET_S``; a timeout does not cancel any close job -- it keeps running and + is only reported as unconfirmed (False). Every close is attempted; one failure never skips another. A + False means a worker MAY still be running: the wait bounds how long we waited, it does not prove native + settlement. A cancelled caller does not prove release either: every job keeps running, observed, so the + producer is still reaped. + """ + borrowing = [c for c in self.cameras if getattr(c, "borrows_rig", False)] + independent = ([self.tracking] if self.tracking is not None else []) + [ + c for c in self.cameras if not getattr(c, "borrows_rig", False) ] - if not closers: - return True - results = await asyncio.gather(*closers, return_exceptions=True) - first_exc = next((r for r in results if isinstance(r, BaseException)), None) - if first_exc is not None: + tasks = [_observe_close(asyncio.ensure_future(a.close())) for a in independent] + if borrowing or self.sensing_rig is not None: + tasks.append(_observe_close(asyncio.ensure_future(self._close_rig_chain(borrowing)))) + pending: set[asyncio.Future] = set() + if tasks: + _done, pending = await asyncio.wait(tasks, timeout=_CLOSE_BUDGET_S) + results: list[bool | BaseException] = [] + for task in tasks: + if task in pending or task.cancelled(): + results.append(False) # unconfirmed: may still be running + elif (exc := task.exception()) is not None: + results.append(exc) + else: + results.append(task.result()) + if (first_exc := next((r for r in results if isinstance(r, BaseException)), None)) is not None: raise first_exc return all(results) + + async def _close_rig_chain(self, adapters: list) -> bool: + """Close the adapters that borrow the producer, then terminate and reap it. + + The producer is reaped whether or not an adapter failed to close. True only if every adapter and the + producer confirmed release; the first failure is raised after both have been attempted. + """ + results = await asyncio.gather(*(a.close() for a in adapters), return_exceptions=True) + rig_result: bool | BaseException = True + if self.sensing_rig is not None: + try: + rig_result = await self.sensing_rig.close() + except Exception as exc: # noqa: BLE001 -- reported below, after the adapters' own results + rig_result = exc + exceptions = [r for r in (*results, rig_result) if isinstance(r, BaseException)] + if exceptions: + # A cancellation takes priority over any other exception collected here: dropping it in favor of + # an unrelated close failure would silently defeat the caller's own cancellation. + cancelled = next((e for e in exceptions if isinstance(e, asyncio.CancelledError)), None) + raise cancelled if cancelled is not None else exceptions[0] + return all(results) and rig_result is True diff --git a/examples/egocap/capture_server/capture_server/acquisition/sensing.py b/examples/egocap/capture_server/capture_server/acquisition/sensing.py new file mode 100644 index 0000000000..5a6bbb0bb8 --- /dev/null +++ b/examples/egocap/capture_server/capture_server/acquisition/sensing.py @@ -0,0 +1,807 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""SENSING camera source: station configuration, producer ownership and the per-stream adapter. + +One native ``camera_plugin_sensing`` producer per rig writes every selected sensor's H.264 stream to its own +file; each stream's adapter tails one file, decodes it in order and JPEG-encodes the newest decoded frame for the +existing CaptureServer image path. Nothing loads the decoder until a stream attaches, so OAK-only and tracking-only +runs never need it. + +The producer's existing interface is used unchanged: each selected sensor's H.264 output file +(``--add-stream=sensor=N,output=PATH``). The adapter tails that file, decodes every unit in order on the CPU, +and JPEG-encodes the newest decoded frame. No CUDA is touched in this process. + +Native configuration is server-owned: a client's Begin selects logical camera names only, never paths, +masks or arguments. +""" + +from __future__ import annotations + +import asyncio +import ctypes +import functools +import os +import re +import shutil +import subprocess +import sys +import tempfile +import threading +import time +from dataclasses import dataclass +from typing import Any, Callable + +from capture_server import protocol as p + +from ._thread_runner import ThreadRunner +from .clock import Clock, MonotonicClock +from .errors import CameraSourceError, SourceError +from .sensing_h264 import DECODERS, SensingStreamError, SensingStreamReader +from .sensing_jpeg import encode_rgb_jpeg + +__all__ = [ + "SensingAdapter", + "SensingDescription", + "SensingRig", + "SensingRigConfig", + "SensingSensor", + "SensingStationCamera", + "inspect_rig", + "parse_sensor_listing", + "load_sensing_station", + "resolve_sensors", +] + +_BACKENDS = ("JETSON", "MOCK") +# Producer startup (camera bring-up) and first-frame budget per stream. +_STARTUP_TIMEOUT_S = 30.0 +_INSPECT_TIMEOUT_S = 10.0 +# A started stream that produces no decoded frame for this long (or five sensor frame periods, if longer) has +# failed, the same rule DepthAIAdapter applies to an OAK camera. +_NO_FRAME_TIMEOUT_S = 5.0 +# Shutdown of the SENSING chain fits inside the server's single 5 s source-release budget: the stream adapters +# settle first (_ADAPTER_RELEASE_S), then the producer gets _RIG_CLOSE_BUDGET_S, of which its own steps use +# _RIG_STEPS_S (half of that is the SIGTERM grace before SIGKILL). The OAK and tracking adapters are not part of +# this chain; they close concurrently on their own bounds. The producer's real teardown time on hardware is not +# measured yet; these are bounds, not measurements. +_ADAPTER_RELEASE_S = 1.5 +_RIG_CLOSE_BUDGET_S = 3.5 +_RIG_STEPS_S = 3.0 +_LOG_TAIL_BYTES = 4096 +# Linux's prctl(2) option that asks the kernel to deliver a signal to this process when its parent dies, so an +# orphaned producer (this server killed hard -- OOM, SIGKILL, a native crash) is reaped by the kernel instead of +# running unsupervised and filling its tmpfs runtime directory forever. No effect on an ordinary close(): that +# path already terminates the producer itself. Linux-only; harmless no-op everywhere else (see _set_pdeathsig). +_PR_SET_PDEATHSIG = 1 + + +@dataclass(frozen=True) +class SensingRigConfig: + """Server-owned settings of the one native producer: which binary, backend, platform and link masks.""" + + executable: str + expected_backend: str + platform_config: str + config_name: str + link_masks: tuple[str, ...] + gpu_id: int = 0 + nito_dir: str | None = None + # Encoder settings passed to the producer only when set; otherwise its own defaults apply. The decoder handles + # whatever GOP the producer uses, so none of these is a decoding requirement. + gop: int | None = None + bitrate_bps: int | None = None + # Whether the producer's NV12 uses the full [0,255] luma range; the decoder must convert with the same. + full_range: bool = False + # Where per-session stream files live; None prefers /dev/shm. tmpfs is memory, not extra capacity: the files + # stay bounded only because consumed bytes are reclaimed as they are read (see sensing_h264.py). + runtime_dir: str | None = None + # How the H.264 is decoded: "auto" uses the Jetson's NVDEC hardware decoder when it is reachable and the CPU + # otherwise (the choice is logged), "hardware" requires NVDEC, "software" forces the CPU. See sensing_h264.py. + decoder: str = "auto" + + def __post_init__(self) -> None: + """Reject an unknown backend or decoder, a missing executable/platform/name, or empty link masks.""" + if self.expected_backend not in _BACKENDS: + raise ValueError(f"SensingRigConfig.expected_backend must be one of {_BACKENDS}") + if self.decoder not in DECODERS: + raise ValueError(f"SensingRigConfig.decoder must be one of {DECODERS}, got {self.decoder!r}") + if not (self.executable and self.platform_config and self.config_name and self.link_masks): + raise ValueError("SensingRigConfig needs executable, platform_config, config_name and link_masks") + + def base_args(self) -> list[str]: + """The producer's rig arguments as an argument list (never a shell command).""" + args = [ + self.executable, + f"--platform-config={self.platform_config}", + f"--config-name={self.config_name}", + f"--link-masks={' '.join(self.link_masks)}", + f"--gpu-id={self.gpu_id}", + ] + if self.nito_dir: + args.append(f"--nito-dir={self.nito_dir}") + if self.gop is not None: + args.append(f"--gop={self.gop}") + if self.bitrate_bps is not None: + args += [f"--bitrate={self.bitrate_bps}", f"--peak-bitrate={self.bitrate_bps * 3 // 2}"] + if self.full_range: + args.append("--full-range") + return args + + +@dataclass(frozen=True) +class SensingStationCamera: + """Server-owned settings of one SENSING sensor, keyed by the logical name clients request.""" + + module_name: str + delivery_fps: float = 30.0 + jpeg_quality: int = 80 + required: bool = True + # Which sensor of the module, counted in pipeline-id order, when a module has several sensors (a stereo + # camera's two eyes share one module name). Required exactly when the module name is not unique. + sensor_index: int | None = None + + def __post_init__(self) -> None: + """Reject an empty module name, a bad sensor index, a non-positive delivery rate or a bad JPEG quality.""" + if not self.module_name: + raise ValueError("SensingStationCamera.module_name must be nonempty") + if self.sensor_index is not None and (type(self.sensor_index) is not int or self.sensor_index < 0): + raise ValueError( + f"SensingStationCamera.sensor_index must be a non-negative integer, got {self.sensor_index!r}" + ) + if not 0 < self.delivery_fps < float("inf"): + raise ValueError(f"SensingStationCamera.delivery_fps must be positive, got {self.delivery_fps}") + if not 1 <= self.jpeg_quality <= 100: + raise ValueError(f"SensingStationCamera.jpeg_quality must be in 1..100, got {self.jpeg_quality}") + + +@dataclass(frozen=True) +class SensingSensor: + """One sensor as the native producer resolved it: module name, pipeline id, native geometry.""" + + module_name: str + pipeline_id: int + width: int + height: int + fps: float + + +@dataclass(frozen=True) +class SensingDescription: + """What the producer reports for the configured platform: each sensor's module, pipeline id, size and rate. + + The producer's existing ``--list-sensors`` does not say which backend it was built with, so the backend is + the station's declared ``expected_backend`` (checked against ``--mock`` at configuration time), not + something read back from the executable. + """ + + sensors: tuple[SensingSensor, ...] + + +_RIG_KEYS = { + "executable", + "expected_backend", + "platform_config", + "config_name", + "link_masks", + "gpu_id", + "nito_dir", + "gop", + "bitrate_bps", + "full_range", + "runtime_dir", + "decoder", +} +_CAMERA_KEYS = {"type", "module_name", "delivery_fps", "jpeg_quality", "required", "sensor_index"} + + +# Scalar types accepted for each key. bool is checked separately because it is a subclass of int in Python. +_RIG_TYPES: dict[str, type] = { + "executable": str, + "expected_backend": str, + "platform_config": str, + "config_name": str, + "gpu_id": int, + "nito_dir": str, + "gop": int, + "bitrate_bps": int, + "full_range": bool, + "runtime_dir": str, + "decoder": str, +} +_CAMERA_TYPES: dict[str, type | tuple[type, ...]] = { + "type": str, + "module_name": str, + "delivery_fps": (int, float), + "jpeg_quality": int, + "required": bool, + "sensor_index": int, +} + + +def _check_types(where: str, mapping: dict, types: dict) -> None: + """Reject a value whose YAML type is not the one the key needs (a bool is never accepted as a number).""" + for key, value in mapping.items(): + wanted = types[key] + accepted = wanted if isinstance(wanted, tuple) else (wanted,) + if (isinstance(value, bool) and bool not in accepted) or not isinstance(value, accepted): + names = "/".join(t.__name__ for t in accepted) + raise ValueError(f"{where}.{key} must be {names}, got {type(value).__name__} {value!r}") + + +def load_sensing_station(path: str) -> tuple[SensingRigConfig, dict[str, SensingStationCamera]]: + """Load the station's SENSING rig and cameras from a YAML file; the one configuration entry point. + + Expected shape: ``sensing_rig:`` (see :class:`SensingRigConfig`) and ``cameras:`` mapping logical names + to ``type: SENSING_COLOR`` entries (see :class:`SensingStationCamera`). Unknown keys, wrongly typed values, + a non-SENSING type, a missing section or a file that is not valid YAML all raise ``ValueError``; nothing is + defaulted silently. Link masks must be quoted strings (``'0x0000'``): an unquoted ``0x0000`` is a YAML + integer and would reach the producer as a decimal number. + """ + import yaml + + try: + with open(path, encoding="utf-8") as f: + data = yaml.safe_load(f) + except yaml.YAMLError as e: + raise ValueError(f"{path}: not valid YAML: {e}") from e + if not isinstance(data, dict) or set(data) - {"sensing_rig", "cameras"} or not data.get("sensing_rig"): + raise ValueError(f"{path}: expected only 'sensing_rig' and 'cameras' sections, with a sensing_rig") + rig = data["sensing_rig"] + if not isinstance(rig, dict) or set(rig) - _RIG_KEYS: + raise ValueError(f"{path}: sensing_rig keys must be among {sorted(_RIG_KEYS)}") + masks = rig.get("link_masks", []) + if not isinstance(masks, list) or not all(isinstance(m, str) for m in masks): + raise ValueError(f"{path}: sensing_rig.link_masks must be a list of quoted strings such as '0x0000'") + scalars = {k: v for k, v in rig.items() if k != "link_masks"} + try: + _check_types(f"{path}: sensing_rig", scalars, _RIG_TYPES) + rig_config = SensingRigConfig(**{**scalars, "link_masks": tuple(masks)}) + except TypeError as e: + raise ValueError(f"{path}: invalid sensing_rig: {e}") from e + raw_cameras = data.get("cameras") + if not isinstance(raw_cameras, dict): + raise ValueError(f"{path}: cameras must be a mapping of logical name to camera settings") + cameras: dict[str, SensingStationCamera] = {} + for name, entry in raw_cameras.items(): + if not isinstance(entry, dict) or set(entry) - _CAMERA_KEYS or entry.get("type") != "SENSING_COLOR": + raise ValueError( + f"{path}: camera {name!r} must be type SENSING_COLOR with keys among {sorted(_CAMERA_KEYS)}" + ) + _check_types(f"{path}: cameras.{name}", entry, _CAMERA_TYPES) + try: + cameras[str(name)] = SensingStationCamera(**{k: v for k, v in entry.items() if k != "type"}) + except TypeError as e: + raise ValueError(f"{path}: invalid camera {name!r}: {e}") from e + if not cameras: + raise ValueError(f"{path}: no SENSING cameras configured") + return rig_config, cameras + + +# One line of the producer's ``--list-sensors``: " sensor= x @ fps". +_SENSOR_LINE = re.compile(r"^\s*sensor=(\d+)\s+(\S+)\s+(\d+)x(\d+)\s+@\s+([0-9.]+)\s+fps\s*$") + + +def parse_sensor_listing(text: str) -> SensingDescription: + """Parse the producer's ``--list-sensors`` output; raises ``ValueError`` on anything unexpected. + + The first line (``CONFIG: N sensor(s)``) states how many sensor lines must follow, so a truncated or + garbled listing is rejected instead of silently describing fewer sensors. + """ + lines = [line for line in text.splitlines() if line.strip()] + header = re.match(r"^\S+: (\d+) sensor\(s\)$", lines[0].strip()) if lines else None + if header is None: + raise ValueError(f"malformed sensor listing: unexpected first line {lines[0] if lines else ''!r}") + sensors = [] + for line in lines[1:]: + match = _SENSOR_LINE.match(line) + if match is None: + raise ValueError(f"malformed sensor listing: unexpected line {line!r}") + pipeline_id, module, width, height, fps = match.groups() + if int(width) <= 0 or int(height) <= 0 or not 0 < float(fps) < float("inf"): + raise ValueError(f"malformed sensor listing: non-positive geometry or rate in {line!r}") + sensors.append(SensingSensor(module, int(pipeline_id), int(width), int(height), float(fps))) + if len(sensors) != int(header.group(1)): + raise ValueError(f"malformed sensor listing: header says {header.group(1)} sensors, found {len(sensors)}") + return SensingDescription(tuple(sensors)) + + +def _producer_env() -> dict[str, str]: + """The environment for every run of the producer binary, listing and capture alike. + + The reduced-size test knob must never reach either, or the listed geometry and the captured geometry could + differ from the station's. + """ + return {k: v for k, v in os.environ.items() if k != "SENSING_MOCK_SCALE_DIV"} + + +@functools.cache +def _bind_prctl() -> Callable[[int, int, int, int, int], int] | None: + """The C library's ``prctl`` with explicit argument types, bound once; ``None`` where it does not exist + (anywhere but Linux). Called in the parent before each spawn (see ``_start_sync``), so the ``preexec_fn`` + below only ever calls an already-resolved C function pointer -- see ``_set_pdeathsig``. + """ + try: + prctl = ctypes.CDLL(None, use_errno=True).prctl + except (OSError, AttributeError): + return None + prctl.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong] + return prctl + + +def _set_pdeathsig() -> None: + """``subprocess.Popen``'s ``preexec_fn``: ask the kernel to SIGTERM the producer if this server dies first. + + Runs in the forked child between ``fork()`` and ``exec()``. It calls only a ctypes function pointer already + bound in the parent, with primitive integer arguments, so it does no dlopen or ctypes construction there. + The kernel delivers the signal when the *spawning thread* (the rig's executor worker), not the process, + exits, and exec clears the setting for a setuid or file-capability binary. A failed or unsupported call (not + Linux, or the kernel refuses it) is silently skipped: PR_SET_PDEATHSIG only narrows the orphaned-producer + window from "unbounded" to "none", it is not the only thing cleaning up the producer on an ordinary close(). + """ + if sys.platform != "linux": + return + prctl = _bind_prctl() + if prctl is not None: + prctl(_PR_SET_PDEATHSIG, 15, 0, 0, 0) # 15 == SIGTERM; avoids importing signal for one constant + + +def inspect_rig( + config: SensingRigConfig, + *, + run: Callable[..., Any] = subprocess.run, + timeout_s: float = _INSPECT_TIMEOUT_S, +) -> SensingDescription: + """Ask the native executable which sensors the configured platform resolves to; blocking and bounded. + + Uses the producer's existing ``--list-sensors`` (hardware-free; it never starts capture). Run it off the + event loop. + """ + try: + done = run( + config.base_args() + ["--list-sensors"], + capture_output=True, + text=True, + timeout=timeout_s, + check=False, + env=_producer_env(), + ) + except (OSError, subprocess.TimeoutExpired) as e: + raise SourceError(f"could not inspect {config.executable}: {e}") from e + if done.returncode != 0: + raise SourceError( + f"{config.executable} --list-sensors failed ({done.returncode}): {done.stderr.strip()[-500:]}" + ) + try: + return parse_sensor_listing(done.stdout) + except ValueError as e: + raise SourceError(str(e)) from e + + +def resolve_sensors( + description: SensingDescription, stations: dict[str, SensingStationCamera] +) -> dict[str, SensingSensor]: + """Map each logical camera name to its resolved sensor by module name. + + A module with several sensors needs ``sensor_index``. Rejects a missing module, an ambiguous or out-of-range + selection, and two logical names selecting the same physical sensor. + """ + resolved: dict[str, SensingSensor] = {} + for name, station in stations.items(): + # Pipeline-id order makes sensor_index stable: the link mask renumbers pipelines, not their order. + matches = sorted( + (s for s in description.sensors if s.module_name == station.module_name), key=lambda s: s.pipeline_id + ) + if not matches: + raise SourceError(f"camera {name!r}: module {station.module_name!r} is not in the platform config") + if station.sensor_index is None: + if len(matches) > 1: + raise SourceError( + f"camera {name!r}: module {station.module_name!r} has {len(matches)} sensors; " + "select one with sensor_index" + ) + resolved[name] = matches[0] + elif station.sensor_index >= len(matches): + raise SourceError( + f"camera {name!r}: module {station.module_name!r} has {len(matches)} sensor(s), " + f"no sensor_index {station.sensor_index}" + ) + else: + resolved[name] = matches[station.sensor_index] + pipelines = [s.pipeline_id for s in resolved.values()] + dup = sorted({i for i in pipelines if pipelines.count(i) > 1}) + if dup: + raise SourceError(f"physical sensor(s) selected by more than one logical camera: pipeline {dup}") + return resolved + + +class SensingRig: + """Owns one producer process, its private runtime directory and per-stream H.264 output file paths. + + ``start`` and ``close`` run on the rig's own dedicated worker, one job at a time, so a spawn still in + flight when Stop or cancellation arrives is always finished and then reaped by the queued close; it can + never escape cleanup. The producer is launched with one ``--add-stream=...,output=FILE`` per selected + sensor, and with PR_SET_PDEATHSIG (Linux; a silent no-op elsewhere) so a hard death of this server + process -- OOM, SIGKILL, a native crash -- still gets the producer reaped by the kernel, rather than + left running unsupervised with nothing left to bound its tmpfs output. Diagnostics are drained + continuously into a byte-bounded tail so a full pipe can never stall capture. A start failure is + recorded in ``start_error`` for each adapter to report under its own required/optional policy rather + than ending the attempt here. + """ + + def __init__( + self, + config: SensingRigConfig, + pipeline_ids: tuple[int, ...], + *, + popen: Callable[..., Any] = subprocess.Popen, + runtime_root: str | None = None, + ) -> None: + """Records the config and selected pipeline ids; nothing is started or created until :meth:`start`.""" + self._config = config + self._pipeline_ids = pipeline_ids + self._popen = popen + self._runtime_root = runtime_root + self._dir: str | None = None + self._proc: Any = None + self._drain: threading.Thread | None = None + self._tail = b"" + self._abort_reason = "" + self._runner = ThreadRunner(release_timeout_s=_RIG_CLOSE_BUDGET_S) + self.start_error: str | None = None + # Sticky: once any close failed to confirm release, no later call reports success. + self.cleanup_ok = True + + def stream_path(self, pipeline_id: int) -> str: + """The H.264 output file for one selected sensor; valid after a successful :meth:`start`.""" + assert self._dir is not None, "rig not started" + return os.path.join(self._dir, f"sensor{pipeline_id}.h264") + + async def start(self) -> None: + """Launch the producer on the rig's worker; a failure is recorded in ``start_error``, not raised.""" + try: + await self._runner.run(self._start_sync) + except SourceError as e: + self.start_error = str(e) + + def _start_sync(self) -> None: + # Local filesystem failures are start failures too (an optional stream then degrades instead of + # aborting healthy OAK/tracking). The directory, once created, stays owned for close() to remove. + """Create the private runtime directory and spawn the producer on the rig worker. + + Raises ``SourceError`` on failure. + """ + try: + root = self._runtime_root or self._config.runtime_dir + if root is None and os.path.isdir("/dev/shm") and os.access("/dev/shm", os.W_OK): + root = "/dev/shm" + self._dir = tempfile.mkdtemp(prefix="sensing-", dir=root) + os.chmod(self._dir, 0o700) + except OSError as e: + raise SourceError(f"could not prepare the SENSING runtime directory: {e}") from e + args = self._config.base_args() + [ + f"--add-stream=sensor={i},output={self.stream_path(i)}" for i in self._pipeline_ids + ] + env = _producer_env() + _bind_prctl() # resolve in the parent, so the post-fork preexec_fn does no dlopen + try: + self._proc = self._popen( + args, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + preexec_fn=_set_pdeathsig, + ) + except OSError as e: + self._remove_dir() + raise SourceError(f"could not start {self._config.executable}: {e}") from e + self._drain = threading.Thread(target=self._drain_output, name="sensing-rig-log", daemon=True) + self._drain.start() + + def _drain_output(self) -> None: + """Log-drain thread body. + + Keeps the producer's most recent output so its pipe can never fill and stall it. + """ + stream = self._proc.stdout + try: + while chunk := stream.read1(4096): + # Byte-bounded (a newline-free flood cannot grow it); one atomic assignment, so readers + # always see a consistent snapshot. + self._tail = (self._tail + chunk)[-_LOG_TAIL_BYTES:] + except (OSError, ValueError): + pass # the pipe was closed under us during shutdown + finally: + stream.close() + + def exit_code(self) -> int | None: + """The producer's exit code once it has exited on its own, else None.""" + return None if self._proc is None else self._proc.poll() + + def diagnostics(self) -> str: + """The producer's most recent output, decoded leniently, for error messages.""" + tail = self._tail.decode("utf-8", errors="replace").strip().replace("\n", " | ") + return f"stopped by the server: {self._abort_reason} | {tail}" if self._abort_reason else tail + + def abort(self, reason: str) -> None: + """Stop the producer because its output can no longer be bounded; ``close`` still reaps it.""" + self._abort_reason = self._abort_reason or reason + if self._proc is not None and self._proc.poll() is None: + self._proc.terminate() + + async def close(self) -> bool: + """Terminate and reap the producer, join the log drain, remove the runtime directory. + + Runs on the rig's worker after any pending start, within one bounded budget. Returns True only when + every step is confirmed, and False on every later call once any attempt failed or timed out. + """ + try: + done = await self._runner.close(self._close_sync) + except Exception: + self.cleanup_ok = False + raise + if not done: + self.cleanup_ok = False + return done and self.cleanup_ok + + def _close_sync(self) -> None: + """Terminate, reap and clean up within ``_RIG_STEPS_S`` on the rig worker. + + Sets ``cleanup_ok`` False if anything is unconfirmed. + """ + deadline = time.monotonic() + _RIG_STEPS_S + ok = True + proc = self._proc + if proc is not None: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=_RIG_STEPS_S / 2) + except subprocess.TimeoutExpired: + proc.kill() + try: + proc.wait(timeout=max(deadline - time.monotonic(), 0.05)) + except subprocess.TimeoutExpired: + ok = False + if ok and self._drain is not None: + self._drain.join(timeout=max(deadline - time.monotonic(), 0.05)) + ok = not self._drain.is_alive() + if ok: + ok = self._remove_dir() + if not ok: + self.cleanup_ok = False + + def _remove_dir(self) -> bool: + """Remove this rig's own runtime directory only; False if it could not be removed.""" + if self._dir is None: + return True + try: + shutil.rmtree(self._dir) + except FileNotFoundError: + pass + except OSError: + return False + self._dir = None + return True + + +class SensingAdapter: + """One SENSING stream: tail its H.264 file, deliver the newest frame as JPEG at the delivery rate. + + The same instance shape the server expects of every camera adapter. All blocking file, decode and + JPEG work runs on this adapter's dedicated worker; the reader is created, used and closed there. + """ + + # Acquisition closes adapters that borrow the shared producer before it reaps the producer. + borrows_rig = True + + def __init__( + self, + *, + rig: SensingRig, + sensor: SensingSensor, + camera: p.CameraStreamConfig, + full_range: bool = False, + decoder: str = "software", + clock: Clock | None = None, + startup_timeout_s: float = _STARTUP_TIMEOUT_S, + ) -> None: + """Bind one sensor of the shared rig to one accepted camera stream. + + ``decoder`` is the reader's decoder choice (see ``SensingStreamReader``); the server passes the rig's + setting, and direct construction defaults to the deterministic CPU decoder. + The reader is created later, in :meth:`enter`. + """ + self._decoder = decoder + self._rig = rig + self._sensor = sensor + self._camera = camera + self._interval = 1.0 / camera.fps + self._clock = clock or MonotonicClock() + self._startup_timeout_s = startup_timeout_s + self._runner = ThreadRunner(release_timeout_s=_ADAPTER_RELEASE_S) + self._full_range = full_range + # Built in enter(): the stream file path exists only once Acquisition has started the rig. + self._reader: SensingStreamReader | None = None + self._deadline: float | None = None + self._dead = False + self._flushed = False + # Set synchronously by close(), before its job queues behind the worker, so startup, reads and the final + # flush stop within a fraction of a second. + self._stop = threading.Event() + # Host time of the last decoded frame (or of the first confirmed one); drives stall detection. + self._last_frame_s: float | None = None + self._degraded = True + self._last_error = "camera not yet started" + # Sticky: a close that failed to confirm release is never forgotten, whatever happens to the stream. + self.cleanup_failed = False + + @property + def stream_id(self) -> int: + """The server-assigned stream id this adapter delivers.""" + return self._camera.stream_id + + @property + def required(self) -> bool: + """Whether this stream's failure ends the whole session.""" + return self._camera.required + + def _fail(self, message: str) -> CameraSourceError: + """Mark the stream dead and degraded, and return the ``CameraSourceError`` for the caller to raise.""" + self._dead, self._degraded, self._last_error = True, True, message + return CameraSourceError(message, camera_name=self._camera.name) + + def _enter_sync(self) -> None: + """Worker-side startup: wait for the stream file and a first complete, decodable frame. + + Raises a camera error if either does not arrive. + """ + if self._rig.start_error is not None: + # A shared-producer failure degrades exactly the streams that depend on it, each under its own + # required/optional policy in Acquisition.enter. + raise self._fail(f"SENSING producer did not start: {self._rig.start_error}") + deadline = time.monotonic() + self._startup_timeout_s + self._reader = reader = SensingStreamReader( + self._rig.stream_path(self._sensor.pipeline_id), + width=self._sensor.width, + height=self._sensor.height, + full_range=self._full_range, + stop=self._stop, + decoder=self._decoder, + ) + # The file existing is not readiness: attach, then wait for a complete, decodable frame. + while True: + if self._stop.is_set(): + raise self._fail("stopped during startup") + if (code := self._rig.exit_code()) is not None: + raise self._fail(f"SENSING producer exited with {code}: {self._rig.diagnostics()}") + left = deadline - time.monotonic() + if left <= 0: + raise self._fail(f"timed out waiting for the SENSING stream: {self._rig.diagnostics()}") + try: + reader.connect(min(left, 2.0)) + break + except OSError: + self._stop.wait(0.1) # the producer has not created the file yet + except SensingStreamError as e: + if self._stop.is_set(): + raise self._fail("stopped during startup") from e + raise self._fail(f"SENSING stream rejected: {e}") from e + try: + while reader.read_latest(min(max(deadline - time.monotonic(), 0.0), 0.5)) is None: + if (code := self._rig.exit_code()) is not None: + raise self._fail(f"SENSING producer exited with {code}: {self._rig.diagnostics()}") + if time.monotonic() >= deadline: + raise self._fail("no frame from the SENSING stream before the startup deadline") + except SensingStreamError as e: + if self._stop.is_set(): + raise self._fail("stopped during startup") from e + raise self._fail(f"SENSING stream failed at startup: {e}") from e + self._last_frame_s = self._clock.monotonic() + + async def enter(self) -> None: + """Attach and confirm a usable frame; the deadline is set first so an optional failure stays pollable.""" + self._deadline = self._clock.monotonic() + self._interval + await self._runner.run(self._enter_sync) + self._degraded = False + self._last_error = "" + + def _poll_sync(self) -> tuple[int, bytes] | None: + """Worker-side delivery: JPEG-encode the newest decoded frame, or None when nothing new has arrived.""" + if self._reader is None: + return None + if self._dead: + # The shared producer keeps writing this sensor's file; keep handing the bytes back (no decoding). + try: + self._reader.reclaim_all() + except SensingStreamError as e: + # Unreclaimable spool: stop the shared producer (every stream then reports its exit) rather + # than let its output fill memory. + self._rig.abort(str(e)) + return None + try: + if (code := self._rig.exit_code()) is not None: + # The producer ended: flush the decoder once for its final frame, then report the exit. + if not self._flushed: + self._flushed = True + if (last := self._reader.finish()) is not None: + return self._to_jpeg(last) + raise self._fail(f"SENSING producer exited with {code}: {self._rig.diagnostics()}") + got = self._reader.read_latest(0.0) + if got is None: + self._check_stall() + return None + return self._to_jpeg(got) + except (SensingStreamError, SourceError, ValueError, OSError) as e: + raise self._fail(str(e) or type(e).__name__) from e + + def _check_stall(self) -> None: + """Fail the stream if the producer is alive but no decoded frame has arrived for too long. + + The clock starts at the first confirmed frame, so startup keeps its own deadline; a producer that stops + writing without exiting (link loss, an encoder or plugin hang) is otherwise indistinguishable from a + quiet camera, and a required camera would never end the session. + """ + if self._last_frame_s is None: + return + silent = self._clock.monotonic() - self._last_frame_s + if silent > max(_NO_FRAME_TIMEOUT_S, 5.0 / self._sensor.fps): + raise SensingStreamError( + f"no frame from the SENSING stream for {silent:.1f}s: {self._rig.diagnostics()}" + ) + + def _to_jpeg(self, got: tuple[Any, int]) -> tuple[int, bytes]: + """Encode a decoded RGB frame at the station quality and mark the stream live: ``(host_ns, jpeg)``.""" + rgb, selected_ns = got + jpeg = encode_rgb_jpeg(rgb, self._camera.jpeg_quality) + self._last_frame_s = self._clock.monotonic() + self._degraded, self._last_error = False, "" + return selected_ns, jpeg + + async def next_camera_jpeg(self) -> tuple[int, bytes] | None: + """Sleep to the next delivery tick, then deliver the newest frame as ``(host_ns, jpeg)`` or None. + + None means nothing new this tick (or the stream already failed; later ticks stay paced and empty + rather than looping on exceptions). The timestamp is host monotonic time at frame selection. + """ + assert self._deadline is not None, "next_camera_jpeg() called before enter()" + await asyncio.sleep(max(0.0, self._deadline - self._clock.monotonic())) + now = self._clock.monotonic() + while self._deadline <= now: + self._deadline += self._interval + return await self._runner.run(self._poll_sync) + + def camera_status(self) -> p.CameraStatus: + """This stream's honest current status; never CONNECTED before a frame was confirmed.""" + return p.CameraStatus( + name=self._camera.name, + stream_id=self._camera.stream_id, + status=p.DeviceState.DEGRADED if self._degraded else p.DeviceState.CONNECTED, + reason=p.StatusReason.NO_CURRENT_DATA if self._degraded else p.StatusReason.NONE, + error=self._last_error if self._degraded else "", + intrinsics=None, + ) + + async def close(self) -> bool: + """Close the reader on its worker; False (sticky) if release could not be confirmed. + + The stop signal is raised here, synchronously, before the close job queues behind whatever the worker is + doing, so a startup or read in progress ends promptly instead of delaying the release. + """ + self._stop.set() + + def _close() -> None: + """Worker-side release of the reader; a failure propagates to the awaiting caller below.""" + if self._reader is not None: + self._reader.close() + + try: + confirmed = await self._runner.close(_close) + except Exception: + self.cleanup_failed = True + raise + if not confirmed: + self.cleanup_failed = True + return confirmed and not self.cleanup_failed diff --git a/examples/egocap/capture_server/capture_server/acquisition/sensing_h264.py b/examples/egocap/capture_server/capture_server/acquisition/sensing_h264.py new file mode 100644 index 0000000000..f1539e7133 --- /dev/null +++ b/examples/egocap/capture_server/capture_server/acquisition/sensing_h264.py @@ -0,0 +1,551 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Reader for the SENSING plugin's existing H.264 file output, decoded to RGB on the CPU. + +The producer (``camera_plugin_sensing``) writes each sensor's H.264 Annex-B elementary stream to the ``output=`` +path of its ``--add-stream``. This module tails that file with one persistent parser and one persistent software +decoder per sensor: every complete access unit is decoded IN ORDER (ordinary H.264 P-frames reference earlier +frames, so units cannot be skipped before decoding), and only the newest DECODED frame is kept for delivery. The +same code serves the real Jetson encoder and the native mock; neither is special-cased. + +Decoding runs on the CPU (PyAV) or, where the Jetson's NVDEC engine is reachable, on the hardware decoder in +``sensing_nvdec`` (``decoder="auto"`` picks hardware when available and says which one it chose). Parsing, the +overload policy, the newest-frame rule and the YUV->RGB conversion are the same for both. + +Everything is blocking and synchronous: run it on one dedicated worker (see ``ThreadRunner``). The parser and +decoder are PyAV's, created lazily so ordinary OAK-only runs do not load them until a SENSING stream attaches. +""" + +from __future__ import annotations + +import collections +import ctypes +import errno +import functools +import logging +import os +import re +import threading +import time +from typing import Any, Callable + +import numpy as np + +from .sensing_nvdec import HardwareDecodeError, HardwareDecoder, hardware_decoder_unavailable_reason + +__all__ = ["DECODERS", "SensingStreamError", "SensingStreamReader"] + +_log = logging.getLogger("isaaccapture.capture_server.sensing_h264") + +DECODERS = ("auto", "hardware", "software") +# Access-unit bytes the hardware decoder may have queued ahead of it. Beyond this the reader keeps units in its +# own pending queue, where the overload policy (resume at the newest restart point) can act on them. +_HW_MAX_INPUT_BYTES = 4 * 1024 * 1024 + +# Bytes consumed from the file before the already-read prefix is handed back to the filesystem. The producer +# writes at the encoder bitrate for the whole session, so without this the file grows without bound. +_PUNCH_CHUNK_BYTES = 8 * 1024 * 1024 +_READ_BYTES = 4 * 1024 * 1024 +# Work done per poll is capped by bytes (a producer appending faster than we parse cannot hold the worker) and +# by the poll's own deadline; the rest of a backlog is simply read on the next poll. +_MAX_POLL_BYTES = 32 * 1024 * 1024 +_MIN_WORK_S = 0.2 +# Bytes fed to the parser without it completing a single access unit before the stream is declared bad: a +# malformed stream must not make the parser retain input without bound. +_MAX_UNIT_BYTES = 32 * 1024 * 1024 +# Complete but not yet decoded stream bytes the reader may hold. A decoder that falls this far behind resumes at +# the newest restart point (an IDR unit carrying its own SPS/PPS; nothing before it is needed), or ends the stream. +# Sized as latency, not memory: at the encoder's default bitrate this is under two seconds of video, so a stalled +# decoder shows a picture that is at most that stale before it jumps to a restart point. +_MAX_PENDING_BYTES = 8 * 1024 * 1024 +# A failed stream's file may stay unreclaimed (a failing fallocate) only this far before the producer stops. +_MAX_UNRECLAIMED_BYTES = 64 * 1024 * 1024 +# Time allowed to read, decode and flush whatever the producer left behind when it ended. +_FINISH_BUDGET_S = 30.0 +_POLL_INTERVAL_S = 0.005 +_FALLOC_FL_KEEP_SIZE = 0x01 +_FALLOC_FL_PUNCH_HOLE = 0x02 + + +# NAL units read straight from the Annex-B bytes (a start code, then a header byte whose low five bits are the +# type): an IDR slice (5), a sequence parameter set (7) and a picture parameter set (8), for any nal_ref_idc. +_IDR_NAL = re.compile(rb"\x00\x00\x01[\x25\x45\x65]") +_SPS_NAL = re.compile(rb"\x00\x00\x01[\x27\x47\x67]") +_PPS_NAL = re.compile(rb"\x00\x00\x01[\x28\x48\x68]") + + +def _is_restart_point(packet: Any) -> bool: + """Whether decoding can safely begin at this access unit with nothing decoded before it. + + That needs an IDR slice (no earlier frame referenced) AND in-band SPS and PPS in the same unit: an IDR alone + says nothing about whether the decoder has the parameter sets it refers to. Read from the bytes themselves, + never from a parser flag. + """ + data = bytes(packet) + return all(nal.search(data) is not None for nal in (_IDR_NAL, _SPS_NAL, _PPS_NAL)) + + +class SensingStreamError(Exception): + """The producer's stream is unreadable, undecodable, or disagrees with the expected geometry.""" + + +@functools.cache +def _bind_fallocate() -> Callable[[int, int, int, int], int]: + """The C library's ``fallocate`` with explicit argument types, bound once; ``-1`` plus ``errno`` on failure.""" + fallocate = ctypes.CDLL(None, use_errno=True).fallocate + fallocate.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_longlong, ctypes.c_longlong] + return fallocate + + +class SensingStreamReader: + """One sensor's H.264 file: attach, then repeatedly take the newest decoded frame as an RGB array. + + A parse, decode or geometry failure is terminal: later reads raise and a new attempt builds a new reader. + The returned array is owned by the reader and valid until the next read. + """ + + def __init__( + self, + path: str, + *, + width: int, + height: int, + full_range: bool = False, + fallocate: Callable[[int, int, int, int], int] | None = None, + stop: threading.Event | None = None, + decoder: str = "software", + hardware_command: list[str] | None = None, + ) -> None: + """Record the expected geometry and colour range; the file is opened later, in :meth:`connect`. + + ``stop`` is set (from any thread) when the stream is being shut down: every wait and loop here checks it + at least every fraction of a second and ends with :class:`SensingStreamError`, so Stop never queues + behind a long startup, read or final flush. + + ``decoder`` is ``"software"`` (PyAV on the CPU), ``"hardware"`` (Jetson NVDEC; an error if it is not + available) or ``"auto"`` (hardware when available, otherwise software). ``hardware_command`` replaces the + hardware decoder's process for tests. + """ + if decoder not in DECODERS: + raise ValueError(f"decoder must be one of {DECODERS}, got {decoder!r}") + self._decoder_choice = decoder + self._hardware_command = hardware_command + self._hw: HardwareDecoder | None = None + self._use_hardware = False + self._path = path + self._width, self._height = width, height + self._full_range = full_range + self._fd: int | None = None + self._offset = 0 + self._punched = 0 + self._closed = False + self._parser: Any = None + self._decoder: Any = None + self._failed = False + # Complete access units (PyAV packets) waiting to be decoded, in stream order, and their total size. + self._pending: collections.deque[Any] = collections.deque() + self._pending_bytes = 0 + self._newest: Any = None # newest decoded frame not yet delivered + self._finished = False + self._fed_since_packet = 0 + # Injectable so tests can make the native call fail; production binds the C library's once. + self._fallocate = fallocate + self._stop = stop + + def _check_stopped(self) -> None: + """End the current operation, with the reason, if the stream is being shut down.""" + if self._stop is not None and self._stop.is_set(): + raise SensingStreamError("stopped") + + def _wait(self, seconds: float) -> None: + """Sleep for ``seconds``, or less if the stream is shut down meanwhile.""" + if self._stop is not None: + self._stop.wait(seconds) + else: + time.sleep(seconds) + + def connect(self, timeout_s: float) -> None: + """Wait up to ``timeout_s`` for the producer to create the file, then open it. + + Raises ``FileNotFoundError`` (an ``OSError``) if it never appears, and :class:`SensingStreamError` + if it exists but cannot be opened for reading and space reclamation: the stream file is a bounded + spool, so a file we cannot reclaim from is an error, never a quietly unbounded one. + """ + deadline = time.monotonic() + timeout_s + while True: + self._check_stopped() + try: + # Read-write: handing consumed bytes back (hole punching) needs a writable descriptor. + self._fd = os.open(self._path, os.O_RDWR) + break + except FileNotFoundError: + if time.monotonic() >= deadline: + raise + self._wait(_POLL_INTERVAL_S * 4) + except OSError as e: + raise SensingStreamError(f"cannot open {self._path} for reading and reclamation: {e}") from e + if self._fallocate is None: + self._fallocate = _bind_fallocate() + import av # noqa: PLC0415 -- lazy: only a SENSING stream needs the decoder + + self._parser = av.CodecContext.create("h264", "r") + self._use_hardware = self._choose_hardware() + if not self._use_hardware: + self._decoder = av.CodecContext.create("h264", "r") + # One thread: frame threading delays output by several frames, which a live preview should not pay. + self._decoder.thread_count = 1 + + def _choose_hardware(self) -> bool: + """Decide between hardware and software decoding for this stream, and log which one is in use. + + ``"hardware"`` that cannot be honoured is an error, never a quiet downgrade; ``"auto"`` falls back and + says why, so a slow preview is traceable to the decoder in use. + """ + if self._decoder_choice == "software": + _log.info("SENSING stream %s: decoding on the CPU (PyAV software), as configured", self._path) + return False + # An injected command stands in for the whole GStreamer stack, so the machine's own capability is moot. + reason = None if self._hardware_command is not None else hardware_decoder_unavailable_reason() + if reason is None: + _log.info("SENSING stream %s: decoding on the Jetson NVDEC hardware decoder", self._path) + return True + if self._decoder_choice == "hardware": + raise SensingStreamError(f"hardware decoding was requested but is unavailable: {reason}") + _log.warning("SENSING stream %s: falling back to CPU (PyAV software) decoding: %s", self._path, reason) + return False + + def _drain_file(self, deadline: float) -> None: + """Read newly written bytes within this poll's work bounds and queue the complete access units.""" + assert self._fd is not None + # A snapshot of the size, not "until EOF": a producer that keeps appending cannot extend this poll. + end = min(os.fstat(self._fd).st_size, self._offset + _MAX_POLL_BYTES) + while self._offset < end: + chunk = os.pread(self._fd, min(_READ_BYTES, end - self._offset), self._offset) + if not chunk: + break + self._offset += len(chunk) + self._fed_since_packet += len(chunk) + # The parser holds a unit back until it sees the next one begin, so a half-written frame is never + # returned; a temporary end of file on a growing file just means waiting for more. + self._queue(self._parser.parse(chunk)) + if self._fed_since_packet > _MAX_UNIT_BYTES: + raise SensingStreamError(f"no complete access unit in {self._fed_since_packet} bytes of stream") + if time.monotonic() >= deadline: + break # always one chunk of progress, then yield to the caller's deadline + self._reclaim() + + def _queue(self, packets: list[Any]) -> None: + """Append parsed access units for decoding, and bound what is held when the decoder is behind.""" + for packet in packets: + self._pending.append(packet) + self._pending_bytes += packet.size + self._fed_since_packet = 0 + if self._pending_bytes <= _MAX_PENDING_BYTES: + return + # Overloaded: resume at the newest restart point, dropping everything older undecoded. This is the only + # place units are skipped before decoding, and it is safe only where nothing earlier is needed: an IDR + # unit that also carries its SPS and PPS. + newest_key = max((i for i, pk in enumerate(self._pending) if _is_restart_point(pk)), default=None) + if newest_key is None: + raise SensingStreamError( + f"decoder cannot keep up: {self._pending_bytes} bytes undecoded and no restart point to resume at" + ) + for _ in range(newest_key): + self._pending_bytes -= self._pending.popleft().size + if self._pending_bytes > _MAX_PENDING_BYTES: + raise SensingStreamError( + f"decoder cannot keep up: {self._pending_bytes} bytes undecoded even from the newest restart point" + ) + + def _decode_pending(self, deadline: float) -> None: + """Decode queued units in order until none are left or ``deadline`` passes (one unit of progress at least). + + A decode that returns nothing is the decoder buffering, not an error; every returned frame is checked + and the newest one is kept. With the hardware decoder, "decoding" means handing units to its process, + which holds back at most a few MiB before this leaves the rest queued for the overload policy. + """ + if self._use_hardware: + self._feed_hardware(deadline) + return + while self._pending: + self._check_stopped() + packet = self._pending.popleft() + self._pending_bytes -= packet.size + for frame in self._decoder.decode(packet): + self._newest = self._checked(frame) + if time.monotonic() >= deadline: + break + + def _start_hardware(self) -> HardwareDecoder: + """Start the hardware decoder once the parser knows the stream's size, and check it against the station's. + + The geometry is read from the parsed stream, not from decoded frames, because the hardware decoder's raw + output carries no header: a mismatch would otherwise be read as garbage pixels. + """ + if (self._parser.width, self._parser.height) != (self._width, self._height): + raise SensingStreamError( + f"decoded {self._parser.width}x{self._parser.height}, expected {self._width}x{self._height}" + ) + hw = HardwareDecoder(self._width, self._height, command=self._hardware_command) + hw.start() + self._hw = hw + return hw + + def _feed_hardware(self, deadline: float) -> None: + """Hand queued access units to the hardware decoder, in order, until it is saturated or ``deadline``. + + Writes never block: when the decoder's input is full, units stay in ``_pending`` and are tried again on + the next poll. A failed or exited decoder surfaces here as :class:`SensingStreamError`. + """ + if not self._pending and self._hw is None: + return + hw = self._hw or self._start_hardware() + try: + hw.pump() + while self._pending and hw.pending_input_bytes < _HW_MAX_INPUT_BYTES: + self._check_stopped() + packet = self._pending.popleft() + self._pending_bytes -= packet.size + hw.write(bytes(packet)) + if time.monotonic() >= deadline: + break + hw.check() + if self._pending: + # Saturated: give the decoder a moment instead of letting a caller's loop spin on this method. + self._wait(_POLL_INTERVAL_S) + except HardwareDecodeError as e: + raise SensingStreamError(str(e)) from e + + def _checked(self, frame: Any) -> Any: + """The decoded frame, if it has the geometry the station and the producer agreed on.""" + if (frame.width, frame.height) != (self._width, self._height): + raise SensingStreamError( + f"decoded {frame.width}x{frame.height}, expected {self._width}x{self._height}" + ) + return frame + + def _has_frame(self) -> bool: + """Whether a decoded frame is waiting to be delivered.""" + if self._use_hardware: + return self._hw is not None and self._hw.has_frame + return self._newest is not None + + def _deliver(self) -> tuple[np.ndarray, int] | None: + """Convert the newest undelivered decoded frame to RGB, or None if there is none.""" + if self._use_hardware: + frame = self._take_hardware_frame() + else: + frame, self._newest = self._newest, None + if frame is None: + return None + selected_ns = time.monotonic_ns() + # The producer converts with BT.601; the range follows its --full-range flag. Stated explicitly so + # colours never depend on a decoder's guess for an unspecified stream. + rgb = frame.reformat( + format="rgb24", src_colorspace="ITU601", src_color_range="JPEG" if self._full_range else "MPEG" + ).to_ndarray() + return rgb, selected_ns + + def _take_hardware_frame(self) -> Any: + """The hardware decoder's newest frame as a PyAV ``yuv420p`` frame, or None. + + Building the PyAV frame copies the pixels, so the decoder's buffer is released at once and its output + thread can go on; the conversion to RGB then goes through the same call as the software path. + """ + import av # noqa: PLC0415 -- already loaded by connect(); named here for the call below + + if self._hw is None: + return None + yuv = self._hw.take_frame() + if yuv is None: + return None + try: + return av.VideoFrame.from_ndarray(yuv, format="yuv420p") + finally: + self._hw.release_frame() + + def _reclaim(self) -> None: + """Hand the consumed prefix back to the filesystem once enough has accumulated; failure is an error. + + Space reclamation is what keeps a long session bounded, so an unsupported or failing call ends the + stream with the reason instead of letting the file grow silently; the reclaimed offset advances only + after the call succeeds. + """ + pending = self._offset - self._punched + if pending < _PUNCH_CHUNK_BYTES: + return + assert self._fd is not None and self._fallocate is not None + ctypes.set_errno(0) + if self._fallocate(self._fd, _FALLOC_FL_PUNCH_HOLE | _FALLOC_FL_KEEP_SIZE, self._punched, pending) != 0: + code = ctypes.get_errno() or errno.EIO + raise SensingStreamError(f"cannot reclaim consumed stream bytes: {os.strerror(code)} (errno {code})") + self._punched = self._offset + + def read_latest(self, timeout_s: float) -> tuple[np.ndarray, int] | None: + """Wait up to ``timeout_s`` for a decoded frame, then return ``(rgb, selected_ns)`` of the newest one. + + Every complete unit is decoded in order; frames decoded but not yet delivered are dropped in favour of + the newest. ``selected_ns`` is host monotonic time when that frame is selected (after decoding; the + protocol's HOST_POLL semantics). Returns None when no frame was produced in time; never fabricates one. + Any failure makes the reader terminal and is raised as :class:`SensingStreamError`. + """ + if self._failed: + raise SensingStreamError("reader failed earlier; a new connection is required") + if self._finished: + raise SensingStreamError("stream already finished; the decoder has been flushed") + if self._fd is None: + raise SensingStreamError("reader is not connected") + deadline = time.monotonic() + max(timeout_s, 0.0) + try: + while True: + self._check_stopped() + # Reading may use a little longer than a zero-timeout poll so it can still make progress. + work_deadline = max(deadline, time.monotonic() + _MIN_WORK_S) + self._drain_file(work_deadline) + self._decode_pending(work_deadline) + if self._has_frame(): + return self._deliver() + if time.monotonic() >= deadline: + return None + self._wait(_POLL_INTERVAL_S) + except SensingStreamError: + self._fail() + raise + except Exception as e: # noqa: BLE001 -- parser/decoder/OS errors all end this stream the same way + self._fail() + raise SensingStreamError(f"{type(e).__name__}: {e}") from e + + def _check_finish_budget(self, deadline: float) -> None: + """Fail the stream, with the reason, if finishing was cancelled or has used up its time budget.""" + self._check_stopped() + if time.monotonic() >= deadline: + raise SensingStreamError( + f"could not finish decoding the remaining stream within {_FINISH_BUDGET_S:g} s" + ) + + def finish(self) -> tuple[np.ndarray, int] | None: + """Flush the parser and decoder once and return the final frame; call only after the producer has ended. + + A temporary end of file on a growing file is not the end of the stream, so this is never called for it. + Reads whatever the producer left in the file first, then drains the parser and the decoder's delayed + output. Returns None if no frame remains. Terminal: the reader cannot be read again. + """ + if self._failed or self._fd is None or self._finished: + return None + self._finished = True + try: + far = time.monotonic() + _FINISH_BUDGET_S + # Everything the producer left behind is read and decoded, in order, before the flush: flushing with + # units still pending would drop or reorder them. The budget bounds the whole sequence. + while self._offset < os.fstat(self._fd).st_size or self._pending: + self._check_finish_budget(far) + self._drain_file(far) + self._decode_pending(far) + self._queue(self._parser.parse(None)) # flush the parser's held-back last unit + while self._pending: + self._check_finish_budget(far) + self._decode_pending(far) + self._flush_decoder(far) # the decoder's delayed output + return self._deliver() + except SensingStreamError: + self._fail() + raise + except Exception as e: # noqa: BLE001 -- parser/decoder/OS errors all end this stream the same way + self._fail() + raise SensingStreamError(f"{type(e).__name__}: {e}") from e + + def _flush_decoder(self, far: float) -> None: + """Drain the decoder's delayed output into the newest-frame slot, within the finish budget ``far``. + + The hardware decoder is flushed by closing its input: it then emits what it still holds and exits, and + the output thread ends. A decoder that never saw a unit has nothing to flush. + """ + if not self._use_hardware: + for frame in self._decoder.decode(None): + self._newest = self._checked(frame) + return + hw = self._hw + if hw is None: + return + try: + # Input still queued must be accepted before the end of stream can be signalled. + while not hw.close_input(): + self._check_finish_budget(far) + self._wait(_POLL_INTERVAL_S) + while not hw.wait_finished(_POLL_INTERVAL_S): + self._check_finish_budget(far) + hw.check() + except HardwareDecodeError as e: + raise SensingStreamError(str(e)) from e + + def _fail(self) -> None: + """Make the reader terminal. The descriptor stays open so :meth:`reclaim_all` can keep reclaiming.""" + self._failed = True + self._pending.clear() + self._pending_bytes = 0 + self._stop_hardware() + + def _stop_hardware(self) -> None: + """End the hardware decoder's process, if one is running; it is never needed again once the reader ends.""" + hw, self._hw = self._hw, None + if hw is not None: + hw.close() + + def reclaim_all(self) -> None: + """For a stream that has failed: keep handing the file's bytes back to the filesystem, without decoding. + + The shared producer keeps appending to every sensor's file for the whole session, so a failed stream's + file would otherwise grow in memory (tmpfs) until Stop. A file the producer created after this stream + failed (before it ever attached) is opened here. A single failing call is simply tried again on the next + tick, but the spool is bounded: once more than ``_MAX_UNRECLAIMED_BYTES`` cannot be handed back, this + raises :class:`SensingStreamError` so the caller can stop the producer rather than let it fill memory. + """ + self._failed = True + if self._fd is None and not self._closed: + try: + self._fd = os.open(self._path, os.O_RDWR) + except FileNotFoundError: + return # the producer has not created the file yet; nothing is accumulating + except OSError: + self._check_unreclaimed(self._path_size(), "cannot open it") + return + if self._fd is None: + return + if self._fallocate is None: + self._fallocate = _bind_fallocate() + size = os.fstat(self._fd).st_size + if size <= self._punched: + return + ctypes.set_errno(0) + mode = _FALLOC_FL_PUNCH_HOLE | _FALLOC_FL_KEEP_SIZE + try: + if self._fallocate(self._fd, mode, self._punched, size - self._punched) == 0: + self._punched = size + return + reason = os.strerror(ctypes.get_errno()) or "fallocate failed" + except OSError as e: + reason = str(e) + self._check_unreclaimed(size - self._punched, reason) + + def _path_size(self) -> int: + try: + return os.stat(self._path).st_size + except OSError: + return 0 + + def _check_unreclaimed(self, unreclaimed: int, reason: str) -> None: + if unreclaimed > _MAX_UNRECLAIMED_BYTES: + raise SensingStreamError( + f"{self._path} holds {unreclaimed >> 20} MiB that cannot be handed back ({reason}); " + "the spool is unbounded" + ) + + def close(self) -> None: + """Close the file and the hardware decoder process; the producer owns the file itself.""" + self._closed = True + self._stop_hardware() + fd, self._fd = self._fd, None + if fd is not None: + os.close(fd) diff --git a/examples/egocap/capture_server/capture_server/acquisition/sensing_jpeg.py b/examples/egocap/capture_server/capture_server/acquisition/sensing_jpeg.py new file mode 100644 index 0000000000..11a3c2a6b9 --- /dev/null +++ b/examples/egocap/capture_server/capture_server/acquisition/sensing_jpeg.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""JPEG encoding for SENSING frames: owned RGB host pixels in, one complete JPEG out. + +The wire protocol carries only independently decodable JPEG images with a hard 2 MiB payload ceiling. +A frame that does not fit is a configuration/source error -- never silently resized, re-quantized or +replaced, so the diagnostic names the dimensions and quality that produced it. + +OpenCV encodes here rather than Pillow: at 5 MP it is about 40% faster, and it releases the GIL for the whole +encode, which keeps the other streams' decoder and file-reading threads running while one frame is compressed. +""" + +from __future__ import annotations + +import numpy as np + +from .errors import SourceError + +__all__ = ["MAX_JPEG_BYTES", "encode_rgb_jpeg"] + +MAX_JPEG_BYTES = 2 * 1024 * 1024 + + +def encode_rgb_jpeg(rgb: np.ndarray, quality: int, *, max_bytes: int = MAX_JPEG_BYTES) -> bytes: + """Encode an ``(H, W, 3)`` uint8 RGB frame as a complete JPEG at ``quality``; returns owned bytes.""" + if rgb.ndim != 3 or rgb.shape[2] != 3 or rgb.dtype != np.uint8: + raise ValueError(f"expected (H, W, 3) uint8 RGB pixels, got {rgb.shape} {rgb.dtype}") + import cv2 # noqa: PLC0415 -- lazy: runs without a SENSING stream never load OpenCV + + height, width = rgb.shape[:2] + # OpenCV's encoder takes BGR; the conversion makes a new array, so nothing here keeps a reference to the + # caller's buffer and the caller may reuse it as soon as this returns. + encoded, buffer = cv2.imencode( + ".jpg", cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR), [cv2.IMWRITE_JPEG_QUALITY, quality] + ) + if not encoded: + raise SourceError(f"encoder failed for {width}x{height} at quality {quality}") + jpeg = buffer.tobytes() + if not (jpeg.startswith(b"\xff\xd8") and jpeg.endswith(b"\xff\xd9")): + raise SourceError(f"encoder produced an incomplete JPEG for {width}x{height} at quality {quality}") + if len(jpeg) > max_bytes: + raise SourceError( + f"{width}x{height} JPEG at quality {quality} is {len(jpeg)} B, over the {max_bytes} B payload ceiling" + ) + return jpeg diff --git a/examples/egocap/capture_server/capture_server/acquisition/sensing_nvdec.py b/examples/egocap/capture_server/capture_server/acquisition/sensing_nvdec.py new file mode 100644 index 0000000000..216d618cf5 --- /dev/null +++ b/examples/egocap/capture_server/capture_server/acquisition/sensing_nvdec.py @@ -0,0 +1,366 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Hardware H.264 decoding for the SENSING reader: Jetson NVDEC driven through a GStreamer subprocess. + +A 5 MP stream at 60 fps is far beyond one CPU core of software decoding, so the reader hands the access units to +``nvv4l2decoder`` instead and takes planar YUV frames back. It is a subprocess rather than a library binding +because the Jetson decoder is only reachable through NVIDIA's GStreamer/V4L2 stack, which neither the PyAV wheel +(its ``h264_v4l2m2m`` cannot open the NVDEC node) nor the system ffmpeg wrapper serve well: the ffmpeg wrapper +holds about 0.6 s of frames internally, while ``nvv4l2decoder disable-dpb=true`` delivers each frame in tens of +milliseconds. ``disable-dpb`` is correct only for streams without frame reordering; the producer encodes with no +B-frames by construction (``setNumBFrames(0)`` in the Jetson encoder, mirrored by the mock encoder). + +Everything here is thread-based and bounded: three frame buffers, a capped input queue, and a close that returns +within a second. The reader owns when frames are taken, so "newest decoded frame wins" holds exactly as it does +for the software path. +""" + +from __future__ import annotations + +import collections +import fcntl +import functools +import logging +import os +import shutil +import subprocess +import threading +from collections.abc import Sequence + +import numpy as np + +__all__ = ["HardwareDecodeError", "HardwareDecoder", "hardware_decoder_unavailable_reason", "pipeline_command"] + +_log = logging.getLogger("isaaccapture.capture_server.sensing_nvdec") + +_DEVICE_NODE = "/dev/v4l2-nvdec" +_GST_LAUNCH = "gst-launch-1.0" +_GST_INSPECT = "gst-inspect-1.0" +_DECODER_ELEMENT = "nvv4l2decoder" +_PROBE_TIMEOUT_S = 10.0 +# Bytes the decoder process may have queued on its input before the caller is asked to wait. The pipe itself +# holds up to _STDIN_PIPE_BYTES; the rest waits in this process, where the reader's own overload policy sees it. +_STDIN_PIPE_BYTES = 1 << 20 # the largest size an unprivileged process may request (/proc/sys/fs/pipe-max-size) +_WRITE_CHUNK_BYTES = 1 << 20 +# Shutdown bounds: the whole SENSING chain must release inside the server's 5 s budget (see sensing.py). +_CLOSE_GRACE_S = 0.4 +_JOIN_S = 0.3 +_STDERR_TAIL_BYTES = 4096 +# One frame being filled, one waiting to be taken, one held by the consumer while it converts it. +_FRAME_BUFFERS = 3 +_F_SETPIPE_SZ = 1031 # Linux fcntl command; not exposed by the fcntl module on every Python + + +class HardwareDecodeError(Exception): + """The hardware decoder process failed, exited early, or produced output that does not fit the geometry.""" + + +@functools.cache +def hardware_decoder_unavailable_reason() -> str | None: + """Why NVDEC decoding cannot be used on this machine, or ``None`` when it can; cached for the process. + + Checks the decoder device node, the GStreamer tools, and that the ``nvv4l2decoder`` element is installed, + so "auto" falls back to software on a desktop or a Jetson without the NVIDIA GStreamer plugins. + """ + # The node exists only on Jetson platforms with an NVDEC engine. + if not os.path.exists(_DEVICE_NODE): + return f"{_DEVICE_NODE} is missing (this is not a Jetson with NVDEC)" + for tool in (_GST_LAUNCH, _GST_INSPECT): + if shutil.which(tool) is None: + return f"{tool} is not installed" + try: + probe = subprocess.run( # noqa: S603 -- fixed argument list, no shell + [_GST_INSPECT, "--exists", _DECODER_ELEMENT], + capture_output=True, + timeout=_PROBE_TIMEOUT_S, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as e: + return f"could not query GStreamer for {_DECODER_ELEMENT}: {e}" + if probe.returncode != 0: + return f"the GStreamer element {_DECODER_ELEMENT} is not installed" + return None + + +def pipeline_command(*, low_delay: bool = True) -> list[str]: + """The ``gst-launch-1.0`` argument list: Annex-B on stdin, tightly packed I420 frames on stdout. + + ``nvvidconv`` only converts NV12 to planar I420 (no scaling and no colour-matrix change), so the reader's own + explicit BT.601 conversion still decides the colours. ``low_delay`` turns off the decoder's reorder buffer: + required for a live preview, valid only for streams without B-frames. + + The one-buffer leaky queue ahead of the converter is what keeps the decoder at full speed. Reading a 7.6 MB + frame through a pipe takes many GIL acquisitions, and when other Python threads (JPEG encoding) hold the GIL + the reads fall behind; without the queue that backpressure stops the decoder and the input backs up + (measured: 11 fps decode against 124 fps with the queue). With it, frames the reader is too slow to take are + dropped inside GStreamer before the costly copy, so the reader always sees the newest one. + """ + decoder = [_DECODER_ELEMENT, "enable-max-performance=1"] + if low_delay: + decoder.append("disable-dpb=true") + return [ + _GST_LAUNCH, + "-q", + "fdsrc", + "fd=0", + "blocksize=4194304", + "!", + "h264parse", + "!", + *decoder, + "!", + "queue", + "max-size-buffers=1", + "max-size-bytes=0", + "max-size-time=0", + "leaky=downstream", + "!", + "nvvidconv", + "!", + "video/x-raw,format=I420", + "!", + "fdsink", + "fd=1", + "sync=false", + ] + + +class HardwareDecoder: + """One decoder process: feed Annex-B access units in, take the newest decoded I420 frame out. + + Frames are ``(height * 3 // 2, width)`` uint8 arrays (planar Y, U, V), the layout PyAV's ``yuv420p`` frames + are built from. Not thread-safe for its own callers: ``start``/``write``/``pump``/``close_input``/``close`` + belong to the reader's single worker; only the internal output and stderr threads run beside it. + """ + + def __init__(self, width: int, height: int, *, command: Sequence[str] | None = None) -> None: + """Record the geometry; the process starts in :meth:`start`. + + ``command`` replaces the GStreamer pipeline (tests inject a stand-in that follows the same pipe contract). + """ + if width <= 0 or height <= 0 or width % 2 or height % 2: + raise ValueError(f"I420 needs positive even dimensions, got {width}x{height}") + self._width, self._height = width, height + self._frame_bytes = width * height * 3 // 2 + self._command = list(command) if command is not None else pipeline_command() + self._proc: subprocess.Popen[bytes] | None = None + self._buffers = [np.empty((height * 3 // 2, width), dtype=np.uint8) for _ in range(_FRAME_BUFFERS)] + self._lock = threading.Condition() + self._ready: int | None = None # newest complete frame not yet taken + self._held: int | None = None # frame the consumer is converting + self._frames_seen = 0 + self._reader_done = threading.Event() + self._reader_error: Exception | None = None + self._input: collections.deque[bytes] = collections.deque() + self._input_offset = 0 # bytes of _input[0] already written + self._input_bytes = 0 + self._input_closed = False + self._closed = False + self._stderr_tail = bytearray() + self._threads: list[threading.Thread] = [] + + @property + def frames_received(self) -> int: + """Frames that reached this process, including ones replaced by a newer frame before being taken. + + The decoder may decode more: its leaky queue drops frames this process is too slow to read. + """ + with self._lock: + return self._frames_seen + + @property + def pending_input_bytes(self) -> int: + """Bytes handed to :meth:`write` that the decoder process has not yet accepted.""" + return self._input_bytes + + def start(self) -> None: + """Launch the decoder process and its output and diagnostics threads.""" + self._proc = proc = subprocess.Popen( # noqa: S603 -- fixed argument list, no shell + self._command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + close_fds=True, + ) + assert proc.stdin is not None and proc.stdout is not None and proc.stderr is not None + # Writes must never block the reader's worker: a stalled decoder would otherwise outlast Stop. A larger + # pipe lets a whole IDR access unit go in one call; failing to resize it only costs more small writes. + os.set_blocking(proc.stdin.fileno(), False) + try: + fcntl.fcntl(proc.stdin.fileno(), _F_SETPIPE_SZ, _STDIN_PIPE_BYTES) + except OSError: + _log.debug("could not enlarge the decoder input pipe", exc_info=True) + self._threads = [ + threading.Thread(target=self._read_frames, name="nvdec-frames", daemon=True), + threading.Thread(target=self._drain_stderr, name="nvdec-stderr", daemon=True), + ] + for thread in self._threads: + thread.start() + + def write(self, data: bytes) -> None: + """Queue one access unit for the decoder and push as much input as the pipe accepts right now.""" + if self._input_closed or self._closed: + raise HardwareDecodeError("the decoder input is already closed") + self._input.append(data) + self._input_bytes += len(data) + self.pump() + + def pump(self) -> None: + """Push queued input into the decoder's pipe without blocking; what does not fit stays queued.""" + proc = self._proc + if proc is None or proc.stdin is None or self._input_closed: + return + fd = proc.stdin.fileno() + while self._input: + head = memoryview(self._input[0])[self._input_offset :] + try: + written = os.write(fd, head[:_WRITE_CHUNK_BYTES]) + except BlockingIOError: + return # the decoder has not consumed what it already has + except BrokenPipeError as e: + raise HardwareDecodeError( + f"the decoder closed its input unexpectedly; {self._diagnostics()}" + ) from e + self._input_bytes -= written + self._input_offset += written + if self._input_offset >= len(self._input[0]): + self._input.popleft() + self._input_offset = 0 + + def close_input(self) -> bool: + """Signal end of stream once all queued input is written; ``False`` while some is still queued. + + Closing the input makes the decoder flush its remaining frames and exit, which ends :meth:`wait_finished`. + """ + self.pump() + if self._input: + return False + proc = self._proc + if proc is not None and proc.stdin is not None and not self._input_closed: + self._input_closed = True + proc.stdin.close() + return True + + def take_frame(self) -> np.ndarray | None: + """The newest complete frame, or ``None``; the array stays valid until :meth:`release_frame`. + + Taking marks the frame as held so the output thread cannot overwrite it while it is converted; every + successful take must be followed by :meth:`release_frame`. + """ + with self._lock: + if self._ready is None: + return None + if self._held is not None: + raise RuntimeError("release_frame() must be called before taking another frame") + self._held, self._ready = self._ready, None + return self._buffers[self._held] + + def release_frame(self) -> None: + """Let the output thread reuse the frame returned by the last :meth:`take_frame`.""" + with self._lock: + self._held = None + + @property + def has_frame(self) -> bool: + """Whether a complete frame is waiting to be taken.""" + with self._lock: + return self._ready is not None + + def wait_finished(self, timeout_s: float) -> bool: + """Wait for the decoder to end its output after :meth:`close_input`; ``False`` if it did not in time.""" + return self._reader_done.wait(timeout_s) + + def check(self) -> None: + """Raise :class:`HardwareDecodeError` if the decoder failed or exited before its input was closed.""" + if self._reader_error is not None: + raise HardwareDecodeError(f"{self._reader_error}; {self._diagnostics()}") from self._reader_error + # The output ending while input is still open means the process died; a closed input ends it on purpose. + if self._reader_done.is_set() and not self._input_closed: + code = self._proc.poll() if self._proc is not None else None + raise HardwareDecodeError(f"the decoder exited unexpectedly (code {code}); {self._diagnostics()}") + + def close(self) -> None: + """Stop the process and its threads within a fraction of a second; safe to call more than once.""" + if self._closed: + return + self._closed = True + proc = self._proc + if proc is None: + return + # Closing the input lets a healthy decoder exit by itself; the signals cover a stalled one. Both waits are + # short because Stop has to finish inside the server's release budget. + try: + if proc.stdin is not None and not proc.stdin.closed: + proc.stdin.close() + except OSError: + pass + if proc.poll() is None: + proc.terminate() + try: + proc.wait(_CLOSE_GRACE_S) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + for thread in self._threads: + thread.join(_JOIN_S) + for stream in (proc.stdout, proc.stderr): + if stream is not None: + stream.close() + + def _claim_free_buffer(self) -> int: + """The index of a buffer neither waiting to be taken nor held; one always exists with three buffers.""" + with self._lock: + return next(i for i in range(_FRAME_BUFFERS) if i not in (self._ready, self._held)) + + def _read_frames(self) -> None: + """Output thread: read whole frames from the process, keeping only the newest one available.""" + proc = self._proc + assert proc is not None and proc.stdout is not None + try: + while True: + index = self._claim_free_buffer() + flat = memoryview(self._buffers[index]).cast("B") + got = 0 + # A pipe read may return less than a frame; the frame is complete only when every byte is here. + while got < self._frame_bytes: + count = proc.stdout.readinto(flat[got:]) + if not count: + break + got += count + if got == 0: + return # clean end of stream + if got < self._frame_bytes: + raise HardwareDecodeError( + f"decoder output ended mid-frame ({got} of {self._frame_bytes} bytes; " + f"expected {self._width}x{self._height} I420)" + ) + with self._lock: + # A frame the consumer never took is replaced: the preview wants the newest, not every frame. + self._ready = index + self._frames_seen += 1 + self._lock.notify_all() + except Exception as e: # noqa: BLE001 -- any read failure ends this decoder; check() reports it + self._reader_error = e + finally: + self._reader_done.set() + with self._lock: + self._lock.notify_all() + + def _drain_stderr(self) -> None: + """Diagnostics thread: keep the last few KiB of the process's error output for failure messages.""" + proc = self._proc + assert proc is not None and proc.stderr is not None + try: + while chunk := proc.stderr.read(1024): + self._stderr_tail += chunk + del self._stderr_tail[:-_STDERR_TAIL_BYTES] + except (OSError, ValueError): + pass # the stream was closed under us by close(); nothing more to collect + + def _diagnostics(self) -> str: + """The decoder's most recent error output, for the message of a failure.""" + text = bytes(self._stderr_tail).decode("utf-8", errors="replace").strip() + return f"decoder output: {text[-500:]!r}" if text else "the decoder printed no error output" diff --git a/examples/egocap/capture_server/capture_server/client.py b/examples/egocap/capture_server/capture_server/client.py index 9776eb38ce..267852433c 100644 --- a/examples/egocap/capture_server/capture_server/client.py +++ b/examples/egocap/capture_server/capture_server/client.py @@ -305,7 +305,7 @@ async def _invoke(callback: Callback[T] | None, value: T) -> None: @dataclass class _StreamRegistry: """Caches SessionInit's stream descriptors for decoding binary - messages -- a tracking stream (ID 0) and up to three camera streams.""" + messages -- a tracking stream (ID 0) and up to four camera streams.""" tracking: p.TrackingStream | None = None cameras: dict[int, p.CameraStream] = dataclass_field(default_factory=dict) diff --git a/examples/egocap/capture_server/capture_server/launch.py b/examples/egocap/capture_server/capture_server/launch.py index 626ebe06be..95b300e2f7 100644 --- a/examples/egocap/capture_server/capture_server/launch.py +++ b/examples/egocap/capture_server/capture_server/launch.py @@ -30,7 +30,15 @@ from typing import TYPE_CHECKING, Any, Callable from . import protocol as p -from .acquisition import Acquisition, DepthAIAdapter, IsaacCaptureAdapter, TrackingSelection +from .acquisition import ( + Acquisition, + DepthAIAdapter, + IsaacCaptureAdapter, + SensingAdapter, + SensingRig, + TrackingSelection, +) +from .acquisition.sensing import SensingStationCamera, resolve_sensors from .server import ServerConfig, StationCamera if TYPE_CHECKING: @@ -86,15 +94,51 @@ def _tracking_adapter( ) +def _sensing_composition( + config: ServerConfig | None, cameras: tuple[p.CameraStreamConfig, ...] +) -> tuple[SensingRig | None, dict[int, SensingAdapter]]: + """The shared SENSING rig and one adapter per SENSING-station camera, keyed by stream id. + + Identical for mock and real: a SENSING camera always runs the native producer (its MOCK build on + desktop) writing H.264 files this process tails. ``(None, {})`` when no requested camera is a SENSING + station camera, so such sessions never construct a rig or load the decoder. + """ + stations = (config.cameras if config else None) or {} + wanted = {c.name: stations[c.name] for c in cameras if isinstance(stations.get(c.name), SensingStationCamera)} + if not wanted: + return None, {} + if config is None or config.sensing_rig is None or config.sensing_description is None: + # Reached only if this factory was built from a different config object than the server's, so the + # server's inspection result never arrived: say so instead of rejecting with an empty message. + raise ValueError("SENSING cameras are requested but the factory has no rig or sensor description") + sensors = resolve_sensors(config.sensing_description, wanted) + rig = SensingRig(config.sensing_rig, tuple(sensors[name].pipeline_id for name in wanted)) + adapters = { + c.stream_id: SensingAdapter( + rig=rig, + sensor=sensors[c.name], + camera=c, + full_range=config.sensing_rig.full_range, + decoder=config.sensing_rig.decoder, + ) + for c in cameras + if c.name in wanted + } + return rig, adapters + + def build_mock_source_factory( *, scenario: str, seed: int, ic_faults: set[str] | frozenset[str] = frozenset(), camera_faults: set[str] | frozenset[str] = frozenset(), + config: ServerConfig | None = None, ) -> SourceFactory: """The default ``CaptureServer`` source factory, wired to the two mock - SDK files. ``ic_faults``/``camera_faults`` are captured here, once, + SDK files for tracking and OAK cameras (SENSING cameras always run the + real plugin executable, its mock backend on a desktop). ``ic_faults``/``camera_faults`` are + captured here, once, from CLI/test configuration -- "[e]xisting documented local CLI controls may delegate to the two mocks without becoming protocol fields" -- and baked into each new mock session/pipeline this @@ -112,8 +156,11 @@ def source_factory(begin: p.BeginRequest, cameras: tuple[p.CameraStreamConfig, . ), ) + rig, sensing_adapters = _sensing_composition(config, cameras) camera_adapters = tuple( - DepthAIAdapter( + sensing_adapters[camera.stream_id] + if camera.stream_id in sensing_adapters + else DepthAIAdapter( sdk=mock_depthai, pipeline_factory=partial( mock_depthai.Pipeline, @@ -125,7 +172,7 @@ def source_factory(begin: p.BeginRequest, cameras: tuple[p.CameraStreamConfig, . ) for camera in cameras ) - return Acquisition(tracking=tracking, cameras=camera_adapters) + return Acquisition(tracking=tracking, cameras=camera_adapters, sensing_rig=rig) # Lets CaptureServer reject a launch-built factory that disagrees with ServerConfig.mock; # untagged custom factories are not checked. @@ -135,11 +182,15 @@ def source_factory(begin: p.BeginRequest, cameras: tuple[p.CameraStreamConfig, . def _real_camera_adapters( config: ServerConfig, cameras: tuple[p.CameraStreamConfig, ...] -) -> tuple[DepthAIAdapter, ...]: - """One real OAK-D adapter per requested camera, each bound to its station-configured physical device.""" +) -> tuple[tuple[DepthAIAdapter | SensingAdapter, ...], SensingRig | None]: + """One adapter per requested camera: real OAK-D by station device ID, or SENSING by module name. + + Also returns the SENSING rig when any camera uses it. The OAK SDK is imported only if an OAK camera + is requested. + """ if not cameras: - return () - # Every camera needs a station device: report that configuration error before importing the optional SDK. + return (), None + # Every camera needs a station entry: report that configuration error before importing optional SDKs. stations = {} for camera in cameras: station = (config.cameras or {}).get(camera.name) @@ -148,17 +199,26 @@ def _real_camera_adapters( f"camera {camera.name!r} has no station device; configure it with --camera NAME=DEVICE_ID" ) stations[camera.name] = station - import depthai as dai - - return tuple( - DepthAIAdapter( - sdk=dai, - pipeline_factory=partial( - _open_pipeline, dai, stations[camera.name].device_id, stations[camera.name].max_usb_speed - ), - camera=camera, - ) - for camera in cameras + rig, sensing_adapters = _sensing_composition(config, cameras) + oak = [c for c in cameras if c.stream_id not in sensing_adapters] + dai = None + if oak: + import depthai as dai + + return ( + tuple( + sensing_adapters[camera.stream_id] + if camera.stream_id in sensing_adapters + else DepthAIAdapter( + sdk=dai, + pipeline_factory=partial( + _open_pipeline, dai, stations[camera.name].device_id, stations[camera.name].max_usb_speed + ), + camera=camera, + ) + for camera in cameras + ), + rig, ) @@ -184,7 +244,8 @@ def source_factory(begin: p.BeginRequest, cameras: tuple[p.CameraStreamConfig, . sdk = load_isaac_capture() if begin.tracking.sources else None tracking = _tracking_adapter(begin, sdk=sdk, session_factory=sdk.TeleopSession) if sdk else None - return Acquisition(tracking=tracking, cameras=_real_camera_adapters(config, cameras)) + camera_adapters, rig = _real_camera_adapters(config, cameras) + return Acquisition(tracking=tracking, cameras=camera_adapters, sensing_rig=rig) source_factory.mock = False # type: ignore[attr-defined] return source_factory @@ -368,5 +429,5 @@ def build_source_factory( if scenario not in SCENARIOS: raise ValueError(f"unknown mock scenario {scenario!r}; choose one of {', '.join(SCENARIOS)}") return build_mock_source_factory( - scenario=scenario, seed=seed, ic_faults=set(faults), camera_faults=set(faults) + scenario=scenario, seed=seed, ic_faults=set(faults), camera_faults=set(faults), config=config ) diff --git a/examples/egocap/capture_server/capture_server/protocol.py b/examples/egocap/capture_server/capture_server/protocol.py index 04ef35bd86..d78f9f84b5 100644 --- a/examples/egocap/capture_server/capture_server/protocol.py +++ b/examples/egocap/capture_server/capture_server/protocol.py @@ -135,6 +135,7 @@ class WireDType(IntEnum): class CameraType(IntEnum): OAK_D_COLOR = 1 + SENSING_COLOR = 2 class ImageCodec(IntEnum): @@ -701,7 +702,7 @@ def tracking_stream_descriptor( # --------------------------------------------------------------------------- -# Camera stream descriptor (stream IDs 1-3). +# Camera stream descriptor (stream IDs 1-4). # --------------------------------------------------------------------------- @@ -745,7 +746,7 @@ class CameraStreamConfig: @dataclass(frozen=True) class CameraStream: - """One camera descriptor (stream_id 1-3) of SessionInit's ``streams`` + """One camera descriptor (stream_id 1-4) of SessionInit's ``streams`` array.""" stream_id: int @@ -798,8 +799,8 @@ def from_json(cls, data: Any) -> CameraStream: "a camera stream descriptor", ) stream_id = data.get("stream_id") - if type(stream_id) is not int or not (1 <= stream_id <= 3): - raise ProtocolError("camera stream descriptor stream_id must be an integer 1-3") + if type(stream_id) is not int or not (1 <= stream_id <= 4): + raise ProtocolError("camera stream descriptor stream_id must be an integer 1-4") name = data.get("name") if not isinstance(name, str) or not name: raise ProtocolError("camera stream.name must be a nonempty string") @@ -855,7 +856,7 @@ def camera_stream_descriptor(camera: CameraStreamConfig) -> CameraStream: _MAX_SESSION_ID_LEN = 128 _MAX_RATE_HZ = 120.0 -_MAX_CAMERAS = 3 +_MAX_CAMERAS = 4 def _require_keys(data: Any, allowed: set[str], context: str) -> dict: @@ -1482,8 +1483,8 @@ def from_json(cls, data: Any) -> CameraStatus: if not isinstance(name, str) or not name: raise ProtocolError("camera status name must be a nonempty string") stream_id = data.get("stream_id") - if type(stream_id) is not int or not (1 <= stream_id <= 3): - raise ProtocolError("camera status stream_id must be an integer 1-3") + if type(stream_id) is not int or not (1 <= stream_id <= 4): + raise ProtocolError("camera status stream_id must be an integer 1-4") error = data.get("error") if not isinstance(error, str): raise ProtocolError("camera status error must be a string") @@ -1738,8 +1739,8 @@ def to_json(self) -> dict[str, Any]: def from_json(cls, data: Any) -> StreamCounters: _require_keys(data, {"stream_id", "produced", "sent", "dropped"}, "stream counters") stream_id = data.get("stream_id") - if type(stream_id) is not int or not (0 <= stream_id <= 3): - raise ProtocolError("stream counters stream_id must be an integer 0-3") + if type(stream_id) is not int or not (0 <= stream_id <= 4): + raise ProtocolError("stream counters stream_id must be an integer 0-4") values = {} for key in ("produced", "sent", "dropped"): value = data.get(key) diff --git a/examples/egocap/capture_server/capture_server/server.py b/examples/egocap/capture_server/capture_server/server.py index 6a46e8c895..0ff53feb58 100644 --- a/examples/egocap/capture_server/capture_server/server.py +++ b/examples/egocap/capture_server/capture_server/server.py @@ -53,6 +53,15 @@ from . import protocol as p from .acquisition import Acquisition, CameraSourceError, SourceExited +from .acquisition.acquisition import _CLOSE_BUDGET_S as _SOURCE_RELEASE_TIMEOUT_S +from .acquisition.sensing import ( + SensingDescription, + SensingRigConfig, + SensingStationCamera, + inspect_rig, + load_sensing_station, + resolve_sensors, +) __all__ = ["CaptureServer", "ServerConfig", "main"] @@ -71,7 +80,6 @@ _CONTROL_QUEUE_CAPACITY = 64 _SEND_DEADLINE_S = 2.0 _CLOSE_TIMEOUT_S = 2.0 -_SOURCE_RELEASE_TIMEOUT_S = 5.0 _COMPLETED_TOMBSTONE_CAPACITY = 16 @@ -153,21 +161,46 @@ class ServerConfig: # registry yet); set to reject an unknown camera name as INVALID_CONFIG # instead of inventing settings for it. known_cameras: tuple[str, ...] | None = None - # Real acquisition: logical camera name -> its station settings. When set, only these names are - # accepted and each stream uses its entry; None keeps the mock's fixed 1280x720@30 defaults. - cameras: dict[str, StationCamera] | None = None + # Logical camera name -> its station settings (OAK-D or SENSING). When set, only these names are + # accepted and each stream uses its entry, including in mock mode: a mock station with SENSING cameras + # therefore cannot also select the OAK mock cameras. None keeps the OAK mock's fixed 1280x720@30 defaults. + cameras: dict[str, StationCamera | SensingStationCamera] | None = None # Operator-facing note appended to the startup banner, saying where the served data comes from. None # prints the mode's default (the mock's SYNTHETIC-data warning, nothing for live); replay sets it so # recorded data is never announced as synthetic. data_note: str | None = None + # The one SENSING producer (at most one rig per session); required iff ``cameras`` has SENSING entries. + sensing_rig: SensingRigConfig | None = None + # Resolved once by the server before the first SENSING Begin and then reused (the platform is fixed). + sensing_description: SensingDescription | None = None def __post_init__(self) -> None: - """Reject one physical camera assigned to more than one logical name.""" + """Reject a physical camera assigned to more than one logical name, within each identity domain. + + OAK devices are identified by MXID and SENSING sensors by (module name, sensor index); the two never + collide. Final physical-pipeline duplicates are caught when the sensors are resolved. A SENSING camera + needs a rig whose declared backend matches ``mock``. + """ if self.cameras is not None: - ids = [c.device_id for c in self.cameras.values()] - duplicates = sorted({i for i in ids if ids.count(i) > 1}) - if duplicates: - raise ValueError(f"physical camera(s) assigned to more than one logical name: {duplicates}") + oak = [c.device_id for c in self.cameras.values() if isinstance(c, StationCamera)] + sensing = [ + (c.module_name, c.sensor_index) + for c in self.cameras.values() + if isinstance(c, SensingStationCamera) + ] + for ids in (oak, sensing): + duplicates = sorted({i for i in ids if ids.count(i) > 1}, key=str) + if duplicates: + raise ValueError(f"physical camera(s) assigned to more than one logical name: {duplicates}") + if sensing and self.sensing_rig is None: + raise ValueError("SENSING cameras are configured but no sensing_rig is") + if self.sensing_rig is not None: + required = "MOCK" if self.mock else "JETSON" + if self.sensing_rig.expected_backend != required: + raise ValueError( + f"sensing_rig.expected_backend must be {required} when mock={self.mock}, " + f"got {self.sensing_rig.expected_backend}" + ) @dataclass @@ -1131,6 +1164,7 @@ def __init__( # ws.send() directly; everything (rejections, replays, session # lifecycle, status, frames) goes through this. self._outbox: _Outbox | None = None + self._sensing_inspection: asyncio.Future[SensingDescription] | None = None self._active_session: _Session | None = None # Scoped to the CURRENT owner connection, reset in _handle_connection # -- a session ID completed under a since-disconnected owner must @@ -1148,6 +1182,19 @@ def request_stop(self) -> None: def _camera_stream_config(self, stream_id: int, camera: Any) -> p.CameraStreamConfig: """The server-owned settings for one requested camera: its station entry, else the mock defaults.""" station = (self.config.cameras or {}).get(camera.name) + if isinstance(station, SensingStationCamera): + assert self.config.sensing_description is not None # resolved by _prepare_sensing before this + sensor = resolve_sensors(self.config.sensing_description, {camera.name: station})[camera.name] + return p.CameraStreamConfig( + stream_id=stream_id, + name=camera.name, + type=camera.type, + required=station.required, + width=sensor.width, + height=sensor.height, + fps=station.delivery_fps, + jpeg_quality=station.jpeg_quality, + ) return p.CameraStreamConfig( stream_id=stream_id, name=camera.name, @@ -1159,6 +1206,61 @@ def _camera_stream_config(self, stream_id: int, camera: Any) -> p.CameraStreamCo jpeg_quality=station.jpeg_quality if station else 80, ) + def _begin_gate_error(self) -> str | None: + """Why a Begin cannot be admitted right now (shutdown, or this connection's writer is gone), else None.""" + if self._stop_event.is_set(): + return "server is shutting down" + if self._outbox is None or self._outbox.closed: + return "this connection is no longer usable" + return None + + async def _prepare_sensing(self, begin: p.BeginRequest) -> str | None: + """Validate requested camera types against the station and resolve SENSING sensors; error text or None. + + The native inspection is blocking, so it runs off the event loop and its result is cached for the + server's lifetime (a failed inspection is not cached). It happens before the admission checks so + nothing awaits between admitting a Begin and recording it as the active session. + """ + stations = self.config.cameras or {} + wanted: dict[str, SensingStationCamera] = {} + for camera in begin.cameras: + station = stations.get(camera.name) + if station is None: + if camera.type == p.CameraType.SENSING_COLOR: + # Never let the permissive fallback build a mock OAK labelled as SENSING. + return f"camera {camera.name!r} requests SENSING_COLOR but has no SENSING station entry" + continue + expected = ( + p.CameraType.SENSING_COLOR + if isinstance(station, SensingStationCamera) + else p.CameraType.OAK_D_COLOR + ) + if camera.type != expected: + return f"camera {camera.name!r} is a {expected.name} station camera, not {camera.type.name}" + if isinstance(station, SensingStationCamera): + wanted[camera.name] = station + # A Begin that is already BUSY (or a replay) builds nothing, so it never triggers an inspection. + if not wanted or self._active_session is not None: + return None + assert self.config.sensing_rig is not None # ServerConfig requires it for SENSING cameras + try: + if self.config.sensing_description is None: + # One shared in-flight inspection: concurrent first Begins await the same blocking call. + if self._sensing_inspection is None: + self._sensing_inspection = asyncio.ensure_future( + asyncio.to_thread(inspect_rig, self.config.sensing_rig) + ) + task = self._sensing_inspection + try: + self.config.sensing_description = await asyncio.shield(task) + finally: + if task.done() and self._sensing_inspection is task: + self._sensing_inspection = None + resolve_sensors(self.config.sensing_description, wanted) + except Exception as exc: # noqa: BLE001 -- reported to the client as a configuration rejection + return f"SENSING camera selection failed: {exc}" + return None + async def run(self) -> None: self._ws_server = await websockets.serve( self._handle_connection, @@ -1374,13 +1476,8 @@ async def _handle_begin(self, ws: ServerConnection, data: dict[str, Any]) -> Non # session onto a dead writer would create a fresh quiesce # acknowledgment that writer will never resolve, hanging that # session's own _finalize forever. - if self._stop_event.is_set() or self._outbox is None or self._outbox.closed: - self._reject( - session_id, - p.MessageType.BEGIN, - p.RejectionReason.BUSY, - "server is shutting down" if self._stop_event.is_set() else "this connection is no longer usable", - ) + if (gate_error := self._begin_gate_error()) is not None: + self._reject(session_id, p.MessageType.BEGIN, p.RejectionReason.BUSY, gate_error) return try: begin = p.parse_begin(data) @@ -1420,6 +1517,16 @@ async def _handle_begin(self, ws: ServerConnection, data: dict[str, Any]) -> Non ) return + sensing_error = await self._prepare_sensing(begin) + if sensing_error is not None: + self._reject(begin.session_id, p.MessageType.BEGIN, p.RejectionReason.INVALID_CONFIG, sensing_error) + return + # The inspection may have awaited for seconds: shutdown or a dead writer can have happened meanwhile, + # so the gate is re-checked before admission rather than trusting the check made before the await. + if (gate_error := self._begin_gate_error()) is not None: + self._reject(begin.session_id, p.MessageType.BEGIN, p.RejectionReason.BUSY, gate_error) + return + if self._active_session is not None: active = self._active_session # Owner identity, not just session_id, gates replay: a session @@ -1645,6 +1752,13 @@ def _build_arg_parser() -> argparse.ArgumentParser: metavar="NAME=DEVICE_ID", help="real only: station camera, logical NAME -> stable physical DEVICE_ID (OAK-D MXID); repeatable", ) + parser.add_argument( + "--sensing-config", + default=None, + metavar="FILE", + help="YAML with the SENSING rig and its cameras (type SENSING_COLOR); works with --mock, " + "where the rig must be a MOCK build, and without it, where it must be a JETSON build", + ) parser.add_argument("--camera-width", type=int, default=1280, help="real only") parser.add_argument("--camera-height", type=int, default=720, help="real only") parser.add_argument("--camera-fps", type=float, default=30.0, help="real only") @@ -1782,12 +1896,21 @@ async def _run_cli(args: argparse.Namespace) -> int: elif args.gmsl_ipc is not None: config, source_factory = _gmsl_setup(args) else: + cameras: dict[str, Any] | None = _station_cameras(args) + sensing_rig = None + if args.sensing_config: + sensing_rig, sensing_cameras = load_sensing_station(args.sensing_config) + clash = sorted(set(cameras or {}) & set(sensing_cameras)) + if clash: + raise ValueError(f"camera name(s) configured both as --camera and SENSING: {clash}") + cameras = {**(cameras or {}), **sensing_cameras} config = ServerConfig( mock=args.mock, host=args.host, port=args.port, connect_delay=args.connect_delay, - cameras=_station_cameras(args), + cameras=cameras, + sensing_rig=sensing_rig, ) # The one place the implementation is chosen, before any session is accepted. source_factory = build_source_factory( diff --git a/examples/egocap/frontend/README.md b/examples/egocap/frontend/README.md index 6d3d502d86..d9314ef698 100644 --- a/examples/egocap/frontend/README.md +++ b/examples/egocap/frontend/README.md @@ -137,7 +137,7 @@ Fixed defaults the backend starts with, set in `backend/egocap_web/config.py`'s | Station label | `Station 01` | the station YAML's label, see below | | Profile label | `Headset and hands` | fixed | | Sources | head, controllers, hands | fixed | -| Cameras | none requested | the station YAML's `video_sources`, see below | +| Cameras | none requested | the station YAML's `video_sources`, see below (`type` `1` = `OAK_D_COLOR`, `2` = `SENSING_COLOR`) | | Plugins | none | `EGOCAP_CAPTURE_PLUGINS`, a comma-separated list of plugin names (case-insensitive); only `manus` exists today, so `manus` requests glove hand tracking. An unknown name stops the backend at startup. | | Tracking sources | head, controllers, hands | `EGOCAP_CAPTURE_SOURCES`, a comma-separated list of `head`, `controllers`, `hands`, or `none` for a cameras-only session that needs no headset and no IsaacCapture (it cannot be combined with `EGOCAP_CAPTURE_PLUGINS`). | | Preview sample rates | 10 Hz cameras, 10 Hz tracking | `EGOCAP_CAMERA_SAMPLE_HZ`, `EGOCAP_TRACKING_SAMPLE_HZ`: how often the page is sent camera frames / tracking samples (positive Hz). Raise to the capture rate (e.g. `60`) to see the native feed; every viewer pays for each sample. | diff --git a/examples/egocap/frontend/src/captureProtocol.test.ts b/examples/egocap/frontend/src/captureProtocol.test.ts index 19fff01897..74cb7130f7 100644 --- a/examples/egocap/frontend/src/captureProtocol.test.ts +++ b/examples/egocap/frontend/src/captureProtocol.test.ts @@ -131,6 +131,24 @@ describe("parseSnapshot", () => { expect(() => parseSnapshot(body)).toThrow(CapturePreviewParseError); }); + it("accepts a camera stream_id of 4 and rejects 5", () => { + const parse = (streamId: number) => + parseSnapshot( + validSnapshot({ + capture: { + connecting: false, + started: startedWith([trackingStreamJson(), cameraStreamJson({ stream_id: streamId })]), + status: null, + stopped: null, + error_message: null, + binary_generation: null, + }, + }), + ); + expect(() => parse(4)).not.toThrow(); + expect(() => parse(5)).toThrow(CapturePreviewParseError); + }); + it("rejects a duplicate field name in a tracking descriptor", () => { const stream = trackingStreamJson(); (stream.layout as { fields: unknown[] }).fields.push({ name: "head.present", dtype: 1, shape: [], offset: 5 }); diff --git a/examples/egocap/frontend/src/captureProtocol.ts b/examples/egocap/frontend/src/captureProtocol.ts index 9519d2a36b..1ed237a76b 100644 --- a/examples/egocap/frontend/src/captureProtocol.ts +++ b/examples/egocap/frontend/src/captureProtocol.ts @@ -328,7 +328,7 @@ function parseStreamDescriptor(value: unknown, field: string): StreamDescriptor }; } // kind === StreamKind.IMAGE (the only other value expectEnum(StreamKind, ...) allows) - const streamId = expectBoundedPositiveInt(value.stream_id, `${field}.stream_id`, 3); + const streamId = expectBoundedPositiveInt(value.stream_id, `${field}.stream_id`, 4); return { kind: "camera", stream_id: streamId, diff --git a/examples/egocap/pyproject.toml b/examples/egocap/pyproject.toml index 8789a81fc6..672964fb25 100644 --- a/examples/egocap/pyproject.toml +++ b/examples/egocap/pyproject.toml @@ -32,6 +32,8 @@ dependencies = [ # decode/encode MP4 files on disk. # --- capture_server --- "websockets>=17.2", + "av>=14", # PyAV: decodes the SENSING producer's H.264 output (capture_server's + # acquisition/sensing_h264.py); loaded lazily, only for SENSING streams. # --- backend (egocap_web) --- "fastapi>=0.115", "uvicorn[standard]>=0.30", diff --git a/examples/egocap/tests/backend/test_station_config.py b/examples/egocap/tests/backend/test_station_config.py index d5df52a5d0..69f60d99a1 100644 --- a/examples/egocap/tests/backend/test_station_config.py +++ b/examples/egocap/tests/backend/test_station_config.py @@ -226,12 +226,48 @@ def test_a_duplicate_video_source_name_is_a_station_config_error(tmp_path): def test_too_many_video_sources_is_a_station_config_error(tmp_path): - entries = "\n".join(f"- type: oak_camera\n name: cam{i}" for i in range(4)) + entries = "\n".join(f"- type: oak_camera\n name: cam{i}" for i in range(5)) path = _write(tmp_path, f"video_sources:\n{entries}\n") - with pytest.raises(StationConfigError, match="at most 3"): + with pytest.raises(StationConfigError, match="at most 4"): load_station_config(path) +def test_sensing_camera_video_source_is_accepted(tmp_path): + path = _write( + tmp_path, + "video_sources:\n- type: sensing_camera\n name: head_left\n", + ) + resolved = load_station_config(path) + assert resolved.cameras == (p.CameraSelection(name="head_left", type=p.CameraType.SENSING_COLOR),) + + +def test_four_sensing_cameras_are_accepted_and_forwarded(tmp_path): + entries = "\n".join(f"- type: sensing_camera\n name: cam{i}" for i in range(4)) + path = _write(tmp_path, f"video_sources:\n{entries}\n") + resolved = load_station_config(path) + assert len(resolved.cameras) == 4 + assert all(c["type"] == p.CameraType.SENSING_COLOR for c in resolved.cameras) + assert [c["name"] for c in resolved.cameras] == [f"cam{i}" for i in range(4)] + + +def test_oak_and_sensing_cameras_can_be_mixed_in_one_station(tmp_path): + path = _write( + tmp_path, + "video_sources:\n" + "- type: oak_camera\n name: ego\n" + "- type: sensing_camera\n name: head_left\n" + "- type: sensing_camera\n name: head_right\n" + "- type: sensing_camera\n name: wrist_left\n", + ) + resolved = load_station_config(path) + assert resolved.cameras == ( + p.CameraSelection(name="ego", type=p.CameraType.OAK_D_COLOR), + p.CameraSelection(name="head_left", type=p.CameraType.SENSING_COLOR), + p.CameraSelection(name="head_right", type=p.CameraType.SENSING_COLOR), + p.CameraSelection(name="wrist_left", type=p.CameraType.SENSING_COLOR), + ) + + def test_station_config_path_from_env(monkeypatch, tmp_path): monkeypatch.delenv(STATION_CONFIG_ENV, raising=False) assert station_config_path_from_env() is None diff --git a/examples/egocap/tests/capture_server/bench_sensing_decode.py b/examples/egocap/tests/capture_server/bench_sensing_decode.py new file mode 100644 index 0000000000..aa6638d701 --- /dev/null +++ b/examples/egocap/tests/capture_server/bench_sensing_decode.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Per-frame cost of the SENSING CPU path at native size: H.264 decode, YUV->RGB, JPEG encode. + +Run on the target (e.g. the Orin) to learn what delivery rate one stream can sustain per core: + + uv run python tests/capture_server/bench_sensing_decode.py [IMAGE.png] [--quality 80] [--frames 30] + +With no image it uses synthetic noise (a worst case for both the H.264 unit size and JPEG size); pass a real +scene capture for realistic numbers. The units here are intra-only fixtures; an inter-coded stream decodes +cheaper per frame. Each iteration decodes exactly one unit, the cost the reader pays per frame. +""" + +from __future__ import annotations + +import argparse +import statistics +import time +from pathlib import Path + +import numpy as np +from PIL import Image + +from capture_server.acquisition.sensing_h264 import SensingStreamReader # noqa: E402 +from capture_server.acquisition.sensing_jpeg import encode_rgb_jpeg # noqa: E402 + +WIDTH, HEIGHT = 2560, 1984 + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("image", nargs="?") + parser.add_argument("--quality", type=int, default=80) + parser.add_argument("--frames", type=int, default=30) + args = parser.parse_args() + + if args.image: + rgb = np.asarray(Image.open(args.image).convert("RGB").resize((WIDTH, HEIGHT))) + else: + rgb = np.random.default_rng(0).integers(0, 256, (HEIGHT, WIDTH, 3), dtype=np.uint8) + # encode_units takes solid colours; encode the real picture the same way through PyAV directly. + from fractions import Fraction + + import av + + enc = av.CodecContext.create("libx264", "w") + enc.width, enc.height, enc.pix_fmt = WIDTH, HEIGHT, "yuv420p" + enc.framerate, enc.time_base, enc.gop_size = 30, Fraction(1, 30), 1 + enc.options = {"preset": "ultrafast", "tune": "zerolatency", "crf": "20", "bframes": "0"} + units = [] + for i in range(3): + frame = av.VideoFrame.from_ndarray(rgb, format="rgb24").reformat( + format="yuv420p", dst_colorspace="ITU601", dst_color_range="MPEG" + ) + frame.pts = i + units += [bytes(p) for p in enc.encode(frame)] + units += [bytes(p) for p in enc.encode(None)] + print(f"intra-only unit: {len(units[0]) / 1e6:.2f} MB; JPEG q{args.quality} cap 2.00 MB") + + path = Path("/tmp/bench_sensing.h264") + decode_ms, jpeg_ms, sizes = [], [], [] + for _ in range(args.frames): + path.write_bytes(b"".join(units[:2])) # the first unit is complete; the parser holds the second back + reader = SensingStreamReader(str(path), width=WIDTH, height=HEIGHT) + reader.connect(1.0) + t = time.perf_counter() + out, _ = reader.read_latest(5.0) + decode_ms.append((time.perf_counter() - t) * 1000) + reader.close() + t = time.perf_counter() + sizes.append(len(encode_rgb_jpeg(out, args.quality, max_bytes=1 << 30))) # measure, do not enforce + jpeg_ms.append((time.perf_counter() - t) * 1000) + d, j = statistics.median(decode_ms), statistics.median(jpeg_ms) + size = statistics.median(sizes) + verdict = "OVER the 2 MiB wire ceiling" if size > 2 * 1024 * 1024 else "within the 2 MiB wire ceiling" + print(f"decode+yuv->rgb: {d:.1f} ms jpeg: {j:.1f} ms JPEG size: {size / 1e6:.2f} MB ({verdict})") + total = d + j + print(f"per frame {total:.1f} ms => ~{1000 / total:.1f} fps per stream per core; two streams need 2x that") + + +if __name__ == "__main__": + main() diff --git a/examples/egocap/tests/capture_server/h264_fixtures.py b/examples/egocap/tests/capture_server/h264_fixtures.py new file mode 100644 index 0000000000..53d289a91d --- /dev/null +++ b/examples/egocap/tests/capture_server/h264_fixtures.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Real H.264 for SENSING reader tests, encoded with libx264 (fixtures, not native output). + +Encoded with PyAV/libx264 (IDR frames carry in-band SPS/PPS; no B-frames), so the reader and adapter +are exercised against genuinely decodable Annex-B data rather than hand-made bytes. +""" + +from __future__ import annotations + +import os +import threading +from fractions import Fraction + +import numpy as np + + +def encode_units(width: int, height: int, colors: list[tuple[int, int, int]], *, gop: int = 1) -> list[bytes]: + """One Annex-B access unit per solid-colour frame; ``gop=1`` makes each independently decodable, a larger GOP + makes later units P-frames that reference earlier ones.""" + import av + + enc = av.CodecContext.create("libx264", "w") + enc.width, enc.height, enc.pix_fmt = width, height, "yuv420p" + enc.framerate, enc.time_base, enc.gop_size = 30, Fraction(1, 30), gop + enc.options = {"preset": "ultrafast", "tune": "zerolatency", "crf": "18", "bframes": "0"} + packets = [] + for i, (r, g, b) in enumerate(colors): + # BT.601 limited-range YUV, the producer's own conversion, so decoded RGB should match the input. + rgb = np.empty((height, width, 3), np.uint8) + rgb[...] = (r, g, b) + frame = av.VideoFrame.from_ndarray(rgb, format="rgb24").reformat( + format="yuv420p", dst_colorspace="ITU601", dst_color_range="MPEG" + ) + frame.pts = i + packets += enc.encode(frame) + packets += enc.encode(None) + return [bytes(p) for p in packets] + + +class FileProducer: + """Appends units to ``path`` like the native ``output=`` writer: buffered, so a unit's tail lags.""" + + def __init__(self, path: str, units: list[bytes], *, interval_s: float = 0.02, loop: bool = False) -> None: + self.path, self.units, self.interval_s, self.loop = path, units, interval_s, loop + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + def start(self) -> "FileProducer": + self._thread.start() + return self + + def _run(self) -> None: + with open(self.path, "wb") as f: + while not self._stop.is_set(): + for unit in self.units: + f.write(unit) + f.flush() + if self._stop.wait(self.interval_s): + return + if not self.loop: + # Like the encoder tail: a final unit only becomes complete when another begins. + f.write(self.units[-1]) + f.flush() + self._stop.wait() + return + + def stop(self) -> None: + self._stop.set() + self._thread.join(timeout=2) + + +def allocated_bytes(path: str) -> int: + return os.stat(path).st_blocks * 512 diff --git a/examples/egocap/tests/capture_server/test_acquisition_adapters.py b/examples/egocap/tests/capture_server/test_acquisition_adapters.py index aaad25f998..ad6e3a6958 100644 --- a/examples/egocap/tests/capture_server/test_acquisition_adapters.py +++ b/examples/egocap/tests/capture_server/test_acquisition_adapters.py @@ -6,16 +6,22 @@ acquisition adapters directly: an object whose own `__enter__()` raised must never see a matching `__exit__()` call, while an object that *did* enter must still be released. See isaac_capture.py's `close()` and -depthai.py's `close()` for the guard these tests cover.""" +depthai.py's `close()` for the guard these tests cover. + +Also exercises Acquisition._close_rig_chain's exception-priority rule: a +cancellation collected alongside another close failure must still be the +one raised, never silently dropped in favor of the other failure.""" from __future__ import annotations +import asyncio from functools import partial from unittest.mock import patch import pytest from capture_server import protocol as p +from capture_server.acquisition.acquisition import Acquisition from capture_server.acquisition.depthai import DepthAIAdapter from capture_server.acquisition.errors import CameraSourceError, SourceError from capture_server.acquisition.isaac_capture import IsaacCaptureAdapter, TrackingSelection @@ -80,3 +86,52 @@ async def test_depthai_skips_pipeline_exit_when_pipeline_enter_raised(): await adapter.close() pipeline_exit.assert_not_called() + + +class _FakeRigAdapter: + """A minimal adapter standing in for a SENSING stream in `_close_rig_chain`.""" + + borrows_rig = True + + def __init__(self, close_raises: BaseException | None = None) -> None: + self._close_raises = close_raises + + async def close(self) -> bool: + if self._close_raises is not None: + raise self._close_raises + return True + + +class _FakeRig: + """A minimal stand-in for `SensingRig`, closed after its adapters in `_close_rig_chain`.""" + + def __init__(self, close_raises: BaseException | None = None) -> None: + self._close_raises = close_raises + + async def close(self) -> bool: + if self._close_raises is not None: + raise self._close_raises + return True + + +async def test_close_rig_chain_raises_cancellation_over_a_concurrent_close_failure(): + """One adapter's close() is cancelled while another's raises an ordinary error. The cancellation must win even + though the ordinary failure is first in iteration order: silently raising it instead would defeat the + caller's own cancellation.""" + acquisition = Acquisition(tracking=None, cameras=(), sensing_rig=_FakeRig()) + failed = _FakeRigAdapter(close_raises=RuntimeError("ordinary close failure")) + cancelled = _FakeRigAdapter(close_raises=asyncio.CancelledError()) + + with pytest.raises(asyncio.CancelledError): + await acquisition._close_rig_chain([failed, cancelled]) + + +async def test_close_rig_chain_raises_cancellation_from_the_rig_itself(): + """The rig's own close() raising CancelledError must take priority over an adapter's ordinary failure too.""" + acquisition = Acquisition( + tracking=None, cameras=(), sensing_rig=_FakeRig(close_raises=asyncio.CancelledError()) + ) + failed = _FakeRigAdapter(close_raises=RuntimeError("ordinary close failure")) + + with pytest.raises(asyncio.CancelledError): + await acquisition._close_rig_chain([failed]) diff --git a/examples/egocap/tests/capture_server/test_protocol.py b/examples/egocap/tests/capture_server/test_protocol.py index b721e2cd35..5acf50783a 100644 --- a/examples/egocap/tests/capture_server/test_protocol.py +++ b/examples/egocap/tests/capture_server/test_protocol.py @@ -218,6 +218,58 @@ def test_camera_stream_descriptor_matches_doc_shape(): assert p.CameraStream.from_json(descriptor) == stream +def test_sensing_camera_descriptor_round_trips_with_jpeg_codec_and_oak_value_unchanged(): + camera = p.CameraStreamConfig( + stream_id=1, + name="camera_left", + type=p.CameraType.SENSING_COLOR, + required=True, + width=2560, + height=1984, + fps=30.0, + jpeg_quality=80, + ) + descriptor = p.camera_stream_descriptor(camera).to_json() + assert descriptor["type"] == 2 and descriptor["codec"] == int(p.ImageCodec.JPEG) + assert (descriptor["width"], descriptor["height"]) == (2560, 1984) + assert p.CameraStream.from_json(descriptor).type is p.CameraType.SENSING_COLOR + assert int(p.CameraType.OAK_D_COLOR) == 1 # the existing value is untouched + # An old peer that only knows OAK_D_COLOR cannot parse the new value; that is stated, not negotiated away. + with pytest.raises(p.ProtocolError): + p.CameraStream.from_json({**descriptor, "type": 3}) + + +def test_four_camera_streams_are_accepted_through_descriptor_status_and_counters(): + camera = p.CameraStreamConfig( + stream_id=4, + name="right_wrist_camera", + type=p.CameraType.SENSING_COLOR, + required=False, + width=1920, + height=1536, + fps=15.0, + jpeg_quality=80, + ) + stream = p.camera_stream_descriptor(camera) + assert p.CameraStream.from_json(stream.to_json()) == stream + counters = p.StreamCounters(stream_id=4, produced=1, sent=1, dropped=0) + assert p.StreamCounters.from_json(counters.to_json()) == counters + with pytest.raises(p.ProtocolError, match="1-4"): + p.CameraStream.from_json({**stream.to_json(), "stream_id": 5}) + status = p.CameraStatus( + name="right_wrist_camera", + stream_id=4, + status=p.DeviceState.CONNECTED, + reason=p.StatusReason.NONE, + error="", + ) + assert p.CameraStatus.from_json(status.to_json()) == status + with pytest.raises(p.ProtocolError, match="1-4"): + p.CameraStatus.from_json({**status.to_json(), "stream_id": 5}) + four = tuple(p.CameraRequest(name=f"camera_{i}", type=p.CameraType.SENSING_COLOR) for i in range(4)) + assert len(p.BeginRequest(tracking=p.TrackingConfig(sources=()), cameras=four).cameras) == 4 + + def test_camera_stream_descriptor_with_intrinsics(): camera = p.CameraStreamConfig( stream_id=2, @@ -357,8 +409,8 @@ def test_begin_request_rejects_duplicate_plugin_kinds(): def test_begin_request_rejects_too_many_cameras(): - cameras = tuple(p.CameraRequest(name=f"camera_{i}", type=p.CameraType.OAK_D_COLOR) for i in range(4)) - with pytest.raises(ValueError, match="at most 3"): + cameras = tuple(p.CameraRequest(name=f"camera_{i}", type=p.CameraType.OAK_D_COLOR) for i in range(5)) + with pytest.raises(ValueError, match="at most 4"): p.BeginRequest(tracking=p.TrackingConfig(sources=()), cameras=cameras) @@ -573,8 +625,8 @@ def test_parse_begin_rejects_cameras_not_a_list(): def test_parse_begin_rejects_too_many_cameras(): - cameras = [{"name": f"camera_{i}", "type": 1} for i in range(4)] - with pytest.raises(p.ProtocolError, match="at most 3"): + cameras = [{"name": f"camera_{i}", "type": 1} for i in range(5)] + with pytest.raises(p.ProtocolError, match="at most 4"): p.parse_begin(_valid_begin_dict(cameras=cameras)) diff --git a/examples/egocap/tests/capture_server/test_sensing_adapter.py b/examples/egocap/tests/capture_server/test_sensing_adapter.py new file mode 100644 index 0000000000..c7ee7672dd --- /dev/null +++ b/examples/egocap/tests/capture_server/test_sensing_adapter.py @@ -0,0 +1,1124 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""SENSING configuration, inspection, rig ownership, adapter lifecycle and Acquisition composition. + +CPU-only: a fake producer writing real intra-only H.264 files stands in for the native binary, +so these do not replace the desktop MOCK-binary or Orin integration runs. +""" + +from __future__ import annotations + +import asyncio +import io +import os +import subprocess +import threading +import time +import types + +import pytest +from PIL import Image + +from capture_server import protocol as p +from capture_server.acquisition import Acquisition, CameraSourceError +from capture_server.acquisition import sensing as sn +from capture_server.acquisition import sensing_h264 as h264 +from capture_server.acquisition.errors import SourceError +from capture_server.server import CaptureServer, ServerConfig, StationCamera + +from .h264_fixtures import FileProducer, allocated_bytes, encode_units + +W, H = 64, 32 + +RIG = dict( + executable="/opt/x/camera_plugin_sensing", + expected_backend="MOCK", + platform_config="/opt/x/p.json", + config_name="CFG", + link_masks=("0x0000", "0x1100"), + decoder="software", # tiny fake streams: the CPU decoder keeps these tests machine-independent +) +DESCRIPTION = sn.SensingDescription((sn.SensingSensor("L", 1, W, H, 30.0), sn.SensingSensor("R", 0, W, H, 30.0))) + + +def _listing(*lines: str, header: str = "CFG: 2 sensor(s)") -> str: + """The native plugin's ``--list-sensors`` text: a header, then one line per sensor.""" + return "\n".join([header, *lines]) + "\n" + + +GOOD_LISTING = _listing( + f" sensor=1 L {W}x{H} @ 30 fps", + f" sensor=0 R {W}x{H} @ 30 fps", +) + + +# --- configuration -------------------- + + +@pytest.mark.parametrize( + "over", [dict(expected_backend="CPU"), dict(executable=""), dict(link_masks=()), dict(config_name="")] +) +def test_rig_config_rejects_bad_values(over): + with pytest.raises(ValueError): + sn.SensingRigConfig(**{**RIG, **over}) + + +@pytest.mark.parametrize("over", [dict(module_name=""), dict(delivery_fps=0), dict(jpeg_quality=101)]) +def test_station_camera_rejects_bad_values(over): + with pytest.raises(ValueError): + sn.SensingStationCamera(**{"module_name": "L", **over}) + + +def test_server_config_dedups_within_each_identity_domain_only(): + rig = sn.SensingRigConfig(**RIG) + cams = { + "a": StationCamera("mxid-1"), + "b": sn.SensingStationCamera("mxid-1"), # same string, different identity domain: fine + } + ServerConfig(mock=True, cameras=cams, sensing_rig=rig) + with pytest.raises(ValueError, match="more than one logical name"): + ServerConfig( + mock=True, + cameras={"a": sn.SensingStationCamera("L"), "b": sn.SensingStationCamera("L")}, + sensing_rig=rig, + ) + + +def test_server_config_allows_both_eyes_of_one_module_but_not_the_same_eye_twice(): + rig = sn.SensingRigConfig(**RIG) + eyes = { + "head_left": sn.SensingStationCamera("HEAD", sensor_index=0), + "head_right": sn.SensingStationCamera("HEAD", sensor_index=1), + } + ServerConfig(mock=True, cameras=eyes, sensing_rig=rig) + twice = { + "a": sn.SensingStationCamera("HEAD", sensor_index=0), + "b": sn.SensingStationCamera("HEAD", sensor_index=0), + } + with pytest.raises(ValueError, match="more than one logical name"): + ServerConfig(mock=True, cameras=twice, sensing_rig=rig) + # One name selecting the module as a whole and another selecting one eye is also a conflict to resolve later. + mixed = {"a": sn.SensingStationCamera("HEAD"), "b": sn.SensingStationCamera("HEAD", sensor_index=1)} + ServerConfig(mock=True, cameras=mixed, sensing_rig=rig) # config-level distinct; resolve_sensors rejects it + + +def test_server_config_requires_rig_and_matching_backend(): + with pytest.raises(ValueError, match="no sensing_rig"): + ServerConfig(mock=True, cameras={"a": sn.SensingStationCamera("L")}) + with pytest.raises(ValueError, match="must be JETSON"): + ServerConfig(mock=False, sensing_rig=sn.SensingRigConfig(**RIG)) + with pytest.raises(ValueError, match="must be MOCK"): + ServerConfig(mock=True, sensing_rig=sn.SensingRigConfig(**{**RIG, "expected_backend": "JETSON"})) + + +def test_every_documented_rig_key_loads_from_yaml(tmp_path): + f = tmp_path / "s.yaml" + f.write_text( + "sensing_rig:\n executable: /x/e\n expected_backend: MOCK\n platform_config: /x/p.json\n" + " config_name: C\n link_masks: ['0x0000']\n gpu_id: 1\n nito_dir: /n\n" + " bitrate_bps: 50000000\n full_range: true\n runtime_dir: /r\n" + "cameras:\n left: {type: SENSING_COLOR, module_name: L}\n" + ) + rig, _ = sn.load_sensing_station(str(f)) + assert (rig.gpu_id, rig.nito_dir, rig.bitrate_bps, rig.full_range, rig.runtime_dir) == ( + 1, + "/n", + 50_000_000, + True, + "/r", + ) + args = rig.base_args() + assert "--full-range" in args and "--bitrate=50000000" in args and "--peak-bitrate=75000000" in args + + +def test_configured_encoder_settings_reach_the_launch_arguments(): + args = sn.SensingRigConfig(**RIG, gop=30, bitrate_bps=20_000_000).base_args() + assert "--gop=30" in args and "--bitrate=20000000" in args and "--peak-bitrate=30000000" in args + + +def test_load_station_yaml(tmp_path): + f = tmp_path / "s.yaml" + f.write_text( + "sensing_rig:\n executable: /x/e\n expected_backend: MOCK\n platform_config: /x/p.json\n" + " config_name: C\n link_masks: ['0x0000', '0x1100']\n" + "cameras:\n left: {type: SENSING_COLOR, module_name: L, delivery_fps: 15}\n" + ) + rig, cams = sn.load_sensing_station(str(f)) + assert rig.link_masks == ("0x0000", "0x1100") and cams["left"].delivery_fps == 15.0 + + +@pytest.mark.parametrize( + "text", + [ + "cameras: {}\n", + "sensing_rig: {executable: e}\ncameras: {}\n", + "sensing_rig: {executable: e, bogus: 1}\ncameras: {l: {type: SENSING_COLOR, module_name: L}}\n", + ], +) +def test_load_station_yaml_rejects_bad_files(tmp_path, text): + f = tmp_path / "s.yaml" + f.write_text(text) + with pytest.raises(ValueError): + sn.load_sensing_station(str(f)) + + +def test_load_station_yaml_rejects_non_sensing_camera(tmp_path): + f = tmp_path / "s.yaml" + f.write_text( + "sensing_rig: {executable: e, expected_backend: MOCK, platform_config: p, config_name: c," + " link_masks: ['0x0']}\n" + "cameras: {l: {type: OAK_D_COLOR, module_name: L}}\n" + ) + with pytest.raises(ValueError, match="SENSING_COLOR"): + sn.load_sensing_station(str(f)) + + +# --- inspection and resolution -------------------- + + +def _fake_run(stdout="", returncode=0, stderr="", exc=None): + def run(args, **kw): + assert args[-1] == "--list-sensors" and kw["timeout"] > 0 and "shell" not in kw + if exc: + raise exc + return subprocess.CompletedProcess(args, returncode, stdout, stderr) + + return run + + +def test_inspect_rig_happy_path_uses_argument_list_and_the_existing_listing(): + d = sn.inspect_rig(sn.SensingRigConfig(**RIG), run=_fake_run(GOOD_LISTING)) + assert [(s.module_name, s.pipeline_id, s.width, s.height) for s in d.sensors] == [ + ("L", 1, W, H), + ("R", 0, W, H), + ] + + +@pytest.mark.parametrize( + "run,msg", + [ + (_fake_run(returncode=2, stderr="bad masks"), "bad masks"), + (_fake_run("not a listing"), "malformed"), + (_fake_run(_listing(f" sensor=0 L {W}x{H} @ 30 fps")), "header says 2 sensors, found 1"), + (_fake_run(_listing(" sensor=0 L garbage", " sensor=1 R 1x1 @ 1 fps")), "unexpected line"), + ( + _fake_run(_listing(f" sensor=0 L 0x{H} @ 30 fps", f" sensor=1 R {W}x{H} @ 30 fps")), + "non-positive", + ), + (_fake_run(exc=subprocess.TimeoutExpired("x", 1)), "could not inspect"), + (_fake_run(exc=FileNotFoundError("nope")), "could not inspect"), + ], +) +def test_inspect_rig_failures(run, msg): + with pytest.raises(SourceError, match=msg): + sn.inspect_rig(sn.SensingRigConfig(**RIG), run=run) + + +@pytest.mark.parametrize( + "bad", ["", "\n\n", "CFG: x sensor(s)\n", "CFG: 1 sensor(s)\n sensor=-1 L 1x1 @ 30 fps\n"] +) +def test_parse_sensor_listing_rejects_instead_of_guessing(bad): + with pytest.raises(ValueError, match="malformed"): + sn.parse_sensor_listing(bad) + + +def test_resolve_sensors_uses_module_names_not_positions(): + got = sn.resolve_sensors( + DESCRIPTION, {"left": sn.SensingStationCamera("L"), "right": sn.SensingStationCamera("R")} + ) + assert got["left"].pipeline_id == 1 and got["right"].pipeline_id == 0 + + +def test_sensor_index_selects_one_eye_of_a_stereo_module_in_pipeline_order(): + stereo = sn.SensingDescription( + ( + sn.SensingSensor("HEAD", 1, W, H, 30.0), + sn.SensingSensor("HEAD", 0, W, H, 30.0), + sn.SensingSensor("WRIST", 2, W, H, 30.0), + ), + ) + got = sn.resolve_sensors( + stereo, + { + "head_left_eye": sn.SensingStationCamera("HEAD", sensor_index=0), + "head_right_eye": sn.SensingStationCamera("HEAD", sensor_index=1), + "wrist": sn.SensingStationCamera("WRIST"), + }, + ) + assert [got[n].pipeline_id for n in ("head_left_eye", "head_right_eye", "wrist")] == [0, 1, 2] + with pytest.raises(SourceError, match="select one with sensor_index"): + sn.resolve_sensors(stereo, {"head": sn.SensingStationCamera("HEAD")}) + with pytest.raises(SourceError, match="no sensor_index 2"): + sn.resolve_sensors(stereo, {"head": sn.SensingStationCamera("HEAD", sensor_index=2)}) + with pytest.raises(SourceError, match="more than one logical"): + sn.resolve_sensors( + stereo, + { + "a": sn.SensingStationCamera("HEAD", sensor_index=0), + "b": sn.SensingStationCamera("HEAD", sensor_index=0), + }, + ) + with pytest.raises(ValueError): + sn.SensingStationCamera("HEAD", sensor_index=-1) + + +def test_resolve_sensors_rejects_missing_ambiguous_and_duplicate(): + with pytest.raises(SourceError, match="not in the platform"): + sn.resolve_sensors(DESCRIPTION, {"x": sn.SensingStationCamera("Z")}) + amb = sn.SensingDescription((sn.SensingSensor("L", 0, W, H, 30.0), sn.SensingSensor("L", 1, W, H, 30.0))) + with pytest.raises(SourceError, match="select one with sensor_index"): + sn.resolve_sensors(amb, {"x": sn.SensingStationCamera("L")}) + with pytest.raises(SourceError, match="more than one logical"): + sn.resolve_sensors(DESCRIPTION, {"a": sn.SensingStationCamera("L"), "b": sn.SensingStationCamera("L")}) + + +# --- rig ownership -------------------- + + +class FakeProc: + def __init__(self, *, ignore_terminate=False, ignore_kill=False) -> None: + self.code: int | None = None + self.stdout = io.BytesIO(b"line one\nline two\n") + self.ignore_terminate, self.ignore_kill = ignore_terminate, ignore_kill + self.calls: list[str] = [] + + def poll(self): + return self.code + + def terminate(self): + self.calls.append("terminate") + if not self.ignore_terminate: + self.code = -15 + + def kill(self): + self.calls.append("kill") + if not self.ignore_kill: + self.code = -9 + + def wait(self, timeout=None): + if self.code is None: + raise subprocess.TimeoutExpired("p", timeout) + return self.code + + +def _rig(tmp_path, proc, popen=None): + captured = {} + + def default_popen(args, **kw): + captured["args"], captured["kw"] = args, kw + return proc + + rig = sn.SensingRig( + sn.SensingRigConfig(**RIG), (1, 0), popen=popen or default_popen, runtime_root=str(tmp_path) + ) + return rig, captured + + +@pytest.mark.asyncio +async def test_rig_start_builds_file_output_argument_list_and_cleans_env(tmp_path, monkeypatch): + monkeypatch.setenv("SENSING_MOCK_SCALE_DIV", "4") + rig, cap = _rig(tmp_path, FakeProc()) + await rig.start() + args = cap["args"] + assert isinstance(args, list) and args[0] == RIG["executable"] and rig.start_error is None + assert "--link-masks=0x0000 0x1100" in args + # Encoder settings are passed only when configured; the decoder does not require any particular GOP. + assert not any(a.startswith(("--gop", "--bitrate", "--peak-bitrate")) for a in args) + streams = [a for a in args if a.startswith("--add-stream=")] + assert len(streams) == 2 and all(",output=" in a and "ipc=" not in a for a in streams) + assert streams[0].startswith("--add-stream=sensor=1,") and rig.stream_path(1) in streams[0] + assert "SENSING_MOCK_SCALE_DIV" not in cap["kw"]["env"] and "shell" not in cap["kw"] + run_dir = os.path.dirname(rig.stream_path(1)) + assert os.stat(run_dir).st_mode & 0o777 == 0o700 + await asyncio.sleep(0.05) # let the log drain read the fake output + assert "line two" in rig.diagnostics() + assert await rig.close() and rig.cleanup_ok + assert not os.path.exists(run_dir) + + +@pytest.mark.asyncio +async def test_rig_close_escalates_to_kill_and_removes_only_its_directory(tmp_path): + other = tmp_path / "keep" + other.mkdir() + proc = FakeProc(ignore_terminate=True) + rig, _ = _rig(tmp_path, proc) + await rig.start() + run_dir = os.path.dirname(rig.stream_path(1)) + assert await rig.close() and proc.calls == ["terminate", "kill"] and other.exists() + assert not os.path.exists(run_dir) + + +@pytest.mark.asyncio +async def test_rig_unconfirmed_exit_stays_false_on_every_later_call(tmp_path, monkeypatch): + monkeypatch.setattr(sn, "_RIG_STEPS_S", 0.1) + rig, _ = _rig(tmp_path, FakeProc(ignore_terminate=True, ignore_kill=True)) + await rig.start() + assert await rig.close() is False and rig.cleanup_ok is False + assert await rig.close() is False # never reported as released just because a wait ended + + +@pytest.mark.asyncio +async def test_rig_close_stays_inside_one_total_budget(tmp_path, monkeypatch): + monkeypatch.setattr(sn, "_RIG_STEPS_S", 0.3) + rig, _ = _rig(tmp_path, FakeProc(ignore_terminate=True, ignore_kill=True)) + await rig.start() + started = time.monotonic() + assert await rig.close() is False + assert time.monotonic() - started < 1.5 + + +@pytest.mark.asyncio +async def test_rig_start_failure_is_recorded_not_raised_and_leaves_no_directory(tmp_path): + def boom(*a, **k): + raise FileNotFoundError("no binary") + + rig, _ = _rig(tmp_path, None, popen=boom) + await rig.start() + assert "could not start" in rig.start_error and list(tmp_path.iterdir()) == [] + assert await rig.close() is True + + +@pytest.mark.asyncio +async def test_slow_spawn_still_in_flight_at_stop_is_reaped_by_the_queued_close(tmp_path): + release = threading.Event() + proc = FakeProc() + + def slow_popen(args, **kw): + release.wait(5) + return proc + + rig, _ = _rig(tmp_path, None, popen=slow_popen) + start = asyncio.ensure_future(rig.start()) + await asyncio.sleep(0.05) + start.cancel() # Stop/cancellation while the spawn is still running on the worker + threading.Timer(0.2, release.set).start() + assert await rig.close() is True + assert proc.calls == ["terminate"] and list(tmp_path.iterdir()) == [] + + +@pytest.mark.asyncio +async def test_log_tail_is_byte_bounded_decoded_leniently_and_closed(tmp_path): + proc = FakeProc() + proc.stdout = io.BytesIO(b"x" * 100_000 + b"\xff\xfe tail-marker") + rig, _ = _rig(tmp_path, proc) + await rig.start() + await asyncio.sleep(0.1) + text = rig.diagnostics() + assert len(text) <= sn._LOG_TAIL_BYTES and text.endswith("tail-marker") + assert proc.stdout.closed + await rig.close() + + +# --- adapter lifecycle ------------------------------------------------------------------------------------------- + + +class FakeRig: + start_error: str | None = None + + def __init__(self, path: str) -> None: + self.path, self.code = path, None + self.aborted: list[str] = [] + + def abort(self, reason: str) -> None: + self.aborted.append(reason) + + def stream_path(self, _pid: int) -> str: + return self.path + + def exit_code(self): + return self.code + + def diagnostics(self) -> str: + return "producer log" + + +def _camera(required=True, fps=200.0) -> p.CameraStreamConfig: + return p.CameraStreamConfig( + stream_id=1, + name="cam", + type=p.CameraType.SENSING_COLOR, + required=required, + width=W, + height=H, + fps=fps, + jpeg_quality=80, + ) + + +def _adapter(tmp_path, **kw): + rig = FakeRig(str(tmp_path / "sensor1.h264")) + adapter = sn.SensingAdapter( + rig=rig, + sensor=sn.SensingSensor("M", 1, W, H, 30.0), + camera=kw.pop("camera", _camera()), + startup_timeout_s=kw.pop("startup_timeout_s", 5.0), + ) + return adapter, rig + + +RED, GREEN, BLUE = (220, 20, 20), (20, 220, 20), (20, 20, 220) + + +@pytest.mark.asyncio +async def test_adapter_delivers_decodable_jpeg_and_closes_cleanly(tmp_path): + adapter, rig = _adapter(tmp_path) + assert adapter.camera_status().status == p.DeviceState.DEGRADED # never CONNECTED before a frame + producer = FileProducer(rig.path, encode_units(W, H, [RED, GREEN, BLUE]), loop=True).start() + try: + await adapter.enter() + assert adapter.camera_status().status == p.DeviceState.CONNECTED + got = None + for _ in range(100): + got = await adapter.next_camera_jpeg() + if got: + break + ts, jpeg = got + img = Image.open(io.BytesIO(jpeg)) + assert img.size == (W, H) and ts > 0 + assert await adapter.close() is True and not adapter.cleanup_failed + finally: + producer.stop() + + +@pytest.mark.asyncio +async def test_producer_exit_during_startup_is_a_camera_error(tmp_path): + adapter, rig = _adapter(tmp_path, startup_timeout_s=3.0) + rig.code = 3 + with pytest.raises(CameraSourceError, match="exited with 3: producer log"): + await adapter.enter() + assert adapter.camera_status().status == p.DeviceState.DEGRADED + assert await adapter.close() is True + + +@pytest.mark.asyncio +async def test_startup_deadline_without_a_frame(tmp_path): + adapter, rig = _adapter(tmp_path, startup_timeout_s=0.5) + open(rig.path, "wb").close() # the file exists but no unit is ever written + with pytest.raises(CameraSourceError, match="no frame"): + await adapter.enter() + await adapter.close() + + +@pytest.mark.asyncio +async def test_undecodable_producer_output_is_a_camera_error_not_a_synthetic_frame(tmp_path): + adapter, rig = _adapter(tmp_path, startup_timeout_s=2.0) + # Annex-B shaped but not video: the placeholder bytes an older mock encoder wrote. + with open(rig.path, "wb") as f: + f.write((b"\x00\x00\x00\x01\x65MOCK" + b"U" * 300) * 6) + with pytest.raises(CameraSourceError, match="failed at startup|no frame"): + await adapter.enter() + assert adapter.camera_status().status == p.DeviceState.DEGRADED + await adapter.close() + + +@pytest.mark.asyncio +async def test_mid_session_producer_exit_then_paced_empty_polls(tmp_path): + adapter, rig = _adapter(tmp_path, camera=_camera(required=False, fps=50.0)) + producer = FileProducer(rig.path, encode_units(W, H, [RED, GREEN]), loop=True).start() + try: + await adapter.enter() + rig.code = 1 + with pytest.raises(CameraSourceError, match="exited with 1"): + for _ in range(20): + await adapter.next_camera_jpeg() + assert adapter.camera_status().status == p.DeviceState.DEGRADED + started = time.monotonic() + for _ in range(3): # after the failure: no exception loop, still one tick per interval + assert await adapter.next_camera_jpeg() is None + assert time.monotonic() - started >= 0.03 + await adapter.close() + finally: + producer.stop() + + +@pytest.mark.asyncio +async def test_failed_close_is_sticky_even_after_stream_degradation(tmp_path): + adapter, rig = _adapter(tmp_path) + producer = FileProducer(rig.path, encode_units(W, H, [RED, GREEN]), loop=True).start() + try: + await adapter.enter() + + def leaky_close(): + raise OSError("could not release") + + adapter._reader.close = leaky_close + with pytest.raises(OSError, match="could not release"): + await adapter.close() + assert adapter.cleanup_failed + finally: + producer.stop() + + +# --- Acquisition ownership -------------------- + + +class SpyAdapter: + stream_id, required = 1, False + + def __init__( + self, + order, + name, + fail=False, + gate: threading.Event | None = None, + enter_error=None, + required=False, + borrows_rig=True, + ): + self.order, self.name, self.fail, self.gate = order, name, fail, gate + self.enter_error, self.required, self.borrows_rig = enter_error, required, borrows_rig + + async def enter(self): + if self.enter_error: + raise CameraSourceError(self.enter_error, camera_name=self.name) + + async def close(self): + self.order.append(f"{self.name}:close-start") + if self.gate is not None: + await asyncio.get_running_loop().run_in_executor(None, self.gate.wait, 5) + self.order.append(f"{self.name}:close-done") + if self.fail: + raise RuntimeError("release failed") + return True + + def camera_status(self): + raise AssertionError + + +class SpyRig: + def __init__(self, order, ok=True, start_error=None): + self.order, self.ok, self.started, self.start_error = order, ok, False, start_error + + async def start(self): + self.started = True + + async def close(self): + self.order.append("rig:close") + return self.ok + + +@pytest.mark.asyncio +async def test_acquisition_starts_rig_then_closes_adapters_before_rig(): + order: list[str] = [] + rig = SpyRig(order) + acq = Acquisition(tracking=None, cameras=(SpyAdapter(order, "a"),), sensing_rig=rig) + await acq.enter() + assert rig.started + assert await acq.close() is True and order == ["a:close-start", "a:close-done", "rig:close"] + + +@pytest.mark.asyncio +async def test_rig_is_not_terminated_while_an_adapter_close_is_still_held(): + order: list[str] = [] + gate = threading.Event() + acq = Acquisition(tracking=None, cameras=(SpyAdapter(order, "a", gate=gate),), sensing_rig=SpyRig(order)) + closing = asyncio.ensure_future(acq.close()) + await asyncio.sleep(0.2) + assert order == ["a:close-start"] # producer teardown has NOT begun while the adapter is still closing + gate.set() + assert await closing is True + assert order == ["a:close-start", "a:close-done", "rig:close"] + + +@pytest.mark.asyncio +async def test_acquisition_reaps_rig_even_if_an_adapter_close_raises_and_reports_unconfirmed_rig(): + order: list[str] = [] + acq = Acquisition(tracking=None, cameras=(SpyAdapter(order, "a", fail=True),), sensing_rig=SpyRig(order)) + with pytest.raises(RuntimeError, match="release failed"): + await acq.close() + assert order[-1] == "rig:close" + acq2 = Acquisition(tracking=None, cameras=(SpyAdapter([], "a"),), sensing_rig=SpyRig([], ok=False)) + assert await acq2.close() is False + + +@pytest.mark.asyncio +async def test_shared_producer_start_failure_degrades_optional_and_fails_required_streams(tmp_path): + class DeadRig(FakeRig): + start_error = "could not start /x: no binary" + + def make(required): + return sn.SensingAdapter( + rig=DeadRig("/nope"), + sensor=sn.SensingSensor("M", 1, W, H, 30.0), + camera=_camera(required=required), + ) + + healthy_oak = SpyAdapter([], "oak") + optional = make(False) + await Acquisition(tracking=None, cameras=(healthy_oak, optional), sensing_rig=None).enter() # tolerated + assert optional.camera_status().status == p.DeviceState.DEGRADED + assert "did not start" in optional.camera_status().error + with pytest.raises(CameraSourceError, match="did not start"): + await Acquisition(tracking=None, cameras=(healthy_oak, make(True)), sensing_rig=None).enter() + for adapter in (optional,): + assert await adapter.close() is True + + +# --- server type validation and launch selection -------------------- + + +def _server(mock=True, cameras=None): + cfg = ServerConfig(mock=mock, cameras=cameras, sensing_rig=sn.SensingRigConfig(**RIG) if cameras else None) + return CaptureServer(cfg, source_factory=lambda *_: None) + + +def _begin(*cams): + return p.BeginRequest( + session_id="s1", + tracking=p.TrackingConfig(sources=(), rate_hz=30.0), + cameras=tuple(p.CameraRequest(name=n, type=t) for n, t in cams), + ) + + +@pytest.mark.asyncio +async def test_prepare_sensing_rejects_type_mismatch_before_inspection(monkeypatch): + monkeypatch.setattr("capture_server.server.inspect_rig", lambda *_: pytest.fail("must not inspect")) + server = _server(cameras={"left": sn.SensingStationCamera("L")}) + err = await server._prepare_sensing(_begin(("left", p.CameraType.OAK_D_COLOR))) + assert "SENSING_COLOR station camera" in err + + +@pytest.mark.asyncio +async def test_prepare_sensing_inspects_once_and_caches(monkeypatch): + calls = [] + + def fake_inspect(cfg): + calls.append(cfg) + return DESCRIPTION + + monkeypatch.setattr("capture_server.server.inspect_rig", fake_inspect) + server = _server(cameras={"left": sn.SensingStationCamera("L")}) + begin = _begin(("left", p.CameraType.SENSING_COLOR)) + assert await server._prepare_sensing(begin) is None and await server._prepare_sensing(begin) is None + assert len(calls) == 1 + cam = server._camera_stream_config(1, begin.cameras[0]) + assert (cam.width, cam.height, cam.fps) == (W, H, 30.0) and cam.type == p.CameraType.SENSING_COLOR + + +@pytest.mark.asyncio +async def test_prepare_sensing_failure_is_reported_and_not_cached(monkeypatch): + def boom(_cfg): + raise SourceError("backend mismatch") + + monkeypatch.setattr("capture_server.server.inspect_rig", boom) + server = _server(cameras={"left": sn.SensingStationCamera("L")}) + err = await server._prepare_sensing(_begin(("left", p.CameraType.SENSING_COLOR))) + assert "backend mismatch" in err and server.config.sensing_description is None + + +@pytest.mark.asyncio +async def test_oak_only_begin_needs_no_sensing(monkeypatch): + monkeypatch.setattr("capture_server.server.inspect_rig", lambda *_: pytest.fail("must not inspect")) + server = _server(cameras={"w": StationCamera("mxid")}) + assert await server._prepare_sensing(_begin(("w", p.CameraType.OAK_D_COLOR))) is None + + +def test_launch_builds_no_rig_for_oak_only_sessions(): + from capture_server.launch import _sensing_composition + + cfg = ServerConfig(mock=True) + assert _sensing_composition(cfg, (_camera(),)) == (None, {}) + + +def test_launch_mixes_sensing_and_mock_oak_in_one_acquisition(): + from capture_server.launch import build_source_factory + + cfg = ServerConfig( + mock=True, + cameras={"left": sn.SensingStationCamera("L")}, + sensing_rig=sn.SensingRigConfig(**RIG), + sensing_description=DESCRIPTION, + ) + factory = build_source_factory(cfg) + begin = _begin(("left", p.CameraType.SENSING_COLOR), ("wrist", p.CameraType.OAK_D_COLOR)) + cams = ( + p.CameraStreamConfig(1, "left", p.CameraType.SENSING_COLOR, True, W, H, 30.0, 80), + p.CameraStreamConfig(2, "wrist", p.CameraType.OAK_D_COLOR, True, 1280, 720, 30.0, 80), + ) + acq = factory(begin, cams) + assert isinstance(acq.cameras[0], sn.SensingAdapter) and not isinstance(acq.cameras[1], sn.SensingAdapter) + assert acq.sensing_rig is not None + + +# --- Begin admission through the real server ----------- + + +async def _begin_rejection(url: str, camera_type: p.CameraType, name: str = "left") -> p.RequestRejected: + """Send one real Begin with a single camera over a raw socket; return the RequestRejected it draws.""" + import json + + import websockets + + begin = p.BeginRequest( + tracking=p.TrackingConfig(sources=()), cameras=(p.CameraRequest(name=name, type=camera_type),) + ) + async with websockets.connect(url) as ws: + await ws.send( + json.dumps( + { + "type": int(p.MessageType.BEGIN), + "version": p.PROTOCOL_VERSION, + "session_id": "sensing-begin-1", + **p.effective_config(begin).to_json(), + } + ) + ) + for _ in range(20): + raw = await asyncio.wait_for(ws.recv(), timeout=3) + if isinstance(raw, bytes): + continue + event = p.parse_server_message(json.loads(raw)) + if isinstance(event, p.RequestRejected): + return event + raise AssertionError("no RequestRejected arrived") + + +@pytest.mark.asyncio +async def test_sensing_request_without_a_station_entry_is_rejected_not_built_as_a_mock_oak(unused_tcp_port): + from tests.support.capture_server import start_capture_server, stop_capture_server + + running = await start_capture_server(unused_tcp_port) # permissive legacy config: cameras=None + try: + rejected = await _begin_rejection(f"ws://127.0.0.1:{unused_tcp_port}", p.CameraType.SENSING_COLOR) + assert rejected.reason == p.RejectionReason.INVALID_CONFIG + assert "no SENSING station entry" in rejected.error_message + assert running.server._active_session is None + finally: + await stop_capture_server(running) + + +@pytest.mark.asyncio +async def test_shutdown_during_a_paused_inspection_never_admits_a_session(unused_tcp_port, monkeypatch): + from tests.support.capture_server import start_capture_server, stop_capture_server + + started, release = threading.Event(), threading.Event() + + def slow_inspect(_cfg): + started.set() + release.wait(5) + return DESCRIPTION + + monkeypatch.setattr("capture_server.server.inspect_rig", slow_inspect) + cfg = ServerConfig( + mock=True, cameras={"left": sn.SensingStationCamera("L")}, sensing_rig=sn.SensingRigConfig(**RIG) + ) + running = await start_capture_server(unused_tcp_port, config=cfg) + try: + pending = asyncio.ensure_future( + _begin_rejection(f"ws://127.0.0.1:{unused_tcp_port}", p.CameraType.SENSING_COLOR) + ) + await asyncio.get_running_loop().run_in_executor(None, started.wait, 5) + running.server.request_stop() # shutdown begins while the Begin is parked in inspection + release.set() + rejected = await pending + assert rejected.reason == p.RejectionReason.BUSY and "shutting down" in rejected.error_message + assert running.server._active_session is None + finally: + release.set() + await stop_capture_server(running) + + +@pytest.mark.asyncio +async def test_concurrent_first_begins_share_one_inspection(monkeypatch): + calls: list[int] = [] + release = threading.Event() + + def slow_inspect(_cfg): + calls.append(1) + release.wait(5) + return DESCRIPTION + + monkeypatch.setattr("capture_server.server.inspect_rig", slow_inspect) + server = _server(cameras={"left": sn.SensingStationCamera("L")}) + begin = _begin(("left", p.CameraType.SENSING_COLOR)) + both = asyncio.gather(server._prepare_sensing(begin), server._prepare_sensing(begin)) + await asyncio.sleep(0.1) + release.set() + assert await both == [None, None] and len(calls) == 1 + + +@pytest.mark.asyncio +async def test_a_begin_that_is_already_busy_triggers_no_inspection(monkeypatch): + monkeypatch.setattr("capture_server.server.inspect_rig", lambda *_: pytest.fail("must not inspect")) + server = _server(cameras={"left": sn.SensingStationCamera("L")}) + server._active_session = object() # a session is already running + assert await server._prepare_sensing(_begin(("left", p.CameraType.SENSING_COLOR))) is None + + +# --- shared shutdown budget and local start failures ------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_a_slow_oak_release_is_not_cut_short_by_the_producer_shutdown(monkeypatch): + from capture_server.acquisition import acquisition as acq_mod + + monkeypatch.setattr(acq_mod, "_CLOSE_BUDGET_S", 0.8) + order: list[str] = [] + gate = threading.Event() + threading.Timer(0.45, gate.set).start() # a release taking 0.45 s of a 0.8 s budget + oak = SpyAdapter(order, "oak", gate=gate, borrows_rig=False) + acq = Acquisition(tracking=None, cameras=(oak, SpyAdapter(order, "sensing")), sensing_rig=SpyRig(order)) + started = time.monotonic() + assert await acq.close() is True # the old shared budget cut OAK at budget minus the producer's reserve + assert time.monotonic() - started < 0.8 + assert order.count("oak:close-done") == 1 and order[-1] in ("rig:close", "oak:close-done") + + +@pytest.mark.asyncio +async def test_a_held_oak_release_does_not_delay_the_producer_reap_and_stays_unconfirmed(monkeypatch): + from capture_server.acquisition import acquisition as acq_mod + + monkeypatch.setattr(acq_mod, "_CLOSE_BUDGET_S", 0.4) + order: list[str] = [] + gate = threading.Event() + oak = SpyAdapter(order, "oak", gate=gate, borrows_rig=False) + acq = Acquisition(tracking=None, cameras=(oak, SpyAdapter(order, "sensing")), sensing_rig=SpyRig(order)) + started = time.monotonic() + assert await acq.close() is False # the OAK release is still pending: unconfirmed, admission stays blocked + assert time.monotonic() - started < 0.9 + assert order.index("rig:close") < order.index("oak:close-done") if "oak:close-done" in order else True + assert "rig:close" in order and "sensing:close-done" in order # the producer chain was not held up by OAK + gate.set() # the retained OAK close finishes later and is observed, not dropped + await asyncio.sleep(0.1) + assert order.count("oak:close-done") == 1 + + +@pytest.mark.asyncio +async def test_runtime_directory_failures_are_start_failures_and_leave_nothing_behind(tmp_path, monkeypatch): + rig = sn.SensingRig( + sn.SensingRigConfig(**RIG), (1,), popen=lambda *a, **k: FakeProc(), runtime_root=str(tmp_path / "missing") + ) + await rig.start() + assert "runtime directory" in rig.start_error and await rig.close() is True + + monkeypatch.setattr(os, "chmod", lambda *a, **k: (_ for _ in ()).throw(PermissionError("denied"))) + rig2 = sn.SensingRig( + sn.SensingRigConfig(**RIG), (1,), popen=lambda *a, **k: FakeProc(), runtime_root=str(tmp_path) + ) + await rig2.start() + assert "denied" in rig2.start_error and list(tmp_path.iterdir()) != [] # created, so retained for cleanup + assert await rig2.close() is True and list(tmp_path.iterdir()) == [] + + +@pytest.mark.asyncio +async def test_cancelled_close_still_reaps_the_producer_and_observes_late_failures(): + import gc + + unobserved: list[dict] = [] + asyncio.get_running_loop().set_exception_handler(lambda _loop, ctx: unobserved.append(ctx)) + order: list[str] = [] + gate = threading.Event() + acq = Acquisition( + tracking=None, + cameras=(SpyAdapter(order, "slow", gate=gate), SpyAdapter(order, "late-fail", fail=True, gate=gate)), + sensing_rig=SpyRig(order), + ) + closing = asyncio.ensure_future(acq.close()) + await asyncio.sleep(0.1) + closing.cancel() # the caller gives up while both adapter closes are still held + with pytest.raises(asyncio.CancelledError): + await closing + gate.set() # the held closes finish later; one of them fails with nobody awaiting it + for _ in range(100): + if "rig:close" in order and "late-fail:close-done" in order: + break + await asyncio.sleep(0.02) + assert "rig:close" in order # the producer was still reaped + del acq, closing + gc.collect() + await asyncio.sleep(0.05) + assert unobserved == [] # no "exception was never retrieved" + + +# --- strict station file, shared producer environment, factory misuse, cancellation precedence ------------------- + + +def _yaml(tmp_path, rig_extra="", camera="{type: SENSING_COLOR, module_name: L}", masks="['0x0000']"): + f = tmp_path / "s.yaml" + f.write_text( + "sensing_rig: {executable: e, expected_backend: MOCK, platform_config: p, config_name: c," + f" link_masks: {masks}{rig_extra}}}\ncameras: {{l: {camera}}}\n" + ) + return str(f) + + +@pytest.mark.parametrize( + "kw,msg", + [ + (dict(masks="[0x0000, 0x1100]"), "quoted strings"), # unquoted masks are YAML integers + (dict(rig_extra=", full_range: yes-please"), "full_range must be bool"), + (dict(rig_extra=", gpu_id: true"), "gpu_id must be int"), + (dict(camera="{type: SENSING_COLOR, module_name: L, jpeg_quality: 80.5}"), "jpeg_quality must be int"), + (dict(camera="{type: SENSING_COLOR, module_name: L, required: 1}"), "required must be bool"), + ( + dict(camera="{type: SENSING_COLOR, module_name: L, delivery_fps: true}"), + "delivery_fps must be int/float", + ), + (dict(camera="{type: SENSING_COLOR, module_name: 7}"), "module_name must be str"), + ], +) +def test_station_yaml_scalars_are_type_checked(tmp_path, kw, msg): + with pytest.raises(ValueError, match=msg): + sn.load_sensing_station(_yaml(tmp_path, **kw)) + + +def test_a_file_that_is_not_yaml_is_a_value_error_not_a_traceback(tmp_path): + f = tmp_path / "s.yaml" + f.write_text("sensing_rig: [unclosed\n") + with pytest.raises(ValueError, match="not valid YAML"): + sn.load_sensing_station(str(f)) + + +def test_the_listing_and_the_capture_run_share_one_sanitized_environment(monkeypatch): + monkeypatch.setenv("SENSING_MOCK_SCALE_DIV", "4") + seen = {} + + def run(args, **kw): + seen["env"] = kw["env"] + return subprocess.CompletedProcess(args, 0, GOOD_LISTING, "") + + sn.inspect_rig(sn.SensingRigConfig(**RIG), run=run) + assert "SENSING_MOCK_SCALE_DIV" not in seen["env"] and seen["env"] == sn._producer_env() + + +def test_a_factory_without_the_servers_sensor_description_says_so(): + from capture_server.launch import _sensing_composition + + cfg = ServerConfig( + mock=True, cameras={"left": sn.SensingStationCamera("L")}, sensing_rig=sn.SensingRigConfig(**RIG) + ) # no sensing_description: what a copied config would look like + camera = p.CameraStreamConfig(1, "left", p.CameraType.SENSING_COLOR, True, W, H, 30.0, 80) + with pytest.raises(ValueError, match="no rig or sensor description"): + _sensing_composition(cfg, (camera,)) + + +@pytest.mark.asyncio +async def test_a_producer_that_goes_quiet_but_stays_alive_fails_the_stream(tmp_path, monkeypatch): + monkeypatch.setattr(sn, "_NO_FRAME_TIMEOUT_S", 0.3) + adapter, rig = _adapter(tmp_path, camera=_camera(required=True, fps=100.0)) + producer = FileProducer(rig.path, encode_units(W, H, [RED, GREEN, BLUE]), loop=False).start() + try: + await adapter.enter() + assert adapter.camera_status().status == p.DeviceState.CONNECTED + with pytest.raises(CameraSourceError, match="no frame from the SENSING stream for"): + for _ in range(200): # rig.exit_code() stays None: the producer is alive, just silent + await adapter.next_camera_jpeg() + assert adapter.camera_status().status == p.DeviceState.DEGRADED + assert await adapter.close() is True + finally: + producer.stop() + + +def test_reclaim_all_hands_back_a_failed_streams_file_without_decoding(tmp_path): + path = tmp_path / "s.h264" + path.write_bytes(os.urandom(4 * 1024 * 1024)) # real, allocated bytes the producer keeps appending to + reader = h264.SensingStreamReader(str(path), width=W, height=H) + reader.connect(1.0) + before = allocated_bytes(str(path)) + reader.reclaim_all() + assert allocated_bytes(str(path)) < before / 2 and os.stat(path).st_size == 4 * 1024 * 1024 + with pytest.raises(h264.SensingStreamError, match="failed earlier"): + reader.read_latest(0.0) # reclaiming never revives the stream + with open(path, "ab") as f: + f.write(os.urandom(2 * 1024 * 1024)) + reader.reclaim_all() # more output arrived: reclaimed again on the next call + assert allocated_bytes(str(path)) < 1024 * 1024 + reader.close() + + +def test_reclaim_all_tolerates_a_failing_native_call_only_up_to_the_spool_bound(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_MAX_UNRECLAIMED_BYTES", 2 * 1024 * 1024) + path = tmp_path / "s.h264" + path.write_bytes(os.urandom(1024 * 1024)) + reader = h264.SensingStreamReader(str(path), width=W, height=H, fallocate=lambda *a: -1) + reader.connect(1.0) + reader.reclaim_all() + assert reader._punched == 0 # nothing marked reclaimed; tried again next tick + with open(path, "ab") as f: + f.write(os.urandom(2 * 1024 * 1024)) + with pytest.raises(h264.SensingStreamError, match="cannot be handed back"): + reader.reclaim_all() # 3 MiB the producer keeps growing and we cannot return: the spool is unbounded + reader.close() + + +def test_reclaim_all_reclaims_a_file_created_after_the_stream_failed_before_attaching(tmp_path): + path = tmp_path / "late.h264" + reader = h264.SensingStreamReader(str(path), width=W, height=H) + reader.reclaim_all() # no file yet: nothing accumulating, nothing raised + path.write_bytes(os.urandom(4 * 1024 * 1024)) # the producer creates it later + reader.reclaim_all() + assert allocated_bytes(str(path)) < 1024 * 1024 + reader.close() + + +def test_reclaim_all_bounds_a_late_file_it_cannot_open(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_MAX_UNRECLAIMED_BYTES", 1024) + path = tmp_path / "late.h264" + path.write_bytes(os.urandom(64 * 1024)) + path.chmod(0o400) # readable only: O_RDWR open fails, so nothing can be handed back + if os.access(path, os.W_OK): + pytest.skip("running as a user that can write read-only files") + reader = h264.SensingStreamReader(str(path), width=W, height=H) + with pytest.raises(h264.SensingStreamError, match="cannot be handed back"): + reader.reclaim_all() + reader.close() + + +@pytest.mark.asyncio +async def test_an_unreclaimable_dead_stream_stops_the_shared_producer(tmp_path): + adapter, rig = _adapter(tmp_path, camera=_camera(required=False, fps=200.0)) + + def unbounded(): + raise h264.SensingStreamError("spool is unbounded") + + adapter._reader = types.SimpleNamespace(reclaim_all=unbounded, close=lambda: None) + adapter._dead = True + adapter._deadline = time.monotonic() + assert await adapter.next_camera_jpeg() is None + assert rig.aborted == ["spool is unbounded"] + await adapter.close() + + +@pytest.mark.asyncio +async def test_a_dead_stream_keeps_reclaiming_on_every_paced_poll(tmp_path): + calls = [] + adapter, _ = _adapter(tmp_path, camera=_camera(required=False, fps=200.0)) + adapter._reader = types.SimpleNamespace(reclaim_all=lambda: calls.append(1), close=lambda: None) + adapter._dead = True + adapter._deadline = time.monotonic() + for _ in range(3): + assert await adapter.next_camera_jpeg() is None + assert len(calls) == 3 + await adapter.close() + + +@pytest.mark.asyncio +async def test_stop_during_bring_up_ends_the_startup_and_the_release_is_confirmed_promptly(tmp_path): + adapter, rig = _adapter(tmp_path, startup_timeout_s=30.0) # the stream file never appears + entering = asyncio.ensure_future(adapter.enter()) + await asyncio.sleep(0.3) + started = time.monotonic() + assert await adapter.close() is True # without the stop signal this queues behind the 30 s startup + assert time.monotonic() - started < 1.0 + with pytest.raises(CameraSourceError, match="stopped during startup"): + await entering + + +def test_a_stopped_reader_ends_every_blocking_operation_with_the_reason(tmp_path): + stop = threading.Event() + path = tmp_path / "s.h264" + path.write_bytes(b"".join(encode_units(W, H, [RED, GREEN, BLUE]) * 50)) + reader = h264.SensingStreamReader(str(path), width=W, height=H, stop=stop) + reader.connect(1.0) + stop.set() + with pytest.raises(h264.SensingStreamError, match="stopped"): + reader.read_latest(5.0) + with pytest.raises(h264.SensingStreamError, match="stopped"): + h264.SensingStreamReader(str(tmp_path / "never"), width=W, height=H, stop=stop).connect(5.0) + finisher = h264.SensingStreamReader(str(path), width=W, height=H, stop=threading.Event()) + finisher.connect(1.0) + finisher._stop.set() + with pytest.raises(h264.SensingStreamError, match="stopped"): + finisher.finish() diff --git a/examples/egocap/tests/capture_server/test_sensing_h264.py b/examples/egocap/tests/capture_server/test_sensing_h264.py new file mode 100644 index 0000000000..60e13a5ac2 --- /dev/null +++ b/examples/egocap/tests/capture_server/test_sensing_h264.py @@ -0,0 +1,345 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""SensingStreamReader: tails the producer's H.264 file, decodes every unit in order, delivers the newest frame. + +CPU-only, against real libx264-encoded units (intra-only and inter-coded GOPs); the native producer's own +output is covered separately. +""" + +from __future__ import annotations + +import ctypes +import errno +import os +import threading +import time + +import numpy as np +import pytest + +from capture_server.acquisition import sensing_h264 as h264 +from capture_server.acquisition.errors import SourceError +from capture_server.acquisition.sensing_jpeg import encode_rgb_jpeg + +from .h264_fixtures import allocated_bytes, encode_units + +W, H = 64, 32 +RED, GREEN, BLUE = (220, 20, 20), (20, 220, 20), (20, 20, 220) + + +def _close(rgb: np.ndarray, expected: tuple[int, int, int], tol: int = 14) -> bool: + return bool(np.abs(rgb[H // 2, W // 2].astype(int) - np.array(expected)).max() <= tol) + + +def _reader(path, **kw) -> h264.SensingStreamReader: + reader = h264.SensingStreamReader(str(path), width=kw.pop("width", W), height=kw.pop("height", H), **kw) + reader.connect(1.0) + return reader + + +def test_decodes_correct_colours_and_geometry(tmp_path): + units = encode_units(W, H, [RED, GREEN]) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) + units[-1]) + reader = _reader(path) + rgb, selected_ns = reader.read_latest(1.0) + assert rgb.shape == (H, W, 3) and selected_ns > 0 and _close(rgb, GREEN) + reader.close() + + +def test_newest_complete_unit_wins_and_older_ones_are_skipped(tmp_path): + units = encode_units(W, H, [RED, GREEN, BLUE]) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) # the last unit is not yet known complete: GREEN is the newest + reader = _reader(path) + rgb, _ = reader.read_latest(1.0) + assert _close(rgb, GREEN) # RED was decoded first but only the newest decoded frame is delivered + assert reader.read_latest(0.05) is None # nothing newer; never invents a frame + reader.close() + + +def test_a_half_written_unit_is_not_delivered_until_complete(tmp_path): + units = encode_units(W, H, [RED, GREEN]) + path = tmp_path / "s.h264" + reader_path = str(path) + with open(path, "wb") as f: + f.write(units[0][: len(units[0]) // 2]) + f.flush() + reader = h264.SensingStreamReader(reader_path, width=W, height=H) + reader.connect(1.0) + assert reader.read_latest(0.05) is None + f.write(units[0][len(units[0]) // 2 :] + units[1]) + f.flush() + rgb, _ = reader.read_latest(1.0) + assert _close(rgb, RED) + reader.close() + + +def test_missing_file_times_out_with_oserror(tmp_path): + reader = h264.SensingStreamReader(str(tmp_path / "never"), width=W, height=H) + with pytest.raises(FileNotFoundError): + reader.connect(0.05) + + +def test_geometry_mismatch_is_terminal(tmp_path): + units = encode_units(W, H, [RED, GREEN]) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) + units[-1]) + reader = _reader(path, width=W * 2) + with pytest.raises(h264.SensingStreamError, match="expected 128x32"): + reader.read_latest(1.0) + with pytest.raises(h264.SensingStreamError, match="failed earlier"): + reader.read_latest(0.0) + + +def test_undecodable_bytes_are_a_terminal_stream_error_not_a_frame(tmp_path): + path = tmp_path / "s.h264" + # Annex-B shaped but not video, like the native mock encoder's units. + path.write_bytes((b"\x00\x00\x00\x01\x67MOCK" + b"U" * 200 + b"\x00\x00\x00\x01\x65MOCK" + b"U" * 200) * 3) + reader = _reader(path) + with pytest.raises(h264.SensingStreamError): + for _ in range(5): + reader.read_latest(0.2) + with pytest.raises(h264.SensingStreamError, match="failed earlier"): + reader.read_latest(0.0) + + +def test_read_before_connect_is_an_error(tmp_path): + reader = h264.SensingStreamReader(str(tmp_path / "x"), width=W, height=H) + with pytest.raises(h264.SensingStreamError, match="not connected"): + reader.read_latest(0.0) + + +def test_consumed_prefix_is_returned_to_the_filesystem_while_reading_continues(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_PUNCH_CHUNK_BYTES", 4096) + units = encode_units(W, H, [RED, GREEN, BLUE]) * 600 # each unit is an independent IDR + path = tmp_path / "s.h264" + with open(path, "wb") as f: + reader = None + for i, unit in enumerate(units): + f.write(unit) + f.flush() + if reader is None: + reader = h264.SensingStreamReader(str(path), width=W, height=H) + reader.connect(1.0) + if i > 2: + assert reader.read_latest(1.0) is not None + size = os.stat(path).st_size + assert allocated_bytes(str(path)) < size # holes punched behind the read offset (best effort) + reader.close() + + +def test_native_size_unit_decodes_independently_and_fits_the_payload_ceiling(tmp_path): + width, height = 2560, 1984 + units = encode_units(width, height, [RED, GREEN, BLUE]) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) + reader = h264.SensingStreamReader(str(path), width=width, height=height) + reader.connect(1.0) + started = time.monotonic() + rgb, _ = reader.read_latest(5.0) # RED then GREEN decoded in order; GREEN delivered + assert rgb.shape == (height, width, 3) + assert bool(np.abs(rgb[900, 1200].astype(int) - np.array(GREEN)).max() <= 14) + assert len(encode_rgb_jpeg(rgb, 80)) < 2 * 1024 * 1024 + assert time.monotonic() - started < 5.0 + reader.close() + + +def test_oversized_jpeg_names_dimensions_and_quality(): + noise = np.random.default_rng(0).integers(0, 256, (64, 64, 3), dtype=np.uint8) + with pytest.raises(SourceError, match=r"64x64 JPEG at quality 95.*ceiling"): + encode_rgb_jpeg(noise, 95, max_bytes=100) + + +def test_jpeg_rejects_wrong_shape(): + with pytest.raises(ValueError): + encode_rgb_jpeg(np.zeros((4, 4, 4), np.uint8), 80) + + +def test_jpeg_keeps_rgb_channel_order(): + rgb = np.zeros((16, 16, 3), np.uint8) + rgb[...] = RED + import cv2 + + decoded = cv2.cvtColor( + cv2.imdecode(np.frombuffer(encode_rgb_jpeg(rgb, 90), np.uint8), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB + ) + assert bool(np.abs(decoded[8, 8].astype(int) - np.array(RED)).max() <= 8) + + +def test_jpeg_reports_an_encoder_failure(monkeypatch): + import cv2 + + monkeypatch.setattr(cv2, "imencode", lambda *a, **k: (False, None)) + with pytest.raises(SourceError, match=r"encoder failed for 4x4 at quality 80"): + encode_rgb_jpeg(np.zeros((4, 4, 3), np.uint8), 80) + + +def test_jpeg_rejects_an_incomplete_stream(monkeypatch): + import cv2 + + monkeypatch.setattr(cv2, "imencode", lambda *a, **k: (True, np.frombuffer(b"\xff\xd8 truncated", np.uint8))) + with pytest.raises(SourceError, match="incomplete JPEG"): + encode_rgb_jpeg(np.zeros((4, 4, 3), np.uint8), 80) + + +def _failing_fallocate(code: int): + """A stand-in for the C call that fails the way the real one does: returns -1 and sets errno.""" + + def fallocate(_fd, _mode, _offset, _length): + ctypes.set_errno(code) + return -1 + + return fallocate + + +def test_a_failing_space_reclamation_ends_the_stream_with_the_reason_and_never_marks_bytes_reclaimed( + tmp_path, monkeypatch +): + monkeypatch.setattr(h264, "_PUNCH_CHUNK_BYTES", 256) + units = encode_units(W, H, [RED, GREEN, BLUE]) * 40 + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) + reader = h264.SensingStreamReader(str(path), width=W, height=H, fallocate=_failing_fallocate(errno.EOPNOTSUPP)) + reader.connect(1.0) + with pytest.raises(h264.SensingStreamError, match=r"cannot reclaim.*errno 95"): + reader.read_latest(1.0) + assert reader._punched == 0 # the reclaimed offset advances only after the call succeeds + with pytest.raises(h264.SensingStreamError, match="failed earlier"): + reader.read_latest(0.0) + + +def test_a_file_we_cannot_reclaim_from_is_an_error_not_a_quiet_read_only_fallback(tmp_path): + path = tmp_path / "s.h264" + path.write_bytes(b"x") + path.chmod(0o444) + if os.access(path, os.W_OK): + pytest.skip("running with privileges that ignore file modes") + reader = h264.SensingStreamReader(str(path), width=W, height=H) + with pytest.raises(h264.SensingStreamError, match="reclamation"): + reader.connect(0.5) + + +def test_one_poll_reads_at_most_its_byte_cap_even_with_a_larger_backlog(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_MAX_POLL_BYTES", 2048) + monkeypatch.setattr(h264, "_READ_BYTES", 512) + units = encode_units(W, H, [RED, GREEN, BLUE]) * 200 + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) + reader = _reader(path) + assert reader.read_latest(1.0) is not None + assert reader._offset <= 2048 and os.stat(path).st_size > 20 * 2048 # the backlog is read over later polls + reader.close() + + +def test_a_producer_that_keeps_appending_cannot_extend_a_poll_past_its_deadline(tmp_path): + units = encode_units(W, H, [RED, GREEN]) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 3) + stop = threading.Event() + + def append() -> None: + with open(path, "ab") as f: + while not stop.is_set(): + f.write(units[0]) + f.flush() + + writer = threading.Thread(target=append, daemon=True) + reader = _reader(path) + writer.start() + started = time.monotonic() + assert reader.read_latest(0.3) is not None + assert time.monotonic() - started < 2.0 + stop.set() + writer.join(timeout=2) + reader.close() + + +def test_input_that_never_completes_a_unit_is_rejected_instead_of_retained_without_bound(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_MAX_UNIT_BYTES", 8192) + path = tmp_path / "s.h264" + path.write_bytes(b"\x01" * 64 * 1024) # no start codes: the parser never completes a unit + reader = _reader(path) + with pytest.raises(h264.SensingStreamError, match="no complete access unit in"): + reader.read_latest(1.0) + + +def test_inter_coded_units_are_decoded_in_order_and_the_newest_decoded_frame_is_delivered(tmp_path): + # gop=30: after the first IDR the units are P-frames that cannot be decoded without their predecessors. + units = encode_units(W, H, [RED] * 3 + [GREEN] * 3 + [BLUE] * 3, gop=30) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) + reader = _reader(path) + rgb, _ = reader.read_latest(2.0) + assert _close(rgb, BLUE) # the last complete unit; the final one is still held back by the parser + reader.close() + + +def test_finish_flushes_the_unit_the_parser_was_holding_back_and_only_once(tmp_path): + units = encode_units(W, H, [RED, GREEN, BLUE]) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) + reader = _reader(path) + rgb, _ = reader.read_latest(1.0) + assert _close(rgb, GREEN) and reader.read_latest(0.05) is None # BLUE is incomplete while the file may grow + last = reader.finish() # the producer ended: now the end of the file is the end of the stream + assert last is not None and _close(last[0], BLUE) + assert reader.finish() is None + reader.close() + + +def test_a_backlog_beyond_the_pending_bound_resumes_at_the_newest_idr(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_MAX_PENDING_BYTES", 400) + units = encode_units(W, H, [RED, GREEN, BLUE] * 17) # 51 independent IDR units + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) + reader = _reader(path) + rgb, _ = reader.read_latest(2.0) + assert _close(rgb, GREEN) # unit 49, the newest complete one; everything older was skipped undecoded + reader.close() + + +def test_an_inter_coded_backlog_that_cannot_be_caught_up_ends_the_stream_with_the_reason(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_MAX_PENDING_BYTES", 100) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(encode_units(W, H, [RED] * 40, gop=1000))) # one IDR, then only P-frames + reader = _reader(path) + with pytest.raises(h264.SensingStreamError, match="decoder cannot keep up"): + reader.read_latest(2.0) + + +def test_a_restart_point_needs_an_idr_slice_and_in_band_parameter_sets(): + idr, p_frame = encode_units(W, H, [RED, RED], gop=30)[:2] + assert h264._is_restart_point(idr) and not h264._is_restart_point(p_frame) + slice_at = idr.index(b"\x00\x00\x01\x65") + assert not h264._is_restart_point(idr[slice_at:]) # the IDR slice alone: no SPS/PPS to restart from + + +def test_finish_straight_away_decodes_every_pending_unit_in_order_before_flushing(tmp_path): + path = tmp_path / "s.h264" + path.write_bytes(b"".join(encode_units(W, H, [RED] * 3 + [GREEN] * 3 + [BLUE] * 3, gop=30))) + reader = _reader(path) + last = reader.finish() # no read_latest first: the units are still queued/held back, all must be decoded + assert last is not None and _close(last[0], BLUE) + reader.close() + + +def test_finish_is_bounded_by_its_budget_and_fails_with_the_reason(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_FINISH_BUDGET_S", 0.0) + monkeypatch.setattr(h264, "_READ_BYTES", 512) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(encode_units(W, H, [RED, GREEN, BLUE]) * 300)) + reader = _reader(path) + with pytest.raises(h264.SensingStreamError, match="could not finish decoding"): + reader.finish() + + +def test_reading_after_finish_is_an_error_not_a_decode_on_a_flushed_decoder(tmp_path): + path = tmp_path / "s.h264" + path.write_bytes(b"".join(encode_units(W, H, [RED, GREEN]))) + reader = _reader(path) + reader.finish() + with pytest.raises(h264.SensingStreamError, match="already finished"): + reader.read_latest(0.0) diff --git a/examples/egocap/tests/capture_server/test_sensing_nvdec.py b/examples/egocap/tests/capture_server/test_sensing_nvdec.py new file mode 100644 index 0000000000..ddf957a1f8 --- /dev/null +++ b/examples/egocap/tests/capture_server/test_sensing_nvdec.py @@ -0,0 +1,538 @@ +# SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Hardware H.264 decoding for the SENSING reader: the decoder process wrapper and the reader's use of it. + +The Jetson decoder is a GStreamer subprocess; these tests replace it with stand-ins that follow the same pipe +contract (Annex-B on stdin, tightly packed I420 frames on stdout), so every code path runs on any machine. A +separate check at the end exercises the real pipeline when this machine has NVDEC. +""" + +from __future__ import annotations + +import logging +import subprocess +import sys +import textwrap +import time + +import numpy as np +import pytest + +from capture_server.acquisition import sensing_h264 as h264 +from capture_server.acquisition import sensing_nvdec as nvdec + +from .h264_fixtures import encode_units + +W, H = 64, 32 +FRAME = W * H * 3 // 2 +RED, GREEN, BLUE = (220, 20, 20), (20, 220, 20), (20, 20, 220) + +# A stand-in that decodes the Annex-B on stdin with PyAV and writes each frame as raw I420: the real decoder's +# contract, so the reader sees correct pixels end to end. +PYAV_DECODER = textwrap.dedent( + """ + import sys + import av + + parser = av.CodecContext.create("h264", "r") + decoder = av.CodecContext.create("h264", "r") + out = sys.stdout.buffer + + def emit(packets): + for packet in packets: + for frame in decoder.decode(packet): + out.write(frame.to_ndarray(format="yuv420p").tobytes()) + out.flush() + + while chunk := sys.stdin.buffer.read1(65536): + emit(parser.parse(chunk)) + emit(parser.parse(None)) + emit([None]) + """ +) + + +def _command(script: str) -> list[str]: + return [sys.executable, "-c", script] + + +def _counting_decoder(unit_bytes: int = 4) -> list[str]: + """A stand-in that writes one frame, filled with its sequence number, per ``unit_bytes`` read.""" + return _command( + textwrap.dedent( + f""" + import sys + n = 0 + while True: + data = sys.stdin.buffer.read({unit_bytes}) + if len(data) < {unit_bytes}: + break + sys.stdout.buffer.write(bytes([n % 251]) * {FRAME}) + sys.stdout.buffer.flush() + n += 1 + """ + ) + ) + + +def _wait_for(predicate, timeout_s: float = 10.0) -> None: + deadline = time.monotonic() + timeout_s + while not predicate(): + assert time.monotonic() < deadline, "timed out waiting for the decoder" + time.sleep(0.01) + + +@pytest.fixture +def decoders(): + """Decoders created by a test, closed afterwards so a failing test never leaks a process.""" + made = [] + + def make(command, width: int = W, height: int = H) -> nvdec.HardwareDecoder: + decoder = nvdec.HardwareDecoder(width, height, command=command) + made.append(decoder) + return decoder + + yield make + for decoder in made: + decoder.close() + + +# --- availability and the pipeline --- + + +@pytest.fixture +def fresh_probe(): + nvdec.hardware_decoder_unavailable_reason.cache_clear() + yield + nvdec.hardware_decoder_unavailable_reason.cache_clear() + + +def _fake_environment(monkeypatch, *, node=True, tools=("gst-launch-1.0", "gst-inspect-1.0"), probe=0): + monkeypatch.setattr(nvdec.os.path, "exists", lambda p: node if p == nvdec._DEVICE_NODE else True) + monkeypatch.setattr(nvdec.shutil, "which", lambda t: f"/usr/bin/{t}" if t in tools else None) + if isinstance(probe, Exception): + + def run(*a, **k): + raise probe + + else: + + def run(*a, **k): + return subprocess.CompletedProcess(a[0], probe) + + monkeypatch.setattr(nvdec.subprocess, "run", run) + + +def test_available_when_node_tools_and_element_exist(monkeypatch, fresh_probe): + _fake_environment(monkeypatch) + assert nvdec.hardware_decoder_unavailable_reason() is None + + +def test_unavailable_without_the_decoder_node(monkeypatch, fresh_probe): + _fake_environment(monkeypatch, node=False) + assert "not a Jetson" in nvdec.hardware_decoder_unavailable_reason() + + +def test_unavailable_without_gstreamer_tools(monkeypatch, fresh_probe): + _fake_environment(monkeypatch, tools=("gst-launch-1.0",)) + assert "gst-inspect-1.0 is not installed" in nvdec.hardware_decoder_unavailable_reason() + + +def test_unavailable_when_the_element_is_missing(monkeypatch, fresh_probe): + _fake_environment(monkeypatch, probe=1) + assert "nvv4l2decoder is not installed" in nvdec.hardware_decoder_unavailable_reason() + + +def test_unavailable_when_the_probe_cannot_run(monkeypatch, fresh_probe): + _fake_environment(monkeypatch, probe=subprocess.TimeoutExpired("gst-inspect-1.0", 10)) + assert "could not query GStreamer" in nvdec.hardware_decoder_unavailable_reason() + + +def test_pipeline_is_low_delay_with_a_leaky_queue_by_default(): + command = nvdec.pipeline_command() + assert "disable-dpb=true" in command and "leaky=downstream" in command + assert command.index("queue") < command.index("nvvidconv") # frames are dropped before the costly copy + + +def test_pipeline_keeps_the_reorder_buffer_when_asked(): + assert "disable-dpb=true" not in nvdec.pipeline_command(low_delay=False) + + +# --- the decoder process --- + + +@pytest.mark.parametrize("width, height", [(0, 32), (64, 0), (63, 32), (64, 31)]) +def test_geometry_must_be_positive_and_even(width, height): + with pytest.raises(ValueError, match="even dimensions"): + nvdec.HardwareDecoder(width, height) + + +def test_frames_flow_and_the_newest_wins(decoders): + decoder = decoders(_counting_decoder()) + decoder.start() + for _ in range(5): + decoder.write(b"\x00\x00\x00\x00") + _wait_for(lambda: decoder.frames_received == 5) + frame = decoder.take_frame() + assert frame.shape == (H * 3 // 2, W) and frame.dtype == np.uint8 + assert (frame == 4).all() # frames 0..3 were replaced before being taken + assert decoder.take_frame() is None and not decoder.has_frame + decoder.release_frame() + + +def test_a_taken_frame_is_not_overwritten_while_held(decoders): + decoder = decoders(_counting_decoder()) + decoder.start() + decoder.write(b"\x00" * 4) + _wait_for(lambda: decoder.has_frame) + held = decoder.take_frame() + for _ in range(6): + decoder.write(b"\x00" * 4) + _wait_for(lambda: decoder.frames_received == 7) + assert (held == 0).all() # the output thread filled other buffers + decoder.release_frame() + assert (decoder.take_frame() == 6).all() + decoder.release_frame() + + +def test_taking_twice_without_releasing_is_a_programming_error(decoders): + decoder = decoders(_counting_decoder()) + decoder.start() + decoder.write(b"\x00" * 8) + _wait_for(lambda: decoder.frames_received == 2) + decoder.take_frame() + decoder.write(b"\x00" * 4) + _wait_for(lambda: decoder.frames_received == 3) + with pytest.raises(RuntimeError, match="release_frame"): + decoder.take_frame() + + +def test_closing_the_input_flushes_and_ends_the_output(decoders): + decoder = decoders(_counting_decoder()) + decoder.start() + decoder.write(b"\x00" * 12) + assert decoder.close_input() is True + assert decoder.wait_finished(10.0) + decoder.check() # a closed input makes the exit expected, not an error + assert decoder.frames_received == 3 + + +def test_writes_after_the_input_is_closed_are_refused(decoders): + decoder = decoders(_counting_decoder()) + decoder.start() + decoder.close_input() + with pytest.raises(nvdec.HardwareDecodeError, match="already closed"): + decoder.write(b"x") + + +def test_close_input_waits_while_input_is_still_queued(decoders): + decoder = decoders(_command("import time\ntime.sleep(60)")) # never reads its input + decoder.start() + decoder.write(b"\x00" * (8 << 20)) + assert decoder.pending_input_bytes > 0 + assert decoder.close_input() is False + + +def test_a_stalled_decoder_never_blocks_the_writer(decoders): + decoder = decoders(_command("import time\ntime.sleep(60)")) + decoder.start() + started = time.monotonic() + for _ in range(8): + decoder.write(b"\x00" * (1 << 20)) + assert time.monotonic() - started < 2.0 + assert decoder.pending_input_bytes >= 6 << 20 # the pipe took about one MiB; the rest is queued here + + +def test_an_early_exit_is_reported_with_its_error_output(decoders): + decoder = decoders(_command("import sys\nsys.stderr.write('no such device')\nsys.exit(3)")) + decoder.start() + _wait_for(lambda: decoder._reader_done.is_set()) + time.sleep(0.2) # let the diagnostics thread collect stderr + with pytest.raises(nvdec.HardwareDecodeError, match=r"exited unexpectedly \(code 3\).*no such device"): + decoder.check() + + +def test_writing_to_a_dead_decoder_is_reported(decoders): + decoder = decoders(_command("import sys\nsys.exit(1)")) + decoder.start() + _wait_for(lambda: decoder._reader_done.is_set()) + with pytest.raises(nvdec.HardwareDecodeError, match="closed its input"): + for _ in range(50): + decoder.write(b"\x00" * 4096) + time.sleep(0.01) + + +def test_output_that_ends_mid_frame_is_an_error(decoders): + decoder = decoders(_command(f"import sys\nsys.stdout.buffer.write(b'x' * {FRAME // 2})\nsys.stdout.flush()")) + decoder.start() + _wait_for(lambda: decoder._reader_done.is_set()) + with pytest.raises(nvdec.HardwareDecodeError, match="mid-frame"): + decoder.check() + + +def test_close_stops_a_stalled_process_quickly_and_is_repeatable(decoders): + decoder = decoders(_command("import time\ntime.sleep(60)")) + decoder.start() + proc = decoder._proc + started = time.monotonic() + decoder.close() + assert time.monotonic() - started < 1.5 and proc.poll() is not None + decoder.close() # idempotent + + +def test_closing_a_decoder_that_never_started_is_harmless(): + nvdec.HardwareDecoder(W, H).close() + + +def test_pumping_before_the_process_starts_does_nothing(): + decoder = nvdec.HardwareDecoder(W, H) + decoder.pump() + assert decoder.pending_input_bytes == 0 + + +def test_close_kills_a_process_that_ignores_termination(decoders): + ignoring = "import signal, time\nsignal.signal(signal.SIGTERM, signal.SIG_IGN)\ntime.sleep(60)" + decoder = decoders(_command(ignoring)) + decoder.start() + proc = decoder._proc + time.sleep(0.5) # the child must have installed its handler before the terminate arrives + started = time.monotonic() + decoder.close() + assert time.monotonic() - started < 2.0 and proc.poll() is not None + + +def test_a_pipe_that_cannot_be_enlarged_is_tolerated(decoders, monkeypatch): + def refuse(*args): + raise OSError("not permitted") + + monkeypatch.setattr(nvdec.fcntl, "fcntl", refuse) + decoder = decoders(_counting_decoder()) + decoder.start() + decoder.write(b"\x00" * 4) + _wait_for(lambda: decoder.has_frame) + + +def test_close_input_twice_is_harmless(decoders): + decoder = decoders(_counting_decoder()) + decoder.start() + assert decoder.close_input() is True + assert decoder.close_input() is True + + +# --- the reader on the hardware path --- + + +def _reader(path, **kw) -> h264.SensingStreamReader: + kw.setdefault("decoder", "hardware") + kw.setdefault("hardware_command", _command(PYAV_DECODER)) + reader = h264.SensingStreamReader(str(path), width=kw.pop("width", W), height=kw.pop("height", H), **kw) + reader.connect(1.0) + return reader + + +def _close(rgb: np.ndarray, expected, tol: int = 14) -> bool: + return bool(np.abs(rgb[H // 2, W // 2].astype(int) - np.array(expected)).max() <= tol) + + +def _read_until(reader, timeout_s: float = 20.0): + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + out = reader.read_latest(0.2) + if out is not None: + return out + raise AssertionError("no frame decoded in time") + + +def test_hardware_reader_delivers_correct_colours(tmp_path, caplog): + units = encode_units(W, H, [RED, GREEN, BLUE], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 4) + with caplog.at_level(logging.INFO, logger="isaaccapture.capture_server.sensing_h264"): + reader = _reader(path) + assert "NVDEC hardware decoder" in caplog.text + rgb, selected_ns = _read_until(reader) + assert rgb.shape == (H, W, 3) and selected_ns > 0 + assert any(_close(rgb, c) for c in (RED, GREEN, BLUE)) + reader.close() + + +def test_hardware_reader_finish_returns_the_last_frame(tmp_path): + units = encode_units(W, H, [RED, GREEN, BLUE], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units)) + reader = _reader(path) + reader.read_latest(0.05) + rgb, _ = reader.finish() + assert _close(rgb, BLUE) # the final unit is only known complete once the stream ends + assert reader.finish() is None # terminal + reader.close() + + +def test_hardware_reader_full_range_flag_changes_the_conversion(tmp_path): + units = encode_units(W, H, [GREEN], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 3) + limited = _reader(path) + full = _reader(path, full_range=True) + a, _ = _read_until(limited) + b, _ = _read_until(full) + assert not np.array_equal(a, b) + limited.close() + full.close() + + +def test_hardware_reader_rejects_a_stream_of_the_wrong_size(tmp_path): + units = encode_units(W, H, [RED, GREEN], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 2) + reader = _reader(path, width=W * 2, height=H * 2) + with pytest.raises(h264.SensingStreamError, match=r"decoded 64x32, expected 128x64"): + _read_until(reader) + reader.close() + + +def test_hardware_reader_reports_a_dead_decoder(tmp_path): + units = encode_units(W, H, [RED, GREEN], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 2) + reader = _reader(path, hardware_command=_command("import sys\nsys.stderr.write('boom')\nsys.exit(2)")) + with pytest.raises(h264.SensingStreamError, match="decoder"): + for _ in range(100): + reader.read_latest(0.1) + with pytest.raises(h264.SensingStreamError, match="failed earlier"): + reader.read_latest(0.0) + reader.close() + + +def test_hardware_reader_close_ends_the_process(tmp_path): + units = encode_units(W, H, [RED, GREEN], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 2) + reader = _reader(path) + _read_until(reader) + proc = reader._hw._proc + reader.close() + assert proc.poll() is not None and reader._hw is None + + +def test_input_beyond_the_decoders_capacity_stays_in_the_readers_queue(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "_HW_MAX_INPUT_BYTES", 1) + units = encode_units(W, H, [RED, GREEN], gop=1) + path = tmp_path / "s.h264" + # More than the decoder's pipe can hold, so the input really is full and later units must wait here. + path.write_bytes(b"".join(units) * (4 * nvdec._STDIN_PIPE_BYTES // sum(map(len, units)))) + reader = _reader(path, hardware_command=_command("import time\ntime.sleep(60)")) + assert reader.read_latest(0.3) is None + assert reader._pending # not handed over while the decoder's input is full + reader.close() + + +# --- choosing the decoder --- + + +def test_unknown_decoder_name_is_rejected(tmp_path): + with pytest.raises(ValueError, match="decoder must be one of"): + h264.SensingStreamReader(str(tmp_path / "s"), width=W, height=H, decoder="gpu") + + +def test_hardware_requested_but_unavailable_is_an_error(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "hardware_decoder_unavailable_reason", lambda: "no NVDEC here") + (tmp_path / "s.h264").write_bytes(b"") + reader = h264.SensingStreamReader(str(tmp_path / "s.h264"), width=W, height=H, decoder="hardware") + with pytest.raises(h264.SensingStreamError, match="requested but is unavailable: no NVDEC here"): + reader.connect(1.0) + reader.close() + + +def test_auto_falls_back_to_software_and_says_why(tmp_path, monkeypatch, caplog): + monkeypatch.setattr(h264, "hardware_decoder_unavailable_reason", lambda: "no NVDEC here") + units = encode_units(W, H, [RED, GREEN], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 2) + with caplog.at_level(logging.INFO, logger="isaaccapture.capture_server.sensing_h264"): + reader = h264.SensingStreamReader(str(path), width=W, height=H, decoder="auto") + reader.connect(1.0) + assert "falling back to CPU" in caplog.text and "no NVDEC here" in caplog.text + rgb, _ = reader.read_latest(1.0) + assert rgb.shape == (H, W, 3) and reader._hw is None + reader.close() + + +def test_auto_uses_hardware_when_available(tmp_path, monkeypatch): + monkeypatch.setattr(h264, "hardware_decoder_unavailable_reason", lambda: None) + units = encode_units(W, H, [RED, GREEN], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 4) + reader = _reader(path, decoder="auto") + _read_until(reader) + assert reader._hw is not None + reader.close() + + +def test_software_choice_is_logged(tmp_path, caplog): + path = tmp_path / "s.h264" + path.write_bytes(b"") + with caplog.at_level(logging.INFO, logger="isaaccapture.capture_server.sensing_h264"): + reader = h264.SensingStreamReader(str(path), width=W, height=H, decoder="software") + reader.connect(1.0) + assert "CPU (PyAV software), as configured" in caplog.text + reader.close() + + +# --- the rig configuration --- + + +def test_rig_decoder_defaults_to_auto_and_rejects_unknown_values(): + from capture_server.acquisition import sensing as sn + + base = dict( + executable="/x", expected_backend="MOCK", platform_config="/p", config_name="C", link_masks=("0x0000",) + ) + assert sn.SensingRigConfig(**base).decoder == "auto" + with pytest.raises(ValueError, match="decoder must be one of"): + sn.SensingRigConfig(**base, decoder="gpu") + + +def test_station_file_accepts_a_decoder_key_and_rejects_a_bad_one(tmp_path): + from capture_server.acquisition import sensing as sn + + def write(decoder: str) -> str: + path = tmp_path / "station.yaml" + path.write_text( + "sensing_rig:\n" + " executable: /x\n expected_backend: MOCK\n platform_config: /p\n config_name: C\n" + f" link_masks: ['0x0000']\n decoder: {decoder}\n" + "cameras:\n left:\n type: SENSING_COLOR\n module_name: M\n" + ) + return str(path) + + rig, _ = sn.load_sensing_station(write("software")) + assert rig.decoder == "software" + with pytest.raises(ValueError, match="decoder"): + sn.load_sensing_station(write("gpu")) + with pytest.raises(ValueError, match="decoder"): + sn.load_sensing_station(write("1")) + + +# --- the real pipeline, when this machine has NVDEC --- + + +@pytest.mark.skipif( + nvdec.hardware_decoder_unavailable_reason() is not None, reason="no Jetson NVDEC on this machine" +) +def test_real_nvdec_pipeline_decodes_correct_colours(tmp_path): + # Hardware decoders have a minimum surface size; a 640x480 stream is well above it. + width, height = 640, 480 + units = encode_units(width, height, [RED, GREEN, BLUE], gop=1) + path = tmp_path / "s.h264" + path.write_bytes(b"".join(units) * 8) + reader = h264.SensingStreamReader(str(path), width=width, height=height, decoder="hardware") + reader.connect(1.0) + try: + rgb, _ = _read_until(reader, timeout_s=30.0) + finally: + reader.close() + centre = rgb[height // 2, width // 2].astype(int) + assert min(np.abs(centre - np.array(c)).max() for c in (RED, GREEN, BLUE)) <= 16 diff --git a/examples/egocap/tests/support/capture_server.py b/examples/egocap/tests/support/capture_server.py index ec92979aa2..e524a3579a 100644 --- a/examples/egocap/tests/support/capture_server.py +++ b/examples/egocap/tests/support/capture_server.py @@ -64,7 +64,12 @@ async def start_capture_server( effective_config, source_factory=source_factory or build_mock_source_factory( - scenario="demo", seed=1, ic_faults=ic_faults or set(), camera_faults=camera_faults or set() + scenario="demo", + seed=1, + ic_faults=ic_faults or set(), + camera_faults=camera_faults or set(), + # The SENSING composition shares this config with the server (its cached sensor description). + config=effective_config, ), ) run_task = asyncio.ensure_future(server.run()) diff --git a/examples/egocap/uv.lock b/examples/egocap/uv.lock index b4a29b14fa..92723f87c0 100644 --- a/examples/egocap/uv.lock +++ b/examples/egocap/uv.lock @@ -42,6 +42,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "av" +version = "19.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/01/76b6bf563fe907597cf58f10d10168e93fe588b6db9f2658db87e87efa7e/av-19.0.2.tar.gz", hash = "sha256:bebbc52598886480f4d861c575dc2b020785d6c04e1308680ddcc545e1b94c66", size = 4277157, upload-time = "2026-10-21T03:45:30.799Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/2f/f51f37820741134516e841053d5ecebf6350fbd451adcbff9ead96aa1244/av-19.0.2-cp312-abi3-macosx_11_0_x86_64.whl", hash = "sha256:f5f7bcf987567cd3f9b01f1c375c59bf59d8a5b98c9c5c154aeda7a11e0c5dce", size = 22627374, upload-time = "2026-10-21T03:44:28.296Z" }, + { url = "https://files.pythonhosted.org/packages/7c/42/067d0bcd2c07103f18cc23b510a8289dd80330e984c49d51ce0bafcee017/av-19.0.2-cp312-abi3-macosx_14_0_arm64.whl", hash = "sha256:b0dff6dfcc6ff9944fbbff15bc4ee2d90a2c1695d493bdea96e86f48823e9feb", size = 18556422, upload-time = "2026-10-21T03:44:31.267Z" }, + { url = "https://files.pythonhosted.org/packages/de/0c/448f89f76cf5e4da35dd6294a1e8155b75daa8839f2f2fa353775855d152/av-19.0.2-cp312-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:04c8e412642c652cb2b1997a9cb2114546ee7d894600c631f4be8ceb26a0d0d2", size = 32683287, upload-time = "2026-10-21T03:44:34.588Z" }, + { url = "https://files.pythonhosted.org/packages/e1/7f/9f1237feb894afa472ef834aa1fbfe4d6398f28dc7e86a1b418650204c36/av-19.0.2-cp312-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c1da7aad0fa6c13b4a5c1e7985be0d13f8526a96284573acaf3d55ead6d3fe8", size = 34990159, upload-time = "2026-10-21T03:44:37.637Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b5/1bde0e54e525d88ad8f5becae657f8142543fd3be9d10b72141f041ca868/av-19.0.2-cp312-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9d4f9f78c0ce1f20deb72626fdb3c723757f61a8cd6fbb7dd44d9401884a91fd", size = 44422922, upload-time = "2026-10-21T03:44:41.847Z" }, + { url = "https://files.pythonhosted.org/packages/08/21/65997929fa4dd7c20f64e742aa87205a8d65485718a2ab51633756078e83/av-19.0.2-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:dfe7af5aa8243afd5d4778f85ad1a7c948e37fad272869da1f8f93f8d57fa7ad", size = 33754699, upload-time = "2026-10-21T03:44:45.772Z" }, + { url = "https://files.pythonhosted.org/packages/30/44/4de8c0ff5a5e25ceb0841efc0a4a7530a22d61800bc0cc19a1d03aa04358/av-19.0.2-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:44d3faae747cf04a33f39db8ae983ff3d93adc864ee663bc186b25b686803c3e", size = 36014845, upload-time = "2026-10-21T03:44:49.64Z" }, + { url = "https://files.pythonhosted.org/packages/32/5a/743a83666ded197df8b9957e2800bef9a39d52b23be2ca211bbd14b5fb22/av-19.0.2-cp312-abi3-win_amd64.whl", hash = "sha256:3ce5a1d7b916d13c16ac58f1756f0fee4af2d53f02bc1403e18550b50d375a22", size = 28150949, upload-time = "2026-10-21T03:44:53.434Z" }, + { url = "https://files.pythonhosted.org/packages/42/fe/e79d786628d9c0428f72214dbe4b90ee0c625e04d552a604d5235ff35515/av-19.0.2-cp312-abi3-win_arm64.whl", hash = "sha256:79a277b077e3768a0ea9f881ca0b3bc96d4135a8d3a7a579938adb58718d0dfd", size = 20708023, upload-time = "2026-10-21T03:44:56.435Z" }, +] + [[package]] name = "certifi" version = "2026.7.22" @@ -201,6 +218,7 @@ name = "egocap" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "av" }, { name = "fastapi" }, { name = "loguru" }, { name = "numpy" }, @@ -232,6 +250,7 @@ oak = [ [package.metadata] requires-dist = [ + { name = "av", specifier = ">=14" }, { name = "cupy-cuda12x", marker = "extra == 'gmsl'", specifier = ">=13,<14" }, { name = "depthai", marker = "extra == 'oak'", specifier = "==3.6.1" }, { name = "fastapi", specifier = ">=0.115" },