sensing: add CUDA IPC transport and a camera_viz source for it

CMakeLists.txt

@@ -71,6 +71,7 @@ option(BUILD_PLUGIN_NOITOM_MOCAP "Build Noitom mocap plugin (downloads MocapApi
 option(BUILD_PLUGIN_OGLO "Build OGLO tactile glove plugin (BLE, Linux only; fetches SimpleBLE + nlohmann/json)" OFF)
 option(BUILD_PLUGIN_WUJI_GLOVE "Build Wuji glove plugin (requires the wuji_sdk C SDK)" OFF)
 option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; skips if the Avatar SDK is missing)" OFF)
+option(BUILD_PLUGIN_SENSING "Build SENSING GMSL camera plugin (Jetson only)" OFF)
 option(BUILD_EXAMPLES "Build examples" ON)
 option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF)
 option(BUILD_TESTING "Build unit tests" ON)

docs/source/references/camera_streaming.rst

@@ -363,6 +363,9 @@ The source kind is selected by the ``type`` field of each entry in the YAML ``ca
    * - ``argus``
      - Jetson libargus cameras through CUDA EGLStream, including GMSL sensors exposed by
        ``nvargus-daemon``. Build setup with ``--with-argus`` first.
+   * - ``cuda_ipc``
+     - RGBA8 frames mapped straight out of another process's CUDA memory — no encode, no host
+       copy. See :ref:`camera-streaming-cuda-ipc`.
    * - ``video``
      - Recorded camera data — video-file replay (anything OpenCV's FFmpeg backend reads). Loops
        by default; ``stereo: true`` splits side-by-side recordings into eyes (viewer only). See
@@ -371,6 +374,45 @@ The source kind is selected by the ``type`` field of each entry in the YAML ``ca
      - Debugging tool — GPU-generated test pattern, no camera or file required. ``stereo: true`` adds
        a ``disparity_px`` offset between the eyes.
 
+.. _camera-streaming-cuda-ipc:
+
+Frames from another process (``cuda_ipc``)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A producer on the same machine publishes captured frames as CUDA device memory; camera_viz maps
+that memory once and renders from it. Nothing is encoded and nothing crosses host RAM, so the
+cost per frame is one 24-byte socket message. Measured on an AGX Orin at 1920×1080, the delay
+from producer timestamp to consumer receipt is about 1 ms, sustained at 60 fps.
+
+.. code-block:: yaml
+
+   cameras:
+     - name: left
+       type: cuda_ipc
+       socket: /tmp/s0.sock   # must match the producer's ipc= path
+       width: 2560            # must match what the producer serves
+       height: 1984
+
+Start either side first: the source retries until the socket appears and survives the producer
+restarting under it. ``width`` and ``height`` are checked during the handshake, and a mismatch is
+refused rather than rendered at the wrong stride. ``stereo: true`` is rejected, because each
+producer serves one sensor — declare one entry per socket.
+
+Any process can be the producer: the wire format is a 24-byte message and a CUDA VMM handle over
+``SCM_RIGHTS``. An animated test pattern ships with it, so the consumer can be developed with no
+camera and no capture SDK:
+
+.. code-block:: bash
+
+   cmake --build build --target sensing_ipc_testsrc
+   ./build/src/plugins/sensing/tools/sensing_ipc_testsrc --socket=/tmp/s0.sock \
+       --width=2560 --height=1984
+
+.. note::
+
+   One consumer at a time. The producer serves whoever connected most recently, so a second
+   camera_viz silently takes the feed from the first.
+
 .. _camera-streaming-live-camera:
 
 Running with a real camera

examples/camera_viz/configs/cuda_ipc.yaml

@@ -0,0 +1,58 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# CUDA IPC source — frames arrive as GPU memory mapped straight out of the
+# producer process. No encode, no decode, no host copy.
+#
+# Producer is the sensing plugin:
+#   camera_plugin_sensing --add-stream=sensor=0,ipc=/tmp/s0.sock \
+#                         --add-stream=sensor=1,ipc=/tmp/s1.sock
+#
+# Or, with no camera attached:
+#   sensing_ipc_testsrc --socket=/tmp/s0.sock --width=2560 --height=1984
+#
+# Then, in a second terminal:
+#   export DISPLAY=:1     # :99 is Xvfb; Tegra EGL cannot drive it
+#   ./camera_viz.sh run configs/cuda_ipc.yaml --mode window   # desktop window
+#   ./camera_viz.sh run configs/cuda_ipc.yaml                 # XR headset
+#
+# ``width``/``height`` must match what the producer serves; the handshake
+# rejects a mismatch rather than rendering garbage. Order does not matter —
+# the source retries until the socket appears, and survives it restarting.
+
+source: local
+
+cameras:
+  - name: left
+    enabled: true
+    type: cuda_ipc
+    socket: /tmp/s0.sock
+    width: 2560
+    height: 1984
+  - name: right
+    enabled: true
+    type: cuda_ipc
+    socket: /tmp/s1.sock
+    width: 2560
+    height: 1984
+
+display:
+  mode: xr                      # xr | window (xr is the default)
+  window:
+    width: 1920
+    height: 1080
+  xr:
+    near_z: 0.05
+    far_z: 100.0
+  clear_color: [0.0, 0.0, 0.0, 0.0]
+  placements:
+    left:
+      lock_mode: lazy
+      distance: 1.0
+      # Orin + CloudXR experimental runtime: the default OpenXR compositor
+      # draws the plane but leaves its contents black. Ignored in window mode.
+      compositor: televiz
+    right:
+      lock_mode: lazy
+      distance: 1.0
+      compositor: televiz

examples/camera_viz/python/isaaccapture_examples/camera_viz/pipeline/interface.py

@@ -38,6 +38,14 @@ class Frame:
 
     Stereo: ``image`` is the left eye; ``image_right`` carries the
     right eye when paired (both from the same capture instant).
+
+    ``image`` may be *borrowed* rather than owned: a source is allowed to
+    hand back a view onto memory it will reuse (``cuda_ipc`` maps producer
+    memory directly). A borrowed frame stays valid only until the next
+    ``latest()`` on the source that produced it, so a consumer must finish
+    reading — or copy — before polling again, and must not retain a Frame
+    across polls. ``ImageLayerBase::submit`` upholds this by synchronizing
+    its copy before returning; keep that guarantee if you change it.
     """
 
     image: Any

examples/camera_viz/python/isaaccapture_examples/camera_viz/sources/__init__.py

@@ -15,6 +15,7 @@
 
 from ._helpers import PairedFrameSource, set_notify_sink, set_verbose
 from .argus import ArgusSource
+from .cuda_ipc import CudaIpcSource
 from .oakd import OakdSource
 from .rtp_h264 import RtpH264Source
 from .synthetic import SyntheticSource, SyntheticStereoSource
@@ -24,6 +25,7 @@
 
 __all__ = [
     "ArgusSource",
+    "CudaIpcSource",
     "OakdSource",
     "PairedFrameSource",
     "RtpH264Source",
@@ -134,6 +136,22 @@ def make_argus(source_name: str, sensor_ids: list[int]) -> ArgusSource:
 
         sensor_id = int(spec.get("sensor_id", spec.get("sensor_id_left", 0)))
         return [make_argus(name, [sensor_id])]
+    if kind == "cuda_ipc":
+        if stereo:
+            # Each publisher serves one sensor, so a stereo rig is two
+            # cameras: entries here, paired by the caller.
+            raise ValueError(
+                f"build_local_camera: cuda_ipc camera {name!r} cannot be stereo — "
+                "declare one entry per socket."
+            )
+        return [
+            CudaIpcSource(
+                name=name,
+                socket_path=spec["socket"],
+                width=int(spec["width"]),
+                height=int(spec["height"]),
+            )
+        ]
     if kind == "oakd":
         # ``stereo: true`` shorthand for ``mode: stereo``; explicit mode wins.
         mode = spec.get("mode", "stereo" if stereo else "mono")
@@ -197,5 +215,5 @@ def make_argus(source_name: str, sensor_ids: list[int]) -> ArgusSource:
         return eyes
     raise ValueError(
         f"build_local_camera: unknown camera type {kind!r} "
-        "(known: synthetic, v4l2, argus, oakd, zed, video)"
+        "(known: synthetic, v4l2, argus, cuda_ipc, oakd, zed, video)"
     )

examples/camera_viz/python/isaaccapture_examples/camera_viz/sources/_cuda_vmm.py

@@ -0,0 +1,195 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Minimal ctypes binding for the CUDA virtual-memory-management API.
+
+Only what importing a shared device allocation needs. ``libcuda.so.1`` ships
+with the driver, so this keeps the ``cuda_ipc`` source free of any new Python
+dependency — CuPy alone is not enough, as it exposes no ``cuMemImport*``.
+
+Legacy CUDA IPC (``cudaIpcOpenMemHandle``) is not an option here: on Tegra it
+fails with ``cudaErrorInvalidValue`` even though the producer's
+``cudaIpcGetMemHandle`` succeeded. The VMM path is the one that works.
+"""
+
+from __future__ import annotations
+
+import ctypes
+
+# CUmemAllocationHandleType
+CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR = 1
+# CUmemLocationType
+CU_MEM_LOCATION_TYPE_DEVICE = 1
+# CUmemAccess_flags
+CU_MEM_ACCESS_FLAGS_PROT_READWRITE = 3
+
+
+class _CUmemLocation(ctypes.Structure):
+    _fields_ = [("type", ctypes.c_int), ("id", ctypes.c_int)]
+
+
+class CUmemAccessDesc(ctypes.Structure):
+    _fields_ = [("location", _CUmemLocation), ("flags", ctypes.c_int)]
+
+
+class CudaDriverError(RuntimeError):
+    pass
+
+
+class CudaDriver:
+    """Lazily-loaded handle onto the few driver entry points we need."""
+
+    def __init__(self) -> None:
+        try:
+            self._lib = ctypes.CDLL("libcuda.so.1")
+        except OSError as e:
+            raise CudaDriverError(
+                "libcuda.so.1 not found — the NVIDIA driver is not installed "
+                "or not visible in this container."
+            ) from e
+
+        # cuMemImportFromShareableHandle takes the fd by value in a void*, not
+        # a pointer to it; passing &fd yields CUDA_ERROR_INVALID_VALUE.
+        self._lib.cuMemImportFromShareableHandle.argtypes = [
+            ctypes.POINTER(ctypes.c_ulonglong),
+            ctypes.c_void_p,
+            ctypes.c_int,
+        ]
+        self._lib.cuMemAddressReserve.argtypes = [
+            ctypes.POINTER(ctypes.c_ulonglong),
+            ctypes.c_size_t,
+            ctypes.c_size_t,
+            ctypes.c_ulonglong,
+            ctypes.c_ulonglong,
+        ]
+        self._lib.cuMemMap.argtypes = [
+            ctypes.c_ulonglong,
+            ctypes.c_size_t,
+            ctypes.c_size_t,
+            ctypes.c_ulonglong,
+            ctypes.c_ulonglong,
+        ]
+        self._lib.cuMemSetAccess.argtypes = [
+            ctypes.c_ulonglong,
+            ctypes.c_size_t,
+            ctypes.POINTER(CUmemAccessDesc),
+            ctypes.c_size_t,
+        ]
+        self._lib.cuMemUnmap.argtypes = [ctypes.c_ulonglong, ctypes.c_size_t]
+        self._lib.cuMemAddressFree.argtypes = [ctypes.c_ulonglong, ctypes.c_size_t]
+        self._lib.cuMemRelease.argtypes = [ctypes.c_ulonglong]
+        self._lib.cuDevicePrimaryCtxRetain.argtypes = [
+            ctypes.POINTER(ctypes.c_void_p),
+            ctypes.c_int,
+        ]
+        self._lib.cuDevicePrimaryCtxRelease.argtypes = [ctypes.c_int]
+        self._lib.cuCtxSetCurrent.argtypes = [ctypes.c_void_p]
+        self._lib.cuInit.argtypes = [ctypes.c_uint]
+        self._lib.cuGetErrorName.argtypes = [
+            ctypes.c_int,
+            ctypes.POINTER(ctypes.c_char_p),
+        ]
+
+        self._check(self._lib.cuInit(0), "cuInit")
+
+    def _check(self, result: int, what: str) -> None:
+        if result == 0:
+            return
+        name = ctypes.c_char_p()
+        self._lib.cuGetErrorName(result, ctypes.byref(name))
+        detail = name.value.decode() if name.value else f"code {result}"
+        raise CudaDriverError(f"{what} failed: {detail}")
+
+    # ── context ───────────────────────────────────────────────────────
+
+    def primary_ctx_retain(self, device_id: int) -> int:
+        ctx = ctypes.c_void_p()
+        self._check(
+            self._lib.cuDevicePrimaryCtxRetain(ctypes.byref(ctx), device_id),
+            "cuDevicePrimaryCtxRetain",
+        )
+        return ctx.value or 0
+
+    def primary_ctx_release(self, device_id: int) -> None:
+        self._lib.cuDevicePrimaryCtxRelease(device_id)
+
+    def ctx_set_current(self, ctx: int) -> None:
+        self._check(self._lib.cuCtxSetCurrent(ctypes.c_void_p(ctx)), "cuCtxSetCurrent")
+
+    # ── shared allocation ─────────────────────────────────────────────
+
+    def import_fd(self, fd: int) -> int:
+        """Import a POSIX fd exported by ``cuMemExportToShareableHandle``."""
+        handle = ctypes.c_ulonglong()
+        self._check(
+            self._lib.cuMemImportFromShareableHandle(
+                ctypes.byref(handle),
+                ctypes.c_void_p(fd),
+                CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR,
+            ),
+            "cuMemImportFromShareableHandle",
+        )
+        return handle.value
+
+    def map_readwrite(self, handle: int, size: int, device_id: int) -> int:
+        """Reserve VA, map the handle into it, and grant this device access.
+
+        Returns the device pointer. On failure everything already acquired is
+        rolled back, so the caller never has to unwind a partial mapping.
+        """
+        ptr = ctypes.c_ulonglong()
+        self._check(
+            self._lib.cuMemAddressReserve(ctypes.byref(ptr), size, 0, 0, 0),
+            "cuMemAddressReserve",
+        )
+        try:
+            self._check(self._lib.cuMemMap(ptr, size, 0, handle, 0), "cuMemMap")
+        except CudaDriverError:
+            self._lib.cuMemAddressFree(ptr, size)
+            raise
+
+        desc = CUmemAccessDesc()
+        desc.location.type = CU_MEM_LOCATION_TYPE_DEVICE
+        desc.location.id = device_id
+        desc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE
+        try:
+            self._check(
+                self._lib.cuMemSetAccess(ptr, size, ctypes.byref(desc), 1),
+                "cuMemSetAccess",
+            )
+        except CudaDriverError:
+            self._lib.cuMemUnmap(ptr, size)
+            self._lib.cuMemAddressFree(ptr, size)
+            raise
+
+        return ptr.value
+
+    def unmap(self, ptr: int, size: int, handle: int) -> list:
+        """Tear down a ``map_readwrite`` mapping. Idempotent.
+
+        Returns the name of each step that failed rather than raising, so a
+        caller unwinding after an error still completes the rest. Silence here
+        would turn a failed release into a slow leak, so callers should log
+        whatever comes back.
+        """
+        failures = []
+        if ptr:
+            for name, fn in (
+                ("cuMemUnmap", self._lib.cuMemUnmap),
+                ("cuMemAddressFree", self._lib.cuMemAddressFree),
+            ):
+                if fn(ptr, size) != 0:
+                    failures.append(name)
+        if handle and self._lib.cuMemRelease(handle) != 0:
+            failures.append("cuMemRelease")
+        return failures
+
+
+_driver: CudaDriver | None = None
+
+
+def driver() -> CudaDriver:
+    """Process-wide driver handle; ``cuInit`` runs once."""
+    global _driver
+    if _driver is None:
+        _driver = CudaDriver()
+    return _driver