Draft: lossless 5K60 path (4:4:4 10-bit, tile-DPCM on the GPU) — seeking guidance on integration

TargetBridge-Receiver/TBReceiverC/Makefile

@@ -24,18 +24,19 @@ export MACOSX_DEPLOYMENT_TARGET
 
 CFLAGS   += -O2 -g -Wall -Wextra -Wno-unused-parameter -std=c11
 CFLAGS   += $(shell $(PKG) --cflags libavcodec libavutil libswscale sdl2)
+CFLAGS   += -I../../TargetBridge-Shared/codec
 CFLAGS   += -DTB_RECEIVER_VERSION=\"$(APP_VERSION)\" -DTB_RECEIVER_BUILD=\"$(APP_BUILD)\"
 CFLAGS   += -DTB_LANGUAGE_SOURCE_DIR=\"$(abspath ../../TargetBridge-Shared/Languages)\"
 
 LDFLAGS  += $(shell $(PKG) --libs libavcodec libavutil libswscale sdl2)
 
 UNAME_S  := $(shell uname -s)
 ifeq ($(UNAME_S),Darwin)
-LDFLAGS  += -framework ApplicationServices -framework AppKit -framework CoreFoundation -framework CoreGraphics -framework CoreText -framework VideoToolbox -framework CoreMedia -framework CoreVideo -framework IOSurface -framework CoreServices -framework CoreAudio -framework SystemConfiguration
+LDFLAGS  += -framework ApplicationServices -framework AppKit -framework CoreFoundation -framework CoreGraphics -framework CoreText -framework VideoToolbox -framework CoreMedia -framework CoreVideo -framework IOSurface -framework CoreServices -framework CoreAudio -framework SystemConfiguration -framework Metal -framework QuartzCore -framework IOKit
 endif
 
-C_SRC = src/main.c src/net.c src/decoder.c src/display.c src/input_queue.c src/idle_watchdog.c src/receiver_profile.c src/tb_i18n.c
-OBJC_SRC = src/tb_gesture_bridge.m src/tb_display_tweaks.m
+C_SRC = src/main.c src/net.c src/decoder.c src/display.c src/input_queue.c src/idle_watchdog.c src/receiver_profile.c src/tb_i18n.c ../../TargetBridge-Shared/codec/tb_dpcm.c
+OBJC_SRC = src/tb_gesture_bridge.m src/tb_display_tweaks.m src/tb_metal_plane.m src/tb_health.m
 OBJ = $(C_SRC:.c=.o) $(OBJC_SRC:.m=.o)
 BIN = tbreceiver
 

TargetBridge-Receiver/TBReceiverC/src/display.c

@@ -7,6 +7,7 @@
  */
 
 #include "display.h"
+#include "tb_metal_plane.h"
 #include "tb_i18n.h"
 #include "tb_gesture_bridge.h"
 
@@ -1139,6 +1140,25 @@ void tb_disp_render_nv12(struct tb_display *d,
     tb_disp_render_current(d);
 }
 
+int tb_disp_render_dpcm(struct tb_display *d, const uint8_t *blob, size_t len) {
+    if (!d) return -1;
+    /* Same lazy creation as the other Metal paths: the plane sits over SDL's
+     * window, so it cannot exist while the status UI is showing. */
+    if (!tb_metal_plane_available()) tb_metal_plane_init(d->win);
+    if (!tb_metal_plane_available()) return -1;
+
+    if (tb_metal_plane_render_dpcm(blob, len) != 0) return -1;
+    tb_disp_set_connection_state(d, 1);
+    d->last_video_frame_time = SDL_GetTicks();
+    return 0;
+}
+
+int tb_disp_dpcm_available(struct tb_display *d) {
+    if (!d) return 0;
+    if (!tb_metal_plane_available()) tb_metal_plane_init(d->win);
+    return tb_metal_plane_available() ? 1 : 0;
+}
+
 void tb_disp_set_cursor(struct tb_display *d,
                         int x, int y,
                         int source_w, int source_h,

TargetBridge-Receiver/TBReceiverC/src/display.h

@@ -48,6 +48,18 @@ void tb_disp_render_nv12(struct tb_display *d,
                          const uint8_t *uv, int uv_stride,
                          int w, int h);
 
+/* Lossless tile-DPCM (TBD2). The blob is self-describing -- dimensions and
+ * sample depth live in its own header -- so there is nothing to pass alongside
+ * it. Returns 0 on success, -1 if there is no usable Metal decode path, in which
+ * case the caller should drop the frame rather than present garbage. */
+int tb_disp_render_dpcm(struct tb_display *d, const uint8_t *blob, size_t len);
+
+/* Whether a DPCM frame could actually be decoded right now. Gates the
+ * "supportsDPCM" capability the receiver advertises: decoding is ~44 million bit
+ * extractions per 5K frame, 166 ms on this receiver's CPU against 6.5 ms on its
+ * GPU, so a receiver without a working Metal path must keep receiving NV12. */
+int tb_disp_dpcm_available(struct tb_display *d);
+
 /* Update low-latency local cursor overlay in source-frame coordinates. */
 void tb_disp_set_cursor(struct tb_display *d,
                         int x, int y,

TargetBridge-Receiver/TBReceiverC/src/main.c

@@ -958,6 +958,19 @@ static void handle_raw_frame(struct app *a, const uint8_t *p, size_t len) {
     a->frames++;
 }
 
+/* A TBD2 blob and nothing else: its own header carries the dimensions and the
+ * sample depth, so unlike TB_PKT_RAW_FRAME there is nothing to parse here. The
+ * decode runs on the GPU -- see tb_metal_plane.m -- because at 5K it is ~44
+ * million bit extractions, which costs 166 ms on this receiver's CPU and 6.5 ms
+ * on its GPU. A failed decode drops the frame rather than presenting garbage;
+ * the codec is lossless, so a partial result is not a degraded picture. */
+static void handle_raw_dpcm(struct app *a, const uint8_t *payload, size_t len) {
+    if (tb_disp_render_dpcm(a->disp, payload, len) != 0) return;
+    a->have_video_frame = 1;
+    tb_copy_i18n(a->status_text, sizeof(a->status_text), "receiver.status.stream_active");
+    a->frames++;
+}
+
 static void ring_read(struct app *a, Uint8 *dst, int len) {
     int first = AUDIO_BUF_CAP - a->audio_buf_tail;
     if (first >= len) {
@@ -1161,6 +1174,10 @@ static void on_packet(uint8_t type, const uint8_t *payload, size_t len, void *ud
         a->session_active = 1;
         handle_raw_frame(a, payload, len);
         break;
+    case TB_PKT_RAW_DPCM:
+        a->session_active = 1;
+        handle_raw_dpcm(a, payload, len);
+        break;
     case TB_PKT_CURSOR:
         {
             int x = 0;
@@ -1828,7 +1845,7 @@ static void send_receiver_info(struct app *a) {
         "{\"receiverName\":\"%s\",\"panelWidth\":%u,\"panelHeight\":%u,"
         "\"modeWidth\":%u,\"modeHeight\":%u,\"refreshRate\":60,"
         "\"hiDPI\":%s,\"captureWidth\":%u,\"captureHeight\":%u,"
-        "\"supportsHEVCDecode\":%s,\"supportsRawNV12\":true,\"inputMonitoringTrusted\":%s,\"accessibilityTrusted\":%s,"
+        "\"supportsHEVCDecode\":%s,\"supportsRawNV12\":true,\"supportsDPCM\":%s,\"inputMonitoringTrusted\":%s,\"accessibilityTrusted\":%s,"
         "\"supportsNightShift\":%s,\"supportsTrueTone\":%s}",
         escaped_name,
         profile.panel_w,
@@ -1839,6 +1856,10 @@ static void send_receiver_info(struct app *a) {
         profile.capture_w,
         profile.capture_h,
         tb_dec_supports_hevc_hwdecode() ? "true" : "false",
+        /* Conditional on a working Metal decoder, not merely on this build
+         * having the code: the CPU fallback is 166 ms/frame at 5K, so a
+         * receiver without a usable GPU path must keep receiving NV12. */
+        tb_disp_dpcm_available(a->disp) ? "true" : "false",
         tb_receiver_input_monitoring_trusted() ? "true" : "false",
         tb_receiver_accessibility_trusted() ? "true" : "false",
         tb_night_shift_supported() ? "true" : "false",

TargetBridge-Receiver/TBReceiverC/src/proto.h

@@ -47,6 +47,20 @@
 #define TB_PKT_FRAME            0x21
 #define TB_PKT_RAW_FRAME        0x22  /* uncompressed NV12 planes (raw passthrough) */
 #define TB_PKT_AUDIO_FRAME      0x23
+/* Full frame, losslessly compressed with tile-DPCM (TBD2; see tb_dpcm.h). The
+ * payload is one TBD2 blob and nothing else -- its own header carries the
+ * dimensions, so none are repeated here.
+ *
+ * Measured 2.96x on near-worst-case photographic content and 4.5-14x on desktop
+ * content, which turns a 23.4 ms receive into ~8 ms on the target hardware.
+ *
+ * Only sent to a receiver that advertised "supportsDPCM" in its display profile,
+ * which is conditional on a working Metal decoder: decoding costs ~44 million bit
+ * extractions per 5K frame, which that receiver's CPU cannot afford (166 ms
+ * single-threaded on its i5, against 6.5 ms on its GPU). A silent peer is an old
+ * peer and keeps getting TB_PKT_RAW_FRAME, so this is additive -- an unpatched
+ * receiver is unaffected. */
+#define TB_PKT_RAW_DPCM         0x25
 #define TB_PKT_HEARTBEAT        0x30
 #define TB_PKT_TEARDOWN         0x31
 #define TB_PKT_CURSOR           0x32

TargetBridge-Receiver/TBReceiverC/src/tb_health.h

@@ -0,0 +1,122 @@
+/* tb_health.h — the receiver machine's own vital signs, shipped to the sender.
+ *
+ * WHY
+ *
+ * Every diagnostic in this project has been lopsided: the sender's numbers are
+ * on the sender, and the receiver runs on a Mac being used as a monitor, with
+ * no keyboard in front of it and its window full-screen. Getting a shell on it
+ * mid-problem is exactly as awkward as it sounds, and that asymmetry is what
+ * kept an intermittent stall alive for days.
+ *
+ * The receiver's stderr is already shipped to the sender and written to
+ * ~/Library/Logs/TargetBridge/receiver.log, so anything printed here lands
+ * beside the sender's own telemetry, in order, on the machine somebody is
+ * actually sitting at.
+ *
+ * WHAT, AND WHY THESE
+ *
+ * The reference receiver is a 2020 Intel iMac driving 5K at 60 Hz — the most
+ * thermally constrained part of the link by a wide margin, and the one we are
+ * about to ask to run at a continuous full rate rather than roughly half.
+ * Video playback already does exactly that, so this is not new load; it is load
+ * we have never watched. Throttling would show up first as the GPU upload
+ * stretching, which reads as "the link got slow" and would send the next
+ * investigation straight back to the network.
+ *
+ *   thermal  NSProcessInfo's thermalState — the OS's own verdict, four levels.
+ *   cpu      this process, as a percentage of one core.
+ *   gpu      accelerator utilisation from IOKit's PerformanceStatistics, which
+ *            is the only place macOS exposes it without private frameworks. Not
+ *            present on every driver, hence optional.
+ *   load     1-minute load average, to separate "we are busy" from "the machine
+ *            is busy".
+ */
+
+#ifndef TB_HEALTH_H
+#define TB_HEALTH_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Start reporting. Call once at startup; idempotent.
+ *
+ * Runs on its own thread rather than piggy-backing the per-second frame tick,
+ * because the readings are not all cheap: the IORegistry walk for GPU
+ * utilisation measured 1.5 ms (1.9 ms worst) on an M4 Pro and will be slower on
+ * the Intel receiver. That fits inside a 16.7 ms frame with room to spare, so
+ * it was not going to drop anything -- but a diagnostic sitting in the render
+ * path is how this project has twice ended up measuring its own instrument. It
+ * costs nothing to put it somewhere it cannot matter. */
+void tb_health_start(void);
+
+/* Hold off App Nap and other background throttling for the whole process.
+ *
+ * A display receiver is doing user-initiated work even when it is not the front
+ * app: the picture must keep arriving while the user clicks something else on
+ * this Mac. Without this, macOS suspends the render loop the moment the app goes
+ * to the background -- measured at 92% of a 23-second window spent outside the
+ * loop, cursor latency 18 SECONDS against a normal 4ms, the video queue filling
+ * to 20/24 with 3600 dropped frames, and Metal left holding surfaces that were
+ * never returned. From the user's side the receiver simply freezes on return and
+ * only a relaunch clears it.
+ *
+ * Safe to call more than once; the assertion is held until the process exits. */
+void tb_health_hold_awake(void);
+
+/* Per-stage CPU accounting for the receive path.
+ *
+ * "63% of a core" is an aggregate, and an aggregate has never once been enough
+ * to find anything in this project. Three stages are plausible and they want
+ * different fixes: the kernel->userspace copy in read(), the memcpy that stages
+ * a blob into a Metal upload buffer, and everything else in submitting a slice.
+ * Splitting them is what named the sender's spike in a single run.
+ *
+ * Call with elapsed milliseconds; totals are folded into the [health] line. */
+void tb_health_note_read(double ms);
+void tb_health_note_upload_copy(double ms);
+void tb_health_note_submit(double ms);
+
+/* Latency samples, as opposed to the occupancy figures above.
+ *
+ * The three note_* calls above accumulate a SHARE of wall clock, which answers
+ * "is this stage busy". Neither of the questions below is about busyness:
+ *
+ *   drawable_wait — how long [layer nextDrawable] blocked. Under vsync this is
+ *   the wait for a free drawable, so it is the direct cost of
+ *   maximumDrawableCount. If it is near zero the drawable count is not what is
+ *   adding latency and lowering it would only risk throughput.
+ *
+ *   cursor_commit — the moment a cursor position reached the compositor. Only
+ *   the CADENCE is recorded, not an absolute latency: the sender samples at
+ *   120 Hz on a different machine, so without clock sync the gap between our own
+ *   commits is the honest measurement. ~8ms means positions flow freely; ~17ms
+ *   means something is gating them to the refresh rate.
+ *
+ * Reported as mean and max, because a mean alone hides exactly the stalls that
+ * make a cursor feel bad. */
+void tb_health_note_drawable_wait(double ms);
+void tb_health_note_cursor_commit(void);
+
+/* Drain the drawable wait for the PHASE REPORT, which is a control signal rather
+ * than a diagnostic — see TB_PKT_PHASE. Returns 0 and writes nothing when no
+ * samples have arrived since the last call, which is the normal state whenever
+ * nothing is being presented; the caller should send no report at all in that
+ * case rather than a mean of zero, since "no frames" and "no wait" would
+ * otherwise look identical and the sender would read the latter as a good phase.
+ *
+ * Independent of the health report's own totals, so calling either does not
+ * disturb the other. */
+int tb_health_take_drawable_phase(double *mean_ms, long *n);
+
+/* Arrival-to-commit for one cursor position: off the socket to on the
+ * compositor, including any wait in the control queue. Excludes the network hop
+ * by design — the sender's clock is on another machine and unsynchronised, so
+ * adding it would be a guess dressed as a measurement. */
+void tb_health_note_cursor_latency(double ms);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif