diff --git a/CHANGELOG.md b/CHANGELOG.md index 28b1041f..e2b7678e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,33 @@ All notable changes to this project will be documented in this file. +## [Unreleased] + +### ✨ Added +- **WebTransport Annex-B + WebCodecs web client**: + - Chromium opens WebTransport on `signaling_port + 2` (`wt_port`) with `serverCertificateHashes` and decodes Annex-B access units with `VideoDecoder` onto a canvas. + - P-frames ride QUIC datagrams (unreliable). Hello, ping/pong, and IDR stay on the control stream. The same port serves the UI over HTTPS; HTTP `/` and `/client/` redirect there. Android keeps HTTP on `:8788`. + - The self-signed certificate is stored in `$XDG_CONFIG_HOME/orbiscreen/wt-cert.pem` and `wt-key.pem` (`0o600`, parent `0o700`) and reused across restarts (14-day WebTransport cap). +- **Reed-Solomon FEC on P-frame datagrams**: + - Systematic Cauchy RS over GF(256). Ladder: 1–3 fragments no FEC; 4–16 +2; 17–64 +3; 65+ +4. IDRs stay on the reliable stream. +- **Reliable IDR + SPS/PPS**: + - Keyframes (SPS/PPS prepended when missing) go on the WebTransport control stream and on TCP `GET /idr` for Android. A reliable IDR resets the datagram assembler so the next P-frame starts a new GOP. +- **One-frame VBV/CPB**: + - Hardware and software H.264 encoders size the coded-picture buffer to one frame of the CBR target so a single IDR cannot occupy hundreds of milliseconds of wire time. +- **Frame transport guide** (`docs/FRAME_TRANSPORT.md`): I/P/IDR/GOP terms, the AU split between datagrams and the reliable stream, and hold-until-IDR recovery. + +### ⚡ Performance & Low Latency +- HTTP MPEG-TS (`GET /stream`) keeps the encoder's infinite GOP: IDR on join, lag, and muxer failure, not every second. +- IDR requests are debounced to 250 ms on HTTP, UDP, and WebTransport until a keyframe arrives. +- A failed UDP datagram no longer aborts the rest of that access unit; only `TooBig` stops the fragment loop. +- Early P-frames after a one-sequence hole are reordered instead of immediately requesting an IDR. + +### 🐛 Fixed +- **Quinn datagram send no longer aborts the daemon**: pin `quinn-proto` to 0.11.16 (0.11.17 double-subtracts dropped datagram bytes). +- **Android no longer drops mid-GOP P-frames**: + - UDP `feedCodec` drains and retries a full input buffer, then holds until IDR. HTTP/ExoPlayer no longer drops late P-frames. +- **Connect IDR is sent after the video path is live**, so the first keyframe is not encoded and dropped during DPLPMTUD. + ## [v0.28.9] - 2026-09-16 Introduce per-client KWin virtual displays and isolated touch scoping to support multiple simultaneous tablets, expand KWin input device introspection beyond event63, add automatic fallback session creation on attach for full backwards compatibility with legacy clients, and bump the release matrix across all platforms. diff --git a/Cargo.lock b/Cargo.lock index 9bd9a245..f8b2a383 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,6 +97,15 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "as-raw-xcb-connection" version = "1.0.1" @@ -119,6 +128,45 @@ dependencies = [ "zbus", ] +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.20", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "async-broadcast" version = "0.7.2" @@ -346,6 +394,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-server" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ab4a3ec9ea8a657c72d99a03a824af695bd0fb5ec639ccbd9cd3543b41a5f9" +dependencies = [ + "arc-swap", + "bytes", + "fs-err", + "http", + "http-body", + "hyper", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + [[package]] name = "base64" version = "0.21.7" @@ -370,7 +440,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec", + "bit-vec 0.8.0", ] [[package]] @@ -379,6 +449,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -403,6 +482,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -604,6 +692,23 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.45" @@ -681,6 +786,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "cookie" version = "0.18.2" @@ -740,6 +851,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.1" @@ -774,6 +894,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "cssparser" version = "0.36.0" @@ -895,6 +1024,20 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.5.8" @@ -942,8 +1085,19 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", ] [[package]] @@ -1232,6 +1386,18 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastbloom" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef975e30683b2d965054bb0a836f8973857c4ebf6acf274fe46617cd285060d8" +dependencies = [ + "foldhash", + "libm", + "portable-atomic", + "siphasher", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -1333,6 +1499,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a" +dependencies = [ + "autocfg", + "tokio", +] + [[package]] name = "futures-channel" version = "0.3.34" @@ -1543,8 +1719,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1566,8 +1744,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -1878,6 +2059,25 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1935,6 +2135,12 @@ dependencies = [ "markup5ever", ] +[[package]] +name = "httlib-huffman" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a9fcbcc408c5526c3ab80d534e5c86e7967c1fb7aa0a8c76abd1edc27deb877" + [[package]] name = "http" version = "1.5.0" @@ -1986,6 +2192,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.11.0" @@ -1996,6 +2211,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -2481,6 +2697,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libredox" version = "0.1.23" @@ -2517,6 +2739,12 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "markup5ever" version = "0.38.0" @@ -2598,6 +2826,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2687,6 +2921,16 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2696,6 +2940,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -2947,6 +3201,21 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "octets" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d59d30d3ad7f7e1c9a66e51c9cb71ae1c794b28ffe7053fd785defb0d180069" + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2959,6 +3228,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "option-ext" version = "0.2.0" @@ -3090,15 +3365,19 @@ name = "orbiscreen-transport" version = "0.28.9" dependencies = [ "axum", + "axum-server", "base64 0.22.1", "gstreamer", "gstreamer-app", "hostname", "libc", "mdns-sd", + "orbiscreen-core", "orbiscreen-encode", "orbiscreen-input", + "quinn-proto", "rand 0.9.5", + "rustls", "serde", "serde_json", "thiserror 2.0.20", @@ -3106,6 +3385,8 @@ dependencies = [ "tokio-stream", "tower-http", "tracing", + "wtransport", + "x509-parser", ] [[package]] @@ -3178,6 +3459,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64 0.22.1", + "serde_core", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3438,6 +3729,63 @@ dependencies = [ "memchr", ] +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.20", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "bytes", + "fastbloom", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.47" @@ -3480,6 +3828,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -3518,12 +3877,40 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "raw-window-handle" version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3627,6 +4014,20 @@ dependencies = [ "web-sys", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -3642,6 +4043,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "1.1.4" @@ -3655,6 +4065,62 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -3676,6 +4142,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "schemars" version = "0.8.22" @@ -3733,6 +4208,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.1", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "selectors" version = "0.36.1" @@ -3951,8 +4449,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -3962,8 +4460,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -4135,6 +4644,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "swift-rs" version = "1.0.8" @@ -4377,7 +4892,7 @@ dependencies = [ "semver", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "syn 2.0.119", "tauri-utils", "thiserror 2.0.20", @@ -4508,7 +5023,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -4656,6 +5171,16 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "tokio-rustls" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.19" @@ -4689,6 +5214,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -5056,6 +5582,12 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "uoctl" version = "1.0.1" @@ -5347,6 +5879,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web_atoms" version = "0.2.6" @@ -5633,6 +6175,15 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -5866,7 +6417,7 @@ dependencies = [ "once_cell", "percent-encoding", "raw-window-handle", - "sha2", + "sha2 0.10.9", "soup3", "tao-macros", "thiserror 2.0.20", @@ -5880,6 +6431,42 @@ dependencies = [ "x11-dl", ] +[[package]] +name = "wtransport" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4273ce3157a3262a68665f8d3f20a0ac0c5b8a69ffd67f05ae986832ebec036" +dependencies = [ + "bytes", + "pem", + "quinn", + "rcgen", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "sha2 0.11.0", + "socket2", + "thiserror 2.0.20", + "time", + "tokio", + "tracing", + "url", + "wtransport-proto", + "x509-parser", +] + +[[package]] +name = "wtransport-proto" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aad9059572c7dbd6901ccef37f3b7321678cd708dcf58a64b1921dbeab7bfede" +dependencies = [ + "httlib-huffman", + "octets", + "thiserror 2.0.20", + "url", +] + [[package]] name = "x11" version = "2.21.0" @@ -5920,6 +6507,24 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acf4d1bc32aa46eec18caa634ec3cf4c05bfa151f12b93b510b15190f69a1ca8" +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.20", + "time", +] + [[package]] name = "xkbcommon" version = "0.9.0" @@ -5937,6 +6542,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + [[package]] name = "yoke" version = "0.8.3" @@ -6072,6 +6687,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerotrie" version = "0.2.5" diff --git a/README.md b/README.md index 092a9a40..ce57e401 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ - **Material 3 Android client**: Jetpack Compose, Catppuccin Mocha / Latte brand palette, light/dark theme - **Auto-Orientation Resolution Adaptation**: Automatically swaps width and height when your phone/tablet rotates between landscape and portrait - **3-Row Spacious Keyboard Overlay**: Top-docked layout ensuring bottom taskbars, terminals, and prompts stay 100% visible -- **Built-in web client**: Watch from any browser at `http://:8788/` (MSE via locally bundled `mpegts.js`, no CDN) +- **Built-in web client**: Watch from any browser at `https://:8790/client/` (HTTP on `:8788` redirects there). WebTransport Annex-B + WebCodecs `VideoDecoder`. - **Live discovery**: NSD scan of nearby hosts, manual `host:port` entry, optional subnet scanner - **Native streaming**: ExoPlayer with `OkHttpDataSource` + tuned load control for ultra-low latency MPEG-TS / H.264 - **Host control panel**: Keyboard, lock, blank, Ctrl+Alt+Del, and retry actions diff --git a/SECURITY.md b/SECURITY.md index 06e5d463..7ccfa2ff 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -166,7 +166,7 @@ The Android release signing key (`orbiscreen-release.keystore`) was removed from ## 🔬 Security Audit -Orbiscreen (v0.25.9) is written in Rust (edition 2021) plus a Kotlin Android client (Material 3 + Jetpack Compose), a small browser web client (MSE via vendored mpegts.js), and a Linux desktop GUI control center (Tauri v2 + WebKitGTK). A running daemon performs: +Orbiscreen (v0.27.4) is written in Rust (edition 2021) plus a Kotlin Android client (Material 3 + Jetpack Compose), a small browser web client (WebTransport + WebCodecs `VideoDecoder`), and a Linux desktop GUI control center (Tauri v2 + WebKitGTK). A running daemon performs: - `open()` on `/dev/dri/card*` evdi nodes for capture - Compositor IPC over session-local Unix sockets: sway i3-ipc (`$SWAYSOCK`) and Hyprland (`HYPRLAND_INSTANCE_SIGNATURE`) to create/destroy headless outputs diff --git a/clients/android/app/src/main/java/com/orbiscreen/android/player/AuReorder.kt b/clients/android/app/src/main/java/com/orbiscreen/android/player/AuReorder.kt new file mode 100644 index 00000000..f44a90db --- /dev/null +++ b/clients/android/app/src/main/java/com/orbiscreen/android/player/AuReorder.kt @@ -0,0 +1,102 @@ +// Orbiscreen - AuReorder.kt (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +package com.orbiscreen.android.player + +/** + * Holds completed access units that arrived ahead of lastSeq+1 so a + * one-frame datagram reorder can play in order. Requests a gap (IDR) + * only after [holeWaitMs] without the missing seq. + */ +class AuReorder( + private val holeWaitMs: Long = HOLE_WAIT_MS, + private val maxHeld: Int = MAX_HELD, +) { + data class Frame( + val seq: Int, + val key: Boolean, + val au: ByteArray, + val sentNs: Long, + ) + + sealed class Out { + data class Video(val frame: Frame) : Out() + data class Gap(val dropped: Int) : Out() + } + + var lastSeq: Int = -1 + private set + private val held = LinkedHashMap() + private var holeSince: Long = 0L + + fun reset() { + lastSeq = -1 + held.clear() + holeSince = 0L + } + + /** Reliable IDR has no datagram seq; the next P starts a new GOP. */ + fun onReliableKeyframe() { + reset() + } + + fun accept(frame: Frame, nowMs: Long): List { + if (lastSeq >= 0) { + val age = seqDelta(frame.seq, lastSeq) + if (age == 0 || age > 32768) return emptyList() + } + if (lastSeq < 0) { + lastSeq = frame.seq + return listOf(Out.Video(frame)) + } + val delta = seqDelta(frame.seq, lastSeq) + if (delta == 1) { + lastSeq = frame.seq + val out = ArrayList(1 + held.size) + out.add(Out.Video(frame)) + drainHeld(out) + return out + } + if (frame.key) { + held.clear() + holeSince = 0L + lastSeq = frame.seq + return listOf(Out.Video(frame)) + } + held[frame.seq] = frame + if (holeSince == 0L) holeSince = nowMs + trimHeld() + return emptyList() + } + + fun expire(nowMs: Long): List { + if (held.isEmpty()) return emptyList() + if (nowMs - holeSince < holeWaitMs) return emptyList() + val dropped = held.size.coerceAtLeast(1) + held.clear() + holeSince = 0L + return listOf(Out.Gap(dropped)) + } + + private fun drainHeld(out: MutableList) { + while (true) { + val next = (lastSeq + 1) and 0xFFFF + val frame = held.remove(next) ?: break + lastSeq = next + out.add(Out.Video(frame)) + } + if (held.isEmpty()) holeSince = 0L + } + + private fun trimHeld() { + if (held.size <= maxHeld) return + val seqs = held.keys.sortedBy { seqDelta(it, lastSeq) } + for (seq in seqs.drop(maxHeld)) held.remove(seq) + } + + companion object { + const val HOLE_WAIT_MS = 48L + const val MAX_HELD = 4 + fun seqDelta(cur: Int, prev: Int): Int = (cur - prev) and 0xFFFF + } +} diff --git a/clients/android/app/src/main/java/com/orbiscreen/android/player/Fec.kt b/clients/android/app/src/main/java/com/orbiscreen/android/player/Fec.kt new file mode 100644 index 00000000..f02aa1a9 --- /dev/null +++ b/clients/android/app/src/main/java/com/orbiscreen/android/player/Fec.kt @@ -0,0 +1,150 @@ +// Orbiscreen - Fec.kt (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +package com.orbiscreen.android.player + +/** Systematic Cauchy Reed-Solomon over GF(256). Matches crates/orbiscreen-transport/src/fec.rs. */ +object Fec { + fun parityCount(k: Int): Int = when { + k <= 3 -> 0 + k <= 16 -> 2 + k <= 64 -> 3 + else -> 4 + } + + fun recover(data: Array, parity: Array): Boolean { + val k = data.size + if (data.all { it != null }) return true + val m = parity.size + if (m == 0) return false + val missing = data.indices.filter { data[it] == null } + val presentP = parity.indices.filter { parity[it] != null } + var width = 0 + for (d in data) if (d != null && d.size > width) width = d.size + for (p in parity) if (p != null && p.size > width) width = p.size + if (width == 0 || missing.size > presentP.size) return false + val usedP = presentP.take(missing.size) + val missN = missing.size + val known = Array(k) { i -> + data[i]?.let { src -> + ByteArray(width).also { it.fill(0); src.copyInto(it) } + } + } + val rhs = Array(missN) { r -> + val p = usedP[r] + val rec = ByteArray(width).also { parity[p]!!.copyInto(it) } + ByteArray(width) { b -> + var s = rec[b].toInt() and 0xff + for (d in 0 until k) { + val kn = known[d] + if (kn != null) s = s xor gfMul(cauchy(p, d, m), kn[b].toInt() and 0xff) + } + s.toByte() + } + } + val mat = Array(missN) { r -> + val p = usedP[r] + ByteArray(missN) { c -> cauchy(p, missing[c], m).toByte() } + } + val inv = invert(mat) ?: return false + for (c in 0 until missN) { + val out = ByteArray(width) + for (b in 0 until width) { + var s = 0 + for (r in 0 until missN) { + s = s xor gfMul(inv[c][r].toInt() and 0xff, rhs[r][b].toInt() and 0xff) + } + out[b] = s.toByte() + } + data[missing[c]] = out + } + return true + } + + fun concatFecAu(parts: Array): ByteArray? { + val n = parts.sumOf { it.size } + if (n < 4) return null + val blob = ByteArray(n) + var o = 0 + for (p in parts) { + p.copyInto(blob, o) + o += p.size + } + val len = (blob[0].toInt() and 0xff) or + ((blob[1].toInt() and 0xff) shl 8) or + ((blob[2].toInt() and 0xff) shl 16) or + ((blob[3].toInt() and 0xff) shl 24) + if (len < 0 || len > blob.size - 4) return null + return blob.copyOfRange(4, 4 + len) + } + + internal fun encode(data: Array, m: Int): Array { + val k = data.size + val width = data[0].size + return Array(m) { p -> + ByteArray(width) { b -> + var s = 0 + for (d in 0 until k) { + s = s xor gfMul(cauchy(p, d, m), data[d][b].toInt() and 0xff) + } + s.toByte() + } + } + } + + private fun cauchy(p: Int, d: Int, m: Int): Int = gfInv(p xor (m + d)) + + private val exp = ByteArray(512) + private val log = ByteArray(256) + + init { + var x = 1 + for (i in 0 until 255) { + exp[i] = x.toByte() + log[x] = i.toByte() + x = x shl 1 + if (x and 0x100 != 0) x = x xor 0x11d + } + for (i in 255 until 512) exp[i] = exp[i - 255] + } + + private fun gfMul(a: Int, b: Int): Int { + if (a == 0 || b == 0) return 0 + return exp[(log[a].toInt() and 0xff) + (log[b].toInt() and 0xff)].toInt() and 0xff + } + + private fun gfInv(a: Int): Int = exp[255 - (log[a].toInt() and 0xff)].toInt() and 0xff + + private fun invert(a: Array): Array? { + val n = a.size + if (n == 0) return emptyArray() + val m = Array(n) { i -> + ByteArray(n * 2).also { row -> + a[i].copyInto(row) + row[n + i] = 1 + } + } + for (col in 0 until n) { + var piv = col + while (piv < n && m[piv][col].toInt() == 0) piv++ + if (piv == n) return null + val tmp = m[col] + m[col] = m[piv] + m[piv] = tmp + val inv = gfInv(m[col][col].toInt() and 0xff) + for (j in 0 until n * 2) { + m[col][j] = gfMul(m[col][j].toInt() and 0xff, inv).toByte() + } + for (row in 0 until n) { + if (row == col) continue + val f = m[row][col].toInt() and 0xff + if (f == 0) continue + for (j in 0 until n * 2) { + val v = (m[row][j].toInt() and 0xff) xor gfMul(f, m[col][j].toInt() and 0xff) + m[row][j] = v.toByte() + } + } + } + return Array(n) { i -> m[i].copyOfRange(n, n * 2) } + } +} diff --git a/clients/android/app/src/main/java/com/orbiscreen/android/player/H264.kt b/clients/android/app/src/main/java/com/orbiscreen/android/player/H264.kt new file mode 100644 index 00000000..83d1d67f --- /dev/null +++ b/clients/android/app/src/main/java/com/orbiscreen/android/player/H264.kt @@ -0,0 +1,135 @@ +// Orbiscreen - H264.kt (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +package com.orbiscreen.android.player + +enum class FrameKind { I, P, B, Other } + +object H264 { + fun classifyAccessUnit(au: ByteArray): FrameKind { + var sawI = false + var sawB = false + var sawP = false + forEachNal(au) { nal -> + when (sliceKind(nal)) { + FrameKind.I -> sawI = true + FrameKind.B -> sawB = true + FrameKind.P -> sawP = true + FrameKind.Other -> {} + } + } + return when { + sawI -> FrameKind.I + sawB -> FrameKind.B + sawP -> FrameKind.P + else -> FrameKind.Other + } + } + + fun forEachNal(au: ByteArray, body: (ByteArray) -> Unit) { + var i = 0 + while (i < au.size) { + val sc = startCodeLen(au, i) ?: break + val nalStart = i + sc + var next = nalStart + while (next < au.size) { + val n = startCodeLen(au, next) + if (n != null && next > nalStart) break + next++ + } + if (nalStart < next) { + body(au.copyOfRange(nalStart, next.coerceAtMost(au.size))) + } + i = next + } + } + + internal fun sliceKind(nal: ByteArray): FrameKind { + if (nal.isEmpty()) return FrameKind.Other + val typ = nal[0].toInt() and 0x1F + if (typ == 5) return FrameKind.I + if (typ != 1 && typ != 2) return FrameKind.Other + val rbsp = unescapeRbsp(nal) + if (rbsp.isEmpty()) return FrameKind.Other + val bits = BitReader(rbsp) + if (bits.ue() < 0) return FrameKind.Other + val sliceType = bits.ue() + if (sliceType < 0) return FrameKind.Other + return when (sliceType % 5) { + 0, 3 -> FrameKind.P + 1 -> FrameKind.B + 2, 4 -> FrameKind.I + else -> FrameKind.Other + } + } + + internal fun unescapeRbsp(nal: ByteArray): ByteArray { + if (nal.size <= 1) return ByteArray(0) + val out = ByteArray(nal.size - 1) + var o = 0 + var i = 1 + while (i < nal.size) { + if (i + 2 < nal.size && + nal[i] == 0.toByte() && + nal[i + 1] == 0.toByte() && + nal[i + 2] == 3.toByte() + ) { + out[o++] = 0 + out[o++] = 0 + i += 3 + } else { + out[o++] = nal[i] + i++ + } + } + return if (o == out.size) out else out.copyOf(o) + } + + private fun startCodeLen(d: ByteArray, i: Int): Int? { + if (i + 3 < d.size && d[i] == 0.toByte() && d[i + 1] == 0.toByte() && d[i + 2] == 1.toByte()) { + return 3 + } + if (i + 4 < d.size && + d[i] == 0.toByte() && + d[i + 1] == 0.toByte() && + d[i + 2] == 0.toByte() && + d[i + 3] == 1.toByte() + ) { + return 4 + } + return null + } + + private class BitReader(private val data: ByteArray) { + private var bitPos = 0 + + fun bit(): Int { + val byteIx = bitPos / 8 + if (byteIx >= data.size) return -1 + val v = (data[byteIx].toInt() and 0xFF) shr (7 - (bitPos % 8)) and 1 + bitPos++ + return v + } + + fun ue(): Int { + var zeros = 0 + while (true) { + when (val b = bit()) { + -1 -> return -1 + 1 -> break + else -> { + zeros++ + if (zeros > 31) return -1 + } + } + } + var suffix = 0 + repeat(zeros) { + val b = bit() + if (b < 0) return -1 + suffix = (suffix shl 1) or b + } + return (1 shl zeros) - 1 + suffix + } + } +} diff --git a/clients/android/app/src/main/java/com/orbiscreen/android/player/Idr.kt b/clients/android/app/src/main/java/com/orbiscreen/android/player/Idr.kt new file mode 100644 index 00000000..dd5e6d8f --- /dev/null +++ b/clients/android/app/src/main/java/com/orbiscreen/android/player/Idr.kt @@ -0,0 +1,10 @@ +// Orbiscreen - Idr.kt (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +package com.orbiscreen.android.player + +object Idr { + const val DEBOUNCE_MS: Long = 250L + + fun due(nowMs: Long, lastMs: Long): Boolean = nowMs - lastMs >= DEBOUNCE_MS +} diff --git a/clients/android/app/src/main/java/com/orbiscreen/android/player/IdrFrames.kt b/clients/android/app/src/main/java/com/orbiscreen/android/player/IdrFrames.kt new file mode 100644 index 00000000..26eb5eda --- /dev/null +++ b/clients/android/app/src/main/java/com/orbiscreen/android/player/IdrFrames.kt @@ -0,0 +1,70 @@ +// Orbiscreen - IdrFrames.kt (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +package com.orbiscreen.android.player + +/** + * Length-prefixed Annex-B video frames as sent on the reliable IDR TCP + * path (`GET /idr`, same codec as WebTransport `encode_video`). + */ +object IdrFrames { + const val TYPE_VIDEO: Int = 1 + const val MAX_FRAME: Int = 4 * 1024 * 1024 + + data class Video( + val key: Boolean, + val ptsNs: Long, + val sentNs: Long, + val au: ByteArray, + ) + + fun split(buf: ByteArray, offset: Int = 0, length: Int = buf.size - offset): Pair? { + if (length < 4) return null + val n = ((buf[offset].toInt() and 0xff) shl 24) or + ((buf[offset + 1].toInt() and 0xff) shl 16) or + ((buf[offset + 2].toInt() and 0xff) shl 8) or + (buf[offset + 3].toInt() and 0xff) + if (n < 0 || n > MAX_FRAME) return null + if (length < 4 + n) return null + return buf.copyOfRange(offset + 4, offset + 4 + n) to (4 + n) + } + + fun decodeVideo(body: ByteArray): Video? { + if (body.isEmpty() || (body[0].toInt() and 0xff) != TYPE_VIDEO) return null + if (body.size < 18) return null + val key = body[1].toInt() != 0 + val pts = le64(body, 2) + val sent = le64(body, 10) + val au = body.copyOfRange(18, body.size) + return Video(key, pts, sent, au) + } + + class Reader { + private var buf = ByteArray(0) + + fun push(chunk: ByteArray) { + if (chunk.isEmpty()) return + val next = ByteArray(buf.size + chunk.size) + System.arraycopy(buf, 0, next, 0, buf.size) + System.arraycopy(chunk, 0, next, buf.size, chunk.size) + buf = next + } + + fun pop(): Video? { + val split = split(buf) ?: return null + val (body, used) = split + buf = buf.copyOfRange(used, buf.size) + return decodeVideo(body) + } + } + + private fun le64(d: ByteArray, o: Int): Long { + var v = 0L + var shift = 0 + for (i in 0 until 8) { + v = v or ((d[o + i].toLong() and 0xffL) shl shift) + shift += 8 + } + return v + } +} diff --git a/clients/android/app/src/main/java/com/orbiscreen/android/player/PendingFecStore.kt b/clients/android/app/src/main/java/com/orbiscreen/android/player/PendingFecStore.kt new file mode 100644 index 00000000..e3fc0a57 --- /dev/null +++ b/clients/android/app/src/main/java/com/orbiscreen/android/player/PendingFecStore.kt @@ -0,0 +1,95 @@ +// Orbiscreen - PendingFecStore.kt (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +package com.orbiscreen.android.player + +/** + * Assembles one seq of FEC-protected datagrams. + * + * Data fragments open (or fill) a pending AU. Parity only attaches to an + * existing one — a late parity after emit, or a parity-first packet, must + * not create a new pending AU. All data present is enough to emit; parity + * is only used to recover missing data shards. + */ +class PendingFecStore { + data class Complete( + val seq: Int, + val key: Boolean, + val sentNs: Long, + val au: ByteArray, + ) + + private class PendingAu( + val k: Int, + val parts: Array, + val parity: Array, + var key: Boolean, + var sentNs: Long, + ) + + private val pending = HashMap() + + val size: Int get() = pending.size + fun contains(seq: Int): Boolean = pending.containsKey(seq) + fun remove(seq: Int) { pending.remove(seq) } + fun clear() { pending.clear() } + fun keys(): Set = pending.keys.toSet() + + fun offer( + seq: Int, + frag: Int, + frags: Int, + key: Boolean, + sentNs: Long, + payload: ByteArray, + ): Complete? { + val m = Fec.parityCount(frags) + if (frags <= 0 || frag < 0 || frag >= frags + m) return null + val isParity = frag >= frags + var slots = pending[seq] + if (slots == null || slots.k != frags) { + if (isParity) return null + slots = PendingAu( + frags, + arrayOfNulls(frags), + arrayOfNulls(m), + key, + sentNs, + ) + pending[seq] = slots + } + if (isParity) { + slots.parity[frag - frags] = payload + } else { + slots.parts[frag] = payload + slots.key = key + slots.sentNs = sentNs + } + if (slots.parts.all { it != null }) { + pending.remove(seq) + val filled = Array(frags) { i -> slots.parts[i]!! } + val au = if (m > 0) Fec.concatFecAu(filled) ?: return null else assemble(slots.parts) + return Complete(seq, slots.key, slots.sentNs, au) + } + val have = slots.parts.count { it != null } + slots.parity.count { it != null } + if (have < frags) return null + if (!Fec.recover(slots.parts, slots.parity)) return null + pending.remove(seq) + val filled = Array(frags) { i -> slots.parts[i]!! } + val au = Fec.concatFecAu(filled) ?: return null + return Complete(seq, slots.key, slots.sentNs, au) + } + + private fun assemble(slots: Array): ByteArray { + var total = 0 + for (p in slots) total += p?.size ?: 0 + val out = ByteArray(total) + var o = 0 + for (p in slots) { + if (p == null) continue + System.arraycopy(p, 0, out, o, p.size) + o += p.size + } + return out + } +} diff --git a/clients/android/app/src/main/java/com/orbiscreen/android/player/PlayerHolder.kt b/clients/android/app/src/main/java/com/orbiscreen/android/player/PlayerHolder.kt index ee51e1d7..7cf894be 100644 --- a/clients/android/app/src/main/java/com/orbiscreen/android/player/PlayerHolder.kt +++ b/clients/android/app/src/main/java/com/orbiscreen/android/player/PlayerHolder.kt @@ -80,6 +80,7 @@ class PlayerHolder( val player: StateFlow get() = _player private val _udp = MutableStateFlow(null) val udpPlayer: StateFlow get() = _udp + val stats = StreamStats() private var reconnectJob: Job? = null private var reconnectDelayMs = 1_000L @@ -145,6 +146,7 @@ class PlayerHolder( retryCount = 0 } lastTarget = StreamTarget(host, port, tokenProvider, session) + stats.reset() val token = try { tokenProvider() @@ -171,9 +173,9 @@ class PlayerHolder( val streamW = session?.width ?: info?.width ?: 1920 val streamH = session?.height ?: info?.height ?: 1080 if (udpPort in 1..65535 && host != "127.0.0.1" && host != "localhost") { - val udp = UdpPlayer() + val udp = UdpPlayer(stats) val started = kotlinx.coroutines.withContext(Dispatchers.IO) { - udp.start(host, udpPort, token, streamW, streamH, session?.id) + udp.start(host, udpPort, token, streamW, streamH, session?.id, port) } if (started) { _udp.value = udp @@ -192,6 +194,31 @@ class PlayerHolder( if (token.isNotBlank()) mapOf("Authorization" to "Bearer $token") else emptyMap() ) val dataSourceFactory = DefaultDataSource.Factory(context, httpFactory) + .setTransferListener(object : androidx.media3.datasource.TransferListener { + override fun onTransferInitializing( + source: androidx.media3.datasource.DataSource, + dataSpec: androidx.media3.datasource.DataSpec, + isNetwork: Boolean, + ) {} + override fun onTransferStart( + source: androidx.media3.datasource.DataSource, + dataSpec: androidx.media3.datasource.DataSpec, + isNetwork: Boolean, + ) {} + override fun onBytesTransferred( + source: androidx.media3.datasource.DataSource, + dataSpec: androidx.media3.datasource.DataSpec, + isNetwork: Boolean, + bytesTransferred: Int, + ) { + if (bytesTransferred > 0) stats.noteBytes(bytesTransferred.toLong()) + } + override fun onTransferEnd( + source: androidx.media3.datasource.DataSource, + dataSpec: androidx.media3.datasource.DataSpec, + isNetwork: Boolean, + ) {} + }) val extractorsFactory = DefaultExtractorsFactory() .setTsExtractorFlags( @@ -365,7 +392,12 @@ class PlayerHolder( enableDecoderFallback, eventHandler, eventListener, - ) { requestIdr() }, + onLagDetected = { + stats.noteDropped() + requestIdr() + }, + onFramePresented = { ptsUs -> stats.notePresentedPts(ptsUs) }, + ), ) } } @@ -420,14 +452,16 @@ class PlayerHolder( fun onAppForegrounded() { val wasBackgrounded = isBackgrounded isBackgrounded = false - if (wasBackgrounded) { - val p = _player.value - if (p != null && p.playbackState != Player.STATE_IDLE && p.playerError == null) { - p.playWhenReady = true - } else { - lastTarget?.let { target -> - retry(target.host, target.port, target.tokenProvider) - } + if (!wasBackgrounded) return + // UDP stays on its own socket; tearing it down here is what a USB + // permission dialog (or any other overlay Activity) used to do. + if (_udp.value != null) return + val p = _player.value + if (p != null && p.playbackState != Player.STATE_IDLE && p.playerError == null) { + p.playWhenReady = true + } else { + lastTarget?.let { target -> + retry(target.host, target.port, target.tokenProvider) } } } @@ -458,6 +492,7 @@ private class LowLatencyVideoRenderer( eventHandler: Handler, eventListener: VideoRendererEventListener, private val onLagDetected: () -> Unit, + private val onFramePresented: (presentationTimeUs: Long) -> Unit, ) : MediaCodecVideoRenderer( context, mediaCodecSelector, @@ -488,6 +523,11 @@ private class LowLatencyVideoRenderer( return config } + override fun onProcessedOutputBuffer(presentationTimeUs: Long) { + super.onProcessedOutputBuffer(presentationTimeUs) + onFramePresented(presentationTimeUs) + } + override fun shouldDropBuffersToKeyframe(earlyUs: Long, elapsedRealtimeUs: Long, isLastBuffer: Boolean): Boolean { if (earlyUs < -300_000) { onLagDetected() @@ -496,6 +536,9 @@ private class LowLatencyVideoRenderer( } override fun shouldDropOutputBuffer(earlyUs: Long, elapsedRealtimeUs: Long, isLastBuffer: Boolean): Boolean { - return earlyUs < -250_000 + // Never drop a single late P-frame. Infinite GOP has no intra-refresh + // on vah264enc; a hole stays as block artifacts until the next IDR. + // Deep lag still snaps to a keyframe via shouldDropBuffersToKeyframe. + return false } } diff --git a/clients/android/app/src/main/java/com/orbiscreen/android/player/StreamStats.kt b/clients/android/app/src/main/java/com/orbiscreen/android/player/StreamStats.kt new file mode 100644 index 00000000..80970041 --- /dev/null +++ b/clients/android/app/src/main/java/com/orbiscreen/android/player/StreamStats.kt @@ -0,0 +1,248 @@ +// Orbiscreen - StreamStats.kt (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +package com.orbiscreen.android.player + +data class StatsSnapshot( + val delayMs: Int?, + val ageMs: Int?, + val bytesPerSec: Long, + val bytesSeries: LongArray, + val frameI: IntArray, + val frameP: IntArray, + val frameB: IntArray, + val frameOther: IntArray, + val frameDropped: IntArray, + val fps: Int, + val lastMinuteI: Int, + val lastMinuteP: Int, + val lastMinuteB: Int, + val lastMinuteOther: Int, + val lastMinuteDropped: Int, +) + +class StreamStats( + private val windowMs: Long = 60_000L, + private val bucketMs: Long = 1_000L, +) { + private val lock = Any() + private val n: Int = (windowMs / bucketMs).toInt().coerceAtLeast(1) + private val bytes = LongArray(n) + private val framesI = IntArray(n) + private val framesP = IntArray(n) + private val framesB = IntArray(n) + private val framesOther = IntArray(n) + private val framesDropped = IntArray(n) + private val presented = IntArray(n) + private var head = 0 + private var cursorBucket = Long.MIN_VALUE + private var delayMs: Int? = null + private var presentedSentNs: Long = 0L + private var presentedAtMs: Long = 0L + private var livePtsOriginUs: Long = Long.MIN_VALUE + private var liveWallOriginMs: Long = 0L + @Volatile var clockOffsetNs: Long = 0L + + fun reset() { + synchronized(lock) { + bytes.fill(0) + framesI.fill(0) + framesP.fill(0) + framesB.fill(0) + framesOther.fill(0) + framesDropped.fill(0) + presented.fill(0) + head = 0 + cursorBucket = Long.MIN_VALUE + delayMs = null + presentedSentNs = 0L + presentedAtMs = 0L + livePtsOriginUs = Long.MIN_VALUE + liveWallOriginMs = 0L + } + } + + fun noteBytes(count: Long, nowMs: Long = System.currentTimeMillis()) { + if (count <= 0) return + synchronized(lock) { + rotateLocked(nowMs) + bytes[head] = bytes[head] + count + } + } + + fun noteFrame(kind: FrameKind, nowMs: Long = System.currentTimeMillis()) { + synchronized(lock) { + rotateLocked(nowMs) + when (kind) { + FrameKind.I -> framesI[head]++ + FrameKind.P -> framesP[head]++ + FrameKind.B -> framesB[head]++ + FrameKind.Other -> framesOther[head]++ + } + } + } + + fun noteDropped(count: Int = 1, nowMs: Long = System.currentTimeMillis()) { + if (count <= 0) return + synchronized(lock) { + rotateLocked(nowMs) + framesDropped[head] += count + } + } + + fun noteDelay(ms: Int) { + if (ms in 0..30_000) delayMs = ms + } + + fun notePresented(sentNs: Long, nowMs: Long = System.currentTimeMillis()) { + presentedSentNs = sentNs + presentedAtMs = nowMs + synchronized(lock) { + rotateLocked(nowMs) + presented[head]++ + } + } + + /** + * HTTP/MPEG-TS has no host send timestamp. Compare the frame PTS + * to wall time from the first presented sample so delay/age still + * populate on USB (where [currentLiveOffset] is TIME_UNSET). + */ + fun notePresentedPts(presentationTimeUs: Long, nowMs: Long = System.currentTimeMillis()) { + if (livePtsOriginUs == Long.MIN_VALUE || presentationTimeUs + 1_000_000L < livePtsOriginUs) { + livePtsOriginUs = presentationTimeUs + liveWallOriginMs = nowMs + } + val ptsElapsedMs = (presentationTimeUs - livePtsOriginUs) / 1000L + val wallElapsedMs = nowMs - liveWallOriginMs + val age = (wallElapsedMs - ptsElapsedMs).toInt() + if (age in 0..30_000) { + noteDelay(age) + notePresented((nowMs - age.toLong()) * 1_000_000L, nowMs) + } else { + livePtsOriginUs = presentationTimeUs + liveWallOriginMs = nowMs + notePresented(0L, nowMs) + } + } + + fun snapshot(nowMs: Long = System.currentTimeMillis()): StatsSnapshot { + synchronized(lock) { + rotateLocked(nowMs) + val series = LongArray(n) + val i = IntArray(n) + val p = IntArray(n) + val b = IntArray(n) + val o = IntArray(n) + val d = IntArray(n) + var sumI = 0 + var sumP = 0 + var sumB = 0 + var sumO = 0 + var sumD = 0 + for (k in 0 until n) { + val idx = (head + 1 + k) % n + series[k] = bytes[idx] + i[k] = framesI[idx] + p[k] = framesP[idx] + b[k] = framesB[idx] + o[k] = framesOther[idx] + d[k] = framesDropped[idx] + sumI += i[k] + sumP += p[k] + sumB += b[k] + sumO += o[k] + sumD += d[k] + } + val elapsed = (nowMs - cursorBucket * bucketMs).coerceIn(1L, bucketMs) + val prev = bytes[(head - 1 + n) % n] + val rate = if (nowMs - cursorBucket * bucketMs < 200L) { + prev + } else { + bytes[head] * bucketMs / elapsed + } + val prevFps = presented[(head - 1 + n) % n] + val curFps = presented[head] + val fps = if (nowMs - cursorBucket * bucketMs < 200L) { + prevFps + } else { + (curFps * bucketMs / elapsed).toInt() + } + return StatsSnapshot( + delayMs = delayMs, + ageMs = ageLocked(nowMs), + bytesPerSec = rate, + bytesSeries = series, + frameI = i, + frameP = p, + frameB = b, + frameOther = o, + frameDropped = d, + fps = fps, + lastMinuteI = sumI, + lastMinuteP = sumP, + lastMinuteB = sumB, + lastMinuteOther = sumO, + lastMinuteDropped = sumD, + ) + } + } + + private fun ageLocked(nowMs: Long): Int? { + if (presentedSentNs > 0L) { + val nowNs = nowMs * 1_000_000L + val age = ((nowNs + clockOffsetNs - presentedSentNs) / 1_000_000L).toInt() + if (age in 0..30_000) return age + } + val delay = delayMs + if (presentedAtMs > 0L && delay != null) { + return delay + (nowMs - presentedAtMs).toInt().coerceAtLeast(0) + } + return null + } + + private fun rotateLocked(nowMs: Long) { + val bucket = nowMs / bucketMs + if (cursorBucket == Long.MIN_VALUE) { + cursorBucket = bucket + return + } + val delta = bucket - cursorBucket + if (delta <= 0L) return + if (delta >= n) { + bytes.fill(0) + framesI.fill(0) + framesP.fill(0) + framesB.fill(0) + framesOther.fill(0) + framesDropped.fill(0) + presented.fill(0) + } else { + for (step in 1..delta.toInt()) { + val idx = (head + step) % n + bytes[idx] = 0 + framesI[idx] = 0 + framesP[idx] = 0 + framesB[idx] = 0 + framesOther[idx] = 0 + framesDropped[idx] = 0 + presented[idx] = 0 + } + } + head = ((head + delta) % n).toInt() + cursorBucket = bucket + } + + companion object { + fun formatRate(bytesPerSec: Long): String { + val n = bytesPerSec.coerceAtLeast(0L) + return when { + n >= 1_000_000L -> String.format("%.1f MB/s", n / 1_000_000.0) + n >= 1_000L -> String.format("%.0f KB/s", n / 1_000.0) + else -> "$n B/s" + } + } + + fun formatMs(ms: Int?): String = if (ms == null) "—" else "${ms} ms" + } +} diff --git a/clients/android/app/src/main/java/com/orbiscreen/android/player/UdpPlayer.kt b/clients/android/app/src/main/java/com/orbiscreen/android/player/UdpPlayer.kt index 5d42ac1e..e3bc4574 100644 --- a/clients/android/app/src/main/java/com/orbiscreen/android/player/UdpPlayer.kt +++ b/clients/android/app/src/main/java/com/orbiscreen/android/player/UdpPlayer.kt @@ -19,12 +19,15 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.isActive import kotlinx.coroutines.launch +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.OkHttpClient +import okhttp3.Request import java.net.DatagramPacket import java.net.DatagramSocket import java.net.InetAddress import java.net.SocketTimeoutException import java.nio.ByteBuffer -import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong private const val TAG = "Orbi.Udp" @@ -43,7 +46,9 @@ private const val RECV_BUF: Int = 65_507 enum class HandshakeAction { Ack, Control, Ignore } -class UdpPlayer { +class UdpPlayer( + val stats: StreamStats = StreamStats(), +) { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) private val _event = MutableStateFlow(StreamEvent.Idle) val event: StateFlow get() = _event @@ -56,16 +61,19 @@ class UdpPlayer { private var recvJob: Job? = null private var pingJob: Job? = null private var watchdogJob: Job? = null + private var idrJob: Job? = null + private var idrCall: okhttp3.Call? = null private var codec: MediaCodec? = null @Volatile private var surface: Surface? = null @Volatile private var running = false - private val pending = ConcurrentHashMap>() + private val pending = PendingFecStore() + private val reorder = AuReorder() private var sps: ByteArray? = null private var pps: ByteArray? = null private var configured = false private var waitKey = true - private var lastEmittedSeq = -1 private val lastIdr = AtomicLong(0) + private val sentQueue = ArrayDeque() private val lastHeardMs = AtomicLong(0) private var hostAddr: InetAddress? = null private var hostPort: Int = 0 @@ -90,8 +98,18 @@ class UdpPlayer { private var sessionId: String? = null - fun start(host: String, port: Int, token: String, width: Int, height: Int, session: String? = null): Boolean { - stop() + fun start( + host: String, + port: Int, + token: String, + width: Int, + height: Int, + session: String? = null, + httpPort: Int = 0, + ): Boolean { + // Replacing the socket is not a leave. Bye would release the + // display session and the next Hello would attach to a dead id. + stop(sendBye = false) this.width = width this.height = height this.sessionId = session @@ -105,6 +123,7 @@ class UdpPlayer { sock.soTimeout = 200 socket = sock running = true + stats.reset() _event.value = StreamEvent.Connecting(Uri.parse("udp://$host:$port")) sendRaw(encodeHello(token, session)) val buf = ByteArray(RECV_BUF) @@ -132,14 +151,15 @@ class UdpPlayer { stop() return false } - sock.soTimeout = 0 + sock.soTimeout = 20 lastHeardMs.set(System.currentTimeMillis()) recvJob = scope.launch { recvLoop() } pingJob = scope.launch { pingLoop() } watchdogJob = scope.launch { watchdogLoop() } - sendRaw(encodeCtrl(TYPE_IDR)) + val idrPort = if (httpPort in 1..65535) httpPort else (port - 1).coerceAtLeast(1) + idrJob = scope.launch { readIdrStream(host, idrPort, token, session) } _event.value = StreamEvent.Buffering - Log.i(TAG, "UDP session up $host:$port") + Log.i(TAG, "UDP session up $host:$port idr=:$idrPort/idr") true } catch (e: Exception) { Log.w(TAG, "UDP start failed: ${e.message}") @@ -148,24 +168,28 @@ class UdpPlayer { } } - fun stop() { - teardown(StreamEvent.Idle) + fun stop(sendBye: Boolean = true) { + teardown(StreamEvent.Idle, sendBye) } private fun fail(reason: String) { - teardown(StreamEvent.Disconnected(reason)) + teardown(StreamEvent.Disconnected(reason), sendBye = true) } - private fun teardown(event: StreamEvent) { + private fun teardown(event: StreamEvent, sendBye: Boolean = true) { val wasRunning = running running = false recvJob?.cancel() pingJob?.cancel() watchdogJob?.cancel() + idrJob?.cancel() + idrCall?.cancel() recvJob = null pingJob = null watchdogJob = null - if (wasRunning) { + idrJob = null + idrCall = null + if (wasRunning && sendBye) { try { sendRaw(encodeBye(sessionId)) } catch (_: Exception) {} } try { socket?.close() } catch (_: Exception) {} @@ -177,7 +201,8 @@ class UdpPlayer { pps = null configured = false waitKey = true - lastEmittedSeq = -1 + reorder.reset() + sentQueue.clear() if (wasRunning || event !is StreamEvent.Idle) { _event.value = event } @@ -216,6 +241,8 @@ class UdpPlayer { prepareReceive(pkt, buf) socket?.receive(pkt) ?: break handle(buf.copyOf(pkt.length)) + } catch (_: SocketTimeoutException) { + applyReorder(reorder.expire(System.currentTimeMillis())) } catch (e: Exception) { if (running) Log.w(TAG, "recv: ${e.message}") break @@ -229,6 +256,7 @@ class UdpPlayer { || data[2] != 'B'.code.toByte() || data[3] != '1'.code.toByte() ) return lastHeardMs.set(System.currentTimeMillis()) + stats.noteBytes(data.size.toLong()) when (data[4]) { TYPE_VIDEO -> handleVideo(data) TYPE_PONG -> { @@ -242,6 +270,7 @@ class UdpPlayer { ewmaClockOffsetNs = if (ewmaClockOffsetNs == Long.MIN_VALUE) raw else ((ewmaClockOffsetNs * 9L + raw) / 10L) clockOffsetNs = ewmaClockOffsetNs + stats.clockOffsetNs = clockOffsetNs } TYPE_PROBE -> { if (data.size < 7) return @@ -252,6 +281,9 @@ class UdpPlayer { TYPE_PMTU -> { if (data.size < 7) return Log.i(TAG, "PMTU confirmed datagram=${le16(data, 5)}") + // Video can flow now; ask for a keyframe so we do not + // start mid-GOP on P-frames that were not sent earlier. + requestIdr() } TYPE_HELLO_ACK -> {} else -> {} @@ -266,50 +298,56 @@ class UdpPlayer { val frags = le16(data, 10) val sentNs = le64(data, 20) val payload = data.copyOfRange(28, data.size) - if (frags <= 0 || frag >= frags) return dropStale(seq) - val slots = pending.getOrPut(seq) { arrayOfNulls(frags) } - if (slots.size != frags) { - pending[seq] = arrayOfNulls(frags).also { it[frag] = payload } - return + val done = pending.offer(seq, frag, frags, key, sentNs, payload) ?: return + val nowMs = System.currentTimeMillis() + applyReorder( + reorder.expire(nowMs) + + reorder.accept(AuReorder.Frame(done.seq, done.key, done.au, done.sentNs), nowMs), + ) + } + + private fun applyReorder(outs: List) { + for (out in outs) { + when (out) { + is AuReorder.Out.Gap -> { + stats.noteDropped(out.dropped) + waitKey = true + requestIdr() + } + is AuReorder.Out.Video -> emitVideo(out.frame) + } } - slots[frag] = payload - if (slots.any { it == null }) return - pending.remove(seq) - val au = assemble(slots) + } + + @Synchronized + private fun emitVideo(frame: AuReorder.Frame) { val now = System.currentTimeMillis() * 1_000_000L - val glass = ((now + clockOffsetNs - sentNs) / 1_000_000L).toInt() - if (glass in 0..5_000) _latencyMs.value = glass - val gap = lastEmittedSeq >= 0 && seqDelta(seq, lastEmittedSeq) != 1 - lastEmittedSeq = seq - if (glass in 75..5_000 && !key) { - pending.keys.toList().forEach { pending.remove(it) } - waitKey = true - requestIdr() - return - } - if (gap && !key) { - waitKey = true - requestIdr() - return + val glass = ((now + clockOffsetNs - frame.sentNs) / 1_000_000L).toInt() + if (glass in 0..5_000) { + _latencyMs.value = glass + stats.noteDelay(glass) } - if (waitKey && !key) { + stats.noteFrame(H264.classifyAccessUnit(frame.au)) + if (waitKey && !frame.key) { + stats.noteDropped() requestIdr() return } - if (key) { - extractSpsPps(au) + if (frame.key) { + extractSpsPps(frame.au) waitKey = false if (_event.value !is StreamEvent.Playing) _event.value = StreamEvent.Playing } - feedCodec(au, key) + feedCodec(frame.au, frame.key, frame.sentNs) } private fun dropStale(seq: Int) { if (pending.size < 4) return - val stale = pending.keys.filter { seqDelta(seq, it) in 2..32767 } + val stale = pending.keys().filter { seqDelta(seq, it) in 2..32767 } if (stale.isEmpty()) return for (s in stale) pending.remove(s) + stats.noteDropped(stale.size) waitKey = true requestIdr() } @@ -336,7 +374,8 @@ class UdpPlayer { } } - private fun feedCodec(au: ByteArray, key: Boolean) { + @Synchronized + private fun feedCodec(au: ByteArray, key: Boolean, sentNs: Long) { val surf = surface ?: return val sps0 = sps val pps0 = pps @@ -372,44 +411,136 @@ class UdpPlayer { } val c = codec ?: return if (waitKey && !key) { + stats.noteDropped() requestIdr() return } try { - val inIx = c.dequeueInputBuffer(8_000) + // Free completed outputs first so a slow dequeue does not + // skip a P-frame. A hole in an infinite GOP (VA-API has no + // intra-refresh) stays on screen as block artifacts until IDR. + drainOutputs(c, sentNs) + var inIx = c.dequeueInputBuffer(8_000) + if (inIx < 0) { + drainOutputs(c, sentNs) + inIx = c.dequeueInputBuffer(0) + } if (inIx < 0) { - if (key) waitKey = true + Log.w(TAG, "codec input full; hold until IDR") + stats.noteDropped() + waitKey = true + requestIdr() return } val inBuf = c.getInputBuffer(inIx) ?: return if (inBuf.remaining() < au.size) { Log.w(TAG, "codec input ${inBuf.remaining()} < au ${au.size}") c.queueInputBuffer(inIx, 0, 0, 0, 0) + stats.noteDropped() waitKey = true requestIdr() + drainOutputs(c, sentNs) return } inBuf.clear() inBuf.put(au) val flags = if (key) MediaCodec.BUFFER_FLAG_KEY_FRAME else 0 - c.queueInputBuffer(inIx, 0, au.size, System.nanoTime() / 1000, flags) - val info = MediaCodec.BufferInfo() - var outIx = c.dequeueOutputBuffer(info, 0) - while (outIx >= 0) { - c.releaseOutputBuffer(outIx, true) - outIx = c.dequeueOutputBuffer(info, 0) + // Real-time PTS so vendor low-latency stays on. A 1,2,3 µs + // counter looks like a file timeline and many SoCs then hold ~200 ms. + val ptsUs = System.nanoTime() / 1000L + if (sentNs > 0L) { + sentQueue.addLast(sentNs) + while (sentQueue.size > 120) sentQueue.removeFirst() } + c.queueInputBuffer(inIx, 0, au.size, ptsUs, flags) + drainOutputs(c, sentNs) } catch (e: Exception) { Log.w(TAG, "codec feed: ${e.message}") + stats.noteDropped() configured = false waitKey = true requestIdr() } } - private fun requestIdr() { + private fun drainOutputs(c: MediaCodec, sentNs: Long) { + val info = MediaCodec.BufferInfo() + var outIx = c.dequeueOutputBuffer(info, 0) + while (outIx >= 0) { + val presentedSent = sentQueue.removeFirstOrNull() ?: sentNs + stats.notePresented(presentedSent) + c.releaseOutputBuffer(outIx, System.nanoTime()) + outIx = c.dequeueOutputBuffer(info, 0) + } + } + + @Synchronized + private fun emitReliableKey(v: IdrFrames.Video) { + val now = System.currentTimeMillis() * 1_000_000L + val glass = ((now + clockOffsetNs - v.sentNs) / 1_000_000L).toInt() + if (glass in 0..5_000) { + _latencyMs.value = glass + stats.noteDelay(glass) + } + stats.noteFrame(H264.classifyAccessUnit(v.au)) + extractSpsPps(v.au) + reorder.onReliableKeyframe() + waitKey = false + if (_event.value !is StreamEvent.Playing) _event.value = StreamEvent.Playing + feedCodec(v.au, true, v.sentNs) + } + + private suspend fun readIdrStream(host: String, httpPort: Int, token: String, session: String?) { + val url = StringBuilder("http://$host:$httpPort/idr") + val q = ArrayList() + if (token.isNotBlank()) q.add("token=$token") + if (!session.isNullOrBlank()) q.add("session=$session") + if (q.isNotEmpty()) url.append('?').append(q.joinToString("&")) + val client = OkHttpClient.Builder() + .connectTimeout(2, TimeUnit.SECONDS) + .readTimeout(0, TimeUnit.MILLISECONDS) + .writeTimeout(2, TimeUnit.SECONDS) + .build() + val req = Request.Builder() + .url(url.toString().toHttpUrl()) + .apply { if (token.isNotBlank()) header("Authorization", "Bearer $token") } + .get() + .build() + val call = client.newCall(req) + idrCall = call + try { + call.execute().use { resp -> + if (!resp.isSuccessful) { + Log.w(TAG, "idr stream HTTP ${resp.code}") + return + } + val src = resp.body?.byteStream() ?: return + val reader = IdrFrames.Reader() + val tmp = ByteArray(8192) + Log.i(TAG, "idr TCP stream open") + while (running) { + val n = src.read(tmp) + if (n < 0) break + if (n == 0) continue + stats.noteBytes(n.toLong()) + reader.push(tmp.copyOf(n)) + var msg = reader.pop() + while (msg != null) { + if (msg.key) emitReliableKey(msg) + msg = reader.pop() + } + } + } + } catch (e: Exception) { + if (running) Log.w(TAG, "idr stream: ${e.message}") + } finally { + if (idrCall === call) idrCall = null + } + } + + fun requestIdr() { val now = System.currentTimeMillis() - if (now - lastIdr.get() < 250) return + if (!Idr.due(now, lastIdr.get())) return lastIdr.set(now) sendRaw(encodeCtrl(TYPE_IDR)) } diff --git a/clients/android/app/src/main/java/com/orbiscreen/android/ui/stream/StreamViewModel.kt b/clients/android/app/src/main/java/com/orbiscreen/android/ui/stream/StreamViewModel.kt index 36e11b12..03fc22f1 100644 --- a/clients/android/app/src/main/java/com/orbiscreen/android/ui/stream/StreamViewModel.kt +++ b/clients/android/app/src/main/java/com/orbiscreen/android/ui/stream/StreamViewModel.kt @@ -9,6 +9,7 @@ import androidx.lifecycle.viewModelScope import com.orbiscreen.android.data.PrefsStore import com.orbiscreen.android.input.InputDispatcher import com.orbiscreen.android.net.HostApi +import com.orbiscreen.android.player.Idr import com.orbiscreen.android.player.PlayerHolder import com.orbiscreen.android.player.StreamEvent import kotlinx.coroutines.Dispatchers @@ -48,6 +49,7 @@ class StreamViewModel( private var sessionToken: String? = null private var displaySessionId: String? = null private var lastIdrAtMs = 0L + private var waitingForKeyframe = false private fun scaleModeFromPref(pref: String): Int = when (pref) { "fill" -> 3 @@ -67,6 +69,7 @@ class StreamViewModel( val player get() = playerHolder.player val udpPlayer get() = playerHolder.udpPlayer + val streamStats get() = playerHolder.stats fun detectNativeDisplay(): Triple { val windowManager = context.getSystemService(Context.WINDOW_SERVICE) as? android.view.WindowManager @@ -111,7 +114,24 @@ class StreamViewModel( viewModelScope.launch { playerHolder.event.collect { ev -> _state.value = _state.value.copy(event = ev) - if (ev is StreamEvent.Error) { + when (ev) { + is StreamEvent.Playing -> waitingForKeyframe = false + is StreamEvent.Error -> { + waitingForKeyframe = true + requestIdr() + } + is StreamEvent.Connecting, is StreamEvent.Buffering -> + waitingForKeyframe = true + is StreamEvent.Disconnected, is StreamEvent.Idle -> + waitingForKeyframe = false + else -> {} + } + } + } + viewModelScope.launch { + while (isActive) { + delay(Idr.DEBOUNCE_MS) + if (waitingForKeyframe && playerHolder.udpPlayer.value == null) { requestIdr() } } @@ -198,8 +218,14 @@ class StreamViewModel( } private fun requestIdr() { + waitingForKeyframe = true + val udp = playerHolder.udpPlayer.value + if (udp != null) { + udp.requestIdr() + return + } val now = android.os.SystemClock.elapsedRealtime() - if (now - lastIdrAtMs < 250L) return + if (!Idr.due(now, lastIdrAtMs)) return lastIdrAtMs = now ensureInput().control("idr") } @@ -219,6 +245,8 @@ class StreamViewModel( } inputDispatcher = it } + // Session is opened in init before the surface calls ensureInput. + // Always refresh so /input is not dropped when several displays exist. dispatcher.sessionId = displaySessionId sessionToken?.let { dispatcher.updateToken(it) } dispatcher.resize(state.value.displayWidth, state.value.displayHeight) diff --git a/clients/android/app/src/test/java/com/orbiscreen/android/player/AuReorderTest.kt b/clients/android/app/src/test/java/com/orbiscreen/android/player/AuReorderTest.kt new file mode 100644 index 00000000..c240efcd --- /dev/null +++ b/clients/android/app/src/test/java/com/orbiscreen/android/player/AuReorderTest.kt @@ -0,0 +1,80 @@ +// Orbiscreen - AuReorderTest.kt (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +package com.orbiscreen.android.player + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class AuReorderTest { + + private fun p(seq: Int, key: Boolean = false) = + AuReorder.Frame(seq, key, byteArrayOf(seq.toByte()), 0L) + + @Test + fun playsHeldFrameWhenHoleFills() { + val r = AuReorder() + val first = r.accept(p(10, key = true), 0) + assertEquals(1, first.size) + assertEquals(10, (first[0] as AuReorder.Out.Video).frame.seq) + + assertTrue(r.accept(p(12), 0).isEmpty()) + val filled = r.accept(p(11), 10) + assertEquals(listOf(11, 12), filled.map { (it as AuReorder.Out.Video).frame.seq }) + assertTrue(r.expire(10 + AuReorder.HOLE_WAIT_MS).isEmpty()) + } + + @Test + fun expireAfterHoleWait() { + val r = AuReorder() + r.accept(p(10, key = true), 0) + assertTrue(r.accept(p(12), 0).isEmpty()) + assertTrue(r.expire(AuReorder.HOLE_WAIT_MS - 1).isEmpty()) + val gap = r.expire(AuReorder.HOLE_WAIT_MS) + assertEquals(1, gap.size) + assertEquals(1, (gap[0] as AuReorder.Out.Gap).dropped) + } + + @Test + fun ignoresOlderSeq() { + val r = AuReorder() + r.accept(p(10, key = true), 0) + assertTrue(r.accept(p(8), 0).isEmpty()) + val next = r.accept(p(11), 0) + assertEquals(11, (next[0] as AuReorder.Out.Video).frame.seq) + assertEquals(11, r.lastSeq) + } + + @Test + fun wrapAroundIsInOrder() { + val r = AuReorder() + r.accept(p(65535, key = true), 0) + val wrapped = r.accept(p(0), 0) + assertEquals(0, (wrapped[0] as AuReorder.Out.Video).frame.seq) + } + + @Test + fun reliableKeyframeLetsNextPPlayAfterAHole() { + val r = AuReorder() + r.accept(p(10, key = true), 0) + assertTrue(r.accept(p(12), 0).isEmpty()) + assertEquals(1, (r.expire(AuReorder.HOLE_WAIT_MS)[0] as AuReorder.Out.Gap).dropped) + + r.onReliableKeyframe() + val next = r.accept(p(40), AuReorder.HOLE_WAIT_MS + 10) + assertEquals(1, next.size) + assertEquals(40, (next[0] as AuReorder.Out.Video).frame.seq) + assertTrue(r.expire(AuReorder.HOLE_WAIT_MS * 2).isEmpty()) + } + + @Test + fun keyWithHoleDropsHeld() { + val r = AuReorder() + r.accept(p(10, key = true), 0) + r.accept(p(12), 0) + val key = r.accept(p(20, key = true), 0) + assertEquals(20, (key[0] as AuReorder.Out.Video).frame.seq) + assertTrue(r.expire(AuReorder.HOLE_WAIT_MS).isEmpty()) + } +} diff --git a/clients/android/app/src/test/java/com/orbiscreen/android/player/FecTest.kt b/clients/android/app/src/test/java/com/orbiscreen/android/player/FecTest.kt new file mode 100644 index 00000000..9c2b1b3a --- /dev/null +++ b/clients/android/app/src/test/java/com/orbiscreen/android/player/FecTest.kt @@ -0,0 +1,51 @@ +// Orbiscreen - FecTest.kt (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +package com.orbiscreen.android.player + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class FecTest { + @Test + fun ladder() { + assertEquals(0, Fec.parityCount(3)) + assertEquals(2, Fec.parityCount(4)) + assertEquals(2, Fec.parityCount(16)) + assertEquals(3, Fec.parityCount(17)) + assertEquals(3, Fec.parityCount(64)) + assertEquals(4, Fec.parityCount(65)) + } + + @Test + fun recoversTwoErasures() { + val k = 4 + val width = 16 + val src = Array(k) { i -> + ByteArray(width) { b -> ((i * 31 + b) and 0xff).toByte() } + } + val parity = Fec.encode(src, 2) + val data: Array = Array(k) { src[it].copyOf() } + data[1] = null + data[3] = null + val pslots: Array = Array(parity.size) { parity[it] } + assertTrue(Fec.recover(data, pslots)) + for (i in 0 until k) { + assertArrayEquals(src[i], data[i]) + } + } + + @Test + fun failsWhenTooManyErasures() { + val src = Array(4) { i -> ByteArray(8) { b -> (i + b).toByte() } } + val parity = Fec.encode(src, 2) + val data: Array = Array(4) { src[it].copyOf() } + data[0] = null + data[1] = null + data[2] = null + assertFalse(Fec.recover(data, Array(2) { parity[it] })) + } +} diff --git a/clients/android/app/src/test/java/com/orbiscreen/android/player/H264Test.kt b/clients/android/app/src/test/java/com/orbiscreen/android/player/H264Test.kt new file mode 100644 index 00000000..9c4a348f --- /dev/null +++ b/clients/android/app/src/test/java/com/orbiscreen/android/player/H264Test.kt @@ -0,0 +1,85 @@ +// Orbiscreen - H264Test.kt (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +package com.orbiscreen.android.player + +import org.junit.Assert.assertEquals +import org.junit.Test + +class H264Test { + + @Test + fun idrIsIFrame() { + val au = withStart(byteArrayOf(0x65, 0x88.toByte())) + assertEquals(FrameKind.I, H264.classifyAccessUnit(au)) + } + + @Test + fun pSliceIsPFrame() { + val au = sliceAu(nalType = 1, sliceType = 0) + assertEquals(FrameKind.P, H264.classifyAccessUnit(au)) + } + + @Test + fun bSliceIsBFrame() { + val au = sliceAu(nalType = 1, sliceType = 1) + assertEquals(FrameKind.B, H264.classifyAccessUnit(au)) + } + + @Test + fun nonIdrISliceIsIFrame() { + val au = sliceAu(nalType = 1, sliceType = 2) + assertEquals(FrameKind.I, H264.classifyAccessUnit(au)) + } + + @Test + fun iAllSliceTypeSevenIsIFrame() { + val au = sliceAu(nalType = 1, sliceType = 7) + assertEquals(FrameKind.I, H264.classifyAccessUnit(au)) + } + + @Test + fun spsPpsOnlyIsOther() { + val sps = withStart(byteArrayOf(0x67, 0x64, 0x00, 0x28)) + val pps = withStart(byteArrayOf(0x68, 0xEE.toByte(), 0x3C)) + assertEquals(FrameKind.Other, H264.classifyAccessUnit(sps + pps)) + } + + @Test + fun keyframeAuWithSpsIsI() { + val sps = withStart(byteArrayOf(0x67, 0x64, 0x00, 0x28)) + val pps = withStart(byteArrayOf(0x68, 0xEE.toByte(), 0x3C)) + val idr = withStart(byteArrayOf(0x65, 0x88.toByte())) + assertEquals(FrameKind.I, H264.classifyAccessUnit(sps + pps + idr)) + } + + @Test + fun emulationPreventionDoesNotBreakSliceType() { + // first_mb=0 (1), slice_type=0 (1) → bits 11, plus leftover zeros. + val nal = byteArrayOf(0x61, 0xC0.toByte()) + assertEquals(FrameKind.P, H264.sliceKind(nal)) + } + + private fun sliceAu(nalType: Int, sliceType: Int): ByteArray { + val bits = encodeUe(0) + encodeUe(sliceType) + val payload = bitsToBytes(bits) + val header = (0x60 or (nalType and 0x1F)).toByte() + return withStart(byteArrayOf(header) + payload) + } + + private fun withStart(nal: ByteArray): ByteArray = byteArrayOf(0, 0, 0, 1) + nal + + private fun encodeUe(codeNum: Int): String { + val x = codeNum + 1 + val zeros = 31 - Integer.numberOfLeadingZeros(x) + val suffix = Integer.toBinaryString(x).substring(1) + return "0".repeat(zeros) + "1" + suffix + } + + private fun bitsToBytes(bits: String): ByteArray { + val padded = bits.padEnd(((bits.length + 7) / 8) * 8, '0') + return ByteArray(padded.length / 8) { i -> + padded.substring(i * 8, i * 8 + 8).toInt(2).toByte() + } + } +} diff --git a/clients/android/app/src/test/java/com/orbiscreen/android/player/IdrFramesTest.kt b/clients/android/app/src/test/java/com/orbiscreen/android/player/IdrFramesTest.kt new file mode 100644 index 00000000..ae1f9781 --- /dev/null +++ b/clients/android/app/src/test/java/com/orbiscreen/android/player/IdrFramesTest.kt @@ -0,0 +1,74 @@ +// Orbiscreen - IdrFramesTest.kt (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +package com.orbiscreen.android.player + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class IdrFramesTest { + + private fun encodeVideo(key: Boolean, pts: Long, sent: Long, au: ByteArray): ByteArray { + val body = ByteArray(18 + au.size) + body[0] = 1 + body[1] = if (key) 1 else 0 + putLe64(body, 2, pts) + putLe64(body, 10, sent) + System.arraycopy(au, 0, body, 18, au.size) + val out = ByteArray(4 + body.size) + val n = body.size + out[0] = ((n ushr 24) and 0xff).toByte() + out[1] = ((n ushr 16) and 0xff).toByte() + out[2] = ((n ushr 8) and 0xff).toByte() + out[3] = (n and 0xff).toByte() + System.arraycopy(body, 0, out, 4, body.size) + return out + } + + private fun putLe64(d: ByteArray, o: Int, v: Long) { + var x = v + for (i in 0 until 8) { + d[o + i] = (x and 0xff).toByte() + x = x ushr 8 + } + } + + @Test + fun splitWaitsForLengthPrefix() { + val frame = encodeVideo(true, 9, 11, byteArrayOf(0, 0, 0, 1, 0x65)) + assertNull(IdrFrames.split(frame.copyOf(3))) + assertNull(IdrFrames.split(frame.copyOf(4))) + val (body, used) = IdrFrames.split(frame)!! + assertEquals(frame.size, used) + assertEquals(frame.size - 4, body.size) + } + + @Test + fun decodeKeyframeAccessUnit() { + val au = byteArrayOf(0, 0, 0, 1, 0x65, 9) + val frame = encodeVideo(true, 33, 99, au) + val (body, _) = IdrFrames.split(frame)!! + val v = IdrFrames.decodeVideo(body)!! + assertTrue(v.key) + assertEquals(33L, v.ptsNs) + assertEquals(99L, v.sentNs) + assertArrayEquals(au, v.au) + } + + @Test + fun pushBufferEmitsCompleteFramesOnly() { + val au = byteArrayOf(0, 0, 0, 1, 0x65) + val frame = encodeVideo(true, 1, 2, au) + val reader = IdrFrames.Reader() + reader.push(frame.copyOfRange(0, 6)) + assertNull(reader.pop()) + reader.push(frame.copyOfRange(6, frame.size)) + val v = reader.pop()!! + assertTrue(v.key) + assertArrayEquals(au, v.au) + assertNull(reader.pop()) + } +} diff --git a/clients/android/app/src/test/java/com/orbiscreen/android/player/IdrTest.kt b/clients/android/app/src/test/java/com/orbiscreen/android/player/IdrTest.kt new file mode 100644 index 00000000..a6131ae3 --- /dev/null +++ b/clients/android/app/src/test/java/com/orbiscreen/android/player/IdrTest.kt @@ -0,0 +1,20 @@ +// Orbiscreen - IdrTest.kt (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen +package com.orbiscreen.android.player + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class IdrTest { + + @Test + fun debounceIs250ms() { + assertEquals(250L, Idr.DEBOUNCE_MS) + assertFalse(Idr.due(249, 0)) + assertTrue(Idr.due(250, 0)) + assertFalse(Idr.due(400, 200)) + assertTrue(Idr.due(451, 200)) + } +} diff --git a/clients/android/app/src/test/java/com/orbiscreen/android/player/PendingFecStoreTest.kt b/clients/android/app/src/test/java/com/orbiscreen/android/player/PendingFecStoreTest.kt new file mode 100644 index 00000000..b92e639f --- /dev/null +++ b/clients/android/app/src/test/java/com/orbiscreen/android/player/PendingFecStoreTest.kt @@ -0,0 +1,86 @@ +// Orbiscreen - PendingFecStoreTest.kt (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +package com.orbiscreen.android.player + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Test + +class PendingFecStoreTest { + private fun shards(au: ByteArray, chunk: Int): Pair, Array> { + val prefix = ByteArray(4) + val n = au.size + prefix[0] = (n and 0xff).toByte() + prefix[1] = ((n shr 8) and 0xff).toByte() + prefix[2] = ((n shr 16) and 0xff).toByte() + prefix[3] = ((n shr 24) and 0xff).toByte() + val blob = prefix + au + val parts = blob.toList().chunked(chunk).map { it.toByteArray() }.toTypedArray() + val padded = Array(parts.size) { i -> + ByteArray(chunk).also { parts[i].copyInto(it) } + } + val parity = Fec.encode(padded, Fec.parityCount(parts.size)) + return parts to parity + } + + @Test + fun allDataEmitsWithoutWaitingForParity() { + val store = PendingFecStore() + val au = ByteArray(36) { it.toByte() } + val (data, _) = shards(au, 10) + assertEquals(4, data.size) + var out: PendingFecStore.Complete? = null + for (i in data.indices) { + out = store.offer(3, i, data.size, false, 9L, data[i]) + } + assertNotNull(out) + assertArrayEquals(au, out!!.au) + assertEquals(0, store.size) + } + + @Test + fun lateParityDoesNotCreateANewPendingAu() { + val store = PendingFecStore() + val au = ByteArray(36) { it.toByte() } + val (data, parity) = shards(au, 10) + for (i in data.indices) store.offer(3, i, data.size, false, 9L, data[i]) + assertEquals(0, store.size) + assertNull(store.offer(3, data.size, data.size, false, 9L, parity[0])) + assertFalse(store.contains(3)) + assertEquals(0, store.size) + } + + @Test + fun parityFirstDoesNotOpenAPendingAu() { + val store = PendingFecStore() + val au = ByteArray(36) { it.toByte() } + val (data, parity) = shards(au, 10) + assertNull(store.offer(5, data.size, data.size, false, 1L, parity[0])) + assertEquals(0, store.size) + var out: PendingFecStore.Complete? = null + for (i in data.indices) { + out = store.offer(5, i, data.size, false, 1L, data[i]) + } + assertNotNull(out) + assertArrayEquals(au, out!!.au) + } + + @Test + fun missingDataPlusParityRecovers() { + val store = PendingFecStore() + val au = ByteArray(36) { it.toByte() } + val (data, parity) = shards(au, 10) + val k = data.size + store.offer(1, 0, k, false, 2L, data[0]) + store.offer(1, 2, k, false, 2L, data[2]) + store.offer(1, 3, k, false, 2L, data[3]) + val out = store.offer(1, k, k, false, 2L, parity[0]) + assertNotNull(out) + assertArrayEquals(au, out!!.au) + assertEquals(0, store.size) + } +} diff --git a/clients/android/app/src/test/java/com/orbiscreen/android/player/StreamStatsTest.kt b/clients/android/app/src/test/java/com/orbiscreen/android/player/StreamStatsTest.kt new file mode 100644 index 00000000..11a543b9 --- /dev/null +++ b/clients/android/app/src/test/java/com/orbiscreen/android/player/StreamStatsTest.kt @@ -0,0 +1,120 @@ +// Orbiscreen - StreamStatsTest.kt (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +package com.orbiscreen.android.player + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class StreamStatsTest { + + @Test + fun bytesLandInCurrentBucket() { + val s = StreamStats(windowMs = 60_000, bucketMs = 1_000) + s.noteBytes(400, nowMs = 10) + s.noteBytes(600, nowMs = 20) + val snap = s.snapshot(nowMs = 50) + assertEquals(1000L, snap.bytesSeries.last()) + } + + @Test + fun bucketsRotateAndKeepLastMinute() { + val s = StreamStats(windowMs = 4_000, bucketMs = 1_000) + s.noteBytes(100, nowMs = 0) + s.noteBytes(200, nowMs = 1_000) + s.noteBytes(300, nowMs = 2_000) + val snap = s.snapshot(nowMs = 2_100) + assertEquals(4, snap.bytesSeries.size) + assertEquals(100L, snap.bytesSeries[1]) + assertEquals(200L, snap.bytesSeries[2]) + assertEquals(300L, snap.bytesSeries[3]) + } + + @Test + fun stackedFramesCountByKind() { + val s = StreamStats(windowMs = 4_000, bucketMs = 1_000) + s.noteFrame(FrameKind.I, nowMs = 0) + s.noteFrame(FrameKind.P, nowMs = 10) + s.noteFrame(FrameKind.P, nowMs = 20) + s.noteFrame(FrameKind.B, nowMs = 30) + val snap = s.snapshot(nowMs = 40) + assertEquals(1, snap.frameI.last()) + assertEquals(2, snap.frameP.last()) + assertEquals(1, snap.frameB.last()) + assertEquals(1, snap.lastMinuteI) + assertEquals(2, snap.lastMinuteP) + assertEquals(1, snap.lastMinuteB) + assertEquals(0, snap.lastMinuteDropped) + } + + @Test + fun droppedFramesStackInRedSeries() { + val s = StreamStats(windowMs = 4_000, bucketMs = 1_000) + s.noteFrame(FrameKind.P, nowMs = 0) + s.noteDropped(2, nowMs = 10) + s.noteDropped(nowMs = 1_000) + val snap = s.snapshot(nowMs = 1_100) + assertEquals(2, snap.frameDropped[snap.frameDropped.lastIndex - 1]) + assertEquals(1, snap.frameDropped.last()) + assertEquals(3, snap.lastMinuteDropped) + } + + @Test + fun ageUsesClockAdjustedSentTime() { + val s = StreamStats() + s.clockOffsetNs = 0 + s.notePresented(sentNs = 1_010_000_000L, nowMs = 1_010) + val age = s.snapshot(nowMs = 1_025).ageMs + assertEquals(15, age) + } + + @Test + fun ageFallsBackToDelayPlusOnScreen() { + val s = StreamStats() + s.noteDelay(8) + s.notePresented(sentNs = 0L, nowMs = 1000) + val age = s.snapshot(nowMs = 1012).ageMs + assertEquals(20, age) + } + + @Test + fun fpsCountsPresentedNotReceived() { + val s = StreamStats(windowMs = 4_000, bucketMs = 1_000) + s.noteFrame(FrameKind.I, nowMs = 0) + s.noteFrame(FrameKind.P, nowMs = 10) + s.noteFrame(FrameKind.P, nowMs = 20) + s.notePresented(sentNs = 1L, nowMs = 30) + s.notePresented(sentNs = 2L, nowMs = 40) + val snap = s.snapshot(nowMs = 1_100) + assertEquals(2, snap.fps) + assertEquals(1, snap.lastMinuteI) + assertEquals(2, snap.lastMinuteP) + } + + @Test + fun httpPtsFillsDelayAndAge() { + val s = StreamStats() + s.notePresentedPts(presentationTimeUs = 0L, nowMs = 1_000) + s.notePresentedPts(presentationTimeUs = 16_000L, nowMs = 1_040) + val snap = s.snapshot(nowMs = 1_040) + assertEquals(24, snap.delayMs) + assertEquals(24, snap.ageMs) + val later = s.snapshot(nowMs = 1_050) + assertEquals(34, later.ageMs) + } + + @Test + fun emptyAgeIsNull() { + val s = StreamStats() + assertNull(s.snapshot(nowMs = 0).ageMs) + } + + @Test + fun formatRate() { + assertEquals("500 B/s", StreamStats.formatRate(500)) + assertTrue(StreamStats.formatRate(12_000).contains("KB")) + assertTrue(StreamStats.formatRate(2_500_000).contains("MB")) + } +} diff --git a/clients/web/annexb.js b/clients/web/annexb.js new file mode 100644 index 00000000..ee3ea9b2 --- /dev/null +++ b/clients/web/annexb.js @@ -0,0 +1,738 @@ +// Orbiscreen - annexb.js (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +(function (root, factory) { + if (typeof module === "object" && module.exports) { + module.exports = factory(); + } else { + root.OrbiAnnexB = factory(); + } +}(typeof self !== "undefined" ? self : this, function () { + const TYPE_VIDEO = 1; + const TYPE_HELLO = 2; + const TYPE_HELLO_ACK = 3; + const TYPE_PING = 4; + const TYPE_PONG = 5; + const TYPE_IDR = 6; + const TYPE_BYE = 10; + const MAX_FRAME = 4 * 1024 * 1024; + const DATAGRAM_HEADER = 24; + + function startCodeLen(data, i) { + if (i + 3 < data.length && data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 1) { + return 3; + } + if (i + 4 < data.length && data[i] === 0 && data[i + 1] === 0 + && data[i + 2] === 0 && data[i + 3] === 1) { + return 4; + } + return 0; + } + + function withStartCode(nal) { + const out = new Uint8Array(4 + nal.length); + out[3] = 1; + out.set(nal, 4); + return out; + } + + function unescapeRbsp(nal) { + if (!nal || nal.length <= 1) return new Uint8Array(0); + const out = []; + let i = 1; + while (i < nal.length) { + if (i + 2 < nal.length && nal[i] === 0 && nal[i + 1] === 0 && nal[i + 2] === 3) { + out.push(0, 0); + i += 3; + } else { + out.push(nal[i]); + i += 1; + } + } + return Uint8Array.from(out); + } + + function readUe(data) { + let bitPos = 0; + function bit() { + const byteIx = (bitPos / 8) | 0; + if (byteIx >= data.length) return -1; + const v = (data[byteIx] >> (7 - (bitPos % 8))) & 1; + bitPos += 1; + return v; + } + function ue() { + let zeros = 0; + while (true) { + const b = bit(); + if (b < 0) return -1; + if (b === 1) break; + zeros += 1; + if (zeros > 31) return -1; + } + let suffix = 0; + for (let i = 0; i < zeros; i += 1) { + const b = bit(); + if (b < 0) return -1; + suffix = (suffix << 1) | b; + } + return (1 << zeros) - 1 + suffix; + } + return { ue }; + } + + function sliceKind(nal) { + if (!nal || !nal.length) return "other"; + const typ = nal[0] & 0x1f; + if (typ === 5) return "I"; + if (typ !== 1 && typ !== 2) return "other"; + const rbsp = unescapeRbsp(nal); + if (!rbsp.length) return "other"; + const bits = readUe(rbsp); + if (bits.ue() < 0) return "other"; + const sliceType = bits.ue(); + if (sliceType < 0) return "other"; + switch (sliceType % 5) { + case 0: + case 3: + return "P"; + case 1: + return "B"; + case 2: + case 4: + return "I"; + default: + return "other"; + } + } + + function classifyAccessUnit(au) { + let sawI = false; + let sawB = false; + let sawP = false; + let i = 0; + while (i < au.length) { + const sc = startCodeLen(au, i); + if (!sc) break; + const nalStart = i + sc; + let next = nalStart; + while (next < au.length) { + if (startCodeLen(au, next) && next > nalStart) break; + next += 1; + } + if (nalStart < next) { + const kind = sliceKind(au.subarray(nalStart, next)); + if (kind === "I") sawI = true; + else if (kind === "B") sawB = true; + else if (kind === "P") sawP = true; + } + i = next; + } + if (sawI) return "I"; + if (sawB) return "B"; + if (sawP) return "P"; + return "other"; + } + + function extractSpsPps(au) { + let sps = null; + let pps = null; + let i = 0; + while (i < au.length) { + const sc = startCodeLen(au, i); + if (!sc) break; + const nalStart = i + sc; + let next = nalStart; + while (next < au.length) { + if (startCodeLen(au, next) && next > nalStart) break; + next += 1; + } + if (nalStart < next) { + const nal = au.subarray(nalStart, next); + const typ = nal[0] & 0x1f; + if (typ === 7) sps = withStartCode(nal); + if (typ === 8) pps = withStartCode(nal); + } + i = next; + } + return { sps, pps }; + } + + function codecStringFromSps(sps) { + if (!sps) return null; + const sc = startCodeLen(sps, 0); + if (!sc || sps.length < sc + 4) return null; + if ((sps[sc] & 0x1f) !== 7) return null; + const hex = (n) => n.toString(16).toUpperCase().padStart(2, "0"); + return `avc1.${hex(sps[sc + 1])}${hex(sps[sc + 2])}${hex(sps[sc + 3])}`; + } + + function spsDimensions(sps) { + if (!sps) return null; + const sc = startCodeLen(sps, 0); + const nal = sps.subarray(sc); + if (nal.length < 5 || (nal[0] & 0x1f) !== 7) return null; + let bit = 32; + const getBits = (n) => { + let v = 0; + for (let i = 0; i < n; i += 1) { + const byte = nal[bit >> 3]; + if (byte === undefined) return 0; + v = (v << 1) | ((byte >> (7 - (bit & 7))) & 1); + bit += 1; + } + return v; + }; + const ue = () => { + let z = 0; + while (getBits(1) === 0) { + z += 1; + if (z > 31) return 0; + } + return z === 0 ? 0 : ((1 << z) | getBits(z)) - 1; + }; + const profile = nal[1]; + ue(); + if (profile === 100 || profile === 110 || profile === 122 || profile === 244 + || profile === 44 || profile === 83 || profile === 86 || profile === 118 + || profile === 128 || profile === 138 || profile === 139 || profile === 134) { + const chroma = ue(); + if (chroma === 3) getBits(1); + ue(); + ue(); + getBits(1); + if (getBits(1)) { + const limit = chroma === 3 ? 12 : 8; + for (let i = 0; i < limit; i += 1) { + if (getBits(1)) { + const last = i < 6 ? 16 : 64; + let next = 8; + for (let j = 0; j < last; j += 1) { + const delta = (() => { + let z = 0; + while (getBits(1) === 0) { + z += 1; + if (z > 31) return 0; + } + const mag = z === 0 ? 0 : ((1 << z) | getBits(z)) - 1; + const signed = (mag % 2 === 0) ? -(mag / 2) : (mag + 1) / 2; + return signed; + })(); + next = (next + delta + 256) % 256; + if (next === 0) break; + } + } + } + } + } + ue(); + const poc = ue(); + if (poc === 0) { + ue(); + } else if (poc === 1) { + getBits(1); + ue(); + ue(); + const n = ue(); + for (let i = 0; i < n; i += 1) ue(); + } + ue(); + getBits(1); + const wMbs = ue() + 1; + const hMap = ue() + 1; + const frameMbsOnly = getBits(1); + if (!frameMbsOnly) getBits(1); + getBits(1); + let cropL = 0; + let cropR = 0; + let cropT = 0; + let cropB = 0; + if (getBits(1)) { + cropL = ue(); + cropR = ue(); + cropT = ue(); + cropB = ue(); + } + const width = wMbs * 16 - (cropL + cropR) * 2; + const height = (2 - frameMbsOnly) * hMap * 16 - (cropT + cropB) * 2; + if (width < 16 || height < 16 || width > 7680 || height > 4320) return null; + return { width, height }; + } + + function concatBytes(parts) { + let n = 0; + for (const p of parts) n += p.length; + const out = new Uint8Array(n); + let o = 0; + for (const p of parts) { + out.set(p, o); + o += p.length; + } + return out; + } + + function u16le(n) { + return new Uint8Array([n & 0xff, (n >> 8) & 0xff]); + } + + function u64le(n) { + const out = new Uint8Array(8); + let x = BigInt(n); + for (let i = 0; i < 8; i += 1) { + out[i] = Number(x & 0xffn); + x >>= 8n; + } + return out; + } + + function readU16le(buf, o) { + return buf[o] | (buf[o + 1] << 8); + } + + function readU64le(buf, o) { + let x = 0n; + for (let i = 0; i < 8; i += 1) { + x |= BigInt(buf[o + i]) << BigInt(8 * i); + } + return x; + } + + function encodeFrame(body) { + if (body.length > MAX_FRAME) throw new Error("frame too large"); + const out = new Uint8Array(4 + body.length); + const len = body.length; + out[0] = (len >>> 24) & 0xff; + out[1] = (len >>> 16) & 0xff; + out[2] = (len >>> 8) & 0xff; + out[3] = len & 0xff; + out.set(body, 4); + return out; + } + + function splitFrame(buf) { + if (buf.length < 4) return null; + const len = ((buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3]) >>> 0; + if (len > MAX_FRAME) throw new Error("frame too large"); + if (buf.length < 4 + len) return null; + return { body: buf.subarray(4, 4 + len), used: 4 + len }; + } + + function encodeHello(token, session) { + const t = new TextEncoder().encode(token || ""); + const s = new TextEncoder().encode(session || ""); + return encodeFrame(concatBytes([ + Uint8Array.of(TYPE_HELLO), + u16le(t.length), + t, + u16le(s.length), + s, + ])); + } + + function encodeCtrl(kind) { + return encodeFrame(Uint8Array.of(kind)); + } + + function encodePing(t0Ns) { + return encodeFrame(concatBytes([Uint8Array.of(TYPE_PING), u64le(t0Ns)])); + } + + function decodeMessage(body) { + if (!body.length) throw new Error("empty"); + const kind = body[0]; + const rest = body.subarray(1); + if (kind === TYPE_HELLO_ACK) { + if (rest.length < 4) throw new Error("truncated"); + return { type: "helloAck", width: readU16le(rest, 0), height: readU16le(rest, 2) }; + } + if (kind === TYPE_VIDEO) { + if (rest.length < 17) throw new Error("truncated"); + return { + type: "video", + key: rest[0] !== 0, + ptsNs: readU64le(rest, 1), + sentNs: readU64le(rest, 9), + au: rest.subarray(17), + }; + } + if (kind === TYPE_PONG) { + if (rest.length < 16) throw new Error("truncated"); + return { type: "pong", t0Ns: readU64le(rest, 0), hostNs: readU64le(rest, 8) }; + } + if (kind === TYPE_IDR) return { type: "idr" }; + if (kind === TYPE_BYE) return { type: "bye" }; + if (kind === TYPE_PING) return { type: "ping", t0Ns: readU64le(rest, 0) }; + if (kind === TYPE_HELLO) return { type: "hello" }; + throw new Error(`unknown type ${kind}`); + } + + function hashFromBase64(b64) { + const bin = atob(b64); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i += 1) out[i] = bin.charCodeAt(i); + return out; + } + + function pickWtHost(cfg, locationHost) { + const h = locationHost || ""; + if (h && h !== "localhost" && h !== "127.0.0.1" && h !== "[::1]") { + return h; + } + const hosts = cfg && Array.isArray(cfg.wt_hosts) ? cfg.wt_hosts : []; + const lan = hosts.find((x) => x && x !== "127.0.0.1" && x !== "::1"); + return lan || hosts[0] || h || "127.0.0.1"; + } + + function parseVideoDatagram(buf) { + if (!buf || buf.length < DATAGRAM_HEADER || buf[0] !== TYPE_VIDEO) return null; + return { + type: "video", + key: buf[1] !== 0, + seq: readU16le(buf, 2), + frag: readU16le(buf, 4), + frags: readU16le(buf, 6), + ptsNs: readU64le(buf, 8), + sentNs: readU64le(buf, 16), + payload: buf.subarray(24), + }; + } + + function encodeVideoDatagram(seq, frag, frags, key, ptsNs, sentNs, payload) { + const out = new Uint8Array(DATAGRAM_HEADER + payload.length); + out[0] = TYPE_VIDEO; + out[1] = key ? 1 : 0; + out.set(u16le(seq), 2); + out.set(u16le(frag), 4); + out.set(u16le(frags), 6); + out.set(u64le(ptsNs), 8); + out.set(u64le(sentNs), 16); + out.set(payload, 24); + return out; + } + + function seqDelta(cur, prev) { + return (cur - prev) & 0xffff; + } + + const HOLE_WAIT_MS = 48; + const MAX_HELD = 4; + + function parityCount(k) { + if (k <= 3) return 0; + if (k <= 16) return 2; + if (k <= 64) return 3; + return 4; + } + + const GF = (() => { + const exp = new Uint8Array(512); + const log = new Uint8Array(256); + let x = 1; + for (let i = 0; i < 255; i += 1) { + exp[i] = x; + log[x] = i; + x <<= 1; + if (x & 0x100) x ^= 0x11d; + } + for (let i = 255; i < 512; i += 1) exp[i] = exp[i - 255]; + return { exp, log }; + })(); + + function gfMul(a, b) { + if (a === 0 || b === 0) return 0; + return GF.exp[GF.log[a] + GF.log[b]]; + } + function gfInv(a) { + return GF.exp[255 - GF.log[a]]; + } + function cauchy(p, d, m) { + return gfInv(p ^ (m + d)); + } + + function invertMatrix(a) { + const n = a.length; + const m = a.map((row, i) => { + const r = new Uint8Array(n * 2); + r.set(row, 0); + r[n + i] = 1; + return r; + }); + for (let col = 0; col < n; col += 1) { + let piv = col; + while (piv < n && m[piv][col] === 0) piv += 1; + if (piv === n) return null; + const tmp = m[col]; + m[col] = m[piv]; + m[piv] = tmp; + const inv = gfInv(m[col][col]); + for (let j = 0; j < n * 2; j += 1) m[col][j] = gfMul(m[col][j], inv); + for (let row = 0; row < n; row += 1) { + if (row === col) continue; + const f = m[row][col]; + if (f === 0) continue; + for (let j = 0; j < n * 2; j += 1) { + m[row][j] ^= gfMul(f, m[col][j]); + } + } + } + return m.map((row) => row.subarray(n)); + } + + function recoverFec(data, parity) { + const k = data.length; + if (data.every((d) => d)) return true; + const m = parity.length; + if (!m) return false; + const missing = []; + for (let i = 0; i < k; i += 1) if (!data[i]) missing.push(i); + const presentP = []; + let width = 0; + for (let i = 0; i < k; i += 1) if (data[i] && data[i].length > width) width = data[i].length; + for (let i = 0; i < m; i += 1) { + if (parity[i]) { + presentP.push(i); + if (parity[i].length > width) width = parity[i].length; + } + } + if (!width || missing.length > presentP.length) return false; + const usedP = presentP.slice(0, missing.length); + const missN = missing.length; + const known = data.map((d) => { + if (!d) return null; + const p = new Uint8Array(width); + p.set(d); + return p; + }); + const rhs = usedP.map((p) => { + const rec = new Uint8Array(width); + rec.set(parity[p]); + const row = new Uint8Array(width); + for (let b = 0; b < width; b += 1) { + let s = rec[b]; + for (let d = 0; d < k; d += 1) { + if (known[d]) s ^= gfMul(cauchy(p, d, m), known[d][b]); + } + row[b] = s; + } + return row; + }); + const mat = usedP.map((p) => { + const row = new Uint8Array(missN); + for (let c = 0; c < missN; c += 1) row[c] = cauchy(p, missing[c], m); + return row; + }); + const inv = invertMatrix(mat); + if (!inv) return false; + for (let c = 0; c < missN; c += 1) { + const out = new Uint8Array(width); + for (let b = 0; b < width; b += 1) { + let s = 0; + for (let r = 0; r < missN; r += 1) s ^= gfMul(inv[c][r], rhs[r][b]); + out[b] = s; + } + data[missing[c]] = out; + } + return true; + } + + function encodeFec(data, m) { + const k = data.length; + const width = data[0].length; + const parity = Array.from({ length: m }, () => new Uint8Array(width)); + for (let p = 0; p < m; p += 1) { + for (let b = 0; b < width; b += 1) { + let s = 0; + for (let d = 0; d < k; d += 1) s ^= gfMul(cauchy(p, d, m), data[d][b]); + parity[p][b] = s; + } + } + return parity; + } + + function concatFecAu(parts) { + let n = 0; + for (const p of parts) n += p.length; + const blob = new Uint8Array(n); + let o = 0; + for (const p of parts) { + blob.set(p, o); + o += p.length; + } + if (blob.length < 4) return null; + const len = (blob[0] | (blob[1] << 8) | (blob[2] << 16) | (blob[3] << 24)) >>> 0; + if (len > blob.length - 4) return null; + return blob.subarray(4, 4 + len); + } + + function videoFromSlots(seq, slots) { + return { + type: "video", + seq, + key: slots.key, + ptsNs: slots.ptsNs, + sentNs: slots.sentNs, + au: concatBytes(slots.parts), + }; + } + + class DatagramAssembler { + constructor() { + this.pending = new Map(); + this.held = new Map(); + this.lastSeq = -1; + this.holeSince = 0; + } + push(buf, now) { + const t = now == null ? Date.now() : now; + const frag = parseVideoDatagram(buf); + const m = frag ? parityCount(frag.frags) : 0; + if (!frag || frag.frags <= 0 || frag.frag >= frag.frags + m) { + return this.expire(t); + } + if (this.lastSeq >= 0) { + const age = seqDelta(frag.seq, this.lastSeq); + if (age === 0 || age > 32768) return this.expire(t); + } + if (this.held.has(frag.seq)) return this.expire(t); + if (this.pending.size >= 4) { + for (const [seq] of this.pending) { + if (seqDelta(frag.seq, seq) > 1 && seqDelta(frag.seq, seq) < 32768) { + this.pending.delete(seq); + } + } + } + let slots = this.pending.get(frag.seq); + const isParity = frag.frag >= frag.frags; + if (!slots || slots.frags !== frag.frags) { + if (isParity) return this.expire(t); + slots = { + frags: frag.frags, + key: frag.key, + ptsNs: frag.ptsNs, + sentNs: frag.sentNs, + parts: Array.from({ length: frag.frags }, () => null), + parity: Array.from({ length: m }, () => null), + }; + this.pending.set(frag.seq, slots); + } + if (isParity) slots.parity[frag.frag - frag.frags] = frag.payload; + else slots.parts[frag.frag] = frag.payload; + const haveData = slots.parts.every((p) => p != null); + const have = slots.parts.filter((p) => p != null).length + + slots.parity.filter((p) => p != null).length; + if (!haveData && have < frag.frags) return this.expire(t); + if (!haveData) { + const data = slots.parts.slice(); + if (!recoverFec(data, slots.parity)) return this.expire(t); + slots.parts = data; + } + this.pending.delete(frag.seq); + const au = m > 0 ? concatFecAu(slots.parts) : concatBytes(slots.parts); + if (!au) return this.expire(t); + slots.parts = [au]; + const expired = this.expire(t); + return expired.concat(this.accept({ + type: "video", + seq: frag.seq, + key: slots.key, + ptsNs: slots.ptsNs, + sentNs: slots.sentNs, + au, + }, t)); + } + accept(frame, now) { + const out = []; + if (this.lastSeq >= 0) { + const age = seqDelta(frame.seq, this.lastSeq); + if (age === 0 || age > 32768) return out; + } + if (this.lastSeq < 0) { + this.lastSeq = frame.seq; + out.push(frame); + return out; + } + const delta = seqDelta(frame.seq, this.lastSeq); + if (delta === 1) { + this.lastSeq = frame.seq; + out.push(frame); + this.drainHeld(out); + return out; + } + if (frame.key) { + this.held.clear(); + this.holeSince = 0; + this.lastSeq = frame.seq; + out.push(frame); + return out; + } + this.held.set(frame.seq, frame); + if (!this.holeSince) this.holeSince = now; + this.trimHeld(); + return out; + } + drainHeld(out) { + for (;;) { + const next = (this.lastSeq + 1) & 0xffff; + const frame = this.held.get(next); + if (!frame) break; + this.held.delete(next); + this.lastSeq = next; + out.push(frame); + } + if (this.held.size === 0) this.holeSince = 0; + } + trimHeld() { + if (this.held.size <= MAX_HELD) return; + const seqs = Array.from(this.held.keys()).sort( + (a, b) => seqDelta(a, this.lastSeq) - seqDelta(b, this.lastSeq), + ); + for (const seq of seqs.slice(MAX_HELD)) this.held.delete(seq); + } + expire(now) { + const t = now == null ? Date.now() : now; + if (this.held.size === 0) return []; + if (t - this.holeSince < HOLE_WAIT_MS) return []; + const dropped = Math.max(1, this.held.size); + this.held.clear(); + this.holeSince = 0; + return [{ type: "gap", dropped }]; + } + // Reliable IDR has no datagram seq. Drop held P-frames from the + // old GOP and accept the next P as the start of the new one. + onReliableKeyframe() { + this.pending.clear(); + this.held.clear(); + this.holeSince = 0; + this.lastSeq = -1; + } + } + + class FrameReader { + constructor() { + this.buf = new Uint8Array(0); + } + push(chunk) { + const next = new Uint8Array(this.buf.length + chunk.length); + next.set(this.buf, 0); + next.set(chunk, this.buf.length); + this.buf = next; + } + pop() { + const split = splitFrame(this.buf); + if (!split) return null; + this.buf = this.buf.subarray(split.used); + return decodeMessage(split.body); + } + } + + return { + TYPE_VIDEO, TYPE_HELLO, TYPE_HELLO_ACK, TYPE_PING, TYPE_PONG, TYPE_IDR, TYPE_BYE, + MAX_FRAME, + startCodeLen, extractSpsPps, codecStringFromSps, spsDimensions, withStartCode, + unescapeRbsp, classifyAccessUnit, sliceKind, + encodeFrame, splitFrame, encodeHello, encodeCtrl, encodePing, decodeMessage, + hashFromBase64, pickWtHost, FrameReader, + parseVideoDatagram, encodeVideoDatagram, DatagramAssembler, seqDelta, DATAGRAM_HEADER, + HOLE_WAIT_MS, parityCount, recoverFec, concatFecAu, encodeFec, + }; +})); diff --git a/clients/web/annexb.test.js b/clients/web/annexb.test.js new file mode 100644 index 00000000..ad67eeca --- /dev/null +++ b/clients/web/annexb.test.js @@ -0,0 +1,250 @@ +// Orbiscreen - annexb.test.js (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const annexb = require("./annexb.js"); + +test("spsDimensions reads High@4.0 1280x800", () => { + const sps = Buffer.from( + "0000000127640028ac11128050065bff0001000110000003001000000789da08042e", + "hex", + ); + assert.deepEqual(annexb.spsDimensions(sps), { width: 1280, height: 800 }); +}); + +test("codec string from High@4.0 SPS", () => { + const sps = annexb.withStartCode(Uint8Array.of(0x67, 0x64, 0x00, 0x28, 0xac)); + assert.equal(annexb.codecStringFromSps(sps), "avc1.640028"); +}); + +test("extracts SPS/PPS from an IDR AU", () => { + const sps = annexb.withStartCode(Uint8Array.of(0x67, 0x64, 0x00, 0x28, 0xac)); + const pps = annexb.withStartCode(Uint8Array.of(0x68, 0xee, 0x3c, 0x80)); + const slice = annexb.withStartCode(Uint8Array.of(0x65, 0x88)); + const au = new Uint8Array(sps.length + pps.length + slice.length); + au.set(sps, 0); + au.set(pps, sps.length); + au.set(slice, sps.length + pps.length); + const found = annexb.extractSpsPps(au); + assert.deepEqual(Array.from(found.sps), Array.from(sps)); + assert.deepEqual(Array.from(found.pps), Array.from(pps)); +}); + +test("hello frame encodes a complete length-prefixed body", () => { + const frame = annexb.encodeHello("tok", "sess-1"); + const split = annexb.splitFrame(frame); + assert.ok(split); + assert.equal(split.used, frame.length); + assert.equal(split.body[0], annexb.TYPE_HELLO); +}); + +test("frame reader reassembles split chunks", () => { + const frame = annexb.encodeCtrl(annexb.TYPE_IDR); + const reader = new annexb.FrameReader(); + reader.push(frame.subarray(0, 3)); + assert.equal(reader.pop(), null); + reader.push(frame.subarray(3)); + const msg = reader.pop(); + assert.equal(msg.type, "idr"); + assert.equal(reader.pop(), null); +}); + +test("video message decode", () => { + const au = Uint8Array.of(0, 0, 0, 1, 0x65, 9); + const body = new Uint8Array(1 + 1 + 8 + 8 + au.length); + body[0] = annexb.TYPE_VIDEO; + body[1] = 1; + body[2] = 42; + body.set(au, 18); + const frame = annexb.encodeFrame(body); + const msg = annexb.decodeMessage(annexb.splitFrame(frame).body); + assert.equal(msg.type, "video"); + assert.equal(msg.key, true); + assert.equal(msg.ptsNs, 42n); + assert.deepEqual(Array.from(msg.au), Array.from(au)); +}); + +test("classifies IDR as I and P-slice as P", () => { + const idr = annexb.withStartCode(Uint8Array.of(0x65, 0x88)); + assert.equal(annexb.classifyAccessUnit(idr), "I"); +}); + +test("pickWtHost prefers the page host when it is not loopback", () => { + assert.equal(annexb.pickWtHost({ wt_hosts: ["10.0.0.5"] }, "192.168.1.8"), "192.168.1.8"); + assert.equal(annexb.pickWtHost({ wt_hosts: ["10.0.0.5", "127.0.0.1"] }, "127.0.0.1"), "10.0.0.5"); + assert.equal(annexb.pickWtHost({ wt_hosts: ["127.0.0.1"] }, "localhost"), "127.0.0.1"); +}); + +test("datagram assembler rebuilds a split AU and reports a gap after the hole wait", () => { + const au = Uint8Array.from({ length: 40 }, (_, i) => i); + const a = annexb.encodeVideoDatagram(4, 0, 2, true, 1n, 2n, au.subarray(0, 20)); + const b = annexb.encodeVideoDatagram(4, 1, 2, true, 1n, 2n, au.subarray(20)); + const asm = new annexb.DatagramAssembler(); + assert.deepEqual(asm.push(a, 0), []); + const msg = asm.push(b, 0); + assert.equal(msg.length, 1); + assert.equal(msg[0].type, "video"); + assert.equal(msg[0].key, true); + assert.deepEqual(Array.from(msg[0].au), Array.from(au)); + + const next = annexb.encodeVideoDatagram(6, 0, 1, false, 3n, 4n, au.subarray(0, 8)); + assert.deepEqual(asm.push(next, 0), []); + assert.deepEqual(asm.expire(annexb.HOLE_WAIT_MS - 1), []); + const gap = asm.expire(annexb.HOLE_WAIT_MS); + assert.equal(gap.length, 1); + assert.equal(gap[0].type, "gap"); + assert.equal(gap[0].dropped, 1); + assert.deepEqual(asm.expire(annexb.HOLE_WAIT_MS * 2), []); +}); + +test("datagram assembler ignores a late AU and does not rewind lastSeq", () => { + const payload = Uint8Array.of(1, 2, 3, 4); + const asm = new annexb.DatagramAssembler(); + const first = asm.push(annexb.encodeVideoDatagram(10, 0, 1, true, 1n, 2n, payload), 0); + assert.equal(first[0].type, "video"); + assert.equal(first[0].key, true); + + const late = annexb.encodeVideoDatagram(8, 0, 1, false, 3n, 4n, payload); + assert.deepEqual(asm.push(late, 0), []); + assert.deepEqual(asm.push(annexb.encodeVideoDatagram(10, 0, 1, true, 1n, 2n, payload), 0), []); + + const inOrder = asm.push(annexb.encodeVideoDatagram(11, 0, 1, false, 5n, 6n, payload), 0); + assert.equal(inOrder[0].type, "video"); + assert.equal(inOrder[0].key, false); +}); + +test("datagram assembler treats wrap from 65535 to 0 as in-order", () => { + const payload = Uint8Array.of(9); + const asm = new annexb.DatagramAssembler(); + assert.equal(asm.push(annexb.encodeVideoDatagram(65535, 0, 1, true, 1n, 2n, payload), 0)[0].type, "video"); + const wrapped = asm.push(annexb.encodeVideoDatagram(0, 0, 1, false, 3n, 4n, payload), 0); + assert.equal(wrapped[0].type, "video"); + assert.equal(wrapped[0].key, false); +}); + +test("datagram assembler plays 11 then 12 when 12 arrives first", () => { + const payload = Uint8Array.of(1, 2, 3, 4); + const asm = new annexb.DatagramAssembler(); + assert.equal(asm.push(annexb.encodeVideoDatagram(10, 0, 1, true, 1n, 2n, payload), 0)[0].seq, 10); + assert.deepEqual(asm.push(annexb.encodeVideoDatagram(12, 0, 1, false, 3n, 4n, payload), 0), []); + const filled = asm.push(annexb.encodeVideoDatagram(11, 0, 1, false, 5n, 6n, payload), 10); + assert.deepEqual(filled.map((m) => m.seq), [11, 12]); + assert.equal(filled.every((m) => m.type === "video"), true); + assert.deepEqual(asm.expire(10 + annexb.HOLE_WAIT_MS), []); +}); + +test("datagram assembler emits gap if the hole never fills", () => { + const payload = Uint8Array.of(7); + const asm = new annexb.DatagramAssembler(); + asm.push(annexb.encodeVideoDatagram(10, 0, 1, true, 1n, 2n, payload), 0); + assert.deepEqual(asm.push(annexb.encodeVideoDatagram(12, 0, 1, false, 3n, 4n, payload), 0), []); + const gap = asm.expire(annexb.HOLE_WAIT_MS); + assert.equal(gap.length, 1); + assert.equal(gap[0].type, "gap"); + const late = asm.push(annexb.encodeVideoDatagram(11, 0, 1, false, 5n, 6n, payload), annexb.HOLE_WAIT_MS + 1); + assert.equal(late[0].type, "video"); + assert.equal(late[0].seq, 11); +}); + +test("after a reliable IDR, the next P-frame plays even if lastSeq had a hole", () => { + const payload = Uint8Array.of(1); + const asm = new annexb.DatagramAssembler(); + asm.push(annexb.encodeVideoDatagram(10, 0, 1, true, 1n, 2n, payload), 0); + assert.deepEqual(asm.push(annexb.encodeVideoDatagram(12, 0, 1, false, 3n, 4n, payload), 0), []); + const gap = asm.expire(annexb.HOLE_WAIT_MS); + assert.equal(gap[0].type, "gap"); + + asm.onReliableKeyframe(); + const next = asm.push(annexb.encodeVideoDatagram(40, 0, 1, false, 5n, 6n, payload), annexb.HOLE_WAIT_MS + 10); + assert.equal(next.length, 1); + assert.equal(next[0].type, "video"); + assert.equal(next[0].seq, 40); + assert.equal(next[0].key, false); + assert.deepEqual(asm.expire(annexb.HOLE_WAIT_MS * 2), []); +}); + +test("hashFromBase64 yields 32 bytes", () => { + const raw = Uint8Array.from({ length: 32 }, (_, i) => i); + const b64 = Buffer.from(raw).toString("base64"); + assert.deepEqual(Array.from(annexb.hashFromBase64(b64)), Array.from(raw)); +}); + +test("parity ladder matches the host", () => { + assert.equal(annexb.parityCount(1), 0); + assert.equal(annexb.parityCount(3), 0); + assert.equal(annexb.parityCount(4), 2); + assert.equal(annexb.parityCount(16), 2); + assert.equal(annexb.parityCount(17), 3); + assert.equal(annexb.parityCount(64), 3); + assert.equal(annexb.parityCount(65), 4); +}); + +test("JS Reed-Solomon recovers two missing data shards", () => { + const k = 4; + const width = 16; + const data = Array.from({ length: k }, (_, i) => + Uint8Array.from({ length: width }, (_, b) => (i * 31 + b) & 0xff), + ); + const parity = annexb.encodeFec(data, 2); + const slots = data.map((d) => new Uint8Array(d)); + slots[1] = null; + slots[3] = null; + assert.equal(annexb.recoverFec(slots, parity), true); + for (let i = 0; i < k; i += 1) { + assert.deepEqual(Array.from(slots[i]), Array.from(data[i])); + } +}); + +test("assembler rebuilds an AU after two data datagrams are lost", () => { + const k = 4; + const width = 8; + const auLen = k * width - 4; + const prefix = new Uint8Array(4); + prefix[0] = auLen & 0xff; + prefix[1] = (auLen >> 8) & 0xff; + const body = Uint8Array.from({ length: auLen }, (_, i) => i & 0xff); + const blob = new Uint8Array(4 + auLen); + blob.set(prefix, 0); + blob.set(body, 4); + const parts = []; + for (let i = 0; i < k; i += 1) parts.push(blob.subarray(i * width, (i + 1) * width)); + const parity = annexb.encodeFec(parts, 2); + const asm = new annexb.DatagramAssembler(); + const pkts = []; + for (let i = 0; i < k; i += 1) { + if (i === 0 || i === 2) continue; + pkts.push(annexb.encodeVideoDatagram(3, i, k, false, 1n, 2n, parts[i])); + } + pkts.push(annexb.encodeVideoDatagram(3, k, k, false, 1n, 2n, parity[0])); + pkts.push(annexb.encodeVideoDatagram(3, k + 1, k, false, 1n, 2n, parity[1])); + let out = []; + for (const p of pkts) out = out.concat(asm.push(p, 0)); + assert.equal(out.length, 1); + assert.equal(out[0].type, "video"); + assert.deepEqual(Array.from(out[0].au), Array.from(body)); +}); + +test("parity first does not open a pending AU; all data still emits", () => { + const k = 4; + const width = 8; + const auLen = k * width - 4; + const prefix = new Uint8Array(4); + prefix[0] = auLen & 0xff; + const body = Uint8Array.from({ length: auLen }, (_, i) => (i + 3) & 0xff); + const blob = new Uint8Array(4 + auLen); + blob.set(prefix, 0); + blob.set(body, 4); + const parts = []; + for (let i = 0; i < k; i += 1) parts.push(blob.subarray(i * width, (i + 1) * width)); + const parity = annexb.encodeFec(parts, 2); + const asm = new annexb.DatagramAssembler(); + assert.deepEqual(asm.push(annexb.encodeVideoDatagram(4, k, k, false, 1n, 2n, parity[0]), 0), []); + let out = []; + for (let i = 0; i < k; i += 1) { + out = out.concat(asm.push(annexb.encodeVideoDatagram(4, i, k, false, 1n, 2n, parts[i]), 0)); + } + assert.equal(out.length, 1); + assert.deepEqual(Array.from(out[0].au), Array.from(body)); + assert.deepEqual(asm.push(annexb.encodeVideoDatagram(4, k + 1, k, false, 1n, 2n, parity[1]), 1), []); +}); diff --git a/clients/web/app.js b/clients/web/app.js index c2df14ef..b246df06 100644 --- a/clients/web/app.js +++ b/clients/web/app.js @@ -10,7 +10,7 @@ const I18N = { btnHideControls: "Hide Toolbar", btnFullscreen: "Fullscreen", btnDisconnect: "Disconnect", - btnRestoreToolbar: "Show Toolbar", + btnShowControls: "Menu", btnCloseKeyboard: "Close", settingsTitle: "Settings", themeTitle: "Theme", @@ -48,7 +48,9 @@ const I18N = { toastLockSent: "Lock sent", toastCadSent: "Ctrl+Alt+Del sent", toastResynced: "Resynced", - toastDisconnected: "Disconnected" + toastDisconnected: "Disconnected", + statusUnsupported: "Unsupported browser", + statusUnsupportedSub: "This client needs the WebCodecs VideoDecoder API. Open the page in Chrome, Brave, Edge, or another Chromium browser." }, ar: { btnInputMode: "وضع الإدخال", @@ -95,7 +97,9 @@ const I18N = { toastLockSent: "تم إرسال أمر القفل", toastCadSent: "تم إرسال Ctrl+Alt+Del", toastResynced: "تمت المزامنة", - toastDisconnected: "تم قطع الاتصال" + toastDisconnected: "تم قطع الاتصال", + statusUnsupported: "متصفح غير مدعوم", + statusUnsupportedSub: "يحتاج هذا العميل إلى WebCodecs VideoDecoder. افتح الصفحة في Chrome أو Brave أو Edge أو متصفح Chromium آخر." } }; @@ -183,10 +187,9 @@ const stageEl = document.getElementById("stage"); const videoEl = document.getElementById("remoteVideo"); const touchIndicator = document.getElementById("touchIndicator"); const controlToolbar = document.getElementById("controlToolbar"); -const miniPill = document.getElementById("miniPill"); +const btnShowControls = document.getElementById("btnShowControls"); const hostNameEl = document.getElementById("hostName"); const hostInfoEl = document.getElementById("hostInfo"); -const miniInfoEl = document.getElementById("miniInfo"); function setOverlayState(state, title, subtitle) { if (overlayEl) overlayEl.classList.remove("hidden"); @@ -194,8 +197,7 @@ function setOverlayState(state, title, subtitle) { if (statusSubtitle && subtitle) statusSubtitle.textContent = subtitle; releaseControl(); - if (controlToolbar) controlToolbar.classList.add("hidden"); - if (miniPill) miniPill.classList.add("hidden"); + hideControlsChrome(); if (keyboardDrawer) keyboardDrawer.classList.add("hidden"); if (settingsModal) settingsModal.classList.add("hidden"); @@ -203,6 +205,10 @@ function setOverlayState(state, title, subtitle) { if (brandLogo) brandLogo.classList.add("hidden"); if (statusSpinner) statusSpinner.classList.remove("hidden"); if (btnReconnect) btnReconnect.classList.add("hidden"); + } else if (state === "unsupported") { + if (statusSpinner) statusSpinner.classList.add("hidden"); + if (brandLogo) brandLogo.classList.remove("hidden"); + if (btnReconnect) btnReconnect.classList.add("hidden"); } else { if (statusSpinner) statusSpinner.classList.add("hidden"); if (brandLogo) brandLogo.classList.remove("hidden"); @@ -248,16 +254,25 @@ let displayWidth = 1920; let displayHeight = 1080; let encoderName = "NVENC"; let authToken = ""; -let mpegtsPlayer = null; +let wtConfig = null; +let wtTransport = null; +let wtWriter = null; +let videoDecoder = null; let reconnectTimer = null; let reconnectDelay = 1000; const MAX_RECONNECT_DELAY = 10000; +let clockOffsetNs = 0n; +let lastFrameAt = 0; let streamActive = false; let isVncFocused = false; let isTouchMode = true; let toastTimer = null; let vncBannerTimer = null; let latencyWatchdog = null; +let lastDelayMs = null; +let controlsVisible = false; +let controlsLockedHidden = false; +let controlsHideTimer = null; const heldKeys = new Set(); const pressedButtons = new Set(); const activeTouches = new Map(); @@ -284,12 +299,56 @@ function showToast(text, duration = 2200) { }, duration); } +window.addEventListener("resize", () => updateInfoDisplay()); + function updateInfoDisplay() { - const infoStr = `${displayWidth}×${displayHeight} ${encoderName}`; + const delayText = (lastDelayMs != null && lastDelayMs >= 0) + ? `delay ${lastDelayMs}ms` + : "delay —"; + const narrow = window.matchMedia("(orientation: portrait), (max-width: 720px)").matches; + const infoStr = narrow + ? delayText + : `${displayWidth}×${displayHeight} ${encoderName} ${delayText}`; if (hostInfoEl) hostInfoEl.textContent = infoStr; - if (miniInfoEl) miniInfoEl.textContent = `${displayWidth}×${displayHeight}`; if (statRes) statRes.textContent = `${displayWidth} × ${displayHeight}`; if (statEncoder) statEncoder.textContent = encoderName; + if (statLatency) { + statLatency.textContent = (lastDelayMs != null && lastDelayMs >= 0) + ? `${lastDelayMs} ms` + : "-- ms"; + } +} + +function hideControlsChrome() { + clearTimeout(controlsHideTimer); + controlsHideTimer = null; + controlsVisible = false; + if (controlToolbar) controlToolbar.classList.add("hidden"); + if (btnShowControls) btnShowControls.classList.add("hidden"); +} + +function setControlsVisible(visible) { + clearTimeout(controlsHideTimer); + controlsHideTimer = null; + if (!streamActive) { + hideControlsChrome(); + return; + } + if (controlsLockedHidden) { + if (controlToolbar) controlToolbar.classList.add("hidden"); + if (btnShowControls) btnShowControls.classList.add("hidden"); + controlsVisible = false; + return; + } + controlsVisible = visible; + if (visible) { + if (controlToolbar) controlToolbar.classList.remove("hidden"); + if (btnShowControls) btnShowControls.classList.add("hidden"); + controlsHideTimer = setTimeout(() => setControlsVisible(false), 12_000); + } else { + if (controlToolbar) controlToolbar.classList.add("hidden"); + if (btnShowControls) btnShowControls.classList.remove("hidden"); + } } function setVncFocus(focused) { @@ -594,7 +653,7 @@ document.querySelectorAll(".chipBtn[data-fit]").forEach((btn) => { btn.addEventListener("click", () => { document.querySelectorAll(".chipBtn[data-fit]").forEach((b) => b.classList.remove("active")); btn.classList.add("active"); - videoEl.style.objectFit = btn.dataset.fit; + if (videoEl) videoEl.style.objectFit = btn.dataset.fit; showToast(`Fit: ${btn.textContent}`); }); }); @@ -630,20 +689,26 @@ document.querySelectorAll("#webLangChips .chipBtn").forEach((btn) => { if (btnHideControls) { btnHideControls.addEventListener("click", (e) => { e.stopPropagation(); - controlToolbar.classList.add("hidden"); - miniPill.classList.remove("hidden"); + controlsLockedHidden = true; + setControlsVisible(false); showToast(t("controlsHidden"), 1800); }); } -if (btnRestoreToolbar) { - btnRestoreToolbar.addEventListener("click", (e) => { +if (btnShowControls) { + btnShowControls.addEventListener("click", (e) => { e.stopPropagation(); - miniPill.classList.add("hidden"); - controlToolbar.classList.remove("hidden"); + setControlsVisible(true); }); } +stageEl.addEventListener("dblclick", (e) => { + if (!streamActive || !controlsLockedHidden) return; + e.preventDefault(); + controlsLockedHidden = false; + setControlsVisible(true); +}); + if (btnFullscreen) { btnFullscreen.addEventListener("click", (e) => { e.stopPropagation(); @@ -671,6 +736,26 @@ async function sendHostAction(action, extra = {}) { } } +const IDR_DEBOUNCE_MS = 250; +let lastIdrAt = 0; +let waitingForKeyframe = false; + +function noteKeyframe() { + waitingForKeyframe = false; +} + +function requestIdr(opts = {}) { + const now = Date.now(); + if (now - lastIdrAt < IDR_DEBOUNCE_MS) return false; + lastIdrAt = now; + if (!opts.keepDecoding) waitingForKeyframe = true; + if (wtWriter) { + try { wtWriter.write(OrbiAnnexB.encodeCtrl(OrbiAnnexB.TYPE_IDR)); } catch (_) {} + } + sendHostAction("idr", displaySessionId ? { session: displaySessionId } : {}); + return true; +} + if (btnLock) { btnLock.addEventListener("click", async (e) => { e.stopPropagation(); @@ -692,6 +777,7 @@ if (btnActionResync) { btnActionResync.addEventListener("click", (e) => { e.stopPropagation(); settingsModal.classList.add("hidden"); + requestIdr(); startStream(); showToast(t("toastResynced")); }); @@ -767,7 +853,7 @@ function sendWheel(deltaY) { } function normalizeWheel(event) { - const vh = videoEl.videoHeight || displayHeight; + const vh = (videoEl && (videoEl.videoHeight || videoEl.height)) || displayHeight; let pixels = event.deltaY; if (event.deltaMode === 1) pixels *= 16; else if (event.deltaMode === 2) pixels *= vh; @@ -998,9 +1084,10 @@ if (keyboardImeInput) { } function mapPointer(event) { - const rect = videoEl.getBoundingClientRect(); - const vw = videoEl.videoWidth || displayWidth; - const vh = videoEl.videoHeight || displayHeight; + const target = videoEl; + const rect = target.getBoundingClientRect(); + const vw = target.videoWidth || target.width || displayWidth; + const vh = target.videoHeight || target.height || displayHeight; const scale = Math.min(rect.width / vw, rect.height / vh); const offsetX = (rect.width - vw * scale) / 2; const offsetY = (rect.height - vh * scale) / 2; @@ -1022,9 +1109,28 @@ function hideTouch() { if (touchIndicator) touchIndicator.classList.add("hidden"); } -function canPlayMpegTs() { - return typeof mpegts !== "undefined" - && mpegts.getFeatureList().mseLivePlayback; +function playbackSupport() { + return { + webTransport: typeof WebTransport === "function", + videoDecoder: typeof VideoDecoder === "function", + annexb: typeof OrbiAnnexB !== "undefined", + }; +} + +function canPlayWebTransport() { + const s = playbackSupport(); + return s.webTransport && s.annexb && s.videoDecoder; +} + +function unsupportedPlayback() { + setOverlayState("unsupported", t("statusUnsupported"), t("statusUnsupportedSub")); +} + +function closeDecoder() { + if (videoDecoder) { + try { videoDecoder.close(); } catch (_) {} + videoDecoder = null; + } } function destroyPlayer() { @@ -1032,15 +1138,15 @@ function destroyPlayer() { clearTimeout(reconnectTimer); reconnectTimer = null; } - if (mpegtsPlayer) { - mpegtsPlayer.destroy(); - mpegtsPlayer = null; + if (wtWriter) { + try { wtWriter.close(); } catch (_) {} + wtWriter = null; } - if (videoEl.src) { - videoEl.pause(); - videoEl.removeAttribute("src"); - videoEl.load(); + if (wtTransport) { + try { wtTransport.close(); } catch (_) {} + wtTransport = null; } + closeDecoder(); streamActive = false; clearInterval(latencyWatchdog); releaseControl(); @@ -1072,95 +1178,343 @@ function scheduleReconnect(reason) { reconnectTimer = null; (async () => { await refreshToken(); - await openDisplaySession(); + const sessionGone = !displaySessionId + || /close|attach|session|no display/i.test(String(reason || "")); + if (sessionGone) { + displaySessionId = ""; + await openDisplaySession(); + } startStream(); })().catch((error) => { console.warn("reconnect attempt failed:", error); + displaySessionId = ""; scheduleReconnect("retry error"); }); }, reconnectDelay); reconnectDelay = Math.min(reconnectDelay * 2, MAX_RECONNECT_DELAY); } -function startStream() { - if (!canPlayMpegTs()) { - setOverlayState("error", "Playback Error", "Browser lacks MSE live video playback support"); +function markPlaying() { + if (!streamActive) { + streamActive = true; + reconnectDelay = 1000; + controlsLockedHidden = false; + if (overlayEl) overlayEl.classList.add("hidden"); + setControlsVisible(false); + } + noteKeyframe(); + lastFrameAt = Date.now(); +} + +function drawVideoFrame(frame) { + try { + if (videoEl.width !== frame.displayWidth || videoEl.height !== frame.displayHeight) { + videoEl.width = frame.displayWidth; + videoEl.height = frame.displayHeight; + displayWidth = frame.displayWidth; + displayHeight = frame.displayHeight; + updateInfoDisplay(); + } + const ctx = videoEl.getContext("2d"); + ctx.drawImage(frame, 0, 0); + markPlaying(); + } finally { + frame.close(); + } +} + +function ensureDecoder(codec) { + if (videoDecoder && videoDecoder.state !== "closed") { + return videoDecoder; + } + closeDecoder(); + videoDecoder = new VideoDecoder({ + output: drawVideoFrame, + error: (err) => { + console.error("VideoDecoder error:", err); + requestIdr(); + scheduleReconnect("decoder"); + }, + }); + videoDecoder.configure({ + codec, + optimizeForLatency: true, + hardwareAcceleration: "prefer-hardware", + }); + return videoDecoder; +} + +function nowNs() { + return BigInt(Date.now()) * 1000000n; +} + +function feedAccessUnit(msg) { + if (typeof VideoDecoder !== "function") { + unsupportedPlayback(); return; } - destroyPlayer(); - setOverlayState("connecting", "Connecting", "Connecting to Linux virtual display…"); + if (msg.key) { + const found = OrbiAnnexB.extractSpsPps(msg.au); + const codec = OrbiAnnexB.codecStringFromSps(found.sps); + if (codec) { + try { + ensureDecoder(codec); + } catch (err) { + console.error("VideoDecoder configure failed:", err); + scheduleReconnect("codec"); + return; + } + } + noteKeyframe(); + } + if (!videoDecoder || videoDecoder.state !== "configured") { + requestIdr(); + return; + } + if (waitingForKeyframe && !msg.key) { + requestIdr(); + return; + } + const chunk = new EncodedVideoChunk({ + type: msg.key ? "key" : "delta", + timestamp: Number(msg.ptsNs / 1000n), + data: msg.au, + }); + try { + videoDecoder.decode(chunk); + } catch (err) { + console.warn("decode failed:", err); + closeDecoder(); + requestIdr(); + } + if (msg.sentNs > 0n) { + const glass = Number((nowNs() + clockOffsetNs - msg.sentNs) / 1000000n); + if (glass >= 0 && glass <= 5000) { + lastDelayMs = glass; + updateInfoDisplay(); + } + } +} +async function startAuStream() { const params = new URLSearchParams(); if (authToken) params.set("token", authToken); if (displaySessionId) params.set("session", displaySessionId); const qs = params.toString(); - const streamUrl = qs ? `/stream?${qs}` : "/stream"; - - mpegtsPlayer = mpegts.createPlayer({ - type: "mpegts", - isLive: true, - url: streamUrl, - }, { - enableStashBuffer: false, - stashInitialSize: 128, - autoCleanupSourceBuffer: true, - autoCleanupMaxBackwardDuration: 2, - autoCleanupMinBackwardDuration: 1, - lazyLoad: false, - lazyLoadMaxDuration: 0, - seekType: "range", - liveBufferLatencyChasing: true, - liveBufferLatencyMaxLatency: 0.08, - liveBufferLatencyMinRemain: 0.016, - liveSync: true, - liveSyncTargetLatency: 0.03, + const res = await fetch(qs ? `/au?${qs}` : "/au", { + headers: authToken ? { authorization: `Bearer ${authToken}` } : {}, }); + if (!res.ok || !res.body) { + throw new Error(`au stream ${res.status}`); + } + const reader = res.body.getReader(); + const frames = new OrbiAnnexB.FrameReader(); + (async () => { + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + frames.push(value); + let msg; + while ((msg = frames.pop())) { + if (msg.type === "video") feedAccessUnit(msg); + } + } + scheduleReconnect("au ended"); + } catch (err) { + scheduleReconnect(err.message || "au"); + } + })(); + waitingForKeyframe = true; + requestIdr(); + if (latencyWatchdog) clearInterval(latencyWatchdog); + latencyWatchdog = setInterval(() => { + if (waitingForKeyframe) requestIdr(); + if (!streamActive) return; + if (lastFrameAt && Date.now() - lastFrameAt > 750) { + requestIdr({ keepDecoding: true }); + if (Date.now() - lastFrameAt > 3000) startStream({ quiet: true }); + } + }, 250); +} - videoEl.muted = true; - mpegtsPlayer.attachMediaElement(videoEl); +async function startStream(opts = {}) { + if (!window.isSecureContext) { + const host = (wtConfig && OrbiAnnexB.pickWtHost(wtConfig, window.location.hostname)) + || window.location.hostname + || "host"; + const port = (wtConfig && wtConfig.wt_port) || 8790; + setOverlayState( + "error", + "Open the HTTPS client", + `WebTransport needs a secure page. Open https://${host}:${port}/client/ and accept the certificate.`, + ); + return; + } + if (typeof VideoDecoder !== "function") { + unsupportedPlayback(); + return; + } + if (!canPlayWebTransport()) { + const s = playbackSupport(); + const missing = [ + !s.webTransport && "WebTransport", + !s.annexb && "protocol helpers", + ].filter(Boolean).join(" and "); + setOverlayState("error", "Playback Error", `This browser is missing ${missing}`); + return; + } + destroyPlayer(); + lastDelayMs = null; + clockOffsetNs = 0n; + lastFrameAt = 0; + if (!opts.quiet) { + setOverlayState("connecting", "Connecting", "Connecting to Linux virtual display…"); + } - mpegtsPlayer.on(mpegts.Events.ERROR, (errorType, errorDetail, errorInfo) => { - console.error("mpegts error:", errorType, errorDetail, errorInfo); - sendHostAction("idr"); - if (errorDetail === "NetworkError" && !authToken && tokenRow) { - tokenRow.classList.remove("hidden"); - } - scheduleReconnect(errorDetail || errorType || "network"); - }); + const cfg = wtConfig || {}; + const port = cfg.wt_port; + const path = cfg.wt_path || "/orbiscreen"; + const hashB64 = cfg.cert_sha256; + if (!port || !hashB64) { + setOverlayState("error", "Playback Error", "Host did not advertise WebTransport"); + return; + } - mpegtsPlayer.on(mpegts.Events.MEDIA_INFO, (mediaInfo) => { - if (mediaInfo && mediaInfo.width && mediaInfo.height) { - displayWidth = mediaInfo.width; - displayHeight = mediaInfo.height; - updateInfoDisplay(); + const host = OrbiAnnexB.pickWtHost(cfg, window.location.hostname); + const url = `https://${host}:${port}${path}`; + waitingForKeyframe = true; + + try { + const hash = OrbiAnnexB.hashFromBase64(hashB64); + let transport; + let lastErr; + for (const opts of [ + { serverCertificateHashes: [{ algorithm: "sha-256", value: hash }] }, + {}, + ]) { + let candidate; + try { + candidate = new WebTransport(url, opts); + await Promise.race([ + candidate.ready, + new Promise((_, reject) => { + setTimeout(() => reject(new Error("webtransport ready timeout")), 4000); + }), + ]); + transport = candidate; + lastErr = null; + break; + } catch (err) { + lastErr = err; + try { if (candidate) candidate.close(); } catch (_) {} + } } - }); + if (!transport) { + console.warn("webtransport failed, using HTTPS AU stream:", lastErr); + await startAuStream(); + return; + } + wtTransport = transport; + const bidi = await transport.createBidirectionalStream(); + const writer = bidi.writable.getWriter(); + wtWriter = writer; + await writer.write(OrbiAnnexB.encodeHello(authToken, displaySessionId || "")); + + const reader = bidi.readable.getReader(); + const frames = new OrbiAnnexB.FrameReader(); + (async () => { + try { + while (wtTransport === transport) { + const { value, done } = await reader.read(); + if (done) break; + frames.push(value); + let msg; + while ((msg = frames.pop())) { + if (msg.type === "helloAck") { + if (msg.width) displayWidth = msg.width; + if (msg.height) displayHeight = msg.height; + updateInfoDisplay(); + requestIdr(); + } else if (msg.type === "video") { + feedAccessUnit(msg); + } else if (msg.type === "pong") { + const now = nowNs(); + const rtt = now - msg.t0Ns; + clockOffsetNs = msg.hostNs + rtt / 2n - now; + } + } + } + } catch (err) { + if (wtTransport === transport) { + console.error("webtransport read:", err); + displaySessionId = ""; + scheduleReconnect(err.message || "transport"); + } + } + })(); - mpegtsPlayer.load(); - const playPromise = mpegtsPlayer.play(); - if (playPromise && typeof playPromise.catch === "function") { - playPromise.catch((error) => { - console.warn("autoplay blocked or rejected:", error); - setOverlayState("connecting", "Ready to Stream", "Click anywhere on the screen to begin"); - const onFirstClick = () => { - window.removeEventListener("pointerdown", onFirstClick); - mpegtsPlayer.play().catch(() => {}); - }; - window.addEventListener("pointerdown", onFirstClick); - }); + const assembler = new OrbiAnnexB.DatagramAssembler(); + let dgramReader = null; + try { + dgramReader = transport.datagrams.readable.getReader(); + } catch (err) { + console.warn("webtransport datagrams unavailable:", err); + } + if (dgramReader) { + (async () => { + try { + while (wtTransport === transport) { + const { value, done } = await dgramReader.read(); + if (done) break; + const msg = assembler.push(value); + if (!msg) continue; + if (msg.type === "gap") { + waitingForKeyframe = true; + requestIdr(); + continue; + } + if (msg.type === "video") feedAccessUnit(msg); + } + } catch (err) { + // Video can still arrive on the control stream or /au. + console.warn("webtransport datagram:", err); + } + })(); + } + } catch (err) { + console.error("webtransport connect:", err); + if (!authToken && tokenRow) tokenRow.classList.remove("hidden"); + scheduleReconnect(err.message || "connect"); + return; } + const wtStartedAt = Date.now(); + let auFallback = false; latencyWatchdog = setInterval(() => { - if (!streamActive || !videoEl || !videoEl.buffered || videoEl.buffered.length === 0) return; - try { - const end = videoEl.buffered.end(videoEl.buffered.length - 1); - const delay = end - videoEl.currentTime; - const ms = Math.max(0, Math.round(delay * 1000)); - if (statLatency) statLatency.textContent = `${ms} ms`; - if (delay > 0.08) { - videoEl.currentTime = end - 0.02; + if (waitingForKeyframe) requestIdr(); + if (wtWriter) { + try { wtWriter.write(OrbiAnnexB.encodePing(nowNs())); } catch (_) {} + } + if (!auFallback && !streamActive && Date.now() - wtStartedAt > 3500) { + auFallback = true; + console.warn("webtransport produced no picture; falling back to /au"); + try { if (wtWriter) wtWriter.close(); } catch (_) {} + try { if (wtTransport) wtTransport.close(); } catch (_) {} + wtWriter = null; + wtTransport = null; + startAuStream().catch((err) => { + scheduleReconnect(err.message || "au"); + }); + return; + } + if (!streamActive) return; + if (lastFrameAt && Date.now() - lastFrameAt > 750) { + requestIdr({ keepDecoding: true }); + if (Date.now() - lastFrameAt > 3000) { + startStream({ quiet: true }); } - } catch (_) {} + } }, 250); } @@ -1222,8 +1576,13 @@ async function openDisplaySession() { async function start() { applyTheme(currentTheme); applyTranslations(); + if (typeof VideoDecoder !== "function") { + unsupportedPlayback(); + return; + } const info = await fetchClientConfig(); if (info) { + wtConfig = info; if (!authToken && typeof info.token === "string" && info.token.length > 0) { authToken = info.token; } @@ -1243,6 +1602,7 @@ async function start() { if (typeof apiInfo?.encoder === "string") { encoderName = apiInfo.encoder.toUpperCase(); } + wtConfig = Object.assign({}, wtConfig || {}, apiInfo); } } catch (error) { console.warn("api/info fetch failed:", error); @@ -1253,25 +1613,7 @@ async function start() { startStream(); } -videoEl.addEventListener("playing", () => { - streamActive = true; - reconnectDelay = 1000; - if (overlayEl) overlayEl.classList.add("hidden"); - if (controlToolbar) controlToolbar.classList.remove("hidden"); - if (miniPill) miniPill.classList.add("hidden"); -}); -videoEl.addEventListener("error", () => { - if (streamActive || mpegtsPlayer) { - scheduleReconnect("media error"); - } -}); - -videoEl.addEventListener("ended", () => { - if (streamActive || mpegtsPlayer) { - scheduleReconnect("stream ended"); - } -}); start().catch((error) => { setOverlayState("error", "Initialization Failed", error.message); diff --git a/clients/web/index.html b/clients/web/index.html index e6d5a1ff..294d0733 100644 --- a/clients/web/index.html +++ b/clients/web/index.html @@ -4,7 +4,7 @@ + content="default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline'; img-src 'self' blob: data:; media-src 'self' blob: data:; connect-src 'self' https:;" /> @@ -82,15 +82,11 @@ - +