From b2d5e3da753f93e8cadd5968b676347f83db672d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awek=20Piotrowski?= Date: Wed, 16 Sep 2026 12:53:25 +0200 Subject: [PATCH 1/2] orbiscreen | feat: WebTransport Annex-B web client Serve the UI over HTTPS on signaling+2 and play Annex-B access units with WebTransport plus WebCodecs. Persist the self-signed certificate so the browser pin survives a daemon restart. Pin quinn-proto 0.11.16 so dropped datagrams do not abort the process. --- CHANGELOG.md | 11 + Cargo.lock | 639 +++++++++++++++++- README.md | 2 +- SECURITY.md | 2 +- clients/web/annexb.js | 389 +++++++++++ clients/web/annexb.test.js | 93 +++ clients/web/app.js | 566 +++++++++++++--- clients/web/index.html | 22 +- clients/web/style.css | 9 +- crates/orbiscreen-core/src/lib.rs | 16 + crates/orbiscreen-daemon/src/ui.rs | 6 +- crates/orbiscreen-transport/Cargo.toml | 11 +- crates/orbiscreen-transport/src/annexb.rs | 123 ++++ crates/orbiscreen-transport/src/lib.rs | 427 +++++++++++- crates/orbiscreen-transport/src/udp_stream.rs | 115 ++-- .../orbiscreen-transport/src/wt_protocol.rs | 424 ++++++++++++ crates/orbiscreen-transport/src/wt_stream.rs | 626 +++++++++++++++++ docs/ARCHITECTURE.md | 6 +- docs/DBUS_SPEC.md | 2 +- docs/TROUBLESHOOTING.md | 11 +- docs/UDP_TRANSPORT.md | 4 +- 21 files changed, 3291 insertions(+), 213 deletions(-) create mode 100644 clients/web/annexb.js create mode 100644 clients/web/annexb.test.js create mode 100644 crates/orbiscreen-transport/src/annexb.rs create mode 100644 crates/orbiscreen-transport/src/wt_protocol.rs create mode 100644 crates/orbiscreen-transport/src/wt_stream.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 28b1041f..c5a0040b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file. +## [Unreleased] + +### โœจ Added +- **WebTransport Annex-B + WebCodecs web client**: + - Chromium opens WebTransport on `signaling_port + 2` (`wt_port`) with `serverCertificateHashes` and decodes Annex-B access units with `VideoDecoder` onto a canvas. + - P-frames ride QUIC datagrams (unreliable). Hello, ping/pong, and IDR stay on the control stream. The same port serves the UI over HTTPS; HTTP `/` and `/client/` redirect there. Android keeps HTTP on `:8788`. + - The self-signed certificate is stored in `$XDG_CONFIG_HOME/orbiscreen/wt-cert.pem` and `wt-key.pem` (`0o600`, parent `0o700`) and reused across restarts (14-day WebTransport cap). + +### ๐Ÿ› Fixed +- **Quinn datagram send no longer aborts the daemon**: pin `quinn-proto` to 0.11.16 (0.11.17 double-subtracts dropped datagram bytes). + ## [v0.28.9] - 2026-09-16 Introduce per-client KWin virtual displays and isolated touch scoping to support multiple simultaneous tablets, expand KWin input device introspection beyond event63, add automatic fallback session creation on attach for full backwards compatibility with legacy clients, and bump the release matrix across all platforms. diff --git a/Cargo.lock b/Cargo.lock index 9bd9a245..af63d855 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,6 +97,15 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "as-raw-xcb-connection" version = "1.0.1" @@ -119,6 +128,45 @@ dependencies = [ "zbus", ] +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.20", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "async-broadcast" version = "0.7.2" @@ -346,6 +394,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-server" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ab4a3ec9ea8a657c72d99a03a824af695bd0fb5ec639ccbd9cd3543b41a5f9" +dependencies = [ + "arc-swap", + "bytes", + "fs-err", + "http", + "http-body", + "hyper", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + [[package]] name = "base64" version = "0.21.7" @@ -370,7 +440,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec", + "bit-vec 0.8.0", ] [[package]] @@ -379,6 +449,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -403,6 +482,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -604,6 +692,23 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.45" @@ -681,6 +786,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "cookie" version = "0.18.2" @@ -740,6 +851,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.1" @@ -774,6 +894,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "cssparser" version = "0.36.0" @@ -895,6 +1024,20 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.5.8" @@ -942,8 +1085,19 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", ] [[package]] @@ -1232,6 +1386,18 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastbloom" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef975e30683b2d965054bb0a836f8973857c4ebf6acf274fe46617cd285060d8" +dependencies = [ + "foldhash", + "libm", + "portable-atomic", + "siphasher", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -1333,6 +1499,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a" +dependencies = [ + "autocfg", + "tokio", +] + [[package]] name = "futures-channel" version = "0.3.34" @@ -1543,8 +1719,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1566,8 +1744,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -1878,6 +2059,25 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1935,6 +2135,12 @@ dependencies = [ "markup5ever", ] +[[package]] +name = "httlib-huffman" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a9fcbcc408c5526c3ab80d534e5c86e7967c1fb7aa0a8c76abd1edc27deb877" + [[package]] name = "http" version = "1.5.0" @@ -1986,6 +2192,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.11.0" @@ -1996,6 +2211,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -2481,6 +2697,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libredox" version = "0.1.23" @@ -2517,6 +2739,12 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "markup5ever" version = "0.38.0" @@ -2598,6 +2826,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2687,6 +2921,16 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2696,6 +2940,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -2947,6 +3201,21 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "octets" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d59d30d3ad7f7e1c9a66e51c9cb71ae1c794b28ffe7053fd785defb0d180069" + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2959,6 +3228,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "option-ext" version = "0.2.0" @@ -3090,15 +3365,19 @@ name = "orbiscreen-transport" version = "0.28.9" dependencies = [ "axum", + "axum-server", "base64 0.22.1", "gstreamer", "gstreamer-app", "hostname", "libc", "mdns-sd", + "orbiscreen-core", "orbiscreen-encode", "orbiscreen-input", + "quinn-proto", "rand 0.9.5", + "rustls", "serde", "serde_json", "thiserror 2.0.20", @@ -3106,6 +3385,8 @@ dependencies = [ "tokio-stream", "tower-http", "tracing", + "wtransport", + "x509-parser", ] [[package]] @@ -3178,6 +3459,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64 0.22.1", + "serde_core", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3438,6 +3729,63 @@ dependencies = [ "memchr", ] +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.20", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "bytes", + "fastbloom", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.47" @@ -3480,6 +3828,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -3518,12 +3877,40 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "raw-window-handle" version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3627,6 +4014,20 @@ dependencies = [ "web-sys", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -3642,6 +4043,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "1.1.4" @@ -3655,6 +4065,62 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -3676,6 +4142,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "schemars" version = "0.8.22" @@ -3733,6 +4208,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.1", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "selectors" version = "0.36.1" @@ -3951,8 +4449,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -3962,8 +4460,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -4135,6 +4644,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "swift-rs" version = "1.0.8" @@ -4377,7 +4892,7 @@ dependencies = [ "semver", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "syn 2.0.119", "tauri-utils", "thiserror 2.0.20", @@ -4656,6 +5171,16 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "tokio-rustls" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.19" @@ -4689,6 +5214,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -5056,6 +5582,12 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "uoctl" version = "1.0.1" @@ -5347,6 +5879,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web_atoms" version = "0.2.6" @@ -5633,6 +6175,15 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -5866,7 +6417,7 @@ dependencies = [ "once_cell", "percent-encoding", "raw-window-handle", - "sha2", + "sha2 0.10.9", "soup3", "tao-macros", "thiserror 2.0.20", @@ -5880,6 +6431,42 @@ dependencies = [ "x11-dl", ] +[[package]] +name = "wtransport" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4273ce3157a3262a68665f8d3f20a0ac0c5b8a69ffd67f05ae986832ebec036" +dependencies = [ + "bytes", + "pem", + "quinn", + "rcgen", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "sha2 0.11.0", + "socket2", + "thiserror 2.0.20", + "time", + "tokio", + "tracing", + "url", + "wtransport-proto", + "x509-parser", +] + +[[package]] +name = "wtransport-proto" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aad9059572c7dbd6901ccef37f3b7321678cd708dcf58a64b1921dbeab7bfede" +dependencies = [ + "httlib-huffman", + "octets", + "thiserror 2.0.20", + "url", +] + [[package]] name = "x11" version = "2.21.0" @@ -5920,6 +6507,24 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acf4d1bc32aa46eec18caa634ec3cf4c05bfa151f12b93b510b15190f69a1ca8" +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.20", + "time", +] + [[package]] name = "xkbcommon" version = "0.9.0" @@ -5937,6 +6542,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + [[package]] name = "yoke" version = "0.8.3" @@ -6072,6 +6687,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerotrie" version = "0.2.5" diff --git a/README.md b/README.md index 092a9a40..ce57e401 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ - **Material 3 Android client**: Jetpack Compose, Catppuccin Mocha / Latte brand palette, light/dark theme - **Auto-Orientation Resolution Adaptation**: Automatically swaps width and height when your phone/tablet rotates between landscape and portrait - **3-Row Spacious Keyboard Overlay**: Top-docked layout ensuring bottom taskbars, terminals, and prompts stay 100% visible -- **Built-in web client**: Watch from any browser at `http://:8788/` (MSE via locally bundled `mpegts.js`, no CDN) +- **Built-in web client**: Watch from any browser at `https://:8790/client/` (HTTP on `:8788` redirects there). WebTransport Annex-B + WebCodecs `VideoDecoder`. - **Live discovery**: NSD scan of nearby hosts, manual `host:port` entry, optional subnet scanner - **Native streaming**: ExoPlayer with `OkHttpDataSource` + tuned load control for ultra-low latency MPEG-TS / H.264 - **Host control panel**: Keyboard, lock, blank, Ctrl+Alt+Del, and retry actions diff --git a/SECURITY.md b/SECURITY.md index 06e5d463..7ccfa2ff 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -166,7 +166,7 @@ The Android release signing key (`orbiscreen-release.keystore`) was removed from ## ๐Ÿ”ฌ Security Audit -Orbiscreen (v0.25.9) is written in Rust (edition 2021) plus a Kotlin Android client (Material 3 + Jetpack Compose), a small browser web client (MSE via vendored mpegts.js), and a Linux desktop GUI control center (Tauri v2 + WebKitGTK). A running daemon performs: +Orbiscreen (v0.27.4) is written in Rust (edition 2021) plus a Kotlin Android client (Material 3 + Jetpack Compose), a small browser web client (WebTransport + WebCodecs `VideoDecoder`), and a Linux desktop GUI control center (Tauri v2 + WebKitGTK). A running daemon performs: - `open()` on `/dev/dri/card*` evdi nodes for capture - Compositor IPC over session-local Unix sockets: sway i3-ipc (`$SWAYSOCK`) and Hyprland (`HYPRLAND_INSTANCE_SIGNATURE`) to create/destroy headless outputs diff --git a/clients/web/annexb.js b/clients/web/annexb.js new file mode 100644 index 00000000..6253f814 --- /dev/null +++ b/clients/web/annexb.js @@ -0,0 +1,389 @@ +// Orbiscreen - annexb.js (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +(function (root, factory) { + if (typeof module === "object" && module.exports) { + module.exports = factory(); + } else { + root.OrbiAnnexB = factory(); + } +}(typeof self !== "undefined" ? self : this, function () { + const TYPE_VIDEO = 1; + const TYPE_HELLO = 2; + const TYPE_HELLO_ACK = 3; + const TYPE_PING = 4; + const TYPE_PONG = 5; + const TYPE_IDR = 6; + const TYPE_BYE = 10; + const MAX_FRAME = 4 * 1024 * 1024; + const DATAGRAM_HEADER = 24; + + function startCodeLen(data, i) { + if (i + 3 < data.length && data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 1) { + return 3; + } + if (i + 4 < data.length && data[i] === 0 && data[i + 1] === 0 + && data[i + 2] === 0 && data[i + 3] === 1) { + return 4; + } + return 0; + } + + function withStartCode(nal) { + const out = new Uint8Array(4 + nal.length); + out[3] = 1; + out.set(nal, 4); + return out; + } + + function extractSpsPps(au) { + let sps = null; + let pps = null; + let i = 0; + while (i < au.length) { + const sc = startCodeLen(au, i); + if (!sc) break; + const nalStart = i + sc; + let next = nalStart; + while (next < au.length) { + if (startCodeLen(au, next) && next > nalStart) break; + next += 1; + } + if (nalStart < next) { + const nal = au.subarray(nalStart, next); + const typ = nal[0] & 0x1f; + if (typ === 7) sps = withStartCode(nal); + if (typ === 8) pps = withStartCode(nal); + } + i = next; + } + return { sps, pps }; + } + + function codecStringFromSps(sps) { + if (!sps) return null; + const sc = startCodeLen(sps, 0); + if (!sc || sps.length < sc + 4) return null; + if ((sps[sc] & 0x1f) !== 7) return null; + const hex = (n) => n.toString(16).toUpperCase().padStart(2, "0"); + return `avc1.${hex(sps[sc + 1])}${hex(sps[sc + 2])}${hex(sps[sc + 3])}`; + } + + function spsDimensions(sps) { + if (!sps) return null; + const sc = startCodeLen(sps, 0); + const nal = sps.subarray(sc); + if (nal.length < 5 || (nal[0] & 0x1f) !== 7) return null; + let bit = 32; + const getBits = (n) => { + let v = 0; + for (let i = 0; i < n; i += 1) { + const byte = nal[bit >> 3]; + if (byte === undefined) return 0; + v = (v << 1) | ((byte >> (7 - (bit & 7))) & 1); + bit += 1; + } + return v; + }; + const ue = () => { + let z = 0; + while (getBits(1) === 0) { + z += 1; + if (z > 31) return 0; + } + return z === 0 ? 0 : ((1 << z) | getBits(z)) - 1; + }; + const profile = nal[1]; + ue(); + if (profile === 100 || profile === 110 || profile === 122 || profile === 244 + || profile === 44 || profile === 83 || profile === 86 || profile === 118 + || profile === 128 || profile === 138 || profile === 139 || profile === 134) { + const chroma = ue(); + if (chroma === 3) getBits(1); + ue(); + ue(); + getBits(1); + if (getBits(1)) { + const limit = chroma === 3 ? 12 : 8; + for (let i = 0; i < limit; i += 1) { + if (getBits(1)) { + const last = i < 6 ? 16 : 64; + let next = 8; + for (let j = 0; j < last; j += 1) { + const delta = (() => { + let z = 0; + while (getBits(1) === 0) { + z += 1; + if (z > 31) return 0; + } + const mag = z === 0 ? 0 : ((1 << z) | getBits(z)) - 1; + const signed = (mag % 2 === 0) ? -(mag / 2) : (mag + 1) / 2; + return signed; + })(); + next = (next + delta + 256) % 256; + if (next === 0) break; + } + } + } + } + } + ue(); + const poc = ue(); + if (poc === 0) { + ue(); + } else if (poc === 1) { + getBits(1); + ue(); + ue(); + const n = ue(); + for (let i = 0; i < n; i += 1) ue(); + } + ue(); + getBits(1); + const wMbs = ue() + 1; + const hMap = ue() + 1; + const frameMbsOnly = getBits(1); + if (!frameMbsOnly) getBits(1); + getBits(1); + let cropL = 0; + let cropR = 0; + let cropT = 0; + let cropB = 0; + if (getBits(1)) { + cropL = ue(); + cropR = ue(); + cropT = ue(); + cropB = ue(); + } + const width = wMbs * 16 - (cropL + cropR) * 2; + const height = (2 - frameMbsOnly) * hMap * 16 - (cropT + cropB) * 2; + if (width < 16 || height < 16 || width > 7680 || height > 4320) return null; + return { width, height }; + } + + function concatBytes(parts) { + let n = 0; + for (const p of parts) n += p.length; + const out = new Uint8Array(n); + let o = 0; + for (const p of parts) { + out.set(p, o); + o += p.length; + } + return out; + } + + function u16le(n) { + return new Uint8Array([n & 0xff, (n >> 8) & 0xff]); + } + + function u64le(n) { + const out = new Uint8Array(8); + let x = BigInt(n); + for (let i = 0; i < 8; i += 1) { + out[i] = Number(x & 0xffn); + x >>= 8n; + } + return out; + } + + function readU16le(buf, o) { + return buf[o] | (buf[o + 1] << 8); + } + + function readU64le(buf, o) { + let x = 0n; + for (let i = 0; i < 8; i += 1) { + x |= BigInt(buf[o + i]) << BigInt(8 * i); + } + return x; + } + + function encodeFrame(body) { + if (body.length > MAX_FRAME) throw new Error("frame too large"); + const out = new Uint8Array(4 + body.length); + const len = body.length; + out[0] = (len >>> 24) & 0xff; + out[1] = (len >>> 16) & 0xff; + out[2] = (len >>> 8) & 0xff; + out[3] = len & 0xff; + out.set(body, 4); + return out; + } + + function splitFrame(buf) { + if (buf.length < 4) return null; + const len = ((buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3]) >>> 0; + if (len > MAX_FRAME) throw new Error("frame too large"); + if (buf.length < 4 + len) return null; + return { body: buf.subarray(4, 4 + len), used: 4 + len }; + } + + function encodeHello(token, session) { + const t = new TextEncoder().encode(token || ""); + const s = new TextEncoder().encode(session || ""); + return encodeFrame(concatBytes([ + Uint8Array.of(TYPE_HELLO), + u16le(t.length), + t, + u16le(s.length), + s, + ])); + } + + function encodeCtrl(kind) { + return encodeFrame(Uint8Array.of(kind)); + } + + function encodePing(t0Ns) { + return encodeFrame(concatBytes([Uint8Array.of(TYPE_PING), u64le(t0Ns)])); + } + + function decodeMessage(body) { + if (!body.length) throw new Error("empty"); + const kind = body[0]; + const rest = body.subarray(1); + if (kind === TYPE_HELLO_ACK) { + if (rest.length < 4) throw new Error("truncated"); + return { type: "helloAck", width: readU16le(rest, 0), height: readU16le(rest, 2) }; + } + if (kind === TYPE_VIDEO) { + if (rest.length < 17) throw new Error("truncated"); + return { + type: "video", + key: rest[0] !== 0, + ptsNs: readU64le(rest, 1), + sentNs: readU64le(rest, 9), + au: rest.subarray(17), + }; + } + if (kind === TYPE_PONG) { + if (rest.length < 16) throw new Error("truncated"); + return { type: "pong", t0Ns: readU64le(rest, 0), hostNs: readU64le(rest, 8) }; + } + if (kind === TYPE_IDR) return { type: "idr" }; + if (kind === TYPE_BYE) return { type: "bye" }; + if (kind === TYPE_PING) return { type: "ping", t0Ns: readU64le(rest, 0) }; + if (kind === TYPE_HELLO) return { type: "hello" }; + throw new Error(`unknown type ${kind}`); + } + + function hashFromBase64(b64) { + const bin = atob(b64); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i += 1) out[i] = bin.charCodeAt(i); + return out; + } + + function pickWtHost(cfg, locationHost) { + const h = locationHost || ""; + if (h && h !== "localhost" && h !== "127.0.0.1" && h !== "[::1]") { + return h; + } + const hosts = cfg && Array.isArray(cfg.wt_hosts) ? cfg.wt_hosts : []; + const lan = hosts.find((x) => x && x !== "127.0.0.1" && x !== "::1"); + return lan || hosts[0] || h || "127.0.0.1"; + } + + function parseVideoDatagram(buf) { + if (!buf || buf.length < DATAGRAM_HEADER || buf[0] !== TYPE_VIDEO) return null; + return { + type: "video", + key: buf[1] !== 0, + seq: readU16le(buf, 2), + frag: readU16le(buf, 4), + frags: readU16le(buf, 6), + ptsNs: readU64le(buf, 8), + sentNs: readU64le(buf, 16), + payload: buf.subarray(24), + }; + } + + function encodeVideoDatagram(seq, frag, frags, key, ptsNs, sentNs, payload) { + const out = new Uint8Array(DATAGRAM_HEADER + payload.length); + out[0] = TYPE_VIDEO; + out[1] = key ? 1 : 0; + out.set(u16le(seq), 2); + out.set(u16le(frag), 4); + out.set(u16le(frags), 6); + out.set(u64le(ptsNs), 8); + out.set(u64le(sentNs), 16); + out.set(payload, 24); + return out; + } + + function seqDelta(cur, prev) { + return (cur - prev) & 0xffff; + } + + class DatagramAssembler { + constructor() { + this.pending = new Map(); + this.lastSeq = -1; + } + push(buf) { + const frag = parseVideoDatagram(buf); + if (!frag || frag.frags <= 0 || frag.frag >= frag.frags) return null; + if (this.pending.size >= 4) { + for (const [seq] of this.pending) { + if (seqDelta(frag.seq, seq) > 1 && seqDelta(frag.seq, seq) < 32768) { + this.pending.delete(seq); + } + } + } + let slots = this.pending.get(frag.seq); + if (!slots || slots.frags !== frag.frags) { + slots = { + frags: frag.frags, + key: frag.key, + ptsNs: frag.ptsNs, + sentNs: frag.sentNs, + parts: Array.from({ length: frag.frags }, () => null), + }; + this.pending.set(frag.seq, slots); + } + slots.parts[frag.frag] = frag.payload; + if (slots.parts.some((p) => p == null)) return null; + this.pending.delete(frag.seq); + const gap = this.lastSeq >= 0 && seqDelta(frag.seq, this.lastSeq) !== 1; + this.lastSeq = frag.seq; + if (gap && !slots.key) { + return { type: "gap" }; + } + return { + type: "video", + key: slots.key, + ptsNs: slots.ptsNs, + sentNs: slots.sentNs, + au: concatBytes(slots.parts), + }; + } + } + + class FrameReader { + constructor() { + this.buf = new Uint8Array(0); + } + push(chunk) { + const next = new Uint8Array(this.buf.length + chunk.length); + next.set(this.buf, 0); + next.set(chunk, this.buf.length); + this.buf = next; + } + pop() { + const split = splitFrame(this.buf); + if (!split) return null; + this.buf = this.buf.subarray(split.used); + return decodeMessage(split.body); + } + } + + return { + TYPE_VIDEO, TYPE_HELLO, TYPE_HELLO_ACK, TYPE_PING, TYPE_PONG, TYPE_IDR, TYPE_BYE, + MAX_FRAME, + startCodeLen, extractSpsPps, codecStringFromSps, spsDimensions, withStartCode, + encodeFrame, splitFrame, encodeHello, encodeCtrl, encodePing, decodeMessage, + hashFromBase64, pickWtHost, FrameReader, + parseVideoDatagram, encodeVideoDatagram, DatagramAssembler, seqDelta, DATAGRAM_HEADER, + }; +})); diff --git a/clients/web/annexb.test.js b/clients/web/annexb.test.js new file mode 100644 index 00000000..d5415949 --- /dev/null +++ b/clients/web/annexb.test.js @@ -0,0 +1,93 @@ +// Orbiscreen - annexb.test.js (GPL-3.0-or-later) +// https://github.com/shadow-x78/orbiscreen + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const annexb = require("./annexb.js"); + +test("spsDimensions reads High@4.0 1280x800", () => { + const sps = Buffer.from( + "0000000127640028ac11128050065bff0001000110000003001000000789da08042e", + "hex", + ); + assert.deepEqual(annexb.spsDimensions(sps), { width: 1280, height: 800 }); +}); + +test("codec string from High@4.0 SPS", () => { + const sps = annexb.withStartCode(Uint8Array.of(0x67, 0x64, 0x00, 0x28, 0xac)); + assert.equal(annexb.codecStringFromSps(sps), "avc1.640028"); +}); + +test("extracts SPS/PPS from an IDR AU", () => { + const sps = annexb.withStartCode(Uint8Array.of(0x67, 0x64, 0x00, 0x28, 0xac)); + const pps = annexb.withStartCode(Uint8Array.of(0x68, 0xee, 0x3c, 0x80)); + const slice = annexb.withStartCode(Uint8Array.of(0x65, 0x88)); + const au = new Uint8Array(sps.length + pps.length + slice.length); + au.set(sps, 0); + au.set(pps, sps.length); + au.set(slice, sps.length + pps.length); + const found = annexb.extractSpsPps(au); + assert.deepEqual(Array.from(found.sps), Array.from(sps)); + assert.deepEqual(Array.from(found.pps), Array.from(pps)); +}); + +test("hello frame encodes a complete length-prefixed body", () => { + const frame = annexb.encodeHello("tok", "sess-1"); + const split = annexb.splitFrame(frame); + assert.ok(split); + assert.equal(split.used, frame.length); + assert.equal(split.body[0], annexb.TYPE_HELLO); +}); + +test("frame reader reassembles split chunks", () => { + const frame = annexb.encodeCtrl(annexb.TYPE_IDR); + const reader = new annexb.FrameReader(); + reader.push(frame.subarray(0, 3)); + assert.equal(reader.pop(), null); + reader.push(frame.subarray(3)); + const msg = reader.pop(); + assert.equal(msg.type, "idr"); + assert.equal(reader.pop(), null); +}); + +test("video message decode", () => { + const au = Uint8Array.of(0, 0, 0, 1, 0x65, 9); + const body = new Uint8Array(1 + 1 + 8 + 8 + au.length); + body[0] = annexb.TYPE_VIDEO; + body[1] = 1; + body[2] = 42; + body.set(au, 18); + const frame = annexb.encodeFrame(body); + const msg = annexb.decodeMessage(annexb.splitFrame(frame).body); + assert.equal(msg.type, "video"); + assert.equal(msg.key, true); + assert.equal(msg.ptsNs, 42n); + assert.deepEqual(Array.from(msg.au), Array.from(au)); +}); + +test("pickWtHost prefers the page host when it is not loopback", () => { + assert.equal(annexb.pickWtHost({ wt_hosts: ["10.0.0.5"] }, "192.168.1.8"), "192.168.1.8"); + assert.equal(annexb.pickWtHost({ wt_hosts: ["10.0.0.5", "127.0.0.1"] }, "127.0.0.1"), "10.0.0.5"); + assert.equal(annexb.pickWtHost({ wt_hosts: ["127.0.0.1"] }, "localhost"), "127.0.0.1"); +}); + +test("datagram assembler rebuilds a split AU and reports a gap", () => { + const au = Uint8Array.from({ length: 40 }, (_, i) => i); + const a = annexb.encodeVideoDatagram(4, 0, 2, true, 1n, 2n, au.subarray(0, 20)); + const b = annexb.encodeVideoDatagram(4, 1, 2, true, 1n, 2n, au.subarray(20)); + const asm = new annexb.DatagramAssembler(); + assert.equal(asm.push(a), null); + const msg = asm.push(b); + assert.equal(msg.type, "video"); + assert.equal(msg.key, true); + assert.deepEqual(Array.from(msg.au), Array.from(au)); + + const next = annexb.encodeVideoDatagram(6, 0, 1, false, 3n, 4n, au.subarray(0, 8)); + assert.equal(asm.push(next).type, "gap"); +}); + +test("hashFromBase64 yields 32 bytes", () => { + const raw = Uint8Array.from({ length: 32 }, (_, i) => i); + const b64 = Buffer.from(raw).toString("base64"); + assert.deepEqual(Array.from(annexb.hashFromBase64(b64)), Array.from(raw)); +}); diff --git a/clients/web/app.js b/clients/web/app.js index c2df14ef..b246df06 100644 --- a/clients/web/app.js +++ b/clients/web/app.js @@ -10,7 +10,7 @@ const I18N = { btnHideControls: "Hide Toolbar", btnFullscreen: "Fullscreen", btnDisconnect: "Disconnect", - btnRestoreToolbar: "Show Toolbar", + btnShowControls: "Menu", btnCloseKeyboard: "Close", settingsTitle: "Settings", themeTitle: "Theme", @@ -48,7 +48,9 @@ const I18N = { toastLockSent: "Lock sent", toastCadSent: "Ctrl+Alt+Del sent", toastResynced: "Resynced", - toastDisconnected: "Disconnected" + toastDisconnected: "Disconnected", + statusUnsupported: "Unsupported browser", + statusUnsupportedSub: "This client needs the WebCodecs VideoDecoder API. Open the page in Chrome, Brave, Edge, or another Chromium browser." }, ar: { btnInputMode: "ูˆุถุน ุงู„ุฅุฏุฎุงู„", @@ -95,7 +97,9 @@ const I18N = { toastLockSent: "ุชู… ุฅุฑุณุงู„ ุฃู…ุฑ ุงู„ู‚ูู„", toastCadSent: "ุชู… ุฅุฑุณุงู„ Ctrl+Alt+Del", toastResynced: "ุชู…ุช ุงู„ู…ุฒุงู…ู†ุฉ", - toastDisconnected: "ุชู… ู‚ุทุน ุงู„ุงุชุตุงู„" + toastDisconnected: "ุชู… ู‚ุทุน ุงู„ุงุชุตุงู„", + statusUnsupported: "ู…ุชุตูุญ ุบูŠุฑ ู…ุฏุนูˆู…", + statusUnsupportedSub: "ูŠุญุชุงุฌ ู‡ุฐุง ุงู„ุนู…ูŠู„ ุฅู„ู‰ WebCodecs VideoDecoder. ุงูุชุญ ุงู„ุตูุญุฉ ููŠ Chrome ุฃูˆ Brave ุฃูˆ Edge ุฃูˆ ู…ุชุตูุญ Chromium ุขุฎุฑ." } }; @@ -183,10 +187,9 @@ const stageEl = document.getElementById("stage"); const videoEl = document.getElementById("remoteVideo"); const touchIndicator = document.getElementById("touchIndicator"); const controlToolbar = document.getElementById("controlToolbar"); -const miniPill = document.getElementById("miniPill"); +const btnShowControls = document.getElementById("btnShowControls"); const hostNameEl = document.getElementById("hostName"); const hostInfoEl = document.getElementById("hostInfo"); -const miniInfoEl = document.getElementById("miniInfo"); function setOverlayState(state, title, subtitle) { if (overlayEl) overlayEl.classList.remove("hidden"); @@ -194,8 +197,7 @@ function setOverlayState(state, title, subtitle) { if (statusSubtitle && subtitle) statusSubtitle.textContent = subtitle; releaseControl(); - if (controlToolbar) controlToolbar.classList.add("hidden"); - if (miniPill) miniPill.classList.add("hidden"); + hideControlsChrome(); if (keyboardDrawer) keyboardDrawer.classList.add("hidden"); if (settingsModal) settingsModal.classList.add("hidden"); @@ -203,6 +205,10 @@ function setOverlayState(state, title, subtitle) { if (brandLogo) brandLogo.classList.add("hidden"); if (statusSpinner) statusSpinner.classList.remove("hidden"); if (btnReconnect) btnReconnect.classList.add("hidden"); + } else if (state === "unsupported") { + if (statusSpinner) statusSpinner.classList.add("hidden"); + if (brandLogo) brandLogo.classList.remove("hidden"); + if (btnReconnect) btnReconnect.classList.add("hidden"); } else { if (statusSpinner) statusSpinner.classList.add("hidden"); if (brandLogo) brandLogo.classList.remove("hidden"); @@ -248,16 +254,25 @@ let displayWidth = 1920; let displayHeight = 1080; let encoderName = "NVENC"; let authToken = ""; -let mpegtsPlayer = null; +let wtConfig = null; +let wtTransport = null; +let wtWriter = null; +let videoDecoder = null; let reconnectTimer = null; let reconnectDelay = 1000; const MAX_RECONNECT_DELAY = 10000; +let clockOffsetNs = 0n; +let lastFrameAt = 0; let streamActive = false; let isVncFocused = false; let isTouchMode = true; let toastTimer = null; let vncBannerTimer = null; let latencyWatchdog = null; +let lastDelayMs = null; +let controlsVisible = false; +let controlsLockedHidden = false; +let controlsHideTimer = null; const heldKeys = new Set(); const pressedButtons = new Set(); const activeTouches = new Map(); @@ -284,12 +299,56 @@ function showToast(text, duration = 2200) { }, duration); } +window.addEventListener("resize", () => updateInfoDisplay()); + function updateInfoDisplay() { - const infoStr = `${displayWidth}ร—${displayHeight} ${encoderName}`; + const delayText = (lastDelayMs != null && lastDelayMs >= 0) + ? `delay ${lastDelayMs}ms` + : "delay โ€”"; + const narrow = window.matchMedia("(orientation: portrait), (max-width: 720px)").matches; + const infoStr = narrow + ? delayText + : `${displayWidth}ร—${displayHeight} ${encoderName} ${delayText}`; if (hostInfoEl) hostInfoEl.textContent = infoStr; - if (miniInfoEl) miniInfoEl.textContent = `${displayWidth}ร—${displayHeight}`; if (statRes) statRes.textContent = `${displayWidth} ร— ${displayHeight}`; if (statEncoder) statEncoder.textContent = encoderName; + if (statLatency) { + statLatency.textContent = (lastDelayMs != null && lastDelayMs >= 0) + ? `${lastDelayMs} ms` + : "-- ms"; + } +} + +function hideControlsChrome() { + clearTimeout(controlsHideTimer); + controlsHideTimer = null; + controlsVisible = false; + if (controlToolbar) controlToolbar.classList.add("hidden"); + if (btnShowControls) btnShowControls.classList.add("hidden"); +} + +function setControlsVisible(visible) { + clearTimeout(controlsHideTimer); + controlsHideTimer = null; + if (!streamActive) { + hideControlsChrome(); + return; + } + if (controlsLockedHidden) { + if (controlToolbar) controlToolbar.classList.add("hidden"); + if (btnShowControls) btnShowControls.classList.add("hidden"); + controlsVisible = false; + return; + } + controlsVisible = visible; + if (visible) { + if (controlToolbar) controlToolbar.classList.remove("hidden"); + if (btnShowControls) btnShowControls.classList.add("hidden"); + controlsHideTimer = setTimeout(() => setControlsVisible(false), 12_000); + } else { + if (controlToolbar) controlToolbar.classList.add("hidden"); + if (btnShowControls) btnShowControls.classList.remove("hidden"); + } } function setVncFocus(focused) { @@ -594,7 +653,7 @@ document.querySelectorAll(".chipBtn[data-fit]").forEach((btn) => { btn.addEventListener("click", () => { document.querySelectorAll(".chipBtn[data-fit]").forEach((b) => b.classList.remove("active")); btn.classList.add("active"); - videoEl.style.objectFit = btn.dataset.fit; + if (videoEl) videoEl.style.objectFit = btn.dataset.fit; showToast(`Fit: ${btn.textContent}`); }); }); @@ -630,20 +689,26 @@ document.querySelectorAll("#webLangChips .chipBtn").forEach((btn) => { if (btnHideControls) { btnHideControls.addEventListener("click", (e) => { e.stopPropagation(); - controlToolbar.classList.add("hidden"); - miniPill.classList.remove("hidden"); + controlsLockedHidden = true; + setControlsVisible(false); showToast(t("controlsHidden"), 1800); }); } -if (btnRestoreToolbar) { - btnRestoreToolbar.addEventListener("click", (e) => { +if (btnShowControls) { + btnShowControls.addEventListener("click", (e) => { e.stopPropagation(); - miniPill.classList.add("hidden"); - controlToolbar.classList.remove("hidden"); + setControlsVisible(true); }); } +stageEl.addEventListener("dblclick", (e) => { + if (!streamActive || !controlsLockedHidden) return; + e.preventDefault(); + controlsLockedHidden = false; + setControlsVisible(true); +}); + if (btnFullscreen) { btnFullscreen.addEventListener("click", (e) => { e.stopPropagation(); @@ -671,6 +736,26 @@ async function sendHostAction(action, extra = {}) { } } +const IDR_DEBOUNCE_MS = 250; +let lastIdrAt = 0; +let waitingForKeyframe = false; + +function noteKeyframe() { + waitingForKeyframe = false; +} + +function requestIdr(opts = {}) { + const now = Date.now(); + if (now - lastIdrAt < IDR_DEBOUNCE_MS) return false; + lastIdrAt = now; + if (!opts.keepDecoding) waitingForKeyframe = true; + if (wtWriter) { + try { wtWriter.write(OrbiAnnexB.encodeCtrl(OrbiAnnexB.TYPE_IDR)); } catch (_) {} + } + sendHostAction("idr", displaySessionId ? { session: displaySessionId } : {}); + return true; +} + if (btnLock) { btnLock.addEventListener("click", async (e) => { e.stopPropagation(); @@ -692,6 +777,7 @@ if (btnActionResync) { btnActionResync.addEventListener("click", (e) => { e.stopPropagation(); settingsModal.classList.add("hidden"); + requestIdr(); startStream(); showToast(t("toastResynced")); }); @@ -767,7 +853,7 @@ function sendWheel(deltaY) { } function normalizeWheel(event) { - const vh = videoEl.videoHeight || displayHeight; + const vh = (videoEl && (videoEl.videoHeight || videoEl.height)) || displayHeight; let pixels = event.deltaY; if (event.deltaMode === 1) pixels *= 16; else if (event.deltaMode === 2) pixels *= vh; @@ -998,9 +1084,10 @@ if (keyboardImeInput) { } function mapPointer(event) { - const rect = videoEl.getBoundingClientRect(); - const vw = videoEl.videoWidth || displayWidth; - const vh = videoEl.videoHeight || displayHeight; + const target = videoEl; + const rect = target.getBoundingClientRect(); + const vw = target.videoWidth || target.width || displayWidth; + const vh = target.videoHeight || target.height || displayHeight; const scale = Math.min(rect.width / vw, rect.height / vh); const offsetX = (rect.width - vw * scale) / 2; const offsetY = (rect.height - vh * scale) / 2; @@ -1022,9 +1109,28 @@ function hideTouch() { if (touchIndicator) touchIndicator.classList.add("hidden"); } -function canPlayMpegTs() { - return typeof mpegts !== "undefined" - && mpegts.getFeatureList().mseLivePlayback; +function playbackSupport() { + return { + webTransport: typeof WebTransport === "function", + videoDecoder: typeof VideoDecoder === "function", + annexb: typeof OrbiAnnexB !== "undefined", + }; +} + +function canPlayWebTransport() { + const s = playbackSupport(); + return s.webTransport && s.annexb && s.videoDecoder; +} + +function unsupportedPlayback() { + setOverlayState("unsupported", t("statusUnsupported"), t("statusUnsupportedSub")); +} + +function closeDecoder() { + if (videoDecoder) { + try { videoDecoder.close(); } catch (_) {} + videoDecoder = null; + } } function destroyPlayer() { @@ -1032,15 +1138,15 @@ function destroyPlayer() { clearTimeout(reconnectTimer); reconnectTimer = null; } - if (mpegtsPlayer) { - mpegtsPlayer.destroy(); - mpegtsPlayer = null; + if (wtWriter) { + try { wtWriter.close(); } catch (_) {} + wtWriter = null; } - if (videoEl.src) { - videoEl.pause(); - videoEl.removeAttribute("src"); - videoEl.load(); + if (wtTransport) { + try { wtTransport.close(); } catch (_) {} + wtTransport = null; } + closeDecoder(); streamActive = false; clearInterval(latencyWatchdog); releaseControl(); @@ -1072,95 +1178,343 @@ function scheduleReconnect(reason) { reconnectTimer = null; (async () => { await refreshToken(); - await openDisplaySession(); + const sessionGone = !displaySessionId + || /close|attach|session|no display/i.test(String(reason || "")); + if (sessionGone) { + displaySessionId = ""; + await openDisplaySession(); + } startStream(); })().catch((error) => { console.warn("reconnect attempt failed:", error); + displaySessionId = ""; scheduleReconnect("retry error"); }); }, reconnectDelay); reconnectDelay = Math.min(reconnectDelay * 2, MAX_RECONNECT_DELAY); } -function startStream() { - if (!canPlayMpegTs()) { - setOverlayState("error", "Playback Error", "Browser lacks MSE live video playback support"); +function markPlaying() { + if (!streamActive) { + streamActive = true; + reconnectDelay = 1000; + controlsLockedHidden = false; + if (overlayEl) overlayEl.classList.add("hidden"); + setControlsVisible(false); + } + noteKeyframe(); + lastFrameAt = Date.now(); +} + +function drawVideoFrame(frame) { + try { + if (videoEl.width !== frame.displayWidth || videoEl.height !== frame.displayHeight) { + videoEl.width = frame.displayWidth; + videoEl.height = frame.displayHeight; + displayWidth = frame.displayWidth; + displayHeight = frame.displayHeight; + updateInfoDisplay(); + } + const ctx = videoEl.getContext("2d"); + ctx.drawImage(frame, 0, 0); + markPlaying(); + } finally { + frame.close(); + } +} + +function ensureDecoder(codec) { + if (videoDecoder && videoDecoder.state !== "closed") { + return videoDecoder; + } + closeDecoder(); + videoDecoder = new VideoDecoder({ + output: drawVideoFrame, + error: (err) => { + console.error("VideoDecoder error:", err); + requestIdr(); + scheduleReconnect("decoder"); + }, + }); + videoDecoder.configure({ + codec, + optimizeForLatency: true, + hardwareAcceleration: "prefer-hardware", + }); + return videoDecoder; +} + +function nowNs() { + return BigInt(Date.now()) * 1000000n; +} + +function feedAccessUnit(msg) { + if (typeof VideoDecoder !== "function") { + unsupportedPlayback(); return; } - destroyPlayer(); - setOverlayState("connecting", "Connecting", "Connecting to Linux virtual displayโ€ฆ"); + if (msg.key) { + const found = OrbiAnnexB.extractSpsPps(msg.au); + const codec = OrbiAnnexB.codecStringFromSps(found.sps); + if (codec) { + try { + ensureDecoder(codec); + } catch (err) { + console.error("VideoDecoder configure failed:", err); + scheduleReconnect("codec"); + return; + } + } + noteKeyframe(); + } + if (!videoDecoder || videoDecoder.state !== "configured") { + requestIdr(); + return; + } + if (waitingForKeyframe && !msg.key) { + requestIdr(); + return; + } + const chunk = new EncodedVideoChunk({ + type: msg.key ? "key" : "delta", + timestamp: Number(msg.ptsNs / 1000n), + data: msg.au, + }); + try { + videoDecoder.decode(chunk); + } catch (err) { + console.warn("decode failed:", err); + closeDecoder(); + requestIdr(); + } + if (msg.sentNs > 0n) { + const glass = Number((nowNs() + clockOffsetNs - msg.sentNs) / 1000000n); + if (glass >= 0 && glass <= 5000) { + lastDelayMs = glass; + updateInfoDisplay(); + } + } +} +async function startAuStream() { const params = new URLSearchParams(); if (authToken) params.set("token", authToken); if (displaySessionId) params.set("session", displaySessionId); const qs = params.toString(); - const streamUrl = qs ? `/stream?${qs}` : "/stream"; - - mpegtsPlayer = mpegts.createPlayer({ - type: "mpegts", - isLive: true, - url: streamUrl, - }, { - enableStashBuffer: false, - stashInitialSize: 128, - autoCleanupSourceBuffer: true, - autoCleanupMaxBackwardDuration: 2, - autoCleanupMinBackwardDuration: 1, - lazyLoad: false, - lazyLoadMaxDuration: 0, - seekType: "range", - liveBufferLatencyChasing: true, - liveBufferLatencyMaxLatency: 0.08, - liveBufferLatencyMinRemain: 0.016, - liveSync: true, - liveSyncTargetLatency: 0.03, + const res = await fetch(qs ? `/au?${qs}` : "/au", { + headers: authToken ? { authorization: `Bearer ${authToken}` } : {}, }); + if (!res.ok || !res.body) { + throw new Error(`au stream ${res.status}`); + } + const reader = res.body.getReader(); + const frames = new OrbiAnnexB.FrameReader(); + (async () => { + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + frames.push(value); + let msg; + while ((msg = frames.pop())) { + if (msg.type === "video") feedAccessUnit(msg); + } + } + scheduleReconnect("au ended"); + } catch (err) { + scheduleReconnect(err.message || "au"); + } + })(); + waitingForKeyframe = true; + requestIdr(); + if (latencyWatchdog) clearInterval(latencyWatchdog); + latencyWatchdog = setInterval(() => { + if (waitingForKeyframe) requestIdr(); + if (!streamActive) return; + if (lastFrameAt && Date.now() - lastFrameAt > 750) { + requestIdr({ keepDecoding: true }); + if (Date.now() - lastFrameAt > 3000) startStream({ quiet: true }); + } + }, 250); +} - videoEl.muted = true; - mpegtsPlayer.attachMediaElement(videoEl); +async function startStream(opts = {}) { + if (!window.isSecureContext) { + const host = (wtConfig && OrbiAnnexB.pickWtHost(wtConfig, window.location.hostname)) + || window.location.hostname + || "host"; + const port = (wtConfig && wtConfig.wt_port) || 8790; + setOverlayState( + "error", + "Open the HTTPS client", + `WebTransport needs a secure page. Open https://${host}:${port}/client/ and accept the certificate.`, + ); + return; + } + if (typeof VideoDecoder !== "function") { + unsupportedPlayback(); + return; + } + if (!canPlayWebTransport()) { + const s = playbackSupport(); + const missing = [ + !s.webTransport && "WebTransport", + !s.annexb && "protocol helpers", + ].filter(Boolean).join(" and "); + setOverlayState("error", "Playback Error", `This browser is missing ${missing}`); + return; + } + destroyPlayer(); + lastDelayMs = null; + clockOffsetNs = 0n; + lastFrameAt = 0; + if (!opts.quiet) { + setOverlayState("connecting", "Connecting", "Connecting to Linux virtual displayโ€ฆ"); + } - mpegtsPlayer.on(mpegts.Events.ERROR, (errorType, errorDetail, errorInfo) => { - console.error("mpegts error:", errorType, errorDetail, errorInfo); - sendHostAction("idr"); - if (errorDetail === "NetworkError" && !authToken && tokenRow) { - tokenRow.classList.remove("hidden"); - } - scheduleReconnect(errorDetail || errorType || "network"); - }); + const cfg = wtConfig || {}; + const port = cfg.wt_port; + const path = cfg.wt_path || "/orbiscreen"; + const hashB64 = cfg.cert_sha256; + if (!port || !hashB64) { + setOverlayState("error", "Playback Error", "Host did not advertise WebTransport"); + return; + } - mpegtsPlayer.on(mpegts.Events.MEDIA_INFO, (mediaInfo) => { - if (mediaInfo && mediaInfo.width && mediaInfo.height) { - displayWidth = mediaInfo.width; - displayHeight = mediaInfo.height; - updateInfoDisplay(); + const host = OrbiAnnexB.pickWtHost(cfg, window.location.hostname); + const url = `https://${host}:${port}${path}`; + waitingForKeyframe = true; + + try { + const hash = OrbiAnnexB.hashFromBase64(hashB64); + let transport; + let lastErr; + for (const opts of [ + { serverCertificateHashes: [{ algorithm: "sha-256", value: hash }] }, + {}, + ]) { + let candidate; + try { + candidate = new WebTransport(url, opts); + await Promise.race([ + candidate.ready, + new Promise((_, reject) => { + setTimeout(() => reject(new Error("webtransport ready timeout")), 4000); + }), + ]); + transport = candidate; + lastErr = null; + break; + } catch (err) { + lastErr = err; + try { if (candidate) candidate.close(); } catch (_) {} + } } - }); + if (!transport) { + console.warn("webtransport failed, using HTTPS AU stream:", lastErr); + await startAuStream(); + return; + } + wtTransport = transport; + const bidi = await transport.createBidirectionalStream(); + const writer = bidi.writable.getWriter(); + wtWriter = writer; + await writer.write(OrbiAnnexB.encodeHello(authToken, displaySessionId || "")); + + const reader = bidi.readable.getReader(); + const frames = new OrbiAnnexB.FrameReader(); + (async () => { + try { + while (wtTransport === transport) { + const { value, done } = await reader.read(); + if (done) break; + frames.push(value); + let msg; + while ((msg = frames.pop())) { + if (msg.type === "helloAck") { + if (msg.width) displayWidth = msg.width; + if (msg.height) displayHeight = msg.height; + updateInfoDisplay(); + requestIdr(); + } else if (msg.type === "video") { + feedAccessUnit(msg); + } else if (msg.type === "pong") { + const now = nowNs(); + const rtt = now - msg.t0Ns; + clockOffsetNs = msg.hostNs + rtt / 2n - now; + } + } + } + } catch (err) { + if (wtTransport === transport) { + console.error("webtransport read:", err); + displaySessionId = ""; + scheduleReconnect(err.message || "transport"); + } + } + })(); - mpegtsPlayer.load(); - const playPromise = mpegtsPlayer.play(); - if (playPromise && typeof playPromise.catch === "function") { - playPromise.catch((error) => { - console.warn("autoplay blocked or rejected:", error); - setOverlayState("connecting", "Ready to Stream", "Click anywhere on the screen to begin"); - const onFirstClick = () => { - window.removeEventListener("pointerdown", onFirstClick); - mpegtsPlayer.play().catch(() => {}); - }; - window.addEventListener("pointerdown", onFirstClick); - }); + const assembler = new OrbiAnnexB.DatagramAssembler(); + let dgramReader = null; + try { + dgramReader = transport.datagrams.readable.getReader(); + } catch (err) { + console.warn("webtransport datagrams unavailable:", err); + } + if (dgramReader) { + (async () => { + try { + while (wtTransport === transport) { + const { value, done } = await dgramReader.read(); + if (done) break; + const msg = assembler.push(value); + if (!msg) continue; + if (msg.type === "gap") { + waitingForKeyframe = true; + requestIdr(); + continue; + } + if (msg.type === "video") feedAccessUnit(msg); + } + } catch (err) { + // Video can still arrive on the control stream or /au. + console.warn("webtransport datagram:", err); + } + })(); + } + } catch (err) { + console.error("webtransport connect:", err); + if (!authToken && tokenRow) tokenRow.classList.remove("hidden"); + scheduleReconnect(err.message || "connect"); + return; } + const wtStartedAt = Date.now(); + let auFallback = false; latencyWatchdog = setInterval(() => { - if (!streamActive || !videoEl || !videoEl.buffered || videoEl.buffered.length === 0) return; - try { - const end = videoEl.buffered.end(videoEl.buffered.length - 1); - const delay = end - videoEl.currentTime; - const ms = Math.max(0, Math.round(delay * 1000)); - if (statLatency) statLatency.textContent = `${ms} ms`; - if (delay > 0.08) { - videoEl.currentTime = end - 0.02; + if (waitingForKeyframe) requestIdr(); + if (wtWriter) { + try { wtWriter.write(OrbiAnnexB.encodePing(nowNs())); } catch (_) {} + } + if (!auFallback && !streamActive && Date.now() - wtStartedAt > 3500) { + auFallback = true; + console.warn("webtransport produced no picture; falling back to /au"); + try { if (wtWriter) wtWriter.close(); } catch (_) {} + try { if (wtTransport) wtTransport.close(); } catch (_) {} + wtWriter = null; + wtTransport = null; + startAuStream().catch((err) => { + scheduleReconnect(err.message || "au"); + }); + return; + } + if (!streamActive) return; + if (lastFrameAt && Date.now() - lastFrameAt > 750) { + requestIdr({ keepDecoding: true }); + if (Date.now() - lastFrameAt > 3000) { + startStream({ quiet: true }); } - } catch (_) {} + } }, 250); } @@ -1222,8 +1576,13 @@ async function openDisplaySession() { async function start() { applyTheme(currentTheme); applyTranslations(); + if (typeof VideoDecoder !== "function") { + unsupportedPlayback(); + return; + } const info = await fetchClientConfig(); if (info) { + wtConfig = info; if (!authToken && typeof info.token === "string" && info.token.length > 0) { authToken = info.token; } @@ -1243,6 +1602,7 @@ async function start() { if (typeof apiInfo?.encoder === "string") { encoderName = apiInfo.encoder.toUpperCase(); } + wtConfig = Object.assign({}, wtConfig || {}, apiInfo); } } catch (error) { console.warn("api/info fetch failed:", error); @@ -1253,25 +1613,7 @@ async function start() { startStream(); } -videoEl.addEventListener("playing", () => { - streamActive = true; - reconnectDelay = 1000; - if (overlayEl) overlayEl.classList.add("hidden"); - if (controlToolbar) controlToolbar.classList.remove("hidden"); - if (miniPill) miniPill.classList.add("hidden"); -}); -videoEl.addEventListener("error", () => { - if (streamActive || mpegtsPlayer) { - scheduleReconnect("media error"); - } -}); - -videoEl.addEventListener("ended", () => { - if (streamActive || mpegtsPlayer) { - scheduleReconnect("stream ended"); - } -}); start().catch((error) => { setOverlayState("error", "Initialization Failed", error.message); diff --git a/clients/web/index.html b/clients/web/index.html index e6d5a1ff..294d0733 100644 --- a/clients/web/index.html +++ b/clients/web/index.html @@ -4,7 +4,7 @@ + content="default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline'; img-src 'self' blob: data:; media-src 'self' blob: data:; connect-src 'self' https:;" /> @@ -82,15 +82,11 @@ - +