diff --git a/src/webrtc/input_bridge.cpp b/src/webrtc/input_bridge.cpp index 493ffb2f4e4..39eaa21c03c 100644 --- a/src/webrtc/input_bridge.cpp +++ b/src/webrtc/input_bridge.cpp @@ -188,6 +188,40 @@ namespace webrtc_stream { } } // namespace + bool is_keyboard_mouse_packet(std::span packet) { + NV_INPUT_HEADER header {}; + if (packet.size() < sizeof(header)) { + return false; + } + std::memcpy(&header, packet.data(), sizeof(header)); + std::size_t expected_size = 0; + switch (util::endian::little(header.magic)) { + case KEY_DOWN_EVENT_MAGIC: + case KEY_UP_EVENT_MAGIC: + expected_size = sizeof(NV_KEYBOARD_PACKET); + break; + case MOUSE_MOVE_REL_MAGIC_GEN5: + expected_size = sizeof(NV_REL_MOUSE_MOVE_PACKET); + break; + case MOUSE_MOVE_ABS_MAGIC: + expected_size = sizeof(NV_ABS_MOUSE_MOVE_PACKET); + break; + case MOUSE_BUTTON_DOWN_EVENT_MAGIC_GEN5: + case MOUSE_BUTTON_UP_EVENT_MAGIC_GEN5: + expected_size = sizeof(NV_MOUSE_BUTTON_PACKET); + break; + case SCROLL_MAGIC_GEN5: + expected_size = sizeof(NV_SCROLL_PACKET); + break; + case SS_HSCROLL_MAGIC: + expected_size = sizeof(SS_HSCROLL_PACKET); + break; + default: + return false; + } + return packet.size() == expected_size && util::endian::big(header.size) == expected_size - sizeof(header.size); + } + input_api_t make_packet_input_api(std::function &&)> sink) { input_api_t api; api.controller_arrival = [sink](std::uint8_t slot, std::uint16_t, std::uint8_t type, std::uint32_t supported_buttons, std::uint16_t capabilities) { @@ -238,13 +272,14 @@ namespace webrtc_stream { packet.scrollAmount = util::endian::big(static_cast(clicks * WHEEL_DELTA_UNITS)); sink(serialize(packet, SS_HSCROLL_MAGIC)); }; + api.keyboard_mouse_packet = sink; return api; } input_bridge_t::input_bridge_t(input_api_t api, control_sender_t control_sender): _api(std::move(api)), _control_sender(std::move(control_sender)) { - if (!_api.controller_arrival || !_api.controller_state || !_api.mouse_move || !_api.mouse_button || !_api.scroll || !_api.horizontal_scroll) { + if (!_api.controller_arrival || !_api.controller_state || !_api.mouse_move || !_api.mouse_button || !_api.scroll || !_api.horizontal_scroll || !_api.keyboard_mouse_packet) { throw std::invalid_argument("Every input callback is required"); } _worker = std::jthread([this](std::stop_token stop_token) { @@ -442,6 +477,21 @@ namespace webrtc_stream { return true; } + bool input_bridge_t::handle_keyboard_mouse_packet(std::span packet) { + if (!is_keyboard_mouse_packet(packet)) { + return false; + } + std::vector bytes(packet.size()); + std::memcpy(bytes.data(), packet.data(), packet.size()); + // Sent under the lock, like controller packets, so nothing reaches a stream that has stopped. + std::lock_guard lock(_mutex); + if (_shutdown || !_stream_active) { + return false; + } + _api.keyboard_mouse_packet(std::move(bytes)); + return true; + } + void input_bridge_t::handle_rumble(std::uint16_t controller_number, std::uint16_t low_frequency_motor, std::uint16_t high_frequency_motor) { std::lock_guard lock(_mutex); if (_shutdown || controller_number >= _slot_to_controller.size() || !_slot_to_controller[controller_number]) { diff --git a/src/webrtc/input_bridge.h b/src/webrtc/input_bridge.h index 6acae118fd9..00772485a5a 100644 --- a/src/webrtc/input_bridge.h +++ b/src/webrtc/input_bridge.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -72,8 +73,20 @@ namespace webrtc_stream { std::function mouse_button; ///< Press flag and Moonlight button. std::function scroll; ///< Vertical wheel clicks. std::function horizontal_scroll; ///< Horizontal wheel clicks. + std::function &&)> keyboard_mouse_packet; ///< Validated keyboard or mouse packet from the client. }; + /** + * @brief Check a keyboard or mouse packet sent by a browser on the input DataChannel. + * + * Only Moonlight keyboard, mouse motion, mouse button and scroll packets of their exact size are + * accepted, so a client cannot inject controller packets that bypass the gamepad bridge. + * + * @param packet Serialized Moonlight input packet, header included. + * @return True when the packet may be passed to Sunshine. + */ + bool is_keyboard_mouse_packet(std::span packet); + /** * @brief Build an input API that feeds Moonlight input packets to a sink. * @param sink Receives each serialized Moonlight input packet. @@ -122,6 +135,13 @@ namespace webrtc_stream { */ bool handle_gamepad_message(std::string_view text, clock::time_point received_at = clock::now()); + /** + * @brief Handle a keyboard or mouse packet from the input DataChannel. + * @param packet Serialized Moonlight input packet, header included. + * @return True when the packet was valid and forwarded to the host. + */ + bool handle_keyboard_mouse_packet(std::span packet); + /** * @brief Start or stop forwarding input to the host. * @param active Whether the host stream is running. diff --git a/src/webrtc/webrtc_stream.cpp b/src/webrtc/webrtc_stream.cpp index 7b47305d613..bdcd83c34c1 100644 --- a/src/webrtc/webrtc_stream.cpp +++ b/src/webrtc/webrtc_stream.cpp @@ -378,6 +378,7 @@ namespace webrtc_stream { std::shared_ptr audio_track; ///< Send-only audio track. std::shared_ptr control_channel; ///< Ordered, reliable control channel. std::shared_ptr gamepad_channel; ///< Unordered, unreliable gamepad snapshots. + std::shared_ptr input_channel; ///< Ordered, reliable keyboard and mouse packets. std::shared_ptr video_rtp; ///< Video RTP state. std::shared_ptr audio_rtp; ///< Audio RTP state. std::shared_ptr input_bridge; ///< Gamepad translation for the whole stream. @@ -1386,6 +1387,18 @@ namespace webrtc_stream { } }); + // Keyboard and mouse travel as Moonlight input packets. Clients that do not know this + // channel close it, which leaves gamepad input untouched. + rtc::DataChannelInit input_options; + input_options.reliability.unordered = false; + stream->input_channel = stream->peer->createDataChannel("input", input_options); + stream->input_channel->onMessage([weak_stream](rtc::message_variant data) { + const auto stream = weak_stream.lock(); + if (stream && std::holds_alternative(data) && !stream->input_bridge->handle_keyboard_mouse_packet(std::get(data))) { + BOOST_LOG(verbose) << "WebRTC: dropped keyboard or mouse packet"sv; + } + }); + rtc::Description::Video video("video", rtc::Description::Direction::SendOnly); switch (settings.codec) { case protocol::video_codec_e::hevc: diff --git a/tests/unit/webrtc/test_webrtc_input_bridge.cpp b/tests/unit/webrtc/test_webrtc_input_bridge.cpp index d4d9826968c..3e348266c5a 100644 --- a/tests/unit/webrtc/test_webrtc_input_bridge.cpp +++ b/tests/unit/webrtc/test_webrtc_input_bridge.cpp @@ -50,6 +50,7 @@ namespace { std::vector states; std::vector> mouse_buttons; std::vector control_messages; + std::vector> keyboard_mouse_packets; input_api_t api() { input_api_t api; @@ -71,6 +72,10 @@ namespace { }; api.horizontal_scroll = [](std::int8_t) { }; + api.keyboard_mouse_packet = [this](std::vector &&packet) { + std::lock_guard lock(mutex); + keyboard_mouse_packets.push_back(std::move(packet)); + }; return api; } @@ -102,6 +107,21 @@ namespace { .dump(); } + /** + * @brief Build a Moonlight packet the way the browser client does. + * @param packet Packet whose header is completed. + * @param magic Packet type. + * @return The serialized packet. + */ + template + std::vector client_packet(T packet, std::uint32_t magic) { + packet.header.size = util::endian::big(sizeof(T) - sizeof(std::uint32_t)); + packet.header.magic = util::endian::little(magic); + std::vector bytes(sizeof(T)); + std::memcpy(bytes.data(), &packet, sizeof(T)); + return bytes; + } + template T packet_as(const std::vector &bytes) { T packet; @@ -156,6 +176,53 @@ TEST(WebrtcInputBridgeTest, BuildsPacketsSunshineAccepts) { EXPECT_EQ(util::endian::big(packet_as(packets[5]).scrollAmount), 120); } +TEST(WebrtcInputBridgeTest, AcceptsOnlyKeyboardAndMousePackets) { + NV_KEYBOARD_PACKET keyboard {}; + keyboard.keyCode = util::endian::little(0x41); + EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(keyboard, KEY_DOWN_EVENT_MAGIC))); + EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(keyboard, KEY_UP_EVENT_MAGIC))); + EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(NV_REL_MOUSE_MOVE_PACKET {}, MOUSE_MOVE_REL_MAGIC_GEN5))); + EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(NV_ABS_MOUSE_MOVE_PACKET {}, MOUSE_MOVE_ABS_MAGIC))); + EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(NV_MOUSE_BUTTON_PACKET {}, MOUSE_BUTTON_DOWN_EVENT_MAGIC_GEN5))); + EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(NV_MOUSE_BUTTON_PACKET {}, MOUSE_BUTTON_UP_EVENT_MAGIC_GEN5))); + EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(NV_SCROLL_PACKET {}, SCROLL_MAGIC_GEN5))); + EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(SS_HSCROLL_PACKET {}, SS_HSCROLL_MAGIC))); + + // Controller packets must go through the gamepad bridge, which owns the controller slots. + EXPECT_FALSE(is_keyboard_mouse_packet(client_packet(NV_MULTI_CONTROLLER_PACKET {}, MULTI_CONTROLLER_MAGIC_GEN5))); + EXPECT_FALSE(is_keyboard_mouse_packet(client_packet(SS_CONTROLLER_ARRIVAL_PACKET {}, SS_CONTROLLER_ARRIVAL_MAGIC))); + + // A known type with the wrong size, a short packet, or a header that disagrees with the size. + EXPECT_FALSE(is_keyboard_mouse_packet(client_packet(NV_SCROLL_PACKET {}, MOUSE_MOVE_REL_MAGIC_GEN5))); + auto truncated = client_packet(NV_REL_MOUSE_MOVE_PACKET {}, MOUSE_MOVE_REL_MAGIC_GEN5); + EXPECT_FALSE(is_keyboard_mouse_packet(std::span(truncated).first(4))); + truncated.pop_back(); + EXPECT_FALSE(is_keyboard_mouse_packet(truncated)); + auto wrong_header = client_packet(NV_REL_MOUSE_MOVE_PACKET {}, MOUSE_MOVE_REL_MAGIC_GEN5); + wrong_header[3] = std::byte {0x7F}; + EXPECT_FALSE(is_keyboard_mouse_packet(wrong_header)); +} + +TEST(WebrtcInputBridgeTest, ForwardsKeyboardAndMousePacketsWhileTheStreamIsActive) { + recorder_t recorder; + input_bridge_t bridge(recorder.api(), recorder.sender()); + NV_REL_MOUSE_MOVE_PACKET move {}; + move.deltaX = util::endian::big(-7); + const auto packet = client_packet(move, MOUSE_MOVE_REL_MAGIC_GEN5); + EXPECT_FALSE(bridge.handle_keyboard_mouse_packet(packet)); + + bridge.set_stream_active(true); + EXPECT_TRUE(bridge.handle_keyboard_mouse_packet(packet)); + EXPECT_FALSE(bridge.handle_keyboard_mouse_packet(client_packet(NV_MULTI_CONTROLLER_PACKET {}, MULTI_CONTROLLER_MAGIC_GEN5))); + ASSERT_EQ(recorder.keyboard_mouse_packets.size(), 1); + EXPECT_TRUE(input::testing::is_valid_input_packet(recorder.keyboard_mouse_packets[0])); + EXPECT_EQ(util::endian::big(packet_as(recorder.keyboard_mouse_packets[0]).deltaX), -7); + + bridge.shutdown(); + EXPECT_FALSE(bridge.handle_keyboard_mouse_packet(packet)); + EXPECT_EQ(recorder.keyboard_mouse_packets.size(), 1); +} + TEST(WebrtcInputBridgeTest, AnnouncesControllersOnceTheStreamIsActive) { recorder_t recorder; input_bridge_t bridge(recorder.api(), recorder.sender()); diff --git a/tests/webrtc_e2e/README.md b/tests/webrtc_e2e/README.md index d49fdbed7a7..78c619911b4 100644 --- a/tests/webrtc_e2e/README.md +++ b/tests/webrtc_e2e/README.md @@ -4,7 +4,8 @@ `client.html` in headless Edge, which plays the TV: it asks to pair showing a PIN, which the driver approves through `/api/pin` like a user in the Web UI would (after a wrong PIN and a cancellation); the page then reconnects and authenticates, lists the applications, fetches -artwork, starts a stream, announces a gamepad, measures the decoded video and audio through `getStats()`, +artwork, starts a stream, announces a gamepad, sends keyboard and mouse input that changes nothing +(F15 and a pointer move it undoes) on the `input` channel, measures the decoded video and audio through `getStats()`, and finally stops the stream and the application. The driver prints the steps and statistics. ```sh diff --git a/tests/webrtc_e2e/client.html b/tests/webrtc_e2e/client.html index c1482af8a0d..7565e90f1bd 100644 --- a/tests/webrtc_e2e/client.html +++ b/tests/webrtc_e2e/client.html @@ -31,7 +31,7 @@ } setTimeout(() => { result.errors.push("timeout"); finish(); }, 60000 + latencySeconds * 1000); -let ws, pc, sessionId, control, gamepad, phase = "auth", credentials = null, appId = null; +let ws, pc, sessionId, control, gamepad, input, phase = "auth", credentials = null, appId = null; const send = (m) => { log.push(">> " + m.type); ws.send(JSON.stringify(Object.assign({ version: 2 }, m))); }; async function hmacHex(secretHex, text) { @@ -136,6 +136,9 @@ } else if (channel.label === "gamepad") { gamepad = channel; channel.onopen = () => note("gamepad-open"); + } else if (channel.label === "input") { + input = channel; + channel.onopen = () => note("input-open"); } }; pc.onconnectionstatechange = () => note("pc-" + pc.connectionState); @@ -243,7 +246,27 @@ send({ type: "stop-session" }); } +// A Moonlight input packet as a browser sends it on the input channel: big-endian size, then +// little-endian type, then the payload the callback fills in. +function inputPacket(magic, payloadSize, fill) { + const view = new DataView(new ArrayBuffer(8 + payloadSize)); + view.setUint32(0, 4 + payloadSize, false); + view.setUint32(4, magic, true); + fill(view); + return view.buffer; +} + async function measure() { + // Keyboard and mouse that change nothing on the desktop: F15, which no application uses, and a + // pointer move that is undone at once. A controller packet on this channel must be dropped. + if (input && input.readyState === "open") { + const key = (magic) => inputPacket(magic, 6, (v) => v.setInt16(9, 0x7e, true)); + const move = (dx, dy) => inputPacket(0x07, 4, (v) => { v.setInt16(8, dx, false); v.setInt16(10, dy, false); }); + for (const packet of [key(0x03), key(0x04), move(40, 30), move(-40, -30), inputPacket(0x0c, 22, () => {})]) { + input.send(packet); + } + note("input-sent"); + } // Neutral gamepad snapshots exercise the unreliable channel without pressing anything. for (let seq = 1; seq <= 20; seq++) { if (gamepad && gamepad.readyState === "open") {