feat(webrtc): accept keyboard and mouse on an input DataChannel

src/webrtc/input_bridge.cpp

@@ -188,6 +188,40 @@ namespace webrtc_stream {
     }
   }  // namespace
 
+  bool is_keyboard_mouse_packet(std::span<const std::byte> packet) {
+    NV_INPUT_HEADER header {};
+    if (packet.size() < sizeof(header)) {
+      return false;
+    }
+    std::memcpy(&header, packet.data(), sizeof(header));
+    std::size_t expected_size = 0;
+    switch (util::endian::little(header.magic)) {
+      case KEY_DOWN_EVENT_MAGIC:
+      case KEY_UP_EVENT_MAGIC:
+        expected_size = sizeof(NV_KEYBOARD_PACKET);
+        break;
+      case MOUSE_MOVE_REL_MAGIC_GEN5:
+        expected_size = sizeof(NV_REL_MOUSE_MOVE_PACKET);
+        break;
+      case MOUSE_MOVE_ABS_MAGIC:
+        expected_size = sizeof(NV_ABS_MOUSE_MOVE_PACKET);
+        break;
+      case MOUSE_BUTTON_DOWN_EVENT_MAGIC_GEN5:
+      case MOUSE_BUTTON_UP_EVENT_MAGIC_GEN5:
+        expected_size = sizeof(NV_MOUSE_BUTTON_PACKET);
+        break;
+      case SCROLL_MAGIC_GEN5:
+        expected_size = sizeof(NV_SCROLL_PACKET);
+        break;
+      case SS_HSCROLL_MAGIC:
+        expected_size = sizeof(SS_HSCROLL_PACKET);
+        break;
+      default:
+        return false;
+    }
+    return packet.size() == expected_size && util::endian::big(header.size) == expected_size - sizeof(header.size);
+  }
+
   input_api_t make_packet_input_api(std::function<void(std::vector<std::uint8_t> &&)> sink) {
     input_api_t api;
     api.controller_arrival = [sink](std::uint8_t slot, std::uint16_t, std::uint8_t type, std::uint32_t supported_buttons, std::uint16_t capabilities) {
@@ -238,13 +272,14 @@ namespace webrtc_stream {
       packet.scrollAmount = util::endian::big(static_cast<std::int16_t>(clicks * WHEEL_DELTA_UNITS));
       sink(serialize(packet, SS_HSCROLL_MAGIC));
     };
+    api.keyboard_mouse_packet = sink;
     return api;
   }
 
   input_bridge_t::input_bridge_t(input_api_t api, control_sender_t control_sender):
       _api(std::move(api)),
       _control_sender(std::move(control_sender)) {
-    if (!_api.controller_arrival || !_api.controller_state || !_api.mouse_move || !_api.mouse_button || !_api.scroll || !_api.horizontal_scroll) {
+    if (!_api.controller_arrival || !_api.controller_state || !_api.mouse_move || !_api.mouse_button || !_api.scroll || !_api.horizontal_scroll || !_api.keyboard_mouse_packet) {
       throw std::invalid_argument("Every input callback is required");
     }
     _worker = std::jthread([this](std::stop_token stop_token) {
@@ -442,6 +477,21 @@ namespace webrtc_stream {
     return true;
   }
 
+  bool input_bridge_t::handle_keyboard_mouse_packet(std::span<const std::byte> packet) {
+    if (!is_keyboard_mouse_packet(packet)) {
+      return false;
+    }
+    std::vector<std::uint8_t> bytes(packet.size());
+    std::memcpy(bytes.data(), packet.data(), packet.size());
+    // Sent under the lock, like controller packets, so nothing reaches a stream that has stopped.
+    std::lock_guard lock(_mutex);
+    if (_shutdown || !_stream_active) {
+      return false;
+    }
+    _api.keyboard_mouse_packet(std::move(bytes));
+    return true;
+  }
+
   void input_bridge_t::handle_rumble(std::uint16_t controller_number, std::uint16_t low_frequency_motor, std::uint16_t high_frequency_motor) {
     std::lock_guard lock(_mutex);
     if (_shutdown || controller_number >= _slot_to_controller.size() || !_slot_to_controller[controller_number]) {

src/webrtc/input_bridge.h

@@ -13,6 +13,7 @@
 #include <functional>
 #include <mutex>
 #include <optional>
+#include <span>
 #include <string>
 #include <string_view>
 #include <thread>
@@ -72,8 +73,20 @@ namespace webrtc_stream {
     std::function<void(bool, std::uint8_t)> mouse_button;  ///< Press flag and Moonlight button.
     std::function<void(std::int8_t)> scroll;  ///< Vertical wheel clicks.
     std::function<void(std::int8_t)> horizontal_scroll;  ///< Horizontal wheel clicks.
+    std::function<void(std::vector<std::uint8_t> &&)> keyboard_mouse_packet;  ///< Validated keyboard or mouse packet from the client.
   };
 
+  /**
+   * @brief Check a keyboard or mouse packet sent by a browser on the input DataChannel.
+   *
+   * Only Moonlight keyboard, mouse motion, mouse button and scroll packets of their exact size are
+   * accepted, so a client cannot inject controller packets that bypass the gamepad bridge.
+   *
+   * @param packet Serialized Moonlight input packet, header included.
+   * @return True when the packet may be passed to Sunshine.
+   */
+  bool is_keyboard_mouse_packet(std::span<const std::byte> packet);
+
   /**
    * @brief Build an input API that feeds Moonlight input packets to a sink.
    * @param sink Receives each serialized Moonlight input packet.
@@ -122,6 +135,13 @@ namespace webrtc_stream {
      */
     bool handle_gamepad_message(std::string_view text, clock::time_point received_at = clock::now());
 
+    /**
+     * @brief Handle a keyboard or mouse packet from the input DataChannel.
+     * @param packet Serialized Moonlight input packet, header included.
+     * @return True when the packet was valid and forwarded to the host.
+     */
+    bool handle_keyboard_mouse_packet(std::span<const std::byte> packet);
+
     /**
      * @brief Start or stop forwarding input to the host.
      * @param active Whether the host stream is running.

src/webrtc/webrtc_stream.cpp

@@ -378,6 +378,7 @@ namespace webrtc_stream {
       std::shared_ptr<rtc::Track> audio_track;  ///< Send-only audio track.
       std::shared_ptr<rtc::DataChannel> control_channel;  ///< Ordered, reliable control channel.
       std::shared_ptr<rtc::DataChannel> gamepad_channel;  ///< Unordered, unreliable gamepad snapshots.
+      std::shared_ptr<rtc::DataChannel> input_channel;  ///< Ordered, reliable keyboard and mouse packets.
       std::shared_ptr<rtc::RtpPacketizationConfig> video_rtp;  ///< Video RTP state.
       std::shared_ptr<rtc::RtpPacketizationConfig> audio_rtp;  ///< Audio RTP state.
       std::shared_ptr<input_bridge_t> input_bridge;  ///< Gamepad translation for the whole stream.
@@ -1386,6 +1387,18 @@ namespace webrtc_stream {
           }
         });
 
+        // Keyboard and mouse travel as Moonlight input packets. Clients that do not know this
+        // channel close it, which leaves gamepad input untouched.
+        rtc::DataChannelInit input_options;
+        input_options.reliability.unordered = false;
+        stream->input_channel = stream->peer->createDataChannel("input", input_options);
+        stream->input_channel->onMessage([weak_stream](rtc::message_variant data) {
+          const auto stream = weak_stream.lock();
+          if (stream && std::holds_alternative<rtc::binary>(data) && !stream->input_bridge->handle_keyboard_mouse_packet(std::get<rtc::binary>(data))) {
+            BOOST_LOG(verbose) << "WebRTC: dropped keyboard or mouse packet"sv;
+          }
+        });
+
         rtc::Description::Video video("video", rtc::Description::Direction::SendOnly);
         switch (settings.codec) {
           case protocol::video_codec_e::hevc:

tests/unit/webrtc/test_webrtc_input_bridge.cpp

@@ -50,6 +50,7 @@ namespace {
     std::vector<state_t> states;
     std::vector<std::pair<bool, std::uint8_t>> mouse_buttons;
     std::vector<std::string> control_messages;
+    std::vector<std::vector<std::uint8_t>> keyboard_mouse_packets;
 
     input_api_t api() {
       input_api_t api;
@@ -71,6 +72,10 @@ namespace {
       };
       api.horizontal_scroll = [](std::int8_t) {
       };
+      api.keyboard_mouse_packet = [this](std::vector<std::uint8_t> &&packet) {
+        std::lock_guard lock(mutex);
+        keyboard_mouse_packets.push_back(std::move(packet));
+      };
       return api;
     }
 
@@ -102,6 +107,21 @@ namespace {
       .dump();
   }
 
+  /**
+   * @brief Build a Moonlight packet the way the browser client does.
+   * @param packet Packet whose header is completed.
+   * @param magic Packet type.
+   * @return The serialized packet.
+   */
+  template<class T>
+  std::vector<std::byte> client_packet(T packet, std::uint32_t magic) {
+    packet.header.size = util::endian::big<std::uint32_t>(sizeof(T) - sizeof(std::uint32_t));
+    packet.header.magic = util::endian::little(magic);
+    std::vector<std::byte> bytes(sizeof(T));
+    std::memcpy(bytes.data(), &packet, sizeof(T));
+    return bytes;
+  }
+
   template<class T>
   T packet_as(const std::vector<std::uint8_t> &bytes) {
     T packet;
@@ -156,6 +176,53 @@ TEST(WebrtcInputBridgeTest, BuildsPacketsSunshineAccepts) {
   EXPECT_EQ(util::endian::big(packet_as<SS_HSCROLL_PACKET>(packets[5]).scrollAmount), 120);
 }
 
+TEST(WebrtcInputBridgeTest, AcceptsOnlyKeyboardAndMousePackets) {
+  NV_KEYBOARD_PACKET keyboard {};
+  keyboard.keyCode = util::endian::little<std::int16_t>(0x41);
+  EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(keyboard, KEY_DOWN_EVENT_MAGIC)));
+  EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(keyboard, KEY_UP_EVENT_MAGIC)));
+  EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(NV_REL_MOUSE_MOVE_PACKET {}, MOUSE_MOVE_REL_MAGIC_GEN5)));
+  EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(NV_ABS_MOUSE_MOVE_PACKET {}, MOUSE_MOVE_ABS_MAGIC)));
+  EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(NV_MOUSE_BUTTON_PACKET {}, MOUSE_BUTTON_DOWN_EVENT_MAGIC_GEN5)));
+  EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(NV_MOUSE_BUTTON_PACKET {}, MOUSE_BUTTON_UP_EVENT_MAGIC_GEN5)));
+  EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(NV_SCROLL_PACKET {}, SCROLL_MAGIC_GEN5)));
+  EXPECT_TRUE(is_keyboard_mouse_packet(client_packet(SS_HSCROLL_PACKET {}, SS_HSCROLL_MAGIC)));
+
+  // Controller packets must go through the gamepad bridge, which owns the controller slots.
+  EXPECT_FALSE(is_keyboard_mouse_packet(client_packet(NV_MULTI_CONTROLLER_PACKET {}, MULTI_CONTROLLER_MAGIC_GEN5)));
+  EXPECT_FALSE(is_keyboard_mouse_packet(client_packet(SS_CONTROLLER_ARRIVAL_PACKET {}, SS_CONTROLLER_ARRIVAL_MAGIC)));
+
+  // A known type with the wrong size, a short packet, or a header that disagrees with the size.
+  EXPECT_FALSE(is_keyboard_mouse_packet(client_packet(NV_SCROLL_PACKET {}, MOUSE_MOVE_REL_MAGIC_GEN5)));
+  auto truncated = client_packet(NV_REL_MOUSE_MOVE_PACKET {}, MOUSE_MOVE_REL_MAGIC_GEN5);
+  EXPECT_FALSE(is_keyboard_mouse_packet(std::span(truncated).first(4)));
+  truncated.pop_back();
+  EXPECT_FALSE(is_keyboard_mouse_packet(truncated));
+  auto wrong_header = client_packet(NV_REL_MOUSE_MOVE_PACKET {}, MOUSE_MOVE_REL_MAGIC_GEN5);
+  wrong_header[3] = std::byte {0x7F};
+  EXPECT_FALSE(is_keyboard_mouse_packet(wrong_header));
+}
+
+TEST(WebrtcInputBridgeTest, ForwardsKeyboardAndMousePacketsWhileTheStreamIsActive) {
+  recorder_t recorder;
+  input_bridge_t bridge(recorder.api(), recorder.sender());
+  NV_REL_MOUSE_MOVE_PACKET move {};
+  move.deltaX = util::endian::big<std::int16_t>(-7);
+  const auto packet = client_packet(move, MOUSE_MOVE_REL_MAGIC_GEN5);
+  EXPECT_FALSE(bridge.handle_keyboard_mouse_packet(packet));
+
+  bridge.set_stream_active(true);
+  EXPECT_TRUE(bridge.handle_keyboard_mouse_packet(packet));
+  EXPECT_FALSE(bridge.handle_keyboard_mouse_packet(client_packet(NV_MULTI_CONTROLLER_PACKET {}, MULTI_CONTROLLER_MAGIC_GEN5)));
+  ASSERT_EQ(recorder.keyboard_mouse_packets.size(), 1);
+  EXPECT_TRUE(input::testing::is_valid_input_packet(recorder.keyboard_mouse_packets[0]));
+  EXPECT_EQ(util::endian::big(packet_as<NV_REL_MOUSE_MOVE_PACKET>(recorder.keyboard_mouse_packets[0]).deltaX), -7);
+
+  bridge.shutdown();
+  EXPECT_FALSE(bridge.handle_keyboard_mouse_packet(packet));
+  EXPECT_EQ(recorder.keyboard_mouse_packets.size(), 1);
+}
+
 TEST(WebrtcInputBridgeTest, AnnouncesControllersOnceTheStreamIsActive) {
   recorder_t recorder;
   input_bridge_t bridge(recorder.api(), recorder.sender());

tests/webrtc_e2e/README.md

@@ -4,7 +4,8 @@
 `client.html` in headless Edge, which plays the TV: it asks to pair showing a PIN, which the driver
 approves through `/api/pin` like a user in the Web UI would (after a wrong PIN and a cancellation); the
 page then reconnects and authenticates, lists the applications, fetches
-artwork, starts a stream, announces a gamepad, measures the decoded video and audio through `getStats()`,
+artwork, starts a stream, announces a gamepad, sends keyboard and mouse input that changes nothing
+(F15 and a pointer move it undoes) on the `input` channel, measures the decoded video and audio through `getStats()`,
 and finally stops the stream and the application. The driver prints the steps and statistics.
 
 ```sh

tests/webrtc_e2e/client.html

@@ -31,7 +31,7 @@
 }
 setTimeout(() => { result.errors.push("timeout"); finish(); }, 60000 + latencySeconds * 1000);
 
-let ws, pc, sessionId, control, gamepad, phase = "auth", credentials = null, appId = null;
+let ws, pc, sessionId, control, gamepad, input, phase = "auth", credentials = null, appId = null;
 const send = (m) => { log.push(">> " + m.type); ws.send(JSON.stringify(Object.assign({ version: 2 }, m))); };
 
 async function hmacHex(secretHex, text) {
@@ -136,6 +136,9 @@
       } else if (channel.label === "gamepad") {
         gamepad = channel;
         channel.onopen = () => note("gamepad-open");
+      } else if (channel.label === "input") {
+        input = channel;
+        channel.onopen = () => note("input-open");
       }
     };
     pc.onconnectionstatechange = () => note("pc-" + pc.connectionState);
@@ -243,7 +246,27 @@
   send({ type: "stop-session" });
 }
 
+// A Moonlight input packet as a browser sends it on the input channel: big-endian size, then
+// little-endian type, then the payload the callback fills in.
+function inputPacket(magic, payloadSize, fill) {
+  const view = new DataView(new ArrayBuffer(8 + payloadSize));
+  view.setUint32(0, 4 + payloadSize, false);
+  view.setUint32(4, magic, true);
+  fill(view);
+  return view.buffer;
+}
+
 async function measure() {
+  // Keyboard and mouse that change nothing on the desktop: F15, which no application uses, and a
+  // pointer move that is undone at once. A controller packet on this channel must be dropped.
+  if (input && input.readyState === "open") {
+    const key = (magic) => inputPacket(magic, 6, (v) => v.setInt16(9, 0x7e, true));
+    const move = (dx, dy) => inputPacket(0x07, 4, (v) => { v.setInt16(8, dx, false); v.setInt16(10, dy, false); });
+    for (const packet of [key(0x03), key(0x04), move(40, 30), move(-40, -30), inputPacket(0x0c, 22, () => {})]) {
+      input.send(packet);
+    }
+    note("input-sent");
+  }
   // Neutral gamepad snapshots exercise the unreliable channel without pressing anything.
   for (let seq = 1; seq <= 20; seq++) {
     if (gamepad && gamepad.readyState === "open") {