From f514980f440cda9db043608eb068e4d522e48862 Mon Sep 17 00:00:00 2001 From: raiseCatError <315733358+raiseCatError@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:29:35 +0530 Subject: [PATCH 1/3] feat(mac): accept a whole keyboard, key repeat and fine scrolling from receivers A receiving Mac's physical keyboard names keys the sender could not inject: add HID usages for the grave key, F1-F20, Help/Insert, Home, End, Page Up/Down, the ISO and JIS keys and the keypad. Older senders ignore them, as an unrecognized usage must be. A keyboard down may now carry repeat: true for the receiver's own key repeat; the sender posts an auto-repeat key-down only while that key is held. An older sender ignores the duplicate down, as before. Scroll deltas carry their rounding remainder into the next event, so slow trackpad scrolls (fractions of a point per event) no longer round away. A single delta still rounds to nearest, as before. Refs #12 --- Mac/InputInjector.swift | 29 +++++--- Mac/InputRouting.swift | 132 ++++++++++++++++++++++++++++++++++++ Mac/MacSender.swift | 3 +- Shared/StreamReceiver.swift | 12 +++- 4 files changed, 166 insertions(+), 10 deletions(-) diff --git a/Mac/InputInjector.swift b/Mac/InputInjector.swift index 0e4b9b8fb..61a9b0206 100644 --- a/Mac/InputInjector.swift +++ b/Mac/InputInjector.swift @@ -78,6 +78,7 @@ final class InputInjector { // Hardware keys currently held down (M4) — released on cancellation so a // dropped session can never leave a modifier or arrow key stuck. private var heldKeys = HeldKeyTracker() + private var scrollRemainder = ScrollRemainder() private var heldModifiers = HeldModifierTracker() init(displayID: CGDirectDisplayID, sessionInputGrant: SessionInputGrantBox) { @@ -96,6 +97,7 @@ final class InputInjector { /// Caller must already hold inputLock. private func cancelActiveInputLocked() { let point = currentCursor() + scrollRemainder.reset() if isDown { if let event = CGEvent( @@ -220,10 +222,12 @@ final class InputInjector { guard inputIsAllowed() else { return } let bounds = CGDisplayBounds(displayID) let scale = bounds.width > 0 ? Double(CGDisplayPixelsWide(displayID)) / bounds.width : 2 + let whole = scrollRemainder.take(dxPoints: dx / scale, dyPoints: dy / scale) + guard whole.dx != 0 || whole.dy != 0 else { return } guard let event = CGEvent(scrollWheelEvent2Source: source, units: .pixel, wheelCount: 2, - wheel1: Int32((dy / scale).rounded()), - wheel2: Int32((dx / scale).rounded()), + wheel1: whole.dy, + wheel2: whole.dx, wheel3: 0) else { return } event.flags = heldModifiers.flags // see `MouseEventFlags`'s doc event.post(tap: .cghidEventTap) @@ -380,13 +384,21 @@ final class InputInjector { } /// A hardware key going down. A duplicate down for an already-held key - /// is ignored rather than re-posted. - func handleKeyboardDown(usage: HIDKeyUsage, modifiers: [String]) { + /// is ignored rather than re-posted — unless it is marked `repeat` (a + /// receiving Mac's own key repeat), which posts an auto-repeat key-down + /// for the held key. + func handleKeyboardDown(usage: HIDKeyUsage, modifiers: [String], isRepeat: Bool = false) { inputLock.lock() defer { inputLock.unlock() } - guard inputIsAllowed(), heldKeys.down(usage) else { return } - postKeyEvent(keyCode: usage.keyCode, keyDown: true, - flags: KeyModifier.flags(named: modifiers).union(heldModifiers.flags)) + guard inputIsAllowed() else { return } + let flags = KeyModifier.flags(named: modifiers).union(heldModifiers.flags) + if isRepeat { + guard heldKeys.allowsRepeat(usage) else { return } + postKeyEvent(keyCode: usage.keyCode, keyDown: true, flags: flags, autorepeat: true) + return + } + guard heldKeys.down(usage) else { return } + postKeyEvent(keyCode: usage.keyCode, keyDown: true, flags: flags) } /// The matching release. A spurious up for a key that isn't held (e.g. @@ -399,9 +411,10 @@ final class InputInjector { flags: KeyModifier.flags(named: modifiers).union(heldModifiers.flags)) } - private func postKeyEvent(keyCode: CGKeyCode, keyDown: Bool, flags: CGEventFlags) { + private func postKeyEvent(keyCode: CGKeyCode, keyDown: Bool, flags: CGEventFlags, autorepeat: Bool = false) { guard let event = CGEvent(keyboardEventSource: source, virtualKey: keyCode, keyDown: keyDown) else { return } event.flags = flags + if autorepeat { event.setIntegerValueField(.keyboardEventAutorepeat, value: 1) } event.post(tap: .cghidEventTap) } diff --git a/Mac/InputRouting.swift b/Mac/InputRouting.swift index e9d090311..5cf7d4289 100644 --- a/Mac/InputRouting.swift +++ b/Mac/InputRouting.swift @@ -96,6 +96,58 @@ enum HIDKeyUsage: Int, CaseIterable { case leftArrow = 80 case downArrow = 81 case upArrow = 82 + // A physical keyboard on a receiving Mac (issue #12) names every key; + // older senders ignore these, as an unrecognized usage must be. + case grave = 53 + case f1 = 58 + case f2 = 59 + case f3 = 60 + case f4 = 61 + case f5 = 62 + case f6 = 63 + case f7 = 64 + case f8 = 65 + case f9 = 66 + case f10 = 67 + case f11 = 68 + case f12 = 69 + case f13 = 104 + case f14 = 105 + case f15 = 106 + case f16 = 107 + case f17 = 108 + case f18 = 109 + case f19 = 110 + case f20 = 111 + case insertOrHelp = 73 + case home = 74 + case pageUp = 75 + case end = 77 + case pageDown = 78 + case nonUSBackslash = 100 + case international1 = 135 + case international3 = 137 + case lang1 = 144 + case lang2 = 145 + case keypadNumLockOrClear = 83 + case keypadSlash = 84 + case keypadAsterisk = 85 + case keypadMinus = 86 + case keypadPlus = 87 + case keypadEnter = 88 + case keypad1 = 89 + case keypad2 = 90 + case keypad3 = 91 + case keypad4 = 92 + case keypad5 = 93 + case keypad6 = 94 + case keypad7 = 95 + case keypad8 = 96 + case keypad9 = 97 + case keypad0 = 98 + case keypadPeriod = 99 + case keypadEqual = 103 + case keypadComma = 133 /// The macOS virtual key code (`CGKeyCode`, US ANSI layout) this usage /// injects as. @@ -157,6 +209,56 @@ enum HIDKeyUsage: Int, CaseIterable { case .leftArrow: return 123 case .downArrow: return 125 case .upArrow: return 126 + case .grave: return 50 + case .f1: return 122 + case .f2: return 120 + case .f3: return 99 + case .f4: return 118 + case .f5: return 96 + case .f6: return 97 + case .f7: return 98 + case .f8: return 100 + case .f9: return 101 + case .f10: return 109 + case .f11: return 103 + case .f12: return 111 + case .f13: return 105 + case .f14: return 107 + case .f15: return 113 + case .f16: return 106 + case .f17: return 64 + case .f18: return 79 + case .f19: return 80 + case .f20: return 90 + case .insertOrHelp: return 114 + case .home: return 115 + case .pageUp: return 116 + case .end: return 119 + case .pageDown: return 121 + case .nonUSBackslash: return 10 + case .international1: return 94 + case .international3: return 93 + case .lang1: return 104 + case .lang2: return 102 + case .keypadNumLockOrClear: return 71 + case .keypadSlash: return 75 + case .keypadAsterisk: return 67 + case .keypadMinus: return 78 + case .keypadPlus: return 69 + case .keypadEnter: return 76 + case .keypad1: return 83 + case .keypad2: return 84 + case .keypad3: return 85 + case .keypad4: return 86 + case .keypad5: return 87 + case .keypad6: return 88 + case .keypad7: return 89 + case .keypad8: return 91 + case .keypad9: return 92 + case .keypad0: return 82 + case .keypadPeriod: return 65 + case .keypadEqual: return 81 + case .keypadComma: return 95 } } @@ -326,6 +428,36 @@ struct HeldKeyTracker { defer { held.removeAll() } return held } + + /// A `down` marked `repeat` (a receiving Mac's own key repeat) posts an + /// auto-repeat key-down only for a key that is actually held — never a + /// fresh press, so a repeat can't outlive its key's release. + func allowsRepeat(_ usage: HIDKeyUsage) -> Bool { + held.contains(usage) + } +} + +/// Scroll arrives in video pixels and posts in whole points, so a single +/// small delta used to round away: a slow trackpad scroll from a receiving +/// Mac (fractions of a point per event) barely moved. The rounding error is +/// carried into the next event instead, so the total scrolled always matches +/// the total sent. Single events still round to nearest, as before. +struct ScrollRemainder { + private(set) var x = 0.0 + private(set) var y = 0.0 + + mutating func take(dxPoints: Double, dyPoints: Double) -> (dx: Int32, dy: Int32) { + let totalX = x + dxPoints, totalY = y + dyPoints + let wholeX = totalX.rounded(), wholeY = totalY.rounded() + x = totalX - wholeX + y = totalY - wholeY + return (Int32(clamping: Int(wholeX)), Int32(clamping: Int(wholeY))) + } + + mutating func reset() { + x = 0 + y = 0 + } } struct SystemGestureShortcut { diff --git a/Mac/MacSender.swift b/Mac/MacSender.swift index 9e15702c3..20785c2ee 100644 --- a/Mac/MacSender.swift +++ b/Mac/MacSender.swift @@ -5110,7 +5110,8 @@ final class MacSender: NSObject, SCStreamOutput, SCStreamDelegate { } case "down": if let usage = HIDKeyUsage.parse(obj["usage"]) { - inputInjector?.handleKeyboardDown(usage: usage, modifiers: obj["modifiers"] as? [String] ?? []) + inputInjector?.handleKeyboardDown(usage: usage, modifiers: obj["modifiers"] as? [String] ?? [], + isRepeat: obj["repeat"] as? Bool == true) } case "up": if let usage = HIDKeyUsage.parse(obj["usage"]) { diff --git a/Shared/StreamReceiver.swift b/Shared/StreamReceiver.swift index 62876818e..364817dd5 100644 --- a/Shared/StreamReceiver.swift +++ b/Shared/StreamReceiver.swift @@ -3324,9 +3324,19 @@ final class StreamReceiver: ObservableObject { /// A hardware key going down, for keys the Mac must hold (arrows, /// modified shortcuts). `modifiers` are named protocol modifiers, not /// raw UIKit flags. + /// + /// `isRepeat`: the receiving Mac's own key repeat for a key already sent + /// down (additive `repeat` field; an older Mac ignores the duplicate + /// down, as it always did). @MainActor func sendKeyboardDown(usage: Int, modifiers: [String]) { + sendKeyboardDown(usage: usage, modifiers: modifiers, isRepeat: false) + } + + @MainActor func sendKeyboardDown(usage: Int, modifiers: [String], isRepeat: Bool) { guard displayState == .running, macSupportsKeyboardWire else { return } - sendUIControl(["type": "keyboard", "action": "down", "usage": usage, "modifiers": modifiers]) + var message: [String: Any] = ["type": "keyboard", "action": "down", "usage": usage, "modifiers": modifiers] + if isRepeat { message["repeat"] = true } + sendUIControl(message) } /// The matching release for `sendKeyboardDown`. From e66337c11846771e03f13e6bfa7de96784648ec2 Mon Sep 17 00:00:00 2001 From: raiseCatError <315733358+raiseCatError@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:30:02 +0530 Subject: [PATCH 2/3] feat(mac-receiver): let this Mac's keyboard and trackpad control the sending Mac The Mac Receiver was display-only. Once the sending Mac grants the session control through the existing per-session request flow, the video window forwards pointer movement, left and right clicks with their click counts, drags (clamped to the picture, released even outside it), precise and wheel scrolling, pinch and rotate, and every key with Shift, Control, Option, Command and Caps Lock, using the existing pointer, scroll, keyboard and nativeAppGesture messages. Forwarding needs the grant, the video window in front and no local pause. Input is never captured globally: only events addressed to this app's video window are looked at, so Command-Tab, Force Quit, Mission Control and other system shortcuts stay on this Mac. A modifier still held from before (the Command of a Command-Tab) is never sent. Everything sent down is released on focus loss, the Control-Option-Escape pause, Release, revocation or window close; a disconnect resets locally. A titlebar indicator shows View Only / Requesting / Controlling / Paused with Request Control or Release, a short hint names the pause shortcut, and the streamed cursor follows the local pointer while controlling. Refs #12 --- MacReceiver/MacReceiver.swift | 129 ++++- MacReceiver/ReceiverInputController.swift | 318 ++++++++++++ MacReceiver/ReceiverInputForwarding.swift | 489 ++++++++++++++++++ MacTests/ReceiverInputForwardingTests.swift | 532 ++++++++++++++++++++ project.yml | 1 + 5 files changed, 1454 insertions(+), 15 deletions(-) create mode 100644 MacReceiver/ReceiverInputController.swift create mode 100644 MacReceiver/ReceiverInputForwarding.swift create mode 100644 MacTests/ReceiverInputForwardingTests.swift diff --git a/MacReceiver/MacReceiver.swift b/MacReceiver/MacReceiver.swift index be4d2c558..93d0335d0 100644 --- a/MacReceiver/MacReceiver.swift +++ b/MacReceiver/MacReceiver.swift @@ -6,8 +6,9 @@ // keeping the machine awake while it serves as a screen. It ships as its own // app (OpenSidecarMacReceiver target, macOS 12+) so old Macs can be displays. // -// Display-only for now: the receiving Mac's keyboard/trackpad are not -// forwarded to the sender (KVM-style input is a follow-up). +// Input (issue #12): once the sending Mac grants this session control, this +// Mac's own keyboard and trackpad drive it while the video window is in +// front — see ReceiverInputForwarding.swift for the rules. import AppKit import AVFoundation @@ -42,6 +43,9 @@ final class ReceiverController: ObservableObject { /// Wake & Connect for a paired Mac (the same shared coordinator the /// iPhone app drives). Lives exactly as long as `receiver`. @Published private(set) var wakeConnect: WakeConnectCoordinator? + /// Keyboard/trackpad forwarding for this receiver's sessions. Lives + /// exactly as long as `receiver`. + @Published private(set) var input: ReceiverInputController? /// The receiver-local manual A/V Sync offset (`AVSyncOffset.range`), /// persisted here and applied to the shared receiver's /// `avSyncPreference` — the one value its audio scheduling and video @@ -87,6 +91,7 @@ final class ReceiverController: ObservableObject { receiver.avSyncPreference.set(avSyncOffsetMs) self.receiver = receiver wakeConnect = WakeConnectCoordinator(receiver: receiver) + input = ReceiverInputController(receiver: receiver) receiver.start() receiver.$connected @@ -192,6 +197,8 @@ final class ReceiverController: ObservableObject { screenSleepObservers = [] wakeConnect?.cancel() wakeConnect = nil + input?.invalidate() + input = nil // The receiver finishes its teardown on its own queue and only calls // back while it still exists, so keep it alive until it has — views // observing it are released as soon as `receiver` goes nil below. @@ -281,7 +288,7 @@ final class ReceiverController: ObservableObject { /// Bring the video window (back) up — also bound to Show Window in Settings for /// when the user closed the window while the stream keeps running. func showWindow() { - guard let receiver, streaming || window != nil else { return } + guard let receiver, let input, streaming || window != nil else { return } // Streaming again within the grace period keeps this window, in // whatever full screen state it's already in. if streaming { windowLifecycle.cancelPendingClose() } @@ -291,12 +298,23 @@ final class ReceiverController: ObservableObject { styleMask: [.titled, .closable, .miniaturizable, .resizable], backing: .buffered, defer: false) w.title = "MeowDisplay" - w.contentView = ReceiverVideoView(receiver: receiver) + let videoView = ReceiverVideoView(receiver: receiver, input: input) + w.contentView = videoView + w.initialFirstResponder = videoView w.isReleasedWhenClosed = false + // A fixed slot (macOS 12 has no hosting-view sizing options): + // the indicator right-aligns inside it as its text changes. + let indicator = NSTitlebarAccessoryViewController() + let indicatorView = NSHostingView(rootView: ReceiverControlIndicator(input: input)) + indicatorView.frame = NSRect(x: 0, y: 0, width: 340, height: 28) + indicator.view = indicatorView + indicator.layoutAttribute = .trailing + w.addTitlebarAccessoryViewController(indicator) w.collectionBehavior.insert(.fullScreenPrimary) w.center() window = w observeWindow(w) + input.attach(to: w) enterFullscreen = windowLifecycle.windowCreated() } // Resizes keep the stream's shape; re-set on every show because a @@ -411,8 +429,10 @@ final class ReceiverController: ObservableObject { /// extended desktop would have no visible pointer. final class ReceiverVideoView: NSView { private weak var receiver: StreamReceiver? + private let input: ReceiverInputController private let displayLayer: AVSampleBufferDisplayLayer private var videoSizeObserver: AnyCancellable? + private var cursorEcho = CursorEchoPolicy() private let cursorLayer: CALayer = { let layer = CALayer() @@ -428,8 +448,9 @@ final class ReceiverVideoView: NSView { private var cursorNorm = CGPoint(x: 0.5, y: 0.5) private var cursorVisible = false - init(receiver: StreamReceiver) { + init(receiver: StreamReceiver, input: ReceiverInputController) { self.receiver = receiver + self.input = input self.displayLayer = receiver.displayLayer super.init(frame: .zero) wantsLayer = true @@ -440,7 +461,16 @@ final class ReceiverVideoView: NSView { layer?.addSublayer(cursorLayer) receiver.onCursor = { [weak self] x, y, visible in - self?.moveCursor(x: x, y: y, visible: visible) + guard let self else { return } + // While this Mac drives the pointer the sprite already sits at + // the local pointer; the sending Mac's (older) reports take over + // once local movement stops. + if self.input.echoesCursor, + !self.cursorEcho.appliesRemotePosition(at: ProcessInfo.processInfo.systemUptime) { + self.setCursorVisible(visible) + return + } + self.moveCursor(x: x, y: y, visible: visible) } receiver.onCursorImage = { [weak self] image, anchor, normSize in self?.setCursorSprite(image, anchor: anchor, normSize: normSize) @@ -464,7 +494,7 @@ final class ReceiverVideoView: NSView { // required constraints that size drove the *window*, which collapsed // to 0 × titlebar with the HUD hidden. Springs and struts never feed // back into the window, on any macOS the app supports. - let overlay = OverlayHostingView(rootView: ReceiverPerfOverlay(receiver: receiver)) + let overlay = OverlayHostingView(rootView: ReceiverPerfOverlay(receiver: receiver, input: input)) overlay.frame = bounds overlay.autoresizingMask = [.width, .height] addSubview(overlay) @@ -476,6 +506,68 @@ final class ReceiverVideoView: NSView { addCursorRect(bounds, cursor: blankCursor) } + // MARK: Input (issue #12) + + private var geometry: ReceiverVideoGeometry { + ReceiverVideoGeometry(viewSize: bounds.size, videoSize: receiver?.videoSize ?? .zero) + } + + override var acceptsFirstResponder: Bool { true } + + override func updateTrackingAreas() { + super.updateTrackingAreas() + trackingAreas.forEach(removeTrackingArea) + addTrackingArea(NSTrackingArea(rect: .zero, options: [.mouseMoved, .activeInKeyWindow, .inVisibleRect], + owner: self, userInfo: nil)) + } + + override func mouseMoved(with event: NSEvent) { pointerMoved(event) } + override func mouseDragged(with event: NSEvent) { pointerMoved(event) } + override func rightMouseDragged(with event: NSEvent) { pointerMoved(event) } + + override func mouseDown(with event: NSEvent) { + input.mouseDown(event, button: .left, geometry: geometry, in: self) + echoCursor(event) + } + + override func mouseUp(with event: NSEvent) { + input.mouseUp(event, button: .left, geometry: geometry, in: self) + } + + override func rightMouseDown(with event: NSEvent) { + input.mouseDown(event, button: .right, geometry: geometry, in: self) + echoCursor(event) + } + + override func rightMouseUp(with event: NSEvent) { + input.mouseUp(event, button: .right, geometry: geometry, in: self) + } + + override func scrollWheel(with event: NSEvent) { + input.scrollWheel(event, geometry: geometry, in: self) + } + + override func magnify(with event: NSEvent) { input.magnify(event) } + override func rotate(with event: NSEvent) { input.rotate(event) } + + private func pointerMoved(_ event: NSEvent) { + input.mouseMoved(event, geometry: geometry, in: self) + echoCursor(event) + } + + /// Draws the streamed cursor at the local pointer while controlling. + private func echoCursor(_ event: NSEvent) { + guard input.echoesCursor, + let (point, _) = geometry.normalized(convert(event.locationInWindow, from: nil)) else { return } + cursorEcho.localMove(at: ProcessInfo.processInfo.systemUptime) + moveCursor(x: Double(point.x), y: Double(point.y), visible: cursorVisible) + } + + private func setCursorVisible(_ visible: Bool) { + guard visible != cursorVisible else { return } + moveCursor(x: Double(cursorNorm.x), y: Double(cursorNorm.y), visible: visible) + } + override func layout() { super.layout() CATransaction.begin() @@ -550,23 +642,30 @@ final class ReceiverVideoView: NSView { /// same "showAnalytics" default the iOS app uses (Settings → System). struct ReceiverPerfOverlay: View { @ObservedObject var receiver: StreamReceiver + let input: ReceiverInputController @AppStorage("showAnalytics") private var showAnalytics = false var body: some View { - if showAnalytics { - VStack { - Spacer() - PerfOverlay(stats: receiver.perf, videoSize: receiver.videoSize) - .padding(.bottom, 10) + ZStack { + ReceiverControlHint(input: input) + if showAnalytics { + VStack { + Spacer() + PerfOverlay(stats: receiver.perf, videoSize: receiver.videoSize) + .padding(.bottom, 10) + } + .allowsHitTesting(false) } - .allowsHitTesting(false) } } } -/// Full-bleed, non-interactive layer above the video. Subclassed only so the -/// blank-cursor rect covers the HUD area too. +/// Full-bleed, non-interactive layer above the video. Subclassed so the +/// blank-cursor rect covers the HUD area too, and so it never takes a click, +/// scroll or drag meant for the video view (and the other Mac) underneath. private final class OverlayHostingView: NSHostingView { + override func hitTest(_ point: NSPoint) -> NSView? { nil } + required init(rootView: ReceiverPerfOverlay) { super.init(rootView: rootView) } diff --git a/MacReceiver/ReceiverInputController.swift b/MacReceiver/ReceiverInputController.swift new file mode 100644 index 000000000..0878674bd --- /dev/null +++ b/MacReceiver/ReceiverInputController.swift @@ -0,0 +1,318 @@ +import AppKit +import Combine +import SwiftUI + +/// The AppKit side of Mac-to-Mac input (issue #12): owns one session's +/// `ReceiverInputForwarder`, turns the video window's events into its calls, +/// and sends what it returns through `StreamReceiver`'s existing input +/// messages. Lives exactly as long as the `StreamReceiver` it serves. +@MainActor +final class ReceiverInputController: ObservableObject { + @Published private(set) var status: ReceiverControlStatus = .unavailable + /// The transient line over the video (`ReceiverControlStatus.hint`). + @Published private(set) var hint: String? + + private weak var receiver: StreamReceiver? + private var forwarder = ReceiverInputForwarder() + private var cancellables = Set() + private var keyMonitor: Any? + private weak var window: NSWindow? + private var windowObservers: [NSObjectProtocol] = [] + private var hintToken = 0 + + /// Only while controlling: the view draws the streamed cursor at the + /// local pointer (see `CursorEchoPolicy`). + var echoesCursor: Bool { status == .controlling } + + init(receiver: StreamReceiver) { + self.receiver = receiver + receiver.onAllowInputStateChange = { [weak self] state in + guard let self else { return } + Log.info("receiverInput: grant \(state.rawValue)") + self.perform(self.forwarder.grantChanged(state)) + } + receiver.$connected.combineLatest(receiver.$macProtocolVersion) + .removeDuplicates { $0 == $1 } + .receive(on: DispatchQueue.main) + .sink { [weak self] connected, version in + guard let self else { return } + // `allowInputRequest` (pv 8) is the newest message this + // needs; pointer (5) and keyboard (4) predate it. + self.forwarder.setConnected(connected, + inputSupported: version >= WireProtocol.allowInputWireVersion) + self.refreshStatus() + } + .store(in: &cancellables) + receiver.$inputResetGeneration.dropFirst() + .receive(on: DispatchQueue.main) + .sink { [weak self] _ in self?.forwarder.senderReset() } + .store(in: &cancellables) + } + + /// Stops listening and releases anything still held. Called when the + /// receiver it serves goes away. + func invalidate() { + detachWindow() + receiver?.onAllowInputStateChange = nil + cancellables.removeAll() + } + + // MARK: Window + + /// The video window. Its key state gates forwarding, and only its key + /// events are ever looked at. + func attach(to window: NSWindow) { + detachWindow() + self.window = window + let center = NotificationCenter.default + windowObservers = [ + center.addObserver(forName: NSWindow.didBecomeKeyNotification, object: window, queue: .main) { [weak self] _ in + MainActor.assumeIsolated { self?.windowKeyChanged(true) } + }, + center.addObserver(forName: NSWindow.didResignKeyNotification, object: window, queue: .main) { [weak self] _ in + MainActor.assumeIsolated { self?.windowKeyChanged(false) } + }, + center.addObserver(forName: NSWindow.willCloseNotification, object: window, queue: .main) { [weak self] _ in + MainActor.assumeIsolated { self?.detachWindow() } + }, + ] + // App-local: sees only events already addressed to this app, never + // other apps' input. Needed because AppKit never delivers a key-up + // while Command is held to a view, and menu key equivalents would + // otherwise win over the sending Mac's shortcuts. + keyMonitor = NSEvent.addLocalMonitorForEvents(matching: [.keyDown, .keyUp, .flagsChanged]) { [weak self] event in + let consumed = MainActor.assumeIsolated { self?.consumeKeyEvent(event) ?? false } + return consumed ? nil : event + } + windowKeyChanged(window.isKeyWindow) + } + + private func detachWindow() { + windowObservers.forEach { NotificationCenter.default.removeObserver($0) } + windowObservers = [] + if let keyMonitor { NSEvent.removeMonitor(keyMonitor) } + keyMonitor = nil + window = nil + windowKeyChanged(false) + } + + private func windowKeyChanged(_ isKey: Bool) { + perform(forwarder.windowKeyChanged(isKey)) + } + + // MARK: Actions + + func requestControl() { perform(forwarder.requestControl()) } + func releaseControl() { perform(forwarder.releaseControl()) } + func pauseControl() { perform(forwarder.pause()) } + + func performStatusAction() { + switch status.action { + case .request: requestControl() + case .release: releaseControl() + case nil: break + } + } + + // MARK: Pointer (from ReceiverVideoView) + + func mouseMoved(_ event: NSEvent, geometry: ReceiverVideoGeometry, in view: NSView) { + guard let (point, inside) = locate(event, geometry: geometry, in: view) else { return } + perform(forwarder.pointerMoved(point, inside: inside)) + } + + func mouseDown(_ event: NSEvent, button: PointerButton, geometry: ReceiverVideoGeometry, in view: NSView) { + guard let (point, inside) = locate(event, geometry: geometry, in: view) else { return } + let wasPaused = forwarder.paused + perform(forwarder.pointerDown(button, clickCount: event.clickCount, at: point, inside: inside)) + if wasPaused, !forwarder.paused { refreshStatus() } + } + + func mouseUp(_ event: NSEvent, button: PointerButton, geometry: ReceiverVideoGeometry, in view: NSView) { + let point = locate(event, geometry: geometry, in: view)?.point + perform(forwarder.pointerUp(button, at: point)) + } + + func scrollWheel(_ event: NSEvent, geometry: ReceiverVideoGeometry, in view: NSView) { + guard let (_, inside) = locate(event, geometry: geometry, in: view), + let pixelsPerPoint = geometry.videoPixelsPerPoint else { return } + // A wheel mouse reports lines; one line is about ten points of + // trackpad travel, which is how macOS itself converts. + let lineScale = event.hasPreciseScrollingDeltas ? 1.0 : 10.0 + perform(forwarder.scrolled(dxPoints: Double(event.scrollingDeltaX) * lineScale, + dyPoints: Double(event.scrollingDeltaY) * lineScale, + inside: inside, videoPixelsPerPoint: pixelsPerPoint)) + } + + func magnify(_ event: NSEvent) { + guard let phase = Self.gesturePhase(event.phase) else { return } + perform(forwarder.gesture(.magnify, phase: phase, delta: Double(event.magnification))) + } + + func rotate(_ event: NSEvent) { + guard let phase = Self.gesturePhase(event.phase) else { return } + // AppKit reports degrees counterclockwise; the wire carries radians. + perform(forwarder.gesture(.rotate, phase: phase, delta: Double(event.rotation) * .pi / 180)) + } + + private static func gesturePhase(_ phase: NSEvent.Phase) -> NativeAppGesturePhase? { + if phase.contains(.began) { return .began } + if phase.contains(.changed) { return .changed } + if phase.contains(.ended) { return .ended } + if phase.contains(.cancelled) { return .cancelled } + return nil + } + + private func locate(_ event: NSEvent, geometry: ReceiverVideoGeometry, + in view: NSView) -> (point: CGPoint, inside: Bool)? { + geometry.normalized(view.convert(event.locationInWindow, from: nil)) + } + + // MARK: Keyboard (app-local monitor) + + /// `false` lets AppKit handle the event here; `true` once it was sent to + /// the other Mac (or consumed, like the pause shortcut). + private func consumeKeyEvent(_ event: NSEvent) -> Bool { + guard let window, event.window === window else { return false } + let modifiers = ReceiverModifierState(rawFlags: event.modifierFlags.rawValue) + let commands: [ReceiverInputCommand]? + switch event.type { + case .keyDown: + commands = forwarder.keyDown(keyCode: event.keyCode, isRepeat: event.isARepeat, modifiers: modifiers) + case .keyUp: + commands = forwarder.keyUp(keyCode: event.keyCode) + case .flagsChanged: + perform(forwarder.modifiersChanged(modifiers)) + // Modifier changes also reach AppKit, so its own key handling + // stays in sync. + return false + default: + return false + } + guard let commands else { return false } + perform(commands) + return true + } + + // MARK: Sending + + private func perform(_ commands: [ReceiverInputCommand]) { + if let receiver { + for command in commands { send(command, through: receiver) } + } + refreshStatus() + } + + private func send(_ command: ReceiverInputCommand, through receiver: StreamReceiver) { + switch command { + case .pointerMove(let x, let y): receiver.sendPointerMove(x: x, y: y) + case .pointerDown(let button, let count): receiver.sendPointerDown(button: button, clickCount: count) + case .pointerUp(let button, let count): receiver.sendPointerUp(button: button, clickCount: count) + case .scroll(let dx, let dy): receiver.sendScroll(dx: dx, dy: dy) + case .keyDown(let usage, let modifiers, let isRepeat): + receiver.sendKeyboardDown(usage: usage, modifiers: modifiers, isRepeat: isRepeat) + case .keyUp(let usage, let modifiers): receiver.sendKeyboardUp(usage: usage, modifiers: modifiers) + case .modifierDown(let modifier): receiver.sendModifier(modifier, down: true) + case .modifierUp(let modifier): receiver.sendModifier(modifier, down: false) + case .gesture(let kind, let phase, let delta): receiver.sendNativeAppGesture(kind: kind, phase: phase, delta: delta) + case .cancelAll: receiver.sendCancelActiveInput() + case .requestControl(let allowed): receiver.requestAllowInput(allowed) + } + } + + private func refreshStatus() { + let next = forwarder.status + guard next != status else { return } + status = next + showHint(next.hint) + } + + private func showHint(_ text: String?) { + hintToken &+= 1 + hint = text + guard text != nil else { return } + let token = hintToken + DispatchQueue.main.asyncAfter(deadline: .now() + 3.5) { [weak self] in + guard let self, self.hintToken == token else { return } + self.hint = nil + } + } +} + +// MARK: - Indicator + +/// The video window's titlebar indicator: View Only / Controlling, and the +/// one action that applies. In full screen it appears with the titlebar +/// when the pointer reaches the top of the screen. +struct ReceiverControlIndicator: View { + @ObservedObject var input: ReceiverInputController + + var body: some View { + if let title = input.status.title { + HStack(spacing: 6) { + Image(systemName: symbol) + .foregroundStyle(tint) + .accessibilityHidden(true) + Text(title) + .font(.callout) + if let action = input.status.action { + Button(action == .request ? String(localized: "Request Control") : String(localized: "Release")) { + input.performStatusAction() + } + .controlSize(.small) + .help(action == .request + ? String(localized: "Ask the other Mac to let this Mac's keyboard and trackpad control it.") + : String(localized: "Give control back. The stream stays on screen.")) + } + } + .padding(.horizontal, 8) + .fixedSize() + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .trailing) + .help(input.status == .controlling + ? String(localized: "Press \(ReceiverInputForwarder.releaseShortcutLabel) to pause control.", + comment: "The placeholder is a keyboard shortcut such as ⌃⌥⎋.") + : "") + } + } + + private var symbol: String { + switch input.status { + case .controlling: return "keyboard" + case .paused, .inactive: return "pause.circle" + case .requesting: return "hourglass" + case .declined, .requestsDisabled: return "hand.raised" + case .viewOnly, .unavailable: return "eye" + } + } + + private var tint: Color { + switch input.status { + case .controlling: return .accentColor + case .declined, .requestsDisabled: return .orange + default: return .secondary + } + } +} + +/// The transient hint over the video. +struct ReceiverControlHint: View { + @ObservedObject var input: ReceiverInputController + + var body: some View { + VStack { + if let hint = input.hint { + Text(hint) + .font(.callout) + .multilineTextAlignment(.center) + .padding(.horizontal, 14) + .padding(.vertical, 8) + .background(.regularMaterial, in: Capsule()) + .padding(.top, 14) + .transition(.opacity) + } + Spacer() + } + .animation(.easeInOut(duration: 0.2), value: input.hint) + .allowsHitTesting(false) + } +} diff --git a/MacReceiver/ReceiverInputForwarding.swift b/MacReceiver/ReceiverInputForwarding.swift new file mode 100644 index 000000000..1f53ac7b7 --- /dev/null +++ b/MacReceiver/ReceiverInputForwarding.swift @@ -0,0 +1,489 @@ +import CoreGraphics +import Foundation + +// Mac-to-Mac input (issue #12): the receiving Mac's own trackpad, mouse and +// keyboard drive the sending Mac while its video window is in front. This +// file is the decision layer — where a point lands, which key a key code is, +// and what may be forwarded right now — kept free of AppKit so every rule is +// unit testable. `ReceiverVideoView` feeds it AppKit events and sends what it +// returns through the existing `StreamReceiver` input messages; no new wire +// message exists for any of it. +// +// Input is never captured globally: only events AppKit delivers to the video +// window while it is key reach this, so the receiving Mac's system shortcuts +// (Command-Tab, Command-Option-Escape, Mission Control, Spotlight, screenshots, +// lock screen) never leave it. The sending Mac stays the authority — nothing +// is forwarded without its per-session grant, and it drops input itself the +// moment that grant ends. + +/// Where the stream sits inside the video view: aspect-fit, centered, in view +/// points with AppKit's bottom-left origin. +struct ReceiverVideoGeometry: Equatable { + let viewSize: CGSize + /// The stream's pixel size (from its SPS). + let videoSize: CGSize + + /// `nil` until both sizes are known. + var fittedRect: CGRect? { + guard videoSize.width > 0, videoSize.height > 0, viewSize.width > 0, viewSize.height > 0 else { return nil } + let scale = min(viewSize.width / videoSize.width, viewSize.height / videoSize.height) + let size = CGSize(width: videoSize.width * scale, height: videoSize.height * scale) + return CGRect(x: (viewSize.width - size.width) / 2, y: (viewSize.height - size.height) / 2, + width: size.width, height: size.height) + } + + /// A view point as normalized video coordinates (PROTOCOL.md section 7: + /// top-left origin, 0...1), clamped onto the video so a drag that leaves + /// the picture (into a letterbox bar, or outside the window) pins to its + /// edge instead of reaching another display on the sending Mac. + /// `inside` reports whether the unclamped point was on the picture. + func normalized(_ viewPoint: CGPoint) -> (point: CGPoint, inside: Bool)? { + guard let rect = fittedRect else { return nil } + let x = (viewPoint.x - rect.minX) / rect.width + let y = (rect.maxY - viewPoint.y) / rect.height + let inside = (0...1).contains(x) && (0...1).contains(y) + return (CGPoint(x: min(max(x, 0), 1), y: min(max(y, 0), 1)), inside) + } + + /// Video pixels per view point — scroll deltas travel in video pixels. + var videoPixelsPerPoint: Double? { + guard let rect = fittedRect else { return nil } + return Double(videoSize.width / rect.width) + } +} + +// MARK: - Keys + +/// macOS virtual key codes (`kVK_*`, physical key positions) to USB HID +/// keyboard-page usages, the vocabulary `keyboard` messages carry. Both name +/// the physical key, so the sending Mac's own keyboard layout decides what +/// gets typed — exactly as if this keyboard were plugged into it. +/// Modifier keys are not here: they travel as `modifierDown`/`modifierUp`. +enum MacVirtualKeyMap { + static func hidUsage(forKeyCode keyCode: UInt16) -> Int? { table[keyCode] } + + static var mappedKeyCodes: [UInt16] { Array(table.keys).sorted() } + + private static let table: [UInt16: Int] = [ + // Letters + 0x00: 4, 0x0B: 5, 0x08: 6, 0x02: 7, 0x0E: 8, 0x03: 9, 0x05: 10, 0x04: 11, 0x22: 12, + 0x26: 13, 0x28: 14, 0x25: 15, 0x2E: 16, 0x2D: 17, 0x1F: 18, 0x23: 19, 0x0C: 20, + 0x0F: 21, 0x01: 22, 0x11: 23, 0x20: 24, 0x09: 25, 0x0D: 26, 0x07: 27, 0x10: 28, 0x06: 29, + // Digits row + 0x12: 30, 0x13: 31, 0x14: 32, 0x15: 33, 0x17: 34, 0x16: 35, 0x1A: 36, 0x1C: 37, 0x19: 38, 0x1D: 39, + // Return, Escape, Delete, Tab, Space + 0x24: 40, 0x35: 41, 0x33: 42, 0x30: 43, 0x31: 44, + // Punctuation + 0x1B: 45, 0x18: 46, 0x21: 47, 0x1E: 48, 0x2A: 49, 0x29: 51, 0x27: 52, 0x32: 53, + 0x2B: 54, 0x2F: 55, 0x2C: 56, + // ISO key left of Z (§ / <), JIS ¥ and _ and keypad comma, Eisū and Kana + 0x0A: 100, 0x5D: 137, 0x5E: 135, 0x5F: 133, 0x66: 145, 0x68: 144, + // F1–F20 + 0x7A: 58, 0x78: 59, 0x63: 60, 0x76: 61, 0x60: 62, 0x61: 63, 0x62: 64, 0x64: 65, + 0x65: 66, 0x6D: 67, 0x67: 68, 0x6F: 69, 0x69: 104, 0x6B: 105, 0x71: 106, 0x6A: 107, + 0x40: 108, 0x4F: 109, 0x50: 110, 0x5A: 111, + // Help/Insert, Home, Page Up, Forward Delete, End, Page Down, arrows + 0x72: 73, 0x73: 74, 0x74: 75, 0x75: 76, 0x77: 77, 0x79: 78, + 0x7C: 79, 0x7B: 80, 0x7D: 81, 0x7E: 82, + // Keypad + 0x47: 83, 0x4B: 84, 0x43: 85, 0x4E: 86, 0x45: 87, 0x4C: 88, 0x53: 89, 0x54: 90, 0x55: 91, + 0x56: 92, 0x57: 93, 0x58: 94, 0x59: 95, 0x5B: 96, 0x5C: 97, 0x52: 98, 0x41: 99, 0x51: 103, + ] + + static let escapeKeyCode: UInt16 = 0x35 +} + +/// The receiving Mac's modifier-key state, reduced to what the wire names. +struct ReceiverModifierState: Equatable { + var held: Set = [] + var capsLock = false + + /// From raw `NSEvent.ModifierFlags` bits, so this file stays AppKit-free. + init(rawFlags: UInt) { + if rawFlags & (1 << 17) != 0 { held.insert(.shift) } + if rawFlags & (1 << 18) != 0 { held.insert(.control) } + if rawFlags & (1 << 19) != 0 { held.insert(.option) } + if rawFlags & (1 << 20) != 0 { held.insert(.command) } + capsLock = rawFlags & (1 << 16) != 0 + } + + init(held: Set = [], capsLock: Bool = false) { + self.held = held + self.capsLock = capsLock + } +} + +// MARK: - Forwarding + +/// What `ReceiverVideoView` sends, in order. Each maps onto one existing +/// `StreamReceiver` send call. +enum ReceiverInputCommand: Equatable { + case pointerMove(x: Double, y: Double) + case pointerDown(PointerButton, clickCount: Int) + case pointerUp(PointerButton, clickCount: Int) + case scroll(dx: Double, dy: Double) + case keyDown(usage: Int, modifiers: [String], isRepeat: Bool) + case keyUp(usage: Int, modifiers: [String]) + case modifierDown(ControlModifier) + case modifierUp(ControlModifier) + case gesture(NativeAppGestureKind, NativeAppGesturePhase, delta: Double) + /// `keyboard` `cancel`: the sending Mac releases everything it holds for + /// this session. Sent after the explicit releases, as a safety net. + case cancelAll + /// `allowInputRequest`. + case requestControl(Bool) +} + +/// What the control indicator shows. +enum ReceiverControlStatus: Equatable { + /// No session, or a sending Mac too old for input requests. + case unavailable + /// View only; Request Control is offered. + case viewOnly + case requesting + /// The sending Mac's owner declined, or the prompt timed out. + case declined + /// The sending Mac doesn't accept control requests from this Mac. + case requestsDisabled + /// Granted, but this window isn't key (another app or window is in front). + case inactive + /// Granted and paused here with the release shortcut; a click resumes. + case paused + /// This Mac's keyboard and pointer are driving the sending Mac. + case controlling +} + +/// The receiver's half of the input contract. Forwards only while the +/// sending Mac granted this session control, the video window is key, and +/// the user hasn't paused it; and it remembers every key, modifier and button +/// it sent down so that ANY way forwarding ends — focus loss, the release +/// shortcut, Release Control, revocation, the window closing — releases them +/// on the sending Mac. A modifier or key already held when forwarding began +/// (Command still down from the Command-Tab that brought this window forward) +/// is never sent: only presses that started while forwarding are. +struct ReceiverInputForwarder { + /// Control-Option-Escape: pauses forwarding without giving up the grant. + /// Not a system shortcut on macOS (Command-Option-Escape is Force Quit). + static let releaseShortcutModifiers: Set = [.control, .option] + static let releaseShortcutLabel = "⌃⌥⎋" + + private(set) var connected = false + /// The sending Mac supports `allowInputRequest` and input messages. + private(set) var inputSupported = false + private(set) var grant: SessionInputWireState = .off + private(set) var windowIsKey = false + private(set) var paused = false + + private(set) var heldKeys: [Int: [String]] = [:] + private(set) var heldModifiers: Set = [] + private(set) var heldButton: (button: PointerButton, clickCount: Int)? + private var activeGestures: Set = [] + private var lastMove: CGPoint? + + var isForwarding: Bool { connected && inputSupported && grant == .allowed && windowIsKey && !paused } + + var status: ReceiverControlStatus { + guard connected, inputSupported else { return .unavailable } + switch grant { + case .off: return .viewOnly + case .requesting: return .requesting + case .notAllowed: return .declined + case .requestsDisabled: return .requestsDisabled + case .allowed: + if paused { return .paused } + return windowIsKey ? .controlling : .inactive + } + } + + var holdsInput: Bool { !heldKeys.isEmpty || !heldModifiers.isEmpty || heldButton != nil || !activeGestures.isEmpty } + + // MARK: Session and focus + + /// A session started or ended. A lost connection takes the sender's + /// state with it (it releases everything itself on session loss), so + /// nothing is sent — local tracking just resets. + mutating func setConnected(_ live: Bool, inputSupported supported: Bool) { + if connected != live || !live { resetTracking() } + connected = live + inputSupported = live && supported + if !live { + grant = .off + paused = false + } + } + + /// The sending Mac's confirmed state. Anything but `.allowed` ends + /// forwarding; the sender has already dropped this session's input, and + /// the explicit releases make sure it holds nothing. + mutating func grantChanged(_ state: SessionInputWireState) -> [ReceiverInputCommand] { + let wasAllowed = grant == .allowed + grant = state + if state == .allowed, !wasAllowed { paused = false } + return state == .allowed ? [] : releaseAll() + } + + /// The sending Mac reset input (`inputReset`): it already released + /// everything, so only local tracking clears. + mutating func senderReset() { + resetTracking() + } + + mutating func windowKeyChanged(_ isKey: Bool) -> [ReceiverInputCommand] { + windowIsKey = isKey + return isKey ? [] : releaseAll() + } + + /// The release shortcut (or the indicator's Pause): stop forwarding but + /// keep the grant. A click on the picture resumes. + mutating func pause() -> [ReceiverInputCommand] { + guard grant == .allowed, !paused else { return [] } + paused = true + return releaseAll() + } + + mutating func resume() { + paused = false + } + + mutating func requestControl() -> [ReceiverInputCommand] { + guard connected, inputSupported, grant != .allowed, grant != .requesting, grant != .requestsDisabled else { + return [] + } + return [.requestControl(true)] + } + + /// Release Control: give the grant back. Effective immediately on the + /// sending Mac — narrowing never needs its owner's decision. + mutating func releaseControl() -> [ReceiverInputCommand] { + guard connected, inputSupported, grant == .allowed || grant == .requesting else { return [] } + let releases = releaseAll() + paused = false + return releases + [.requestControl(false)] + } + + // MARK: Pointer + + mutating func pointerMoved(_ normalized: CGPoint, inside: Bool) -> [ReceiverInputCommand] { + guard isForwarding, inside || heldButton != nil else { return [] } + guard normalized != lastMove else { return [] } + lastMove = normalized + return [.pointerMove(x: Double(normalized.x), y: Double(normalized.y))] + } + + /// A press on the video. While paused, a press on the picture resumes + /// forwarding and is consumed (it was a "click back in", not a click on + /// the sending Mac). Presses outside the picture (letterbox) do nothing. + mutating func pointerDown(_ button: PointerButton, clickCount: Int, at normalized: CGPoint, + inside: Bool) -> [ReceiverInputCommand] { + guard inside else { return [] } + if paused, connected, grant == .allowed, windowIsKey { + paused = false + return [] + } + guard isForwarding, heldButton == nil else { return [] } + heldButton = (button, max(clickCount, 1)) + lastMove = normalized + // Position first: the sending Mac clicks at its own cursor. + return [.pointerMove(x: Double(normalized.x), y: Double(normalized.y)), + .pointerDown(button, clickCount: max(clickCount, 1))] + } + + /// A release is forwarded only for a press this forwarder sent — never + /// gated on forwarding, so a release always matches its press. + mutating func pointerUp(_ button: PointerButton, at normalized: CGPoint?) -> [ReceiverInputCommand] { + guard let held = heldButton, held.button == button else { return [] } + heldButton = nil + var commands: [ReceiverInputCommand] = [] + if isForwarding, let normalized, normalized != lastMove { + lastMove = normalized + commands.append(.pointerMove(x: Double(normalized.x), y: Double(normalized.y))) + } + commands.append(.pointerUp(button, clickCount: held.clickCount)) + return commands + } + + /// Scroll deltas in view points with the receiving Mac's own scroll + /// direction already applied (`NSEvent.scrollingDelta*`), so this Mac's + /// natural-scrolling setting is what its user feels. + mutating func scrolled(dxPoints: Double, dyPoints: Double, inside: Bool, + videoPixelsPerPoint: Double) -> [ReceiverInputCommand] { + guard isForwarding, inside, dxPoints != 0 || dyPoints != 0 else { return [] } + return [.scroll(dx: dxPoints * videoPixelsPerPoint, dy: dyPoints * videoPixelsPerPoint)] + } + + /// Trackpad pinch/rotate, as the existing `nativeAppGesture` lifecycle. + mutating func gesture(_ kind: NativeAppGestureKind, phase: NativeAppGesturePhase, + delta: Double) -> [ReceiverInputCommand] { + switch phase { + case .began: + guard isForwarding, !activeGestures.contains(kind) else { return [] } + activeGestures.insert(kind) + case .changed: + guard isForwarding, activeGestures.contains(kind) else { return [] } + case .ended, .cancelled: + guard activeGestures.remove(kind) != nil else { return [] } + } + return [.gesture(kind, phase, delta: delta)] + } + + // MARK: Keyboard + + /// A key press. Returns `nil` when the event is NOT for the sending Mac + /// and AppKit should handle it locally (not forwarding, or a key the + /// wire can't name); an empty array when it was consumed here (the + /// release shortcut, a repeat of a key that was never sent). + mutating func keyDown(keyCode: UInt16, isRepeat: Bool, + modifiers: ReceiverModifierState) -> [ReceiverInputCommand]? { + if keyCode == MacVirtualKeyMap.escapeKeyCode, modifiers.held == Self.releaseShortcutModifiers { + return grant == .allowed && connected ? pause() : nil + } + guard isForwarding else { return nil } + guard let usage = MacVirtualKeyMap.hidUsage(forKeyCode: keyCode) else { return [] } + let names = wireModifiers(capsLock: modifiers.capsLock) + if isRepeat { + // Only repeat what this forwarder pressed. + guard heldKeys[usage] != nil else { return [] } + return [.keyDown(usage: usage, modifiers: names, isRepeat: true)] + } + guard heldKeys[usage] == nil else { return [] } + heldKeys[usage] = names + return [.keyDown(usage: usage, modifiers: names, isRepeat: false)] + } + + /// A key release: forwarded only for a key this forwarder pressed, and + /// never gated on forwarding, so every press gets its release. + mutating func keyUp(keyCode: UInt16) -> [ReceiverInputCommand]? { + guard let usage = MacVirtualKeyMap.hidUsage(forKeyCode: keyCode), + let names = heldKeys.removeValue(forKey: usage) else { + return isForwarding ? [] : nil + } + return [.keyUp(usage: usage, modifiers: names)] + } + + /// `flagsChanged`: modifiers pressed while forwarding go down on the + /// sending Mac; releases of modifiers this forwarder sent go up. + mutating func modifiersChanged(_ state: ReceiverModifierState) -> [ReceiverInputCommand] { + var commands: [ReceiverInputCommand] = [] + for modifier in ControlModifier.allCases where heldModifiers.contains(modifier) && !state.held.contains(modifier) { + heldModifiers.remove(modifier) + commands.append(.modifierUp(modifier)) + } + guard isForwarding else { return commands } + for modifier in ControlModifier.allCases where !heldModifiers.contains(modifier) && state.held.contains(modifier) { + heldModifiers.insert(modifier) + commands.append(.modifierDown(modifier)) + } + return commands + } + + // MARK: Private + + private func wireModifiers(capsLock: Bool) -> [String] { + var names = ControlModifier.allCases.filter(heldModifiers.contains).map(\.rawValue) + if capsLock { names.append("capsLock") } + return names + } + + /// Everything this forwarder holds goes up on the sending Mac, then a + /// `cancel` as the safety net. Nothing is sent when nothing is held. + private mutating func releaseAll() -> [ReceiverInputCommand] { + guard connected, holdsInput else { + resetTracking() + return [] + } + var commands: [ReceiverInputCommand] = [] + for kind in NativeAppGestureKind.allCases where activeGestures.contains(kind) { + commands.append(.gesture(kind, .cancelled, delta: 0)) + } + if let held = heldButton { + commands.append(.pointerUp(held.button, clickCount: held.clickCount)) + } + for usage in heldKeys.keys.sorted() { + commands.append(.keyUp(usage: usage, modifiers: heldKeys[usage] ?? [])) + } + for modifier in ControlModifier.allCases where heldModifiers.contains(modifier) { + commands.append(.modifierUp(modifier)) + } + commands.append(.cancelAll) + resetTracking() + return commands + } + + private mutating func resetTracking() { + heldKeys = [:] + heldModifiers = [] + heldButton = nil + activeGestures = [] + lastMove = nil + } +} + +// MARK: - Presentation + +extension ReceiverControlStatus { + /// The indicator's short label; `nil` hides it. + var title: String? { + switch self { + case .unavailable: return nil + case .viewOnly: return String(localized: "View Only", comment: "Receiver control status: the stream can be watched but not controlled.") + case .requesting: return String(localized: "Requesting Control…") + case .declined: return String(localized: "Control Declined") + case .requestsDisabled: return String(localized: "Control Not Available") + case .inactive: return String(localized: "Control Allowed") + case .paused: return String(localized: "Control Paused") + case .controlling: return String(localized: "Controlling") + } + } + + /// The one action the indicator offers. + enum Action: Equatable { + case request + case release + } + + var action: Action? { + switch self { + case .viewOnly, .declined: return .request + case .requesting, .inactive, .paused, .controlling: return .release + case .unavailable, .requestsDisabled: return nil + } + } + + /// A short-lived line over the video when the state changes, so the way + /// back out is never a secret. + var hint: String? { + switch self { + case .controlling: + return String(localized: "This Mac's keyboard and trackpad now control the other Mac. Press \(ReceiverInputForwarder.releaseShortcutLabel) to pause.", + comment: "The placeholder is a keyboard shortcut such as ⌃⌥⎋.") + case .paused: return String(localized: "Control paused. Click the picture to resume.") + case .declined: return String(localized: "The other Mac didn't allow control.") + case .requestsDisabled: return String(localized: "The other Mac doesn't accept control requests from this Mac.") + case .unavailable, .viewOnly, .requesting, .inactive: return nil + } + } +} + +/// While this Mac drives the pointer, the streamed cursor is drawn where the +/// local pointer is right away instead of waiting a network round trip and a +/// video frame for the sending Mac to report it. The sending Mac's reports +/// take over again once local movement stops, so the two can never disagree +/// for long (the sending Mac's own trackpad, a cursor clamped at a screen +/// edge). +struct CursorEchoPolicy { + static let holdAfterLocalMove: TimeInterval = 0.25 + + private(set) var localUntil: TimeInterval = 0 + + mutating func localMove(at now: TimeInterval) { + localUntil = now + Self.holdAfterLocalMove + } + + func appliesRemotePosition(at now: TimeInterval) -> Bool { + now >= localUntil + } + + mutating func reset() { + localUntil = 0 + } +} diff --git a/MacTests/ReceiverInputForwardingTests.swift b/MacTests/ReceiverInputForwardingTests.swift new file mode 100644 index 000000000..5c90286a5 --- /dev/null +++ b/MacTests/ReceiverInputForwardingTests.swift @@ -0,0 +1,532 @@ +import CoreGraphics +import XCTest + +/// Issue #12: a receiving Mac's keyboard and trackpad driving the sending +/// Mac. These drive the real `ReceiverInputForwarder` with synthetic AppKit +/// event sequences and check every command it would send. They prove the +/// receiver's rules; they do not prove how a physical trackpad or keyboard +/// feels (see RECEIVER_INPUT.md's physical checklist). +final class ReceiverInputForwardingTests: XCTestCase { + + // MARK: - Geometry + + func testExactFitMapsCornersAndFlipsY() { + let geometry = ReceiverVideoGeometry(viewSize: CGSize(width: 1440, height: 900), + videoSize: CGSize(width: 2880, height: 1800)) + XCTAssertEqual(geometry.fittedRect, CGRect(x: 0, y: 0, width: 1440, height: 900)) + // AppKit's origin is bottom-left; the wire's is top-left. + XCTAssertEqual(geometry.normalized(CGPoint(x: 0, y: 900))?.point, CGPoint(x: 0, y: 0)) + XCTAssertEqual(geometry.normalized(CGPoint(x: 1440, y: 0))?.point, CGPoint(x: 1, y: 1)) + XCTAssertEqual(geometry.normalized(CGPoint(x: 360, y: 675))?.point, CGPoint(x: 0.25, y: 0.25)) + XCTAssertEqual(geometry.videoPixelsPerPoint, 2) + } + + func testWideWindowLetterboxesAndLetterboxIsOutsideThePicture() { + // A 16:10 stream in a 2:1 window: bars left and right. + let geometry = ReceiverVideoGeometry(viewSize: CGSize(width: 2000, height: 1000), + videoSize: CGSize(width: 3200, height: 2000)) + XCTAssertEqual(geometry.fittedRect, CGRect(x: 200, y: 0, width: 1600, height: 1000)) + let inBar = geometry.normalized(CGPoint(x: 100, y: 500))! + XCTAssertFalse(inBar.inside) + XCTAssertEqual(inBar.point, CGPoint(x: 0, y: 0.5)) + XCTAssertTrue(geometry.normalized(CGPoint(x: 1000, y: 500))!.inside) + } + + func testTallWindowPillarboxes() { + let geometry = ReceiverVideoGeometry(viewSize: CGSize(width: 800, height: 1000), + videoSize: CGSize(width: 1600, height: 1000)) + XCTAssertEqual(geometry.fittedRect, CGRect(x: 0, y: 250, width: 800, height: 500)) + XCTAssertEqual(geometry.normalized(CGPoint(x: 400, y: 500))?.point, CGPoint(x: 0.5, y: 0.5)) + XCTAssertFalse(geometry.normalized(CGPoint(x: 400, y: 900))!.inside) + } + + func testResizingKeepsTheSameNormalizedPointAndRescalesScrollUnits() { + let video = CGSize(width: 2880, height: 1800) + for (width, expectedScale) in [(1440.0, 2.0), (720.0, 4.0), (2880.0, 1.0)] { + let geometry = ReceiverVideoGeometry(viewSize: CGSize(width: width, height: width * 0.625), videoSize: video) + XCTAssertEqual(geometry.normalized(CGPoint(x: width / 2, y: width * 0.625 / 4))?.point, + CGPoint(x: 0.5, y: 0.75)) + XCTAssertEqual(geometry.videoPixelsPerPoint!, expectedScale, accuracy: 1e-9) + } + } + + func testMixedResolutionFullScreenStreamFitsInsideTheScreen() { + // A 1920×1200 stream full screen on a 2560×1440-point display. + let geometry = ReceiverVideoGeometry(viewSize: CGSize(width: 2560, height: 1440), + videoSize: CGSize(width: 1920, height: 1200)) + let rect = geometry.fittedRect! + XCTAssertEqual(rect.height, 1440) + XCTAssertEqual(rect.width, 2304) + XCTAssertEqual(rect.minX, 128) + XCTAssertEqual(geometry.videoPixelsPerPoint!, 1920.0 / 2304.0, accuracy: 1e-9) + } + + func testDragFarOutsideTheWindowClampsToTheEdge() { + let geometry = ReceiverVideoGeometry(viewSize: CGSize(width: 1000, height: 500), + videoSize: CGSize(width: 2000, height: 1000)) + let result = geometry.normalized(CGPoint(x: 5000, y: -300))! + XCTAssertEqual(result.point, CGPoint(x: 1, y: 1)) + XCTAssertFalse(result.inside) + } + + func testUnknownVideoSizeMapsNothing() { + XCTAssertNil(ReceiverVideoGeometry(viewSize: CGSize(width: 100, height: 100), videoSize: .zero).normalized(.zero)) + XCTAssertNil(ReceiverVideoGeometry(viewSize: .zero, videoSize: CGSize(width: 10, height: 10)).videoPixelsPerPoint) + } + + // MARK: - Key map + + func testEveryMappedKeyRoundTripsThroughTheSendersInjectionTable() { + for keyCode in MacVirtualKeyMap.mappedKeyCodes { + let usage = MacVirtualKeyMap.hidUsage(forKeyCode: keyCode)! + guard let senderUsage = HIDKeyUsage(rawValue: usage) else { + return XCTFail("sender has no injection for usage \(usage) (key code \(keyCode))") + } + XCTAssertEqual(senderUsage.keyCode, CGKeyCode(keyCode), "usage \(usage)") + } + } + + func testMappingIsOneToOne() { + let usages = MacVirtualKeyMap.mappedKeyCodes.compactMap(MacVirtualKeyMap.hidUsage(forKeyCode:)) + XCTAssertEqual(Set(usages).count, usages.count) + XCTAssertGreaterThanOrEqual(usages.count, 100, "a full keyboard, F-keys, navigation and keypad") + } + + func testModifierKeysTravelAsModifiersNotKeys() { + // Command, Shift, Option, Control (both sides), Caps Lock, Fn. + for keyCode: UInt16 in [0x37, 0x36, 0x38, 0x3C, 0x3A, 0x3D, 0x3B, 0x3E, 0x39, 0x3F] { + XCTAssertNil(MacVirtualKeyMap.hidUsage(forKeyCode: keyCode)) + } + } + + func testRepresentativeKeys() { + XCTAssertEqual(MacVirtualKeyMap.hidUsage(forKeyCode: 0x00), 4) // A + XCTAssertEqual(MacVirtualKeyMap.hidUsage(forKeyCode: 0x32), 53) // ` + XCTAssertEqual(MacVirtualKeyMap.hidUsage(forKeyCode: 0x60), 62) // F5 + XCTAssertEqual(MacVirtualKeyMap.hidUsage(forKeyCode: 0x73), 74) // Home + XCTAssertEqual(MacVirtualKeyMap.hidUsage(forKeyCode: 0x0A), 100) // ISO § + XCTAssertEqual(MacVirtualKeyMap.hidUsage(forKeyCode: 0x4C), 88) // keypad Enter + } + + func testModifierStateFromRawFlags() { + // NSEvent.ModifierFlags: capsLock 1<<16, shift 17, control 18, option 19, command 20. + let state = ReceiverModifierState(rawFlags: (1 << 16) | (1 << 17) | (1 << 20) | (1 << 23)) + XCTAssertEqual(state.held, [.shift, .command]) + XCTAssertTrue(state.capsLock) + XCTAssertEqual(ReceiverModifierState(rawFlags: 0), ReceiverModifierState()) + } + + // MARK: - Helpers + + private let center = CGPoint(x: 0.5, y: 0.5) + private let none = ReceiverModifierState() + + private func controlling() -> ReceiverInputForwarder { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + _ = forwarder.windowKeyChanged(true) + _ = forwarder.grantChanged(.allowed) + XCTAssertEqual(forwarder.status, .controlling) + return forwarder + } + + private func mods(_ held: Set, capsLock: Bool = false) -> ReceiverModifierState { + ReceiverModifierState(held: held, capsLock: capsLock) + } + + // MARK: - View only by default + + func testANewSessionIsViewOnlyAndForwardsNothing() { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + _ = forwarder.windowKeyChanged(true) + XCTAssertEqual(forwarder.status, .viewOnly) + XCTAssertEqual(forwarder.pointerMoved(center, inside: true), []) + XCTAssertEqual(forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true), []) + XCTAssertNil(forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none), "typed keys stay local") + XCTAssertEqual(forwarder.modifiersChanged(mods([.command])), []) + XCTAssertEqual(forwarder.scrolled(dxPoints: 0, dyPoints: 5, inside: true, videoPixelsPerPoint: 2), []) + XCTAssertEqual(forwarder.gesture(.magnify, phase: .began, delta: 0.1), []) + } + + func testRequestControlUsesTheExistingRequestOnce() { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + XCTAssertEqual(forwarder.requestControl(), [.requestControl(true)]) + _ = forwarder.grantChanged(.requesting) + XCTAssertEqual(forwarder.status, .requesting) + XCTAssertEqual(forwarder.requestControl(), [], "already waiting") + _ = forwarder.grantChanged(.notAllowed) + XCTAssertEqual(forwarder.status, .declined) + XCTAssertEqual(forwarder.requestControl(), [.requestControl(true)], "may ask again") + _ = forwarder.grantChanged(.requestsDisabled) + XCTAssertEqual(forwarder.requestControl(), [], "the Mac refuses requests from this device") + } + + func testAnOldSendingMacOffersNoControl() { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: false) + _ = forwarder.windowKeyChanged(true) + _ = forwarder.grantChanged(.allowed) + XCTAssertEqual(forwarder.status, .unavailable) + XCTAssertEqual(forwarder.requestControl(), []) + XCTAssertNil(forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none)) + } + + func testGrantedButNotFrontmostStaysLocal() { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + _ = forwarder.grantChanged(.allowed) + XCTAssertEqual(forwarder.status, .inactive) + XCTAssertNil(forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none)) + XCTAssertEqual(forwarder.pointerMoved(center, inside: true), []) + } + + // MARK: - Pointer + + func testHoverMovesForwardOnlyOverThePictureAndDeduplicate() { + var forwarder = controlling() + XCTAssertEqual(forwarder.pointerMoved(CGPoint(x: 0.2, y: 0.3), inside: true), [.pointerMove(x: 0.2, y: 0.3)]) + XCTAssertEqual(forwarder.pointerMoved(CGPoint(x: 0.2, y: 0.3), inside: true), []) + XCTAssertEqual(forwarder.pointerMoved(CGPoint(x: 0, y: 0.3), inside: false), [], "letterbox hover") + } + + func testClickDragReleaseOutsideTheVideo() { + var forwarder = controlling() + XCTAssertEqual(forwarder.pointerDown(.left, clickCount: 1, at: CGPoint(x: 0.4, y: 0.4), inside: true), + [.pointerMove(x: 0.4, y: 0.4), .pointerDown(.left, clickCount: 1)]) + XCTAssertEqual(forwarder.pointerMoved(CGPoint(x: 0.9, y: 0.4), inside: true), [.pointerMove(x: 0.9, y: 0.4)]) + // Dragged past the edge: still forwarded, clamped by the geometry. + XCTAssertEqual(forwarder.pointerMoved(CGPoint(x: 1, y: 0.4), inside: false), [.pointerMove(x: 1, y: 0.4)]) + XCTAssertEqual(forwarder.pointerUp(.left, at: CGPoint(x: 1, y: 0.4)), [.pointerUp(.left, clickCount: 1)]) + XCTAssertFalse(forwarder.holdsInput) + } + + func testReleaseAtANewPositionMovesFirst() { + var forwarder = controlling() + _ = forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true) + XCTAssertEqual(forwarder.pointerUp(.left, at: CGPoint(x: 0.6, y: 0.5)), + [.pointerMove(x: 0.6, y: 0.5), .pointerUp(.left, clickCount: 1)]) + } + + func testDoubleClickAndRightClickCarryTheirCountsAndButton() { + var forwarder = controlling() + _ = forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true) + _ = forwarder.pointerUp(.left, at: center) + XCTAssertEqual(forwarder.pointerDown(.left, clickCount: 2, at: center, inside: true).last, + .pointerDown(.left, clickCount: 2)) + XCTAssertEqual(forwarder.pointerUp(.left, at: center), [.pointerUp(.left, clickCount: 2)]) + XCTAssertEqual(forwarder.pointerDown(.right, clickCount: 1, at: center, inside: true).last, + .pointerDown(.right, clickCount: 1)) + XCTAssertEqual(forwarder.pointerUp(.right, at: center), [.pointerUp(.right, clickCount: 1)]) + } + + func testASecondButtonDuringADragIsIgnoredAndItsReleaseNeverSent() { + var forwarder = controlling() + _ = forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true) + XCTAssertEqual(forwarder.pointerDown(.right, clickCount: 1, at: center, inside: true), []) + XCTAssertEqual(forwarder.pointerUp(.right, at: center), []) + XCTAssertEqual(forwarder.pointerUp(.left, at: center), [.pointerUp(.left, clickCount: 1)]) + } + + func testClickInTheLetterboxDoesNothing() { + var forwarder = controlling() + XCTAssertEqual(forwarder.pointerDown(.left, clickCount: 1, at: CGPoint(x: 0, y: 0.5), inside: false), []) + XCTAssertEqual(forwarder.pointerUp(.left, at: nil), []) + } + + func testWindowLosingFocusMidDragReleasesTheButton() { + var forwarder = controlling() + _ = forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true) + XCTAssertEqual(forwarder.windowKeyChanged(false), [.pointerUp(.left, clickCount: 1), .cancelAll]) + XCTAssertEqual(forwarder.status, .inactive) + XCTAssertEqual(forwarder.pointerUp(.left, at: center), [], "AppKit's late mouse-up is not sent twice") + } + + func testScrollConvertsPointsToVideoPixels() { + var forwarder = controlling() + XCTAssertEqual(forwarder.scrolled(dxPoints: -1.5, dyPoints: 4, inside: true, videoPixelsPerPoint: 2), + [.scroll(dx: -3, dy: 8)]) + XCTAssertEqual(forwarder.scrolled(dxPoints: 0, dyPoints: 4, inside: false, videoPixelsPerPoint: 2), []) + XCTAssertEqual(forwarder.scrolled(dxPoints: 0, dyPoints: 0, inside: true, videoPixelsPerPoint: 2), []) + } + + func testPinchAndRotateFollowTheirLifecycle() { + var forwarder = controlling() + XCTAssertEqual(forwarder.gesture(.rotate, phase: .changed, delta: 0.1), [], "no began") + XCTAssertEqual(forwarder.gesture(.magnify, phase: .began, delta: 0), [.gesture(.magnify, .began, delta: 0)]) + XCTAssertEqual(forwarder.gesture(.magnify, phase: .changed, delta: 0.05), [.gesture(.magnify, .changed, delta: 0.05)]) + XCTAssertEqual(forwarder.gesture(.magnify, phase: .ended, delta: 0), [.gesture(.magnify, .ended, delta: 0)]) + XCTAssertEqual(forwarder.gesture(.magnify, phase: .ended, delta: 0), []) + } + + func testFocusLossMidPinchCancelsIt() { + var forwarder = controlling() + _ = forwarder.gesture(.magnify, phase: .began, delta: 0) + XCTAssertEqual(forwarder.windowKeyChanged(false), [.gesture(.magnify, .cancelled, delta: 0), .cancelAll]) + XCTAssertEqual(forwarder.gesture(.magnify, phase: .ended, delta: 0), []) + } + + // MARK: - Keyboard + + func testTypingWithModifiersFollowsTheDownUpLifecycle() { + var forwarder = controlling() + XCTAssertEqual(forwarder.modifiersChanged(mods([.command])), [.modifierDown(.command)]) + XCTAssertEqual(forwarder.keyDown(keyCode: 0x08, isRepeat: false, modifiers: mods([.command])), + [.keyDown(usage: 6, modifiers: ["command"], isRepeat: false)]) + XCTAssertEqual(forwarder.keyUp(keyCode: 0x08), [.keyUp(usage: 6, modifiers: ["command"])]) + XCTAssertEqual(forwarder.modifiersChanged(mods([])), [.modifierUp(.command)]) + XCTAssertFalse(forwarder.holdsInput) + } + + func testShortcutWithSeveralModifiers() { + var forwarder = controlling() + _ = forwarder.modifiersChanged(mods([.command])) + XCTAssertEqual(forwarder.modifiersChanged(mods([.command, .shift])), [.modifierDown(.shift)]) + XCTAssertEqual(forwarder.keyDown(keyCode: 0x14, isRepeat: false, modifiers: mods([.command, .shift])), + [.keyDown(usage: 32, modifiers: ["command", "shift"], isRepeat: false)]) + } + + func testModifierHeldFromCommandTabIsNeverSent() { + // Command is still down from the Command-Tab that brought this window + // forward: the sending Mac must not see a Command-Q. + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + _ = forwarder.grantChanged(.allowed) + _ = forwarder.modifiersChanged(mods([.command])) // pressed while another app was in front + _ = forwarder.windowKeyChanged(true) + XCTAssertEqual(forwarder.keyDown(keyCode: 0x0C, isRepeat: false, modifiers: mods([.command])), + [.keyDown(usage: 20, modifiers: [], isRepeat: false)]) + XCTAssertEqual(forwarder.modifiersChanged(mods([])), [], "its release isn't sent either") + } + + func testKeyRepeatIsForwardedOnlyForAKeyThisMacPressed() { + var forwarder = controlling() + _ = forwarder.keyDown(keyCode: 0x33, isRepeat: false, modifiers: none) + XCTAssertEqual(forwarder.keyDown(keyCode: 0x33, isRepeat: true, modifiers: none), + [.keyDown(usage: 42, modifiers: [], isRepeat: true)]) + XCTAssertEqual(forwarder.keyDown(keyCode: 0x7B, isRepeat: true, modifiers: none), [], + "a repeat of a key pressed before forwarding began") + } + + func testDuplicateDownWithoutRepeatIsNotSentTwice() { + var forwarder = controlling() + _ = forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none) + XCTAssertEqual(forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none), []) + } + + func testKeyUpForAKeyPressedBeforeForwardingStaysLocal() { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + _ = forwarder.windowKeyChanged(true) + XCTAssertNil(forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none)) + _ = forwarder.grantChanged(.allowed) + XCTAssertEqual(forwarder.keyUp(keyCode: 0x00), [], "consumed but not sent: no down reached the other Mac") + } + + func testCapsLockTravelsAsANamedModifier() { + var forwarder = controlling() + XCTAssertEqual(forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: mods([], capsLock: true)), + [.keyDown(usage: 4, modifiers: ["capsLock"], isRepeat: false)]) + } + + func testAKeyWithNoUsageIsConsumedNotLeaked() { + var forwarder = controlling() + XCTAssertEqual(forwarder.keyDown(keyCode: 0xFF, isRepeat: false, modifiers: none), []) + } + + // MARK: - Pause shortcut and release + + func testPauseShortcutReleasesEverythingAndKeepsTheGrant() { + var forwarder = controlling() + _ = forwarder.modifiersChanged(mods([.control])) + _ = forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: mods([.control])) + _ = forwarder.modifiersChanged(mods([.control, .option])) + let commands = forwarder.keyDown(keyCode: MacVirtualKeyMap.escapeKeyCode, isRepeat: false, + modifiers: mods([.control, .option])) + XCTAssertEqual(commands, [.keyUp(usage: 4, modifiers: ["control"]), .modifierUp(.option), + .modifierUp(.control), .cancelAll], + "Escape itself never reaches the other Mac") + XCTAssertEqual(forwarder.status, .paused) + XCTAssertEqual(forwarder.grant, .allowed) + XCTAssertNil(forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none), "paused: keys stay local") + XCTAssertEqual(forwarder.modifiersChanged(mods([])), [], "released modifiers aren't released twice") + } + + func testAClickOnThePictureResumesAndIsNotForwarded() { + var forwarder = controlling() + _ = forwarder.pause() + XCTAssertEqual(forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true), []) + XCTAssertEqual(forwarder.status, .controlling) + XCTAssertEqual(forwarder.pointerUp(.left, at: center), [], "the resume click's release isn't sent") + XCTAssertEqual(forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true).last, + .pointerDown(.left, clickCount: 1)) + } + + func testPauseShortcutWithoutControlIsAnOrdinaryLocalKey() { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + _ = forwarder.windowKeyChanged(true) + XCTAssertNil(forwarder.keyDown(keyCode: MacVirtualKeyMap.escapeKeyCode, isRepeat: false, + modifiers: mods([.control, .option]))) + } + + func testPlainEscapeIsForwarded() { + var forwarder = controlling() + XCTAssertEqual(forwarder.keyDown(keyCode: MacVirtualKeyMap.escapeKeyCode, isRepeat: false, modifiers: none), + [.keyDown(usage: 41, modifiers: [], isRepeat: false)]) + } + + func testReleaseControlGivesTheGrantBack() { + var forwarder = controlling() + _ = forwarder.keyDown(keyCode: 0x31, isRepeat: false, modifiers: none) + XCTAssertEqual(forwarder.releaseControl(), + [.keyUp(usage: 44, modifiers: []), .cancelAll, .requestControl(false)]) + _ = forwarder.grantChanged(.off) + XCTAssertEqual(forwarder.status, .viewOnly) + } + + func testReleaseWhileRequestingWithdrawsTheRequest() { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + _ = forwarder.grantChanged(.requesting) + XCTAssertEqual(forwarder.releaseControl(), [.requestControl(false)]) + } + + func testFocusLossWithNothingHeldSendsNothing() { + var forwarder = controlling() + XCTAssertEqual(forwarder.windowKeyChanged(false), []) + } + + // MARK: - Revocation, disconnects, reconnects + + func testRevocationMidTypingReleasesAndStopsForwarding() { + var forwarder = controlling() + _ = forwarder.modifiersChanged(mods([.shift])) + _ = forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: mods([.shift])) + _ = forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true) + XCTAssertEqual(forwarder.grantChanged(.off), + [.pointerUp(.left, clickCount: 1), .keyUp(usage: 4, modifiers: ["shift"]), + .modifierUp(.shift), .cancelAll]) + XCTAssertEqual(forwarder.status, .viewOnly) + XCTAssertNil(forwarder.keyUp(keyCode: 0x00), "the release after revocation stays local") + XCTAssertEqual(forwarder.pointerMoved(center, inside: true), []) + } + + func testDisconnectDuringInputSendsNothingAndClearsState() { + var forwarder = controlling() + _ = forwarder.modifiersChanged(mods([.command])) + _ = forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: mods([.command])) + forwarder.setConnected(false, inputSupported: false) + XCTAssertFalse(forwarder.holdsInput) + XCTAssertEqual(forwarder.status, .unavailable) + } + + func testReconnectStartsViewOnlyWithNoStaleKeys() { + var forwarder = controlling() + _ = forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none) + _ = forwarder.pause() + forwarder.setConnected(false, inputSupported: false) + forwarder.setConnected(true, inputSupported: true) + XCTAssertEqual(forwarder.status, .viewOnly, "a new session's grant starts off") + XCTAssertFalse(forwarder.paused) + XCTAssertNil(forwarder.keyUp(keyCode: 0x00)) + _ = forwarder.grantChanged(.allowed) + XCTAssertEqual(forwarder.status, .controlling) + } + + func testSenderResetClearsTrackingWithoutSending() { + var forwarder = controlling() + _ = forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none) + forwarder.senderReset() + XCTAssertFalse(forwarder.holdsInput) + XCTAssertEqual(forwarder.windowKeyChanged(false), []) + } + + func testRegrantAfterPauseClearsThePause() { + var forwarder = controlling() + _ = forwarder.pause() + _ = forwarder.grantChanged(.off) + _ = forwarder.grantChanged(.allowed) + XCTAssertEqual(forwarder.status, .controlling) + } + + // MARK: - Presentation + + func testIndicatorOffersTheOneRelevantAction() { + XCTAssertNil(ReceiverControlStatus.unavailable.title) + XCTAssertEqual(ReceiverControlStatus.viewOnly.action, .request) + XCTAssertEqual(ReceiverControlStatus.declined.action, .request) + XCTAssertNil(ReceiverControlStatus.requestsDisabled.action) + for status: ReceiverControlStatus in [.requesting, .inactive, .paused, .controlling] { + XCTAssertEqual(status.action, .release) + } + XCTAssertTrue(ReceiverControlStatus.controlling.hint!.contains(ReceiverInputForwarder.releaseShortcutLabel)) + } + + func testCursorEchoHandsBackToTheSendingMacAfterLocalMovementStops() { + var echo = CursorEchoPolicy() + XCTAssertTrue(echo.appliesRemotePosition(at: 10)) + echo.localMove(at: 10) + XCTAssertFalse(echo.appliesRemotePosition(at: 10.1)) + XCTAssertTrue(echo.appliesRemotePosition(at: 10 + CursorEchoPolicy.holdAfterLocalMove)) + echo.localMove(at: 20) + echo.reset() + XCTAssertTrue(echo.appliesRemotePosition(at: 20)) + } + + // MARK: - Sender side + + func testScrollRemainderKeepsSlowTrackpadScrolls() { + var remainder = ScrollRemainder() + var total: Int32 = 0 + for _ in 0..<10 { total += remainder.take(dxPoints: 0, dyPoints: 0.3).dy } + XCTAssertEqual(total, 3, "ten 0.3-point deltas scroll three points, not zero") + } + + func testScrollRemainderStillRoundsASingleDeltaToNearest() { + var remainder = ScrollRemainder() + XCTAssertEqual(remainder.take(dxPoints: 0.6, dyPoints: -2.4).dx, 1) + remainder.reset() + XCTAssertEqual(remainder.take(dxPoints: 0, dyPoints: -2.4).dy, -2) + } + + func testScrollRemainderNeverDriftsAcrossDirectionChanges() { + var remainder = ScrollRemainder() + var total: Int32 = 0 + let deltas = [0.4, 0.4, -0.3, 1.7, -0.9, 0.2, 0.25] + for delta in deltas { total += remainder.take(dxPoints: 0, dyPoints: delta).dy } + XCTAssertEqual(Double(total) + remainder.y, deltas.reduce(0, +), accuracy: 1e-9) + XCTAssertLessThanOrEqual(abs(remainder.y), 0.5) + } + + func testSenderRepeatsOnlyHeldKeys() { + var held = HeldKeyTracker() + XCTAssertFalse(held.allowsRepeat(.deleteOrBackspace)) + held.down(.deleteOrBackspace) + XCTAssertTrue(held.allowsRepeat(.deleteOrBackspace)) + held.up(.deleteOrBackspace) + XCTAssertFalse(held.allowsRepeat(.deleteOrBackspace), "a repeat can't outlive its key") + } + + func testSenderParsesTheNewUsagesAndOldOnesUnchanged() { + XCTAssertEqual(HIDKeyUsage.parse(62 as NSNumber)?.keyCode, 96) // F5 + XCTAssertEqual(HIDKeyUsage.parse(74 as NSNumber)?.keyCode, 115) // Home + XCTAssertEqual(HIDKeyUsage.parse(42 as NSNumber)?.keyCode, 51) // Backspace, as before + XCTAssertNil(HIDKeyUsage.parse(200 as NSNumber), "still ignores unknown usages") + } + + // MARK: - Cost + + /// Per-event cost of the receiver's decision layer, for the record (no + /// threshold): a 120 Hz trackpad produces ~120 moves a second. + func testPointerMoveDecisionCost() { + measure { + var forwarder = controlling() + for index in 0..<10_000 { + let x = Double(index % 1000) / 1000 + _ = forwarder.pointerMoved(CGPoint(x: x, y: 0.5), inside: true) + } + } + } +} diff --git a/project.yml b/project.yml index 3ee9d8291..f6c2927f3 100644 --- a/project.yml +++ b/project.yml @@ -179,6 +179,7 @@ targets: - Mac/SmartTouchTargetClassifier.swift - MacReceiver/ReceiverWindowLifecycle.swift - MacReceiver/ReceiverSettingsCategory.swift + - MacReceiver/ReceiverInputForwarding.swift - Shared/ReceiverGesture.swift - Mac/Log.swift - Mac/LogPolicies.swift From ffd2b6b64ffacd2e17e1840b03e21846950843c1 Mon Sep 17 00:00:00 2001 From: raiseCatError <315733358+raiseCatError@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:31:54 +0530 Subject: [PATCH 3/3] docs: describe Mac Receiver keyboard and trackpad input Document what is forwarded and what stays local, the pause shortcut, the release guarantees and a physical test checklist; the additive keyboard repeat field and the full key range in PROTOCOL.md; and drop the display-only wording from README, SETUP and ARCHITECTURE. Refs #12 --- ARCHITECTURE.md | 11 ++-- PROTOCOL.md | 15 +++++- README.md | 7 +-- RECEIVER_INPUT.md | 130 ++++++++++++++++++++++++++++++++++++++++++++++ SETUP.md | 4 +- 5 files changed, 159 insertions(+), 8 deletions(-) create mode 100644 RECEIVER_INPUT.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 36ec60ffc..932808793 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -25,7 +25,7 @@ Mac. A second receiver app lets another Mac act as a display too. │ video (+ optional audio) + control ▼ ┌──────────────┐ - │ Mac Receiver │ display only: its keyboard/trackpad are NOT forwarded + │ Mac Receiver │ its keyboard/trackpad drive the sender once control is granted └──────────────┘ ``` @@ -100,8 +100,12 @@ drawing, keeping the Mac awake, and its own settings/pairing UI `MacReceiver/ReceiverPairingPanel.swift`). It advertises its own panel size like an iPad does. -It is **display-only**: the receiving Mac's keyboard and trackpad are not -sent back to the sender. +Once the sending Mac grants the session control, the receiving Mac's own +keyboard and trackpad drive it while the video window is in front +(`MacReceiver/ReceiverInputForwarding.swift`, `ReceiverInputController.swift`; +see [RECEIVER_INPUT.md](RECEIVER_INPUT.md)). Input is never captured globally, +and it uses the same `pointer`/`scroll`/`keyboard` messages and per-session +grant as the iPhone and iPad. ## Shared code (`Shared/`) @@ -384,3 +388,4 @@ hints and an in-memory cooldown. USB never uses QUIC. - [SUPPORT.md](SUPPORT.md) — where to get help - [TECHNICAL_NOTES.md](TECHNICAL_NOTES.md) — platform constraints behind the design - [MULTI_DEVICE.md](MULTI_DEVICE.md) — per-device Mirror/Extend and its physical test checklist +- [RECEIVER_INPUT.md](RECEIVER_INPUT.md) — Mac Receiver keyboard and trackpad input, and its physical test checklist diff --git a/PROTOCOL.md b/PROTOCOL.md index 83a940967..235d5d911 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -681,6 +681,12 @@ and the stylus would go dead; the fallback keeps it usable.) receiver sends MUST be followed by a matching `"up"` (or by disconnect — senders MUST release a still-held key on session loss regardless). `{"type":"keyboard","action":"down","usage":80,"modifiers":["shift"]}` + A `"down"` MAY carry `repeat` (bool, additive, no `pv` bump): `true` + marks the receiver's own key repeat for a key it already sent down and + has not released. A sender SHOULD post it as an auto-repeat key-down + only while that key is held and MUST ignore it otherwise; a sender that + doesn't know the field treats it as the duplicate down it already + ignores. * `"modifierDown"` / `"modifierUp"` (pv 6) — a synthetic modifier's held lifecycle: `modifier` is one of `"command"`, `"option"`, `"control"`, or `"shift"`. Every down MUST be followed by an up; senders MUST also release @@ -696,7 +702,13 @@ unrecognized modifier names. `usage` values are USB HID Usage Tables keyboard-page (0x07) usage numbers. Senders MUST validate `usage` (an integral value in `0...65535`) and MUST ignore an out-of-range, non-integral, or unrecognized value rather than -treat it as fatal. The keys the official apps exchange today: +treat it as fatal. The official Mac Receiver sends any key of a physical +keyboard: letters, digits and punctuation (4–56), F1–F12 (58–69) and +F13–F20 (104–111), Insert/Help (73), Home (74), Page Up (75), End (77), +Page Down (78), the keypad (83–99, 103, 133), the ISO key (100) and the JIS +keys (135, 137, 144, 145). The usage names a physical key, so the +sender's own keyboard layout decides the character. The special keys the +iPhone and iPad apps send: | Usage | Key | |---|---| @@ -1471,3 +1483,4 @@ This file is versioned by git; the authoritative change log is | 2026-09-27 | `pv` 22: optional reliable-stream QUIC transport (section 2.4) with authenticated `transports`/`qv` capability | | 2026-09-30 | Documented optional HEVC video (section 5.4, `pv` 19) and the `pv` 15/16/18 rows; no wire change. The sender now also falls back to H.264 when an HEVC encoder produces no frames or a new `hello` drops `"hevc"` | | 2026-09-26 | Hardening, no `pv` change: session admission and input grants are per connection (section 6.9); the same `hello` identity/current-pin check on USB (section 2.2); the Remote Access knock declares its intent | +| 2026-10-25 | Additive, no `pv` change: `keyboard` `"down"` may carry `repeat`; the Mac Receiver sends the full physical-keyboard usage range | diff --git a/README.md b/README.md index e68035406..4cb126ae3 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ later). Both are free. | **Connections** | USB · local Wi-Fi · Remote Access · Auto-Reconnect · Wake & Connect | | **Audio** | Mac system audio on the receiver · Mac keeps playing locally · A/V sync and Resync | | **Security** | Trusted pairing with a matching code · TLS 1.3 · pinned peer identity | -| **Receivers** | iPhone and iPad · another Mac with the display-only Mac Receiver | +| **Receivers** | iPhone and iPad · another Mac with the Mac Receiver, whose keyboard and trackpad can control the sending Mac | ## Why MeowDisplay? @@ -312,8 +312,9 @@ Either side can start a session, and the Mac sender always streams. When connection approval is required, an explicit Connect waits for acceptance; background reconnects never prompt. -Receivers include **iPhone**, **iPad**, and the **display-only Mac Receiver**. -The Mac Receiver does not forward its keyboard or trackpad to the other Mac. +Receivers include **iPhone**, **iPad**, and the **Mac Receiver**. Once the +sending Mac allows control, the receiving Mac's own keyboard and trackpad drive +it while the video window is in front; press ⌃⌥⎋ to pause. More on [Devices](https://meowdisplay.app/features.html#devices). ## Connections diff --git a/RECEIVER_INPUT.md b/RECEIVER_INPUT.md new file mode 100644 index 000000000..b0c936bc2 --- /dev/null +++ b/RECEIVER_INPUT.md @@ -0,0 +1,130 @@ +# Mac Receiver input + +A Mac running the Mac Receiver is a display **and** a control surface +([issue #12](https://github.com/raiseCatError/meowdisplay/issues/12)). Once the +sending Mac allows it, the receiving Mac's own trackpad, mouse and keyboard +drive the sending Mac while the video window is in front, in Mirror and in +Extend. Drag a window onto the receiving MacBook's extended desktop and use +that MacBook's trackpad and keyboard on it. + +## Using it + +1. Connect as usual. Every session starts **View Only**. +2. Click **Request Control** in the video window's titlebar. In full screen, + move the pointer to the top of the screen to show the titlebar. The + sending Mac asks its owner, or allows it straight away if this device is + set to **Always Allow Requests**. +3. While the titlebar says **Controlling**, the pointer, clicks, scrolling, + pinch/rotate and keyboard go to the sending Mac. +4. To get this Mac back: + - **⌃⌥⎋ (Control-Option-Escape)** pauses control. Click the picture to + resume; that click only resumes and isn't sent. + - Switching to another app or window (Command-Tab, a click outside the + window) stops forwarding until the video window is in front again. + - **Release** in the titlebar gives control back to the sending Mac. A + new request is needed to control it again. + +The sending Mac can revoke control at any time. Its own Allow Input switch, +per-device policy and request prompt all work exactly as for an iPhone or +iPad. + +## What is forwarded + +| Input | Sent as | Notes | +| --- | --- | --- | +| Pointer movement | `pointer` `move` (absolute, normalized) | Over the picture only, or anywhere during a drag (clamped to the picture's edge). | +| Click, double/triple click | `pointer` `down`/`up`, `clickCount` | Left and right (two-finger click, Control-click via the Control modifier). | +| Drag | `down`, moves, `up` | Released correctly even when the button goes up outside the window. | +| Scroll | `scroll` (video pixels) | Trackpad precise deltas and momentum; wheel lines ≈ 10 pt. Direction follows **this** Mac's natural-scrolling setting. | +| Pinch, rotate | `nativeAppGesture` | Full began/changed/ended lifecycle; cancelled if focus is lost. | +| Keys | `keyboard` `down`/`up` with HID usage | Every key of a full keyboard, F-keys, navigation and keypad. The **sending** Mac's keyboard layout and input method decide the character, as with a keyboard plugged into it. | +| Key repeat | `keyboard` `down` + `repeat: true` | This Mac's own repeat rate. Needs a sender from this branch; an older sender simply doesn't repeat. | +| Shift, Control, Option, Command | `keyboard` `modifierDown`/`modifierUp` | Applied to keys, clicks and scrolls on the sending Mac. | +| Caps Lock | `capsLock` in each key's `modifiers` | Not toggled on the sending Mac. | + +**Not forwarded**: +- **Middle and other mouse buttons**: the protocol names left and right only. +- **Fn and Globe-key shortcuts**: macOS turns Fn+arrow into Home/End/Page keys before the app sees it, and those are forwarded. +- **Three- and four-finger swipes, Mission Control, Launchpad and other trackpad system gestures**: macOS keeps these on this Mac, by design. +- **Force Touch pressure**, and **text composed by this Mac's input method**: the sending Mac's own input method composes instead. + +## What stays on this Mac + +Input is never captured globally. Only events macOS delivers to the video +window while it is in front are looked at (an app-local event monitor scoped +to that window), so: + +- System shortcuts macOS handles before any app — **Command-Tab, Command-Option-Escape (Force Quit), Command-Space, Mission Control, screenshots, Control-Command-Q (Lock Screen)** — always act on this Mac. +- **⌃⌥⎋** is reserved for pausing and is never sent. +- While controlling, app shortcuts such as Command-Q, Command-W or Command-, go + to the **sending** Mac (it's what typing into that window means). Pause + first (⌃⌥⎋) or switch away to use them on this Mac. +- A modifier still held from before forwarding started (the Command of the + Command-Tab that brought the window forward) is never sent, so it can't + turn the next key into a shortcut on the sending Mac. + +## Never stuck + +The receiver remembers every key, modifier, button and gesture it sent down. +Pausing, focus loss, Release, revocation by the sending Mac and closing the +window all send the matching releases, then `keyboard` `cancel` as a safety +net. A disconnect resets locally: the sending Mac releases everything on +session loss itself (PROTOCOL.md), and a reconnect starts View Only with +nothing held. The sending Mac ignores input the moment its grant ends, +whatever a receiver sends. + +## Pointer feel + +While controlling, the streamed cursor is drawn at this Mac's pointer +immediately instead of after a network round trip and a video frame. The +sending Mac's reported position takes over 0.25 s after local movement stops, +so the two can't disagree for long (for example when the sending Mac's own +trackpad moves the cursor). + +## Code + +| Piece | Where | +| --- | --- | +| Rules: geometry, key map, forwarding state machine, cursor echo | `MacReceiver/ReceiverInputForwarding.swift` (AppKit-free, unit tested) | +| AppKit events, window focus, sending, titlebar indicator, hint | `MacReceiver/ReceiverInputController.swift` | +| Video view event overrides | `ReceiverVideoView` in `MacReceiver/MacReceiver.swift` | +| Sender: extra HID usages, key repeat, scroll remainder | `Mac/InputRouting.swift`, `Mac/InputInjector.swift` | + +No new wire message exists for any of this. The only addition is the optional +`repeat` field (PROTOCOL.md section 6.1). + +## Automated coverage + +`MacTests/ReceiverInputForwardingTests.swift` drives the forwarder with +synthetic event sequences: +- **Geometry**: exact fit, letterbox, pillarbox, resize, mixed-resolution full screen, clamped drags. +- **Key map**: a full round trip through the sender's injection table. +- **Pointer**: view only by default; request, decline and requests-disabled; old senders; hover; click, drag and release outside; double and right click; letterbox clicks. +- **Focus and pause**: focus loss mid-drag and mid-pinch; the pause shortcut and click-to-resume; Release. +- **Keyboard**: modifiers and shortcuts; Command held from Command-Tab; key repeat; Caps Lock. +- **Session**: revocation mid-typing; disconnect and reconnect with no stale state. +- **Sender side**: scroll remainder and repeat. +- **Cost**: a measured per-move decision cost. + +These prove the receiver's rules and the sender's pure helpers. Nothing here +posts events into the live system. They do not prove how a physical +trackpad or keyboard feels, real latency, or the sending Mac's injected +events in real apps. + +## Physical test checklist + +Needs two Macs on the same network, paired, both built from this branch: +the sender (for example a MacBook Air) and the receiver (an older MacBook). + +1. **View Only.** Connect: the titlebar says View Only; moving, clicking and typing in the window do nothing on the sender. +2. **Request.** Request Control → the sender prompts → Allow: the titlebar says Controlling and the hint names ⌃⌥⎋. +3. **Pointer, Extend.** Drag a window from the sender onto the receiver's extended desktop. With the receiver's trackpad: move, click, double-click a word, right-click (two fingers and Control-click), drag the window by its title bar and drop it with the button released outside the receiver window. +4. **Pointer, Mirror.** Same in Mirror; check the cursor lands exactly under the receiver's pointer at all four corners, windowed and full screen. +5. **Scroll and gestures.** Slow and fast two-finger scrolls (including momentum) in Safari and Finder; the direction matches the receiver's natural-scrolling setting. Pinch-zoom a photo in Preview. +6. **Keyboard.** Type a sentence with Shift; Command-C/Command-V; hold Delete (repeats); arrows, Home/End, F-keys, Escape; Caps Lock. A non-US layout on the sender types its own characters. +7. **Local shortcuts.** While controlling: Command-Tab, Command-Space, Mission Control, Command-Shift-4 and Command-Option-Escape act on the receiver. +8. **Pause and resume.** ⌃⌥⎋ → Paused; typing stays local; click the picture → Controlling again, without a click reaching the sender. +9. **No stuck keys.** Hold Shift (or a mouse button mid-drag), then Command-Tab away, then check on the sender that nothing is held: typing there is lower-case and no drag continues. Repeat with revoking control on the sender, and with unplugging the network mid-drag. +10. **Release and revoke.** Release → View Only, nothing reaches the sender. Request again, then revoke from the sender's device settings: input stops immediately. +11. **Reconnect.** Quit and relaunch the receiver: it starts View Only. +12. **iPhone/iPad unchanged.** With the same sender, an iPhone's Direct Touch, trackpad mode, keyboard, modifiers and scrolling behave as before. Slow two-finger scrolls should be no worse (the sender now carries the scroll remainder instead of rounding each event). diff --git a/SETUP.md b/SETUP.md index 3a80fdc4c..0b75864d9 100644 --- a/SETUP.md +++ b/SETUP.md @@ -186,7 +186,9 @@ Settings. **Always Allow Requests** skips the Mac prompt but still requires that request each session. See [Allow Input & control requests](README.md#allow-input--control-requests). Receivers can still view the Mac with input off. The standalone Mac Receiver -is display-only; its keyboard and trackpad are not forwarded. +uses the same request: once the sending Mac allows it, the receiving Mac's own +keyboard and trackpad control it while the video window is in front. See +[RECEIVER_INPUT.md](RECEIVER_INPUT.md). ### Connecting and reconnecting