diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 36ec60ffc..932808793 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -25,7 +25,7 @@ Mac. A second receiver app lets another Mac act as a display too. │ video (+ optional audio) + control ▼ ┌──────────────┐ - │ Mac Receiver │ display only: its keyboard/trackpad are NOT forwarded + │ Mac Receiver │ its keyboard/trackpad drive the sender once control is granted └──────────────┘ ``` @@ -100,8 +100,12 @@ drawing, keeping the Mac awake, and its own settings/pairing UI `MacReceiver/ReceiverPairingPanel.swift`). It advertises its own panel size like an iPad does. -It is **display-only**: the receiving Mac's keyboard and trackpad are not -sent back to the sender. +Once the sending Mac grants the session control, the receiving Mac's own +keyboard and trackpad drive it while the video window is in front +(`MacReceiver/ReceiverInputForwarding.swift`, `ReceiverInputController.swift`; +see [RECEIVER_INPUT.md](RECEIVER_INPUT.md)). Input is never captured globally, +and it uses the same `pointer`/`scroll`/`keyboard` messages and per-session +grant as the iPhone and iPad. ## Shared code (`Shared/`) @@ -384,3 +388,4 @@ hints and an in-memory cooldown. USB never uses QUIC. - [SUPPORT.md](SUPPORT.md) — where to get help - [TECHNICAL_NOTES.md](TECHNICAL_NOTES.md) — platform constraints behind the design - [MULTI_DEVICE.md](MULTI_DEVICE.md) — per-device Mirror/Extend and its physical test checklist +- [RECEIVER_INPUT.md](RECEIVER_INPUT.md) — Mac Receiver keyboard and trackpad input, and its physical test checklist diff --git a/Mac/InputInjector.swift b/Mac/InputInjector.swift index 0e4b9b8fb..61a9b0206 100644 --- a/Mac/InputInjector.swift +++ b/Mac/InputInjector.swift @@ -78,6 +78,7 @@ final class InputInjector { // Hardware keys currently held down (M4) — released on cancellation so a // dropped session can never leave a modifier or arrow key stuck. private var heldKeys = HeldKeyTracker() + private var scrollRemainder = ScrollRemainder() private var heldModifiers = HeldModifierTracker() init(displayID: CGDirectDisplayID, sessionInputGrant: SessionInputGrantBox) { @@ -96,6 +97,7 @@ final class InputInjector { /// Caller must already hold inputLock. private func cancelActiveInputLocked() { let point = currentCursor() + scrollRemainder.reset() if isDown { if let event = CGEvent( @@ -220,10 +222,12 @@ final class InputInjector { guard inputIsAllowed() else { return } let bounds = CGDisplayBounds(displayID) let scale = bounds.width > 0 ? Double(CGDisplayPixelsWide(displayID)) / bounds.width : 2 + let whole = scrollRemainder.take(dxPoints: dx / scale, dyPoints: dy / scale) + guard whole.dx != 0 || whole.dy != 0 else { return } guard let event = CGEvent(scrollWheelEvent2Source: source, units: .pixel, wheelCount: 2, - wheel1: Int32((dy / scale).rounded()), - wheel2: Int32((dx / scale).rounded()), + wheel1: whole.dy, + wheel2: whole.dx, wheel3: 0) else { return } event.flags = heldModifiers.flags // see `MouseEventFlags`'s doc event.post(tap: .cghidEventTap) @@ -380,13 +384,21 @@ final class InputInjector { } /// A hardware key going down. A duplicate down for an already-held key - /// is ignored rather than re-posted. - func handleKeyboardDown(usage: HIDKeyUsage, modifiers: [String]) { + /// is ignored rather than re-posted — unless it is marked `repeat` (a + /// receiving Mac's own key repeat), which posts an auto-repeat key-down + /// for the held key. + func handleKeyboardDown(usage: HIDKeyUsage, modifiers: [String], isRepeat: Bool = false) { inputLock.lock() defer { inputLock.unlock() } - guard inputIsAllowed(), heldKeys.down(usage) else { return } - postKeyEvent(keyCode: usage.keyCode, keyDown: true, - flags: KeyModifier.flags(named: modifiers).union(heldModifiers.flags)) + guard inputIsAllowed() else { return } + let flags = KeyModifier.flags(named: modifiers).union(heldModifiers.flags) + if isRepeat { + guard heldKeys.allowsRepeat(usage) else { return } + postKeyEvent(keyCode: usage.keyCode, keyDown: true, flags: flags, autorepeat: true) + return + } + guard heldKeys.down(usage) else { return } + postKeyEvent(keyCode: usage.keyCode, keyDown: true, flags: flags) } /// The matching release. A spurious up for a key that isn't held (e.g. @@ -399,9 +411,10 @@ final class InputInjector { flags: KeyModifier.flags(named: modifiers).union(heldModifiers.flags)) } - private func postKeyEvent(keyCode: CGKeyCode, keyDown: Bool, flags: CGEventFlags) { + private func postKeyEvent(keyCode: CGKeyCode, keyDown: Bool, flags: CGEventFlags, autorepeat: Bool = false) { guard let event = CGEvent(keyboardEventSource: source, virtualKey: keyCode, keyDown: keyDown) else { return } event.flags = flags + if autorepeat { event.setIntegerValueField(.keyboardEventAutorepeat, value: 1) } event.post(tap: .cghidEventTap) } diff --git a/Mac/InputRouting.swift b/Mac/InputRouting.swift index e9d090311..5cf7d4289 100644 --- a/Mac/InputRouting.swift +++ b/Mac/InputRouting.swift @@ -96,6 +96,58 @@ enum HIDKeyUsage: Int, CaseIterable { case leftArrow = 80 case downArrow = 81 case upArrow = 82 + // A physical keyboard on a receiving Mac (issue #12) names every key; + // older senders ignore these, as an unrecognized usage must be. + case grave = 53 + case f1 = 58 + case f2 = 59 + case f3 = 60 + case f4 = 61 + case f5 = 62 + case f6 = 63 + case f7 = 64 + case f8 = 65 + case f9 = 66 + case f10 = 67 + case f11 = 68 + case f12 = 69 + case f13 = 104 + case f14 = 105 + case f15 = 106 + case f16 = 107 + case f17 = 108 + case f18 = 109 + case f19 = 110 + case f20 = 111 + case insertOrHelp = 73 + case home = 74 + case pageUp = 75 + case end = 77 + case pageDown = 78 + case nonUSBackslash = 100 + case international1 = 135 + case international3 = 137 + case lang1 = 144 + case lang2 = 145 + case keypadNumLockOrClear = 83 + case keypadSlash = 84 + case keypadAsterisk = 85 + case keypadMinus = 86 + case keypadPlus = 87 + case keypadEnter = 88 + case keypad1 = 89 + case keypad2 = 90 + case keypad3 = 91 + case keypad4 = 92 + case keypad5 = 93 + case keypad6 = 94 + case keypad7 = 95 + case keypad8 = 96 + case keypad9 = 97 + case keypad0 = 98 + case keypadPeriod = 99 + case keypadEqual = 103 + case keypadComma = 133 /// The macOS virtual key code (`CGKeyCode`, US ANSI layout) this usage /// injects as. @@ -157,6 +209,56 @@ enum HIDKeyUsage: Int, CaseIterable { case .leftArrow: return 123 case .downArrow: return 125 case .upArrow: return 126 + case .grave: return 50 + case .f1: return 122 + case .f2: return 120 + case .f3: return 99 + case .f4: return 118 + case .f5: return 96 + case .f6: return 97 + case .f7: return 98 + case .f8: return 100 + case .f9: return 101 + case .f10: return 109 + case .f11: return 103 + case .f12: return 111 + case .f13: return 105 + case .f14: return 107 + case .f15: return 113 + case .f16: return 106 + case .f17: return 64 + case .f18: return 79 + case .f19: return 80 + case .f20: return 90 + case .insertOrHelp: return 114 + case .home: return 115 + case .pageUp: return 116 + case .end: return 119 + case .pageDown: return 121 + case .nonUSBackslash: return 10 + case .international1: return 94 + case .international3: return 93 + case .lang1: return 104 + case .lang2: return 102 + case .keypadNumLockOrClear: return 71 + case .keypadSlash: return 75 + case .keypadAsterisk: return 67 + case .keypadMinus: return 78 + case .keypadPlus: return 69 + case .keypadEnter: return 76 + case .keypad1: return 83 + case .keypad2: return 84 + case .keypad3: return 85 + case .keypad4: return 86 + case .keypad5: return 87 + case .keypad6: return 88 + case .keypad7: return 89 + case .keypad8: return 91 + case .keypad9: return 92 + case .keypad0: return 82 + case .keypadPeriod: return 65 + case .keypadEqual: return 81 + case .keypadComma: return 95 } } @@ -326,6 +428,36 @@ struct HeldKeyTracker { defer { held.removeAll() } return held } + + /// A `down` marked `repeat` (a receiving Mac's own key repeat) posts an + /// auto-repeat key-down only for a key that is actually held — never a + /// fresh press, so a repeat can't outlive its key's release. + func allowsRepeat(_ usage: HIDKeyUsage) -> Bool { + held.contains(usage) + } +} + +/// Scroll arrives in video pixels and posts in whole points, so a single +/// small delta used to round away: a slow trackpad scroll from a receiving +/// Mac (fractions of a point per event) barely moved. The rounding error is +/// carried into the next event instead, so the total scrolled always matches +/// the total sent. Single events still round to nearest, as before. +struct ScrollRemainder { + private(set) var x = 0.0 + private(set) var y = 0.0 + + mutating func take(dxPoints: Double, dyPoints: Double) -> (dx: Int32, dy: Int32) { + let totalX = x + dxPoints, totalY = y + dyPoints + let wholeX = totalX.rounded(), wholeY = totalY.rounded() + x = totalX - wholeX + y = totalY - wholeY + return (Int32(clamping: Int(wholeX)), Int32(clamping: Int(wholeY))) + } + + mutating func reset() { + x = 0 + y = 0 + } } struct SystemGestureShortcut { diff --git a/Mac/MacSender.swift b/Mac/MacSender.swift index 9e15702c3..20785c2ee 100644 --- a/Mac/MacSender.swift +++ b/Mac/MacSender.swift @@ -5110,7 +5110,8 @@ final class MacSender: NSObject, SCStreamOutput, SCStreamDelegate { } case "down": if let usage = HIDKeyUsage.parse(obj["usage"]) { - inputInjector?.handleKeyboardDown(usage: usage, modifiers: obj["modifiers"] as? [String] ?? []) + inputInjector?.handleKeyboardDown(usage: usage, modifiers: obj["modifiers"] as? [String] ?? [], + isRepeat: obj["repeat"] as? Bool == true) } case "up": if let usage = HIDKeyUsage.parse(obj["usage"]) { diff --git a/MacReceiver/MacReceiver.swift b/MacReceiver/MacReceiver.swift index be4d2c558..93d0335d0 100644 --- a/MacReceiver/MacReceiver.swift +++ b/MacReceiver/MacReceiver.swift @@ -6,8 +6,9 @@ // keeping the machine awake while it serves as a screen. It ships as its own // app (OpenSidecarMacReceiver target, macOS 12+) so old Macs can be displays. // -// Display-only for now: the receiving Mac's keyboard/trackpad are not -// forwarded to the sender (KVM-style input is a follow-up). +// Input (issue #12): once the sending Mac grants this session control, this +// Mac's own keyboard and trackpad drive it while the video window is in +// front — see ReceiverInputForwarding.swift for the rules. import AppKit import AVFoundation @@ -42,6 +43,9 @@ final class ReceiverController: ObservableObject { /// Wake & Connect for a paired Mac (the same shared coordinator the /// iPhone app drives). Lives exactly as long as `receiver`. @Published private(set) var wakeConnect: WakeConnectCoordinator? + /// Keyboard/trackpad forwarding for this receiver's sessions. Lives + /// exactly as long as `receiver`. + @Published private(set) var input: ReceiverInputController? /// The receiver-local manual A/V Sync offset (`AVSyncOffset.range`), /// persisted here and applied to the shared receiver's /// `avSyncPreference` — the one value its audio scheduling and video @@ -87,6 +91,7 @@ final class ReceiverController: ObservableObject { receiver.avSyncPreference.set(avSyncOffsetMs) self.receiver = receiver wakeConnect = WakeConnectCoordinator(receiver: receiver) + input = ReceiverInputController(receiver: receiver) receiver.start() receiver.$connected @@ -192,6 +197,8 @@ final class ReceiverController: ObservableObject { screenSleepObservers = [] wakeConnect?.cancel() wakeConnect = nil + input?.invalidate() + input = nil // The receiver finishes its teardown on its own queue and only calls // back while it still exists, so keep it alive until it has — views // observing it are released as soon as `receiver` goes nil below. @@ -281,7 +288,7 @@ final class ReceiverController: ObservableObject { /// Bring the video window (back) up — also bound to Show Window in Settings for /// when the user closed the window while the stream keeps running. func showWindow() { - guard let receiver, streaming || window != nil else { return } + guard let receiver, let input, streaming || window != nil else { return } // Streaming again within the grace period keeps this window, in // whatever full screen state it's already in. if streaming { windowLifecycle.cancelPendingClose() } @@ -291,12 +298,23 @@ final class ReceiverController: ObservableObject { styleMask: [.titled, .closable, .miniaturizable, .resizable], backing: .buffered, defer: false) w.title = "MeowDisplay" - w.contentView = ReceiverVideoView(receiver: receiver) + let videoView = ReceiverVideoView(receiver: receiver, input: input) + w.contentView = videoView + w.initialFirstResponder = videoView w.isReleasedWhenClosed = false + // A fixed slot (macOS 12 has no hosting-view sizing options): + // the indicator right-aligns inside it as its text changes. + let indicator = NSTitlebarAccessoryViewController() + let indicatorView = NSHostingView(rootView: ReceiverControlIndicator(input: input)) + indicatorView.frame = NSRect(x: 0, y: 0, width: 340, height: 28) + indicator.view = indicatorView + indicator.layoutAttribute = .trailing + w.addTitlebarAccessoryViewController(indicator) w.collectionBehavior.insert(.fullScreenPrimary) w.center() window = w observeWindow(w) + input.attach(to: w) enterFullscreen = windowLifecycle.windowCreated() } // Resizes keep the stream's shape; re-set on every show because a @@ -411,8 +429,10 @@ final class ReceiverController: ObservableObject { /// extended desktop would have no visible pointer. final class ReceiverVideoView: NSView { private weak var receiver: StreamReceiver? + private let input: ReceiverInputController private let displayLayer: AVSampleBufferDisplayLayer private var videoSizeObserver: AnyCancellable? + private var cursorEcho = CursorEchoPolicy() private let cursorLayer: CALayer = { let layer = CALayer() @@ -428,8 +448,9 @@ final class ReceiverVideoView: NSView { private var cursorNorm = CGPoint(x: 0.5, y: 0.5) private var cursorVisible = false - init(receiver: StreamReceiver) { + init(receiver: StreamReceiver, input: ReceiverInputController) { self.receiver = receiver + self.input = input self.displayLayer = receiver.displayLayer super.init(frame: .zero) wantsLayer = true @@ -440,7 +461,16 @@ final class ReceiverVideoView: NSView { layer?.addSublayer(cursorLayer) receiver.onCursor = { [weak self] x, y, visible in - self?.moveCursor(x: x, y: y, visible: visible) + guard let self else { return } + // While this Mac drives the pointer the sprite already sits at + // the local pointer; the sending Mac's (older) reports take over + // once local movement stops. + if self.input.echoesCursor, + !self.cursorEcho.appliesRemotePosition(at: ProcessInfo.processInfo.systemUptime) { + self.setCursorVisible(visible) + return + } + self.moveCursor(x: x, y: y, visible: visible) } receiver.onCursorImage = { [weak self] image, anchor, normSize in self?.setCursorSprite(image, anchor: anchor, normSize: normSize) @@ -464,7 +494,7 @@ final class ReceiverVideoView: NSView { // required constraints that size drove the *window*, which collapsed // to 0 × titlebar with the HUD hidden. Springs and struts never feed // back into the window, on any macOS the app supports. - let overlay = OverlayHostingView(rootView: ReceiverPerfOverlay(receiver: receiver)) + let overlay = OverlayHostingView(rootView: ReceiverPerfOverlay(receiver: receiver, input: input)) overlay.frame = bounds overlay.autoresizingMask = [.width, .height] addSubview(overlay) @@ -476,6 +506,68 @@ final class ReceiverVideoView: NSView { addCursorRect(bounds, cursor: blankCursor) } + // MARK: Input (issue #12) + + private var geometry: ReceiverVideoGeometry { + ReceiverVideoGeometry(viewSize: bounds.size, videoSize: receiver?.videoSize ?? .zero) + } + + override var acceptsFirstResponder: Bool { true } + + override func updateTrackingAreas() { + super.updateTrackingAreas() + trackingAreas.forEach(removeTrackingArea) + addTrackingArea(NSTrackingArea(rect: .zero, options: [.mouseMoved, .activeInKeyWindow, .inVisibleRect], + owner: self, userInfo: nil)) + } + + override func mouseMoved(with event: NSEvent) { pointerMoved(event) } + override func mouseDragged(with event: NSEvent) { pointerMoved(event) } + override func rightMouseDragged(with event: NSEvent) { pointerMoved(event) } + + override func mouseDown(with event: NSEvent) { + input.mouseDown(event, button: .left, geometry: geometry, in: self) + echoCursor(event) + } + + override func mouseUp(with event: NSEvent) { + input.mouseUp(event, button: .left, geometry: geometry, in: self) + } + + override func rightMouseDown(with event: NSEvent) { + input.mouseDown(event, button: .right, geometry: geometry, in: self) + echoCursor(event) + } + + override func rightMouseUp(with event: NSEvent) { + input.mouseUp(event, button: .right, geometry: geometry, in: self) + } + + override func scrollWheel(with event: NSEvent) { + input.scrollWheel(event, geometry: geometry, in: self) + } + + override func magnify(with event: NSEvent) { input.magnify(event) } + override func rotate(with event: NSEvent) { input.rotate(event) } + + private func pointerMoved(_ event: NSEvent) { + input.mouseMoved(event, geometry: geometry, in: self) + echoCursor(event) + } + + /// Draws the streamed cursor at the local pointer while controlling. + private func echoCursor(_ event: NSEvent) { + guard input.echoesCursor, + let (point, _) = geometry.normalized(convert(event.locationInWindow, from: nil)) else { return } + cursorEcho.localMove(at: ProcessInfo.processInfo.systemUptime) + moveCursor(x: Double(point.x), y: Double(point.y), visible: cursorVisible) + } + + private func setCursorVisible(_ visible: Bool) { + guard visible != cursorVisible else { return } + moveCursor(x: Double(cursorNorm.x), y: Double(cursorNorm.y), visible: visible) + } + override func layout() { super.layout() CATransaction.begin() @@ -550,23 +642,30 @@ final class ReceiverVideoView: NSView { /// same "showAnalytics" default the iOS app uses (Settings → System). struct ReceiverPerfOverlay: View { @ObservedObject var receiver: StreamReceiver + let input: ReceiverInputController @AppStorage("showAnalytics") private var showAnalytics = false var body: some View { - if showAnalytics { - VStack { - Spacer() - PerfOverlay(stats: receiver.perf, videoSize: receiver.videoSize) - .padding(.bottom, 10) + ZStack { + ReceiverControlHint(input: input) + if showAnalytics { + VStack { + Spacer() + PerfOverlay(stats: receiver.perf, videoSize: receiver.videoSize) + .padding(.bottom, 10) + } + .allowsHitTesting(false) } - .allowsHitTesting(false) } } } -/// Full-bleed, non-interactive layer above the video. Subclassed only so the -/// blank-cursor rect covers the HUD area too. +/// Full-bleed, non-interactive layer above the video. Subclassed so the +/// blank-cursor rect covers the HUD area too, and so it never takes a click, +/// scroll or drag meant for the video view (and the other Mac) underneath. private final class OverlayHostingView: NSHostingView { + override func hitTest(_ point: NSPoint) -> NSView? { nil } + required init(rootView: ReceiverPerfOverlay) { super.init(rootView: rootView) } diff --git a/MacReceiver/ReceiverInputController.swift b/MacReceiver/ReceiverInputController.swift new file mode 100644 index 000000000..0878674bd --- /dev/null +++ b/MacReceiver/ReceiverInputController.swift @@ -0,0 +1,318 @@ +import AppKit +import Combine +import SwiftUI + +/// The AppKit side of Mac-to-Mac input (issue #12): owns one session's +/// `ReceiverInputForwarder`, turns the video window's events into its calls, +/// and sends what it returns through `StreamReceiver`'s existing input +/// messages. Lives exactly as long as the `StreamReceiver` it serves. +@MainActor +final class ReceiverInputController: ObservableObject { + @Published private(set) var status: ReceiverControlStatus = .unavailable + /// The transient line over the video (`ReceiverControlStatus.hint`). + @Published private(set) var hint: String? + + private weak var receiver: StreamReceiver? + private var forwarder = ReceiverInputForwarder() + private var cancellables = Set() + private var keyMonitor: Any? + private weak var window: NSWindow? + private var windowObservers: [NSObjectProtocol] = [] + private var hintToken = 0 + + /// Only while controlling: the view draws the streamed cursor at the + /// local pointer (see `CursorEchoPolicy`). + var echoesCursor: Bool { status == .controlling } + + init(receiver: StreamReceiver) { + self.receiver = receiver + receiver.onAllowInputStateChange = { [weak self] state in + guard let self else { return } + Log.info("receiverInput: grant \(state.rawValue)") + self.perform(self.forwarder.grantChanged(state)) + } + receiver.$connected.combineLatest(receiver.$macProtocolVersion) + .removeDuplicates { $0 == $1 } + .receive(on: DispatchQueue.main) + .sink { [weak self] connected, version in + guard let self else { return } + // `allowInputRequest` (pv 8) is the newest message this + // needs; pointer (5) and keyboard (4) predate it. + self.forwarder.setConnected(connected, + inputSupported: version >= WireProtocol.allowInputWireVersion) + self.refreshStatus() + } + .store(in: &cancellables) + receiver.$inputResetGeneration.dropFirst() + .receive(on: DispatchQueue.main) + .sink { [weak self] _ in self?.forwarder.senderReset() } + .store(in: &cancellables) + } + + /// Stops listening and releases anything still held. Called when the + /// receiver it serves goes away. + func invalidate() { + detachWindow() + receiver?.onAllowInputStateChange = nil + cancellables.removeAll() + } + + // MARK: Window + + /// The video window. Its key state gates forwarding, and only its key + /// events are ever looked at. + func attach(to window: NSWindow) { + detachWindow() + self.window = window + let center = NotificationCenter.default + windowObservers = [ + center.addObserver(forName: NSWindow.didBecomeKeyNotification, object: window, queue: .main) { [weak self] _ in + MainActor.assumeIsolated { self?.windowKeyChanged(true) } + }, + center.addObserver(forName: NSWindow.didResignKeyNotification, object: window, queue: .main) { [weak self] _ in + MainActor.assumeIsolated { self?.windowKeyChanged(false) } + }, + center.addObserver(forName: NSWindow.willCloseNotification, object: window, queue: .main) { [weak self] _ in + MainActor.assumeIsolated { self?.detachWindow() } + }, + ] + // App-local: sees only events already addressed to this app, never + // other apps' input. Needed because AppKit never delivers a key-up + // while Command is held to a view, and menu key equivalents would + // otherwise win over the sending Mac's shortcuts. + keyMonitor = NSEvent.addLocalMonitorForEvents(matching: [.keyDown, .keyUp, .flagsChanged]) { [weak self] event in + let consumed = MainActor.assumeIsolated { self?.consumeKeyEvent(event) ?? false } + return consumed ? nil : event + } + windowKeyChanged(window.isKeyWindow) + } + + private func detachWindow() { + windowObservers.forEach { NotificationCenter.default.removeObserver($0) } + windowObservers = [] + if let keyMonitor { NSEvent.removeMonitor(keyMonitor) } + keyMonitor = nil + window = nil + windowKeyChanged(false) + } + + private func windowKeyChanged(_ isKey: Bool) { + perform(forwarder.windowKeyChanged(isKey)) + } + + // MARK: Actions + + func requestControl() { perform(forwarder.requestControl()) } + func releaseControl() { perform(forwarder.releaseControl()) } + func pauseControl() { perform(forwarder.pause()) } + + func performStatusAction() { + switch status.action { + case .request: requestControl() + case .release: releaseControl() + case nil: break + } + } + + // MARK: Pointer (from ReceiverVideoView) + + func mouseMoved(_ event: NSEvent, geometry: ReceiverVideoGeometry, in view: NSView) { + guard let (point, inside) = locate(event, geometry: geometry, in: view) else { return } + perform(forwarder.pointerMoved(point, inside: inside)) + } + + func mouseDown(_ event: NSEvent, button: PointerButton, geometry: ReceiverVideoGeometry, in view: NSView) { + guard let (point, inside) = locate(event, geometry: geometry, in: view) else { return } + let wasPaused = forwarder.paused + perform(forwarder.pointerDown(button, clickCount: event.clickCount, at: point, inside: inside)) + if wasPaused, !forwarder.paused { refreshStatus() } + } + + func mouseUp(_ event: NSEvent, button: PointerButton, geometry: ReceiverVideoGeometry, in view: NSView) { + let point = locate(event, geometry: geometry, in: view)?.point + perform(forwarder.pointerUp(button, at: point)) + } + + func scrollWheel(_ event: NSEvent, geometry: ReceiverVideoGeometry, in view: NSView) { + guard let (_, inside) = locate(event, geometry: geometry, in: view), + let pixelsPerPoint = geometry.videoPixelsPerPoint else { return } + // A wheel mouse reports lines; one line is about ten points of + // trackpad travel, which is how macOS itself converts. + let lineScale = event.hasPreciseScrollingDeltas ? 1.0 : 10.0 + perform(forwarder.scrolled(dxPoints: Double(event.scrollingDeltaX) * lineScale, + dyPoints: Double(event.scrollingDeltaY) * lineScale, + inside: inside, videoPixelsPerPoint: pixelsPerPoint)) + } + + func magnify(_ event: NSEvent) { + guard let phase = Self.gesturePhase(event.phase) else { return } + perform(forwarder.gesture(.magnify, phase: phase, delta: Double(event.magnification))) + } + + func rotate(_ event: NSEvent) { + guard let phase = Self.gesturePhase(event.phase) else { return } + // AppKit reports degrees counterclockwise; the wire carries radians. + perform(forwarder.gesture(.rotate, phase: phase, delta: Double(event.rotation) * .pi / 180)) + } + + private static func gesturePhase(_ phase: NSEvent.Phase) -> NativeAppGesturePhase? { + if phase.contains(.began) { return .began } + if phase.contains(.changed) { return .changed } + if phase.contains(.ended) { return .ended } + if phase.contains(.cancelled) { return .cancelled } + return nil + } + + private func locate(_ event: NSEvent, geometry: ReceiverVideoGeometry, + in view: NSView) -> (point: CGPoint, inside: Bool)? { + geometry.normalized(view.convert(event.locationInWindow, from: nil)) + } + + // MARK: Keyboard (app-local monitor) + + /// `false` lets AppKit handle the event here; `true` once it was sent to + /// the other Mac (or consumed, like the pause shortcut). + private func consumeKeyEvent(_ event: NSEvent) -> Bool { + guard let window, event.window === window else { return false } + let modifiers = ReceiverModifierState(rawFlags: event.modifierFlags.rawValue) + let commands: [ReceiverInputCommand]? + switch event.type { + case .keyDown: + commands = forwarder.keyDown(keyCode: event.keyCode, isRepeat: event.isARepeat, modifiers: modifiers) + case .keyUp: + commands = forwarder.keyUp(keyCode: event.keyCode) + case .flagsChanged: + perform(forwarder.modifiersChanged(modifiers)) + // Modifier changes also reach AppKit, so its own key handling + // stays in sync. + return false + default: + return false + } + guard let commands else { return false } + perform(commands) + return true + } + + // MARK: Sending + + private func perform(_ commands: [ReceiverInputCommand]) { + if let receiver { + for command in commands { send(command, through: receiver) } + } + refreshStatus() + } + + private func send(_ command: ReceiverInputCommand, through receiver: StreamReceiver) { + switch command { + case .pointerMove(let x, let y): receiver.sendPointerMove(x: x, y: y) + case .pointerDown(let button, let count): receiver.sendPointerDown(button: button, clickCount: count) + case .pointerUp(let button, let count): receiver.sendPointerUp(button: button, clickCount: count) + case .scroll(let dx, let dy): receiver.sendScroll(dx: dx, dy: dy) + case .keyDown(let usage, let modifiers, let isRepeat): + receiver.sendKeyboardDown(usage: usage, modifiers: modifiers, isRepeat: isRepeat) + case .keyUp(let usage, let modifiers): receiver.sendKeyboardUp(usage: usage, modifiers: modifiers) + case .modifierDown(let modifier): receiver.sendModifier(modifier, down: true) + case .modifierUp(let modifier): receiver.sendModifier(modifier, down: false) + case .gesture(let kind, let phase, let delta): receiver.sendNativeAppGesture(kind: kind, phase: phase, delta: delta) + case .cancelAll: receiver.sendCancelActiveInput() + case .requestControl(let allowed): receiver.requestAllowInput(allowed) + } + } + + private func refreshStatus() { + let next = forwarder.status + guard next != status else { return } + status = next + showHint(next.hint) + } + + private func showHint(_ text: String?) { + hintToken &+= 1 + hint = text + guard text != nil else { return } + let token = hintToken + DispatchQueue.main.asyncAfter(deadline: .now() + 3.5) { [weak self] in + guard let self, self.hintToken == token else { return } + self.hint = nil + } + } +} + +// MARK: - Indicator + +/// The video window's titlebar indicator: View Only / Controlling, and the +/// one action that applies. In full screen it appears with the titlebar +/// when the pointer reaches the top of the screen. +struct ReceiverControlIndicator: View { + @ObservedObject var input: ReceiverInputController + + var body: some View { + if let title = input.status.title { + HStack(spacing: 6) { + Image(systemName: symbol) + .foregroundStyle(tint) + .accessibilityHidden(true) + Text(title) + .font(.callout) + if let action = input.status.action { + Button(action == .request ? String(localized: "Request Control") : String(localized: "Release")) { + input.performStatusAction() + } + .controlSize(.small) + .help(action == .request + ? String(localized: "Ask the other Mac to let this Mac's keyboard and trackpad control it.") + : String(localized: "Give control back. The stream stays on screen.")) + } + } + .padding(.horizontal, 8) + .fixedSize() + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .trailing) + .help(input.status == .controlling + ? String(localized: "Press \(ReceiverInputForwarder.releaseShortcutLabel) to pause control.", + comment: "The placeholder is a keyboard shortcut such as ⌃⌥⎋.") + : "") + } + } + + private var symbol: String { + switch input.status { + case .controlling: return "keyboard" + case .paused, .inactive: return "pause.circle" + case .requesting: return "hourglass" + case .declined, .requestsDisabled: return "hand.raised" + case .viewOnly, .unavailable: return "eye" + } + } + + private var tint: Color { + switch input.status { + case .controlling: return .accentColor + case .declined, .requestsDisabled: return .orange + default: return .secondary + } + } +} + +/// The transient hint over the video. +struct ReceiverControlHint: View { + @ObservedObject var input: ReceiverInputController + + var body: some View { + VStack { + if let hint = input.hint { + Text(hint) + .font(.callout) + .multilineTextAlignment(.center) + .padding(.horizontal, 14) + .padding(.vertical, 8) + .background(.regularMaterial, in: Capsule()) + .padding(.top, 14) + .transition(.opacity) + } + Spacer() + } + .animation(.easeInOut(duration: 0.2), value: input.hint) + .allowsHitTesting(false) + } +} diff --git a/MacReceiver/ReceiverInputForwarding.swift b/MacReceiver/ReceiverInputForwarding.swift new file mode 100644 index 000000000..1f53ac7b7 --- /dev/null +++ b/MacReceiver/ReceiverInputForwarding.swift @@ -0,0 +1,489 @@ +import CoreGraphics +import Foundation + +// Mac-to-Mac input (issue #12): the receiving Mac's own trackpad, mouse and +// keyboard drive the sending Mac while its video window is in front. This +// file is the decision layer — where a point lands, which key a key code is, +// and what may be forwarded right now — kept free of AppKit so every rule is +// unit testable. `ReceiverVideoView` feeds it AppKit events and sends what it +// returns through the existing `StreamReceiver` input messages; no new wire +// message exists for any of it. +// +// Input is never captured globally: only events AppKit delivers to the video +// window while it is key reach this, so the receiving Mac's system shortcuts +// (Command-Tab, Command-Option-Escape, Mission Control, Spotlight, screenshots, +// lock screen) never leave it. The sending Mac stays the authority — nothing +// is forwarded without its per-session grant, and it drops input itself the +// moment that grant ends. + +/// Where the stream sits inside the video view: aspect-fit, centered, in view +/// points with AppKit's bottom-left origin. +struct ReceiverVideoGeometry: Equatable { + let viewSize: CGSize + /// The stream's pixel size (from its SPS). + let videoSize: CGSize + + /// `nil` until both sizes are known. + var fittedRect: CGRect? { + guard videoSize.width > 0, videoSize.height > 0, viewSize.width > 0, viewSize.height > 0 else { return nil } + let scale = min(viewSize.width / videoSize.width, viewSize.height / videoSize.height) + let size = CGSize(width: videoSize.width * scale, height: videoSize.height * scale) + return CGRect(x: (viewSize.width - size.width) / 2, y: (viewSize.height - size.height) / 2, + width: size.width, height: size.height) + } + + /// A view point as normalized video coordinates (PROTOCOL.md section 7: + /// top-left origin, 0...1), clamped onto the video so a drag that leaves + /// the picture (into a letterbox bar, or outside the window) pins to its + /// edge instead of reaching another display on the sending Mac. + /// `inside` reports whether the unclamped point was on the picture. + func normalized(_ viewPoint: CGPoint) -> (point: CGPoint, inside: Bool)? { + guard let rect = fittedRect else { return nil } + let x = (viewPoint.x - rect.minX) / rect.width + let y = (rect.maxY - viewPoint.y) / rect.height + let inside = (0...1).contains(x) && (0...1).contains(y) + return (CGPoint(x: min(max(x, 0), 1), y: min(max(y, 0), 1)), inside) + } + + /// Video pixels per view point — scroll deltas travel in video pixels. + var videoPixelsPerPoint: Double? { + guard let rect = fittedRect else { return nil } + return Double(videoSize.width / rect.width) + } +} + +// MARK: - Keys + +/// macOS virtual key codes (`kVK_*`, physical key positions) to USB HID +/// keyboard-page usages, the vocabulary `keyboard` messages carry. Both name +/// the physical key, so the sending Mac's own keyboard layout decides what +/// gets typed — exactly as if this keyboard were plugged into it. +/// Modifier keys are not here: they travel as `modifierDown`/`modifierUp`. +enum MacVirtualKeyMap { + static func hidUsage(forKeyCode keyCode: UInt16) -> Int? { table[keyCode] } + + static var mappedKeyCodes: [UInt16] { Array(table.keys).sorted() } + + private static let table: [UInt16: Int] = [ + // Letters + 0x00: 4, 0x0B: 5, 0x08: 6, 0x02: 7, 0x0E: 8, 0x03: 9, 0x05: 10, 0x04: 11, 0x22: 12, + 0x26: 13, 0x28: 14, 0x25: 15, 0x2E: 16, 0x2D: 17, 0x1F: 18, 0x23: 19, 0x0C: 20, + 0x0F: 21, 0x01: 22, 0x11: 23, 0x20: 24, 0x09: 25, 0x0D: 26, 0x07: 27, 0x10: 28, 0x06: 29, + // Digits row + 0x12: 30, 0x13: 31, 0x14: 32, 0x15: 33, 0x17: 34, 0x16: 35, 0x1A: 36, 0x1C: 37, 0x19: 38, 0x1D: 39, + // Return, Escape, Delete, Tab, Space + 0x24: 40, 0x35: 41, 0x33: 42, 0x30: 43, 0x31: 44, + // Punctuation + 0x1B: 45, 0x18: 46, 0x21: 47, 0x1E: 48, 0x2A: 49, 0x29: 51, 0x27: 52, 0x32: 53, + 0x2B: 54, 0x2F: 55, 0x2C: 56, + // ISO key left of Z (§ / <), JIS ¥ and _ and keypad comma, Eisū and Kana + 0x0A: 100, 0x5D: 137, 0x5E: 135, 0x5F: 133, 0x66: 145, 0x68: 144, + // F1–F20 + 0x7A: 58, 0x78: 59, 0x63: 60, 0x76: 61, 0x60: 62, 0x61: 63, 0x62: 64, 0x64: 65, + 0x65: 66, 0x6D: 67, 0x67: 68, 0x6F: 69, 0x69: 104, 0x6B: 105, 0x71: 106, 0x6A: 107, + 0x40: 108, 0x4F: 109, 0x50: 110, 0x5A: 111, + // Help/Insert, Home, Page Up, Forward Delete, End, Page Down, arrows + 0x72: 73, 0x73: 74, 0x74: 75, 0x75: 76, 0x77: 77, 0x79: 78, + 0x7C: 79, 0x7B: 80, 0x7D: 81, 0x7E: 82, + // Keypad + 0x47: 83, 0x4B: 84, 0x43: 85, 0x4E: 86, 0x45: 87, 0x4C: 88, 0x53: 89, 0x54: 90, 0x55: 91, + 0x56: 92, 0x57: 93, 0x58: 94, 0x59: 95, 0x5B: 96, 0x5C: 97, 0x52: 98, 0x41: 99, 0x51: 103, + ] + + static let escapeKeyCode: UInt16 = 0x35 +} + +/// The receiving Mac's modifier-key state, reduced to what the wire names. +struct ReceiverModifierState: Equatable { + var held: Set = [] + var capsLock = false + + /// From raw `NSEvent.ModifierFlags` bits, so this file stays AppKit-free. + init(rawFlags: UInt) { + if rawFlags & (1 << 17) != 0 { held.insert(.shift) } + if rawFlags & (1 << 18) != 0 { held.insert(.control) } + if rawFlags & (1 << 19) != 0 { held.insert(.option) } + if rawFlags & (1 << 20) != 0 { held.insert(.command) } + capsLock = rawFlags & (1 << 16) != 0 + } + + init(held: Set = [], capsLock: Bool = false) { + self.held = held + self.capsLock = capsLock + } +} + +// MARK: - Forwarding + +/// What `ReceiverVideoView` sends, in order. Each maps onto one existing +/// `StreamReceiver` send call. +enum ReceiverInputCommand: Equatable { + case pointerMove(x: Double, y: Double) + case pointerDown(PointerButton, clickCount: Int) + case pointerUp(PointerButton, clickCount: Int) + case scroll(dx: Double, dy: Double) + case keyDown(usage: Int, modifiers: [String], isRepeat: Bool) + case keyUp(usage: Int, modifiers: [String]) + case modifierDown(ControlModifier) + case modifierUp(ControlModifier) + case gesture(NativeAppGestureKind, NativeAppGesturePhase, delta: Double) + /// `keyboard` `cancel`: the sending Mac releases everything it holds for + /// this session. Sent after the explicit releases, as a safety net. + case cancelAll + /// `allowInputRequest`. + case requestControl(Bool) +} + +/// What the control indicator shows. +enum ReceiverControlStatus: Equatable { + /// No session, or a sending Mac too old for input requests. + case unavailable + /// View only; Request Control is offered. + case viewOnly + case requesting + /// The sending Mac's owner declined, or the prompt timed out. + case declined + /// The sending Mac doesn't accept control requests from this Mac. + case requestsDisabled + /// Granted, but this window isn't key (another app or window is in front). + case inactive + /// Granted and paused here with the release shortcut; a click resumes. + case paused + /// This Mac's keyboard and pointer are driving the sending Mac. + case controlling +} + +/// The receiver's half of the input contract. Forwards only while the +/// sending Mac granted this session control, the video window is key, and +/// the user hasn't paused it; and it remembers every key, modifier and button +/// it sent down so that ANY way forwarding ends — focus loss, the release +/// shortcut, Release Control, revocation, the window closing — releases them +/// on the sending Mac. A modifier or key already held when forwarding began +/// (Command still down from the Command-Tab that brought this window forward) +/// is never sent: only presses that started while forwarding are. +struct ReceiverInputForwarder { + /// Control-Option-Escape: pauses forwarding without giving up the grant. + /// Not a system shortcut on macOS (Command-Option-Escape is Force Quit). + static let releaseShortcutModifiers: Set = [.control, .option] + static let releaseShortcutLabel = "⌃⌥⎋" + + private(set) var connected = false + /// The sending Mac supports `allowInputRequest` and input messages. + private(set) var inputSupported = false + private(set) var grant: SessionInputWireState = .off + private(set) var windowIsKey = false + private(set) var paused = false + + private(set) var heldKeys: [Int: [String]] = [:] + private(set) var heldModifiers: Set = [] + private(set) var heldButton: (button: PointerButton, clickCount: Int)? + private var activeGestures: Set = [] + private var lastMove: CGPoint? + + var isForwarding: Bool { connected && inputSupported && grant == .allowed && windowIsKey && !paused } + + var status: ReceiverControlStatus { + guard connected, inputSupported else { return .unavailable } + switch grant { + case .off: return .viewOnly + case .requesting: return .requesting + case .notAllowed: return .declined + case .requestsDisabled: return .requestsDisabled + case .allowed: + if paused { return .paused } + return windowIsKey ? .controlling : .inactive + } + } + + var holdsInput: Bool { !heldKeys.isEmpty || !heldModifiers.isEmpty || heldButton != nil || !activeGestures.isEmpty } + + // MARK: Session and focus + + /// A session started or ended. A lost connection takes the sender's + /// state with it (it releases everything itself on session loss), so + /// nothing is sent — local tracking just resets. + mutating func setConnected(_ live: Bool, inputSupported supported: Bool) { + if connected != live || !live { resetTracking() } + connected = live + inputSupported = live && supported + if !live { + grant = .off + paused = false + } + } + + /// The sending Mac's confirmed state. Anything but `.allowed` ends + /// forwarding; the sender has already dropped this session's input, and + /// the explicit releases make sure it holds nothing. + mutating func grantChanged(_ state: SessionInputWireState) -> [ReceiverInputCommand] { + let wasAllowed = grant == .allowed + grant = state + if state == .allowed, !wasAllowed { paused = false } + return state == .allowed ? [] : releaseAll() + } + + /// The sending Mac reset input (`inputReset`): it already released + /// everything, so only local tracking clears. + mutating func senderReset() { + resetTracking() + } + + mutating func windowKeyChanged(_ isKey: Bool) -> [ReceiverInputCommand] { + windowIsKey = isKey + return isKey ? [] : releaseAll() + } + + /// The release shortcut (or the indicator's Pause): stop forwarding but + /// keep the grant. A click on the picture resumes. + mutating func pause() -> [ReceiverInputCommand] { + guard grant == .allowed, !paused else { return [] } + paused = true + return releaseAll() + } + + mutating func resume() { + paused = false + } + + mutating func requestControl() -> [ReceiverInputCommand] { + guard connected, inputSupported, grant != .allowed, grant != .requesting, grant != .requestsDisabled else { + return [] + } + return [.requestControl(true)] + } + + /// Release Control: give the grant back. Effective immediately on the + /// sending Mac — narrowing never needs its owner's decision. + mutating func releaseControl() -> [ReceiverInputCommand] { + guard connected, inputSupported, grant == .allowed || grant == .requesting else { return [] } + let releases = releaseAll() + paused = false + return releases + [.requestControl(false)] + } + + // MARK: Pointer + + mutating func pointerMoved(_ normalized: CGPoint, inside: Bool) -> [ReceiverInputCommand] { + guard isForwarding, inside || heldButton != nil else { return [] } + guard normalized != lastMove else { return [] } + lastMove = normalized + return [.pointerMove(x: Double(normalized.x), y: Double(normalized.y))] + } + + /// A press on the video. While paused, a press on the picture resumes + /// forwarding and is consumed (it was a "click back in", not a click on + /// the sending Mac). Presses outside the picture (letterbox) do nothing. + mutating func pointerDown(_ button: PointerButton, clickCount: Int, at normalized: CGPoint, + inside: Bool) -> [ReceiverInputCommand] { + guard inside else { return [] } + if paused, connected, grant == .allowed, windowIsKey { + paused = false + return [] + } + guard isForwarding, heldButton == nil else { return [] } + heldButton = (button, max(clickCount, 1)) + lastMove = normalized + // Position first: the sending Mac clicks at its own cursor. + return [.pointerMove(x: Double(normalized.x), y: Double(normalized.y)), + .pointerDown(button, clickCount: max(clickCount, 1))] + } + + /// A release is forwarded only for a press this forwarder sent — never + /// gated on forwarding, so a release always matches its press. + mutating func pointerUp(_ button: PointerButton, at normalized: CGPoint?) -> [ReceiverInputCommand] { + guard let held = heldButton, held.button == button else { return [] } + heldButton = nil + var commands: [ReceiverInputCommand] = [] + if isForwarding, let normalized, normalized != lastMove { + lastMove = normalized + commands.append(.pointerMove(x: Double(normalized.x), y: Double(normalized.y))) + } + commands.append(.pointerUp(button, clickCount: held.clickCount)) + return commands + } + + /// Scroll deltas in view points with the receiving Mac's own scroll + /// direction already applied (`NSEvent.scrollingDelta*`), so this Mac's + /// natural-scrolling setting is what its user feels. + mutating func scrolled(dxPoints: Double, dyPoints: Double, inside: Bool, + videoPixelsPerPoint: Double) -> [ReceiverInputCommand] { + guard isForwarding, inside, dxPoints != 0 || dyPoints != 0 else { return [] } + return [.scroll(dx: dxPoints * videoPixelsPerPoint, dy: dyPoints * videoPixelsPerPoint)] + } + + /// Trackpad pinch/rotate, as the existing `nativeAppGesture` lifecycle. + mutating func gesture(_ kind: NativeAppGestureKind, phase: NativeAppGesturePhase, + delta: Double) -> [ReceiverInputCommand] { + switch phase { + case .began: + guard isForwarding, !activeGestures.contains(kind) else { return [] } + activeGestures.insert(kind) + case .changed: + guard isForwarding, activeGestures.contains(kind) else { return [] } + case .ended, .cancelled: + guard activeGestures.remove(kind) != nil else { return [] } + } + return [.gesture(kind, phase, delta: delta)] + } + + // MARK: Keyboard + + /// A key press. Returns `nil` when the event is NOT for the sending Mac + /// and AppKit should handle it locally (not forwarding, or a key the + /// wire can't name); an empty array when it was consumed here (the + /// release shortcut, a repeat of a key that was never sent). + mutating func keyDown(keyCode: UInt16, isRepeat: Bool, + modifiers: ReceiverModifierState) -> [ReceiverInputCommand]? { + if keyCode == MacVirtualKeyMap.escapeKeyCode, modifiers.held == Self.releaseShortcutModifiers { + return grant == .allowed && connected ? pause() : nil + } + guard isForwarding else { return nil } + guard let usage = MacVirtualKeyMap.hidUsage(forKeyCode: keyCode) else { return [] } + let names = wireModifiers(capsLock: modifiers.capsLock) + if isRepeat { + // Only repeat what this forwarder pressed. + guard heldKeys[usage] != nil else { return [] } + return [.keyDown(usage: usage, modifiers: names, isRepeat: true)] + } + guard heldKeys[usage] == nil else { return [] } + heldKeys[usage] = names + return [.keyDown(usage: usage, modifiers: names, isRepeat: false)] + } + + /// A key release: forwarded only for a key this forwarder pressed, and + /// never gated on forwarding, so every press gets its release. + mutating func keyUp(keyCode: UInt16) -> [ReceiverInputCommand]? { + guard let usage = MacVirtualKeyMap.hidUsage(forKeyCode: keyCode), + let names = heldKeys.removeValue(forKey: usage) else { + return isForwarding ? [] : nil + } + return [.keyUp(usage: usage, modifiers: names)] + } + + /// `flagsChanged`: modifiers pressed while forwarding go down on the + /// sending Mac; releases of modifiers this forwarder sent go up. + mutating func modifiersChanged(_ state: ReceiverModifierState) -> [ReceiverInputCommand] { + var commands: [ReceiverInputCommand] = [] + for modifier in ControlModifier.allCases where heldModifiers.contains(modifier) && !state.held.contains(modifier) { + heldModifiers.remove(modifier) + commands.append(.modifierUp(modifier)) + } + guard isForwarding else { return commands } + for modifier in ControlModifier.allCases where !heldModifiers.contains(modifier) && state.held.contains(modifier) { + heldModifiers.insert(modifier) + commands.append(.modifierDown(modifier)) + } + return commands + } + + // MARK: Private + + private func wireModifiers(capsLock: Bool) -> [String] { + var names = ControlModifier.allCases.filter(heldModifiers.contains).map(\.rawValue) + if capsLock { names.append("capsLock") } + return names + } + + /// Everything this forwarder holds goes up on the sending Mac, then a + /// `cancel` as the safety net. Nothing is sent when nothing is held. + private mutating func releaseAll() -> [ReceiverInputCommand] { + guard connected, holdsInput else { + resetTracking() + return [] + } + var commands: [ReceiverInputCommand] = [] + for kind in NativeAppGestureKind.allCases where activeGestures.contains(kind) { + commands.append(.gesture(kind, .cancelled, delta: 0)) + } + if let held = heldButton { + commands.append(.pointerUp(held.button, clickCount: held.clickCount)) + } + for usage in heldKeys.keys.sorted() { + commands.append(.keyUp(usage: usage, modifiers: heldKeys[usage] ?? [])) + } + for modifier in ControlModifier.allCases where heldModifiers.contains(modifier) { + commands.append(.modifierUp(modifier)) + } + commands.append(.cancelAll) + resetTracking() + return commands + } + + private mutating func resetTracking() { + heldKeys = [:] + heldModifiers = [] + heldButton = nil + activeGestures = [] + lastMove = nil + } +} + +// MARK: - Presentation + +extension ReceiverControlStatus { + /// The indicator's short label; `nil` hides it. + var title: String? { + switch self { + case .unavailable: return nil + case .viewOnly: return String(localized: "View Only", comment: "Receiver control status: the stream can be watched but not controlled.") + case .requesting: return String(localized: "Requesting Control…") + case .declined: return String(localized: "Control Declined") + case .requestsDisabled: return String(localized: "Control Not Available") + case .inactive: return String(localized: "Control Allowed") + case .paused: return String(localized: "Control Paused") + case .controlling: return String(localized: "Controlling") + } + } + + /// The one action the indicator offers. + enum Action: Equatable { + case request + case release + } + + var action: Action? { + switch self { + case .viewOnly, .declined: return .request + case .requesting, .inactive, .paused, .controlling: return .release + case .unavailable, .requestsDisabled: return nil + } + } + + /// A short-lived line over the video when the state changes, so the way + /// back out is never a secret. + var hint: String? { + switch self { + case .controlling: + return String(localized: "This Mac's keyboard and trackpad now control the other Mac. Press \(ReceiverInputForwarder.releaseShortcutLabel) to pause.", + comment: "The placeholder is a keyboard shortcut such as ⌃⌥⎋.") + case .paused: return String(localized: "Control paused. Click the picture to resume.") + case .declined: return String(localized: "The other Mac didn't allow control.") + case .requestsDisabled: return String(localized: "The other Mac doesn't accept control requests from this Mac.") + case .unavailable, .viewOnly, .requesting, .inactive: return nil + } + } +} + +/// While this Mac drives the pointer, the streamed cursor is drawn where the +/// local pointer is right away instead of waiting a network round trip and a +/// video frame for the sending Mac to report it. The sending Mac's reports +/// take over again once local movement stops, so the two can never disagree +/// for long (the sending Mac's own trackpad, a cursor clamped at a screen +/// edge). +struct CursorEchoPolicy { + static let holdAfterLocalMove: TimeInterval = 0.25 + + private(set) var localUntil: TimeInterval = 0 + + mutating func localMove(at now: TimeInterval) { + localUntil = now + Self.holdAfterLocalMove + } + + func appliesRemotePosition(at now: TimeInterval) -> Bool { + now >= localUntil + } + + mutating func reset() { + localUntil = 0 + } +} diff --git a/MacTests/ReceiverInputForwardingTests.swift b/MacTests/ReceiverInputForwardingTests.swift new file mode 100644 index 000000000..5c90286a5 --- /dev/null +++ b/MacTests/ReceiverInputForwardingTests.swift @@ -0,0 +1,532 @@ +import CoreGraphics +import XCTest + +/// Issue #12: a receiving Mac's keyboard and trackpad driving the sending +/// Mac. These drive the real `ReceiverInputForwarder` with synthetic AppKit +/// event sequences and check every command it would send. They prove the +/// receiver's rules; they do not prove how a physical trackpad or keyboard +/// feels (see RECEIVER_INPUT.md's physical checklist). +final class ReceiverInputForwardingTests: XCTestCase { + + // MARK: - Geometry + + func testExactFitMapsCornersAndFlipsY() { + let geometry = ReceiverVideoGeometry(viewSize: CGSize(width: 1440, height: 900), + videoSize: CGSize(width: 2880, height: 1800)) + XCTAssertEqual(geometry.fittedRect, CGRect(x: 0, y: 0, width: 1440, height: 900)) + // AppKit's origin is bottom-left; the wire's is top-left. + XCTAssertEqual(geometry.normalized(CGPoint(x: 0, y: 900))?.point, CGPoint(x: 0, y: 0)) + XCTAssertEqual(geometry.normalized(CGPoint(x: 1440, y: 0))?.point, CGPoint(x: 1, y: 1)) + XCTAssertEqual(geometry.normalized(CGPoint(x: 360, y: 675))?.point, CGPoint(x: 0.25, y: 0.25)) + XCTAssertEqual(geometry.videoPixelsPerPoint, 2) + } + + func testWideWindowLetterboxesAndLetterboxIsOutsideThePicture() { + // A 16:10 stream in a 2:1 window: bars left and right. + let geometry = ReceiverVideoGeometry(viewSize: CGSize(width: 2000, height: 1000), + videoSize: CGSize(width: 3200, height: 2000)) + XCTAssertEqual(geometry.fittedRect, CGRect(x: 200, y: 0, width: 1600, height: 1000)) + let inBar = geometry.normalized(CGPoint(x: 100, y: 500))! + XCTAssertFalse(inBar.inside) + XCTAssertEqual(inBar.point, CGPoint(x: 0, y: 0.5)) + XCTAssertTrue(geometry.normalized(CGPoint(x: 1000, y: 500))!.inside) + } + + func testTallWindowPillarboxes() { + let geometry = ReceiverVideoGeometry(viewSize: CGSize(width: 800, height: 1000), + videoSize: CGSize(width: 1600, height: 1000)) + XCTAssertEqual(geometry.fittedRect, CGRect(x: 0, y: 250, width: 800, height: 500)) + XCTAssertEqual(geometry.normalized(CGPoint(x: 400, y: 500))?.point, CGPoint(x: 0.5, y: 0.5)) + XCTAssertFalse(geometry.normalized(CGPoint(x: 400, y: 900))!.inside) + } + + func testResizingKeepsTheSameNormalizedPointAndRescalesScrollUnits() { + let video = CGSize(width: 2880, height: 1800) + for (width, expectedScale) in [(1440.0, 2.0), (720.0, 4.0), (2880.0, 1.0)] { + let geometry = ReceiverVideoGeometry(viewSize: CGSize(width: width, height: width * 0.625), videoSize: video) + XCTAssertEqual(geometry.normalized(CGPoint(x: width / 2, y: width * 0.625 / 4))?.point, + CGPoint(x: 0.5, y: 0.75)) + XCTAssertEqual(geometry.videoPixelsPerPoint!, expectedScale, accuracy: 1e-9) + } + } + + func testMixedResolutionFullScreenStreamFitsInsideTheScreen() { + // A 1920×1200 stream full screen on a 2560×1440-point display. + let geometry = ReceiverVideoGeometry(viewSize: CGSize(width: 2560, height: 1440), + videoSize: CGSize(width: 1920, height: 1200)) + let rect = geometry.fittedRect! + XCTAssertEqual(rect.height, 1440) + XCTAssertEqual(rect.width, 2304) + XCTAssertEqual(rect.minX, 128) + XCTAssertEqual(geometry.videoPixelsPerPoint!, 1920.0 / 2304.0, accuracy: 1e-9) + } + + func testDragFarOutsideTheWindowClampsToTheEdge() { + let geometry = ReceiverVideoGeometry(viewSize: CGSize(width: 1000, height: 500), + videoSize: CGSize(width: 2000, height: 1000)) + let result = geometry.normalized(CGPoint(x: 5000, y: -300))! + XCTAssertEqual(result.point, CGPoint(x: 1, y: 1)) + XCTAssertFalse(result.inside) + } + + func testUnknownVideoSizeMapsNothing() { + XCTAssertNil(ReceiverVideoGeometry(viewSize: CGSize(width: 100, height: 100), videoSize: .zero).normalized(.zero)) + XCTAssertNil(ReceiverVideoGeometry(viewSize: .zero, videoSize: CGSize(width: 10, height: 10)).videoPixelsPerPoint) + } + + // MARK: - Key map + + func testEveryMappedKeyRoundTripsThroughTheSendersInjectionTable() { + for keyCode in MacVirtualKeyMap.mappedKeyCodes { + let usage = MacVirtualKeyMap.hidUsage(forKeyCode: keyCode)! + guard let senderUsage = HIDKeyUsage(rawValue: usage) else { + return XCTFail("sender has no injection for usage \(usage) (key code \(keyCode))") + } + XCTAssertEqual(senderUsage.keyCode, CGKeyCode(keyCode), "usage \(usage)") + } + } + + func testMappingIsOneToOne() { + let usages = MacVirtualKeyMap.mappedKeyCodes.compactMap(MacVirtualKeyMap.hidUsage(forKeyCode:)) + XCTAssertEqual(Set(usages).count, usages.count) + XCTAssertGreaterThanOrEqual(usages.count, 100, "a full keyboard, F-keys, navigation and keypad") + } + + func testModifierKeysTravelAsModifiersNotKeys() { + // Command, Shift, Option, Control (both sides), Caps Lock, Fn. + for keyCode: UInt16 in [0x37, 0x36, 0x38, 0x3C, 0x3A, 0x3D, 0x3B, 0x3E, 0x39, 0x3F] { + XCTAssertNil(MacVirtualKeyMap.hidUsage(forKeyCode: keyCode)) + } + } + + func testRepresentativeKeys() { + XCTAssertEqual(MacVirtualKeyMap.hidUsage(forKeyCode: 0x00), 4) // A + XCTAssertEqual(MacVirtualKeyMap.hidUsage(forKeyCode: 0x32), 53) // ` + XCTAssertEqual(MacVirtualKeyMap.hidUsage(forKeyCode: 0x60), 62) // F5 + XCTAssertEqual(MacVirtualKeyMap.hidUsage(forKeyCode: 0x73), 74) // Home + XCTAssertEqual(MacVirtualKeyMap.hidUsage(forKeyCode: 0x0A), 100) // ISO § + XCTAssertEqual(MacVirtualKeyMap.hidUsage(forKeyCode: 0x4C), 88) // keypad Enter + } + + func testModifierStateFromRawFlags() { + // NSEvent.ModifierFlags: capsLock 1<<16, shift 17, control 18, option 19, command 20. + let state = ReceiverModifierState(rawFlags: (1 << 16) | (1 << 17) | (1 << 20) | (1 << 23)) + XCTAssertEqual(state.held, [.shift, .command]) + XCTAssertTrue(state.capsLock) + XCTAssertEqual(ReceiverModifierState(rawFlags: 0), ReceiverModifierState()) + } + + // MARK: - Helpers + + private let center = CGPoint(x: 0.5, y: 0.5) + private let none = ReceiverModifierState() + + private func controlling() -> ReceiverInputForwarder { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + _ = forwarder.windowKeyChanged(true) + _ = forwarder.grantChanged(.allowed) + XCTAssertEqual(forwarder.status, .controlling) + return forwarder + } + + private func mods(_ held: Set, capsLock: Bool = false) -> ReceiverModifierState { + ReceiverModifierState(held: held, capsLock: capsLock) + } + + // MARK: - View only by default + + func testANewSessionIsViewOnlyAndForwardsNothing() { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + _ = forwarder.windowKeyChanged(true) + XCTAssertEqual(forwarder.status, .viewOnly) + XCTAssertEqual(forwarder.pointerMoved(center, inside: true), []) + XCTAssertEqual(forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true), []) + XCTAssertNil(forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none), "typed keys stay local") + XCTAssertEqual(forwarder.modifiersChanged(mods([.command])), []) + XCTAssertEqual(forwarder.scrolled(dxPoints: 0, dyPoints: 5, inside: true, videoPixelsPerPoint: 2), []) + XCTAssertEqual(forwarder.gesture(.magnify, phase: .began, delta: 0.1), []) + } + + func testRequestControlUsesTheExistingRequestOnce() { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + XCTAssertEqual(forwarder.requestControl(), [.requestControl(true)]) + _ = forwarder.grantChanged(.requesting) + XCTAssertEqual(forwarder.status, .requesting) + XCTAssertEqual(forwarder.requestControl(), [], "already waiting") + _ = forwarder.grantChanged(.notAllowed) + XCTAssertEqual(forwarder.status, .declined) + XCTAssertEqual(forwarder.requestControl(), [.requestControl(true)], "may ask again") + _ = forwarder.grantChanged(.requestsDisabled) + XCTAssertEqual(forwarder.requestControl(), [], "the Mac refuses requests from this device") + } + + func testAnOldSendingMacOffersNoControl() { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: false) + _ = forwarder.windowKeyChanged(true) + _ = forwarder.grantChanged(.allowed) + XCTAssertEqual(forwarder.status, .unavailable) + XCTAssertEqual(forwarder.requestControl(), []) + XCTAssertNil(forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none)) + } + + func testGrantedButNotFrontmostStaysLocal() { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + _ = forwarder.grantChanged(.allowed) + XCTAssertEqual(forwarder.status, .inactive) + XCTAssertNil(forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none)) + XCTAssertEqual(forwarder.pointerMoved(center, inside: true), []) + } + + // MARK: - Pointer + + func testHoverMovesForwardOnlyOverThePictureAndDeduplicate() { + var forwarder = controlling() + XCTAssertEqual(forwarder.pointerMoved(CGPoint(x: 0.2, y: 0.3), inside: true), [.pointerMove(x: 0.2, y: 0.3)]) + XCTAssertEqual(forwarder.pointerMoved(CGPoint(x: 0.2, y: 0.3), inside: true), []) + XCTAssertEqual(forwarder.pointerMoved(CGPoint(x: 0, y: 0.3), inside: false), [], "letterbox hover") + } + + func testClickDragReleaseOutsideTheVideo() { + var forwarder = controlling() + XCTAssertEqual(forwarder.pointerDown(.left, clickCount: 1, at: CGPoint(x: 0.4, y: 0.4), inside: true), + [.pointerMove(x: 0.4, y: 0.4), .pointerDown(.left, clickCount: 1)]) + XCTAssertEqual(forwarder.pointerMoved(CGPoint(x: 0.9, y: 0.4), inside: true), [.pointerMove(x: 0.9, y: 0.4)]) + // Dragged past the edge: still forwarded, clamped by the geometry. + XCTAssertEqual(forwarder.pointerMoved(CGPoint(x: 1, y: 0.4), inside: false), [.pointerMove(x: 1, y: 0.4)]) + XCTAssertEqual(forwarder.pointerUp(.left, at: CGPoint(x: 1, y: 0.4)), [.pointerUp(.left, clickCount: 1)]) + XCTAssertFalse(forwarder.holdsInput) + } + + func testReleaseAtANewPositionMovesFirst() { + var forwarder = controlling() + _ = forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true) + XCTAssertEqual(forwarder.pointerUp(.left, at: CGPoint(x: 0.6, y: 0.5)), + [.pointerMove(x: 0.6, y: 0.5), .pointerUp(.left, clickCount: 1)]) + } + + func testDoubleClickAndRightClickCarryTheirCountsAndButton() { + var forwarder = controlling() + _ = forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true) + _ = forwarder.pointerUp(.left, at: center) + XCTAssertEqual(forwarder.pointerDown(.left, clickCount: 2, at: center, inside: true).last, + .pointerDown(.left, clickCount: 2)) + XCTAssertEqual(forwarder.pointerUp(.left, at: center), [.pointerUp(.left, clickCount: 2)]) + XCTAssertEqual(forwarder.pointerDown(.right, clickCount: 1, at: center, inside: true).last, + .pointerDown(.right, clickCount: 1)) + XCTAssertEqual(forwarder.pointerUp(.right, at: center), [.pointerUp(.right, clickCount: 1)]) + } + + func testASecondButtonDuringADragIsIgnoredAndItsReleaseNeverSent() { + var forwarder = controlling() + _ = forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true) + XCTAssertEqual(forwarder.pointerDown(.right, clickCount: 1, at: center, inside: true), []) + XCTAssertEqual(forwarder.pointerUp(.right, at: center), []) + XCTAssertEqual(forwarder.pointerUp(.left, at: center), [.pointerUp(.left, clickCount: 1)]) + } + + func testClickInTheLetterboxDoesNothing() { + var forwarder = controlling() + XCTAssertEqual(forwarder.pointerDown(.left, clickCount: 1, at: CGPoint(x: 0, y: 0.5), inside: false), []) + XCTAssertEqual(forwarder.pointerUp(.left, at: nil), []) + } + + func testWindowLosingFocusMidDragReleasesTheButton() { + var forwarder = controlling() + _ = forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true) + XCTAssertEqual(forwarder.windowKeyChanged(false), [.pointerUp(.left, clickCount: 1), .cancelAll]) + XCTAssertEqual(forwarder.status, .inactive) + XCTAssertEqual(forwarder.pointerUp(.left, at: center), [], "AppKit's late mouse-up is not sent twice") + } + + func testScrollConvertsPointsToVideoPixels() { + var forwarder = controlling() + XCTAssertEqual(forwarder.scrolled(dxPoints: -1.5, dyPoints: 4, inside: true, videoPixelsPerPoint: 2), + [.scroll(dx: -3, dy: 8)]) + XCTAssertEqual(forwarder.scrolled(dxPoints: 0, dyPoints: 4, inside: false, videoPixelsPerPoint: 2), []) + XCTAssertEqual(forwarder.scrolled(dxPoints: 0, dyPoints: 0, inside: true, videoPixelsPerPoint: 2), []) + } + + func testPinchAndRotateFollowTheirLifecycle() { + var forwarder = controlling() + XCTAssertEqual(forwarder.gesture(.rotate, phase: .changed, delta: 0.1), [], "no began") + XCTAssertEqual(forwarder.gesture(.magnify, phase: .began, delta: 0), [.gesture(.magnify, .began, delta: 0)]) + XCTAssertEqual(forwarder.gesture(.magnify, phase: .changed, delta: 0.05), [.gesture(.magnify, .changed, delta: 0.05)]) + XCTAssertEqual(forwarder.gesture(.magnify, phase: .ended, delta: 0), [.gesture(.magnify, .ended, delta: 0)]) + XCTAssertEqual(forwarder.gesture(.magnify, phase: .ended, delta: 0), []) + } + + func testFocusLossMidPinchCancelsIt() { + var forwarder = controlling() + _ = forwarder.gesture(.magnify, phase: .began, delta: 0) + XCTAssertEqual(forwarder.windowKeyChanged(false), [.gesture(.magnify, .cancelled, delta: 0), .cancelAll]) + XCTAssertEqual(forwarder.gesture(.magnify, phase: .ended, delta: 0), []) + } + + // MARK: - Keyboard + + func testTypingWithModifiersFollowsTheDownUpLifecycle() { + var forwarder = controlling() + XCTAssertEqual(forwarder.modifiersChanged(mods([.command])), [.modifierDown(.command)]) + XCTAssertEqual(forwarder.keyDown(keyCode: 0x08, isRepeat: false, modifiers: mods([.command])), + [.keyDown(usage: 6, modifiers: ["command"], isRepeat: false)]) + XCTAssertEqual(forwarder.keyUp(keyCode: 0x08), [.keyUp(usage: 6, modifiers: ["command"])]) + XCTAssertEqual(forwarder.modifiersChanged(mods([])), [.modifierUp(.command)]) + XCTAssertFalse(forwarder.holdsInput) + } + + func testShortcutWithSeveralModifiers() { + var forwarder = controlling() + _ = forwarder.modifiersChanged(mods([.command])) + XCTAssertEqual(forwarder.modifiersChanged(mods([.command, .shift])), [.modifierDown(.shift)]) + XCTAssertEqual(forwarder.keyDown(keyCode: 0x14, isRepeat: false, modifiers: mods([.command, .shift])), + [.keyDown(usage: 32, modifiers: ["command", "shift"], isRepeat: false)]) + } + + func testModifierHeldFromCommandTabIsNeverSent() { + // Command is still down from the Command-Tab that brought this window + // forward: the sending Mac must not see a Command-Q. + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + _ = forwarder.grantChanged(.allowed) + _ = forwarder.modifiersChanged(mods([.command])) // pressed while another app was in front + _ = forwarder.windowKeyChanged(true) + XCTAssertEqual(forwarder.keyDown(keyCode: 0x0C, isRepeat: false, modifiers: mods([.command])), + [.keyDown(usage: 20, modifiers: [], isRepeat: false)]) + XCTAssertEqual(forwarder.modifiersChanged(mods([])), [], "its release isn't sent either") + } + + func testKeyRepeatIsForwardedOnlyForAKeyThisMacPressed() { + var forwarder = controlling() + _ = forwarder.keyDown(keyCode: 0x33, isRepeat: false, modifiers: none) + XCTAssertEqual(forwarder.keyDown(keyCode: 0x33, isRepeat: true, modifiers: none), + [.keyDown(usage: 42, modifiers: [], isRepeat: true)]) + XCTAssertEqual(forwarder.keyDown(keyCode: 0x7B, isRepeat: true, modifiers: none), [], + "a repeat of a key pressed before forwarding began") + } + + func testDuplicateDownWithoutRepeatIsNotSentTwice() { + var forwarder = controlling() + _ = forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none) + XCTAssertEqual(forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none), []) + } + + func testKeyUpForAKeyPressedBeforeForwardingStaysLocal() { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + _ = forwarder.windowKeyChanged(true) + XCTAssertNil(forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none)) + _ = forwarder.grantChanged(.allowed) + XCTAssertEqual(forwarder.keyUp(keyCode: 0x00), [], "consumed but not sent: no down reached the other Mac") + } + + func testCapsLockTravelsAsANamedModifier() { + var forwarder = controlling() + XCTAssertEqual(forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: mods([], capsLock: true)), + [.keyDown(usage: 4, modifiers: ["capsLock"], isRepeat: false)]) + } + + func testAKeyWithNoUsageIsConsumedNotLeaked() { + var forwarder = controlling() + XCTAssertEqual(forwarder.keyDown(keyCode: 0xFF, isRepeat: false, modifiers: none), []) + } + + // MARK: - Pause shortcut and release + + func testPauseShortcutReleasesEverythingAndKeepsTheGrant() { + var forwarder = controlling() + _ = forwarder.modifiersChanged(mods([.control])) + _ = forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: mods([.control])) + _ = forwarder.modifiersChanged(mods([.control, .option])) + let commands = forwarder.keyDown(keyCode: MacVirtualKeyMap.escapeKeyCode, isRepeat: false, + modifiers: mods([.control, .option])) + XCTAssertEqual(commands, [.keyUp(usage: 4, modifiers: ["control"]), .modifierUp(.option), + .modifierUp(.control), .cancelAll], + "Escape itself never reaches the other Mac") + XCTAssertEqual(forwarder.status, .paused) + XCTAssertEqual(forwarder.grant, .allowed) + XCTAssertNil(forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none), "paused: keys stay local") + XCTAssertEqual(forwarder.modifiersChanged(mods([])), [], "released modifiers aren't released twice") + } + + func testAClickOnThePictureResumesAndIsNotForwarded() { + var forwarder = controlling() + _ = forwarder.pause() + XCTAssertEqual(forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true), []) + XCTAssertEqual(forwarder.status, .controlling) + XCTAssertEqual(forwarder.pointerUp(.left, at: center), [], "the resume click's release isn't sent") + XCTAssertEqual(forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true).last, + .pointerDown(.left, clickCount: 1)) + } + + func testPauseShortcutWithoutControlIsAnOrdinaryLocalKey() { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + _ = forwarder.windowKeyChanged(true) + XCTAssertNil(forwarder.keyDown(keyCode: MacVirtualKeyMap.escapeKeyCode, isRepeat: false, + modifiers: mods([.control, .option]))) + } + + func testPlainEscapeIsForwarded() { + var forwarder = controlling() + XCTAssertEqual(forwarder.keyDown(keyCode: MacVirtualKeyMap.escapeKeyCode, isRepeat: false, modifiers: none), + [.keyDown(usage: 41, modifiers: [], isRepeat: false)]) + } + + func testReleaseControlGivesTheGrantBack() { + var forwarder = controlling() + _ = forwarder.keyDown(keyCode: 0x31, isRepeat: false, modifiers: none) + XCTAssertEqual(forwarder.releaseControl(), + [.keyUp(usage: 44, modifiers: []), .cancelAll, .requestControl(false)]) + _ = forwarder.grantChanged(.off) + XCTAssertEqual(forwarder.status, .viewOnly) + } + + func testReleaseWhileRequestingWithdrawsTheRequest() { + var forwarder = ReceiverInputForwarder() + forwarder.setConnected(true, inputSupported: true) + _ = forwarder.grantChanged(.requesting) + XCTAssertEqual(forwarder.releaseControl(), [.requestControl(false)]) + } + + func testFocusLossWithNothingHeldSendsNothing() { + var forwarder = controlling() + XCTAssertEqual(forwarder.windowKeyChanged(false), []) + } + + // MARK: - Revocation, disconnects, reconnects + + func testRevocationMidTypingReleasesAndStopsForwarding() { + var forwarder = controlling() + _ = forwarder.modifiersChanged(mods([.shift])) + _ = forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: mods([.shift])) + _ = forwarder.pointerDown(.left, clickCount: 1, at: center, inside: true) + XCTAssertEqual(forwarder.grantChanged(.off), + [.pointerUp(.left, clickCount: 1), .keyUp(usage: 4, modifiers: ["shift"]), + .modifierUp(.shift), .cancelAll]) + XCTAssertEqual(forwarder.status, .viewOnly) + XCTAssertNil(forwarder.keyUp(keyCode: 0x00), "the release after revocation stays local") + XCTAssertEqual(forwarder.pointerMoved(center, inside: true), []) + } + + func testDisconnectDuringInputSendsNothingAndClearsState() { + var forwarder = controlling() + _ = forwarder.modifiersChanged(mods([.command])) + _ = forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: mods([.command])) + forwarder.setConnected(false, inputSupported: false) + XCTAssertFalse(forwarder.holdsInput) + XCTAssertEqual(forwarder.status, .unavailable) + } + + func testReconnectStartsViewOnlyWithNoStaleKeys() { + var forwarder = controlling() + _ = forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none) + _ = forwarder.pause() + forwarder.setConnected(false, inputSupported: false) + forwarder.setConnected(true, inputSupported: true) + XCTAssertEqual(forwarder.status, .viewOnly, "a new session's grant starts off") + XCTAssertFalse(forwarder.paused) + XCTAssertNil(forwarder.keyUp(keyCode: 0x00)) + _ = forwarder.grantChanged(.allowed) + XCTAssertEqual(forwarder.status, .controlling) + } + + func testSenderResetClearsTrackingWithoutSending() { + var forwarder = controlling() + _ = forwarder.keyDown(keyCode: 0x00, isRepeat: false, modifiers: none) + forwarder.senderReset() + XCTAssertFalse(forwarder.holdsInput) + XCTAssertEqual(forwarder.windowKeyChanged(false), []) + } + + func testRegrantAfterPauseClearsThePause() { + var forwarder = controlling() + _ = forwarder.pause() + _ = forwarder.grantChanged(.off) + _ = forwarder.grantChanged(.allowed) + XCTAssertEqual(forwarder.status, .controlling) + } + + // MARK: - Presentation + + func testIndicatorOffersTheOneRelevantAction() { + XCTAssertNil(ReceiverControlStatus.unavailable.title) + XCTAssertEqual(ReceiverControlStatus.viewOnly.action, .request) + XCTAssertEqual(ReceiverControlStatus.declined.action, .request) + XCTAssertNil(ReceiverControlStatus.requestsDisabled.action) + for status: ReceiverControlStatus in [.requesting, .inactive, .paused, .controlling] { + XCTAssertEqual(status.action, .release) + } + XCTAssertTrue(ReceiverControlStatus.controlling.hint!.contains(ReceiverInputForwarder.releaseShortcutLabel)) + } + + func testCursorEchoHandsBackToTheSendingMacAfterLocalMovementStops() { + var echo = CursorEchoPolicy() + XCTAssertTrue(echo.appliesRemotePosition(at: 10)) + echo.localMove(at: 10) + XCTAssertFalse(echo.appliesRemotePosition(at: 10.1)) + XCTAssertTrue(echo.appliesRemotePosition(at: 10 + CursorEchoPolicy.holdAfterLocalMove)) + echo.localMove(at: 20) + echo.reset() + XCTAssertTrue(echo.appliesRemotePosition(at: 20)) + } + + // MARK: - Sender side + + func testScrollRemainderKeepsSlowTrackpadScrolls() { + var remainder = ScrollRemainder() + var total: Int32 = 0 + for _ in 0..<10 { total += remainder.take(dxPoints: 0, dyPoints: 0.3).dy } + XCTAssertEqual(total, 3, "ten 0.3-point deltas scroll three points, not zero") + } + + func testScrollRemainderStillRoundsASingleDeltaToNearest() { + var remainder = ScrollRemainder() + XCTAssertEqual(remainder.take(dxPoints: 0.6, dyPoints: -2.4).dx, 1) + remainder.reset() + XCTAssertEqual(remainder.take(dxPoints: 0, dyPoints: -2.4).dy, -2) + } + + func testScrollRemainderNeverDriftsAcrossDirectionChanges() { + var remainder = ScrollRemainder() + var total: Int32 = 0 + let deltas = [0.4, 0.4, -0.3, 1.7, -0.9, 0.2, 0.25] + for delta in deltas { total += remainder.take(dxPoints: 0, dyPoints: delta).dy } + XCTAssertEqual(Double(total) + remainder.y, deltas.reduce(0, +), accuracy: 1e-9) + XCTAssertLessThanOrEqual(abs(remainder.y), 0.5) + } + + func testSenderRepeatsOnlyHeldKeys() { + var held = HeldKeyTracker() + XCTAssertFalse(held.allowsRepeat(.deleteOrBackspace)) + held.down(.deleteOrBackspace) + XCTAssertTrue(held.allowsRepeat(.deleteOrBackspace)) + held.up(.deleteOrBackspace) + XCTAssertFalse(held.allowsRepeat(.deleteOrBackspace), "a repeat can't outlive its key") + } + + func testSenderParsesTheNewUsagesAndOldOnesUnchanged() { + XCTAssertEqual(HIDKeyUsage.parse(62 as NSNumber)?.keyCode, 96) // F5 + XCTAssertEqual(HIDKeyUsage.parse(74 as NSNumber)?.keyCode, 115) // Home + XCTAssertEqual(HIDKeyUsage.parse(42 as NSNumber)?.keyCode, 51) // Backspace, as before + XCTAssertNil(HIDKeyUsage.parse(200 as NSNumber), "still ignores unknown usages") + } + + // MARK: - Cost + + /// Per-event cost of the receiver's decision layer, for the record (no + /// threshold): a 120 Hz trackpad produces ~120 moves a second. + func testPointerMoveDecisionCost() { + measure { + var forwarder = controlling() + for index in 0..<10_000 { + let x = Double(index % 1000) / 1000 + _ = forwarder.pointerMoved(CGPoint(x: x, y: 0.5), inside: true) + } + } + } +} diff --git a/PROTOCOL.md b/PROTOCOL.md index 83a940967..235d5d911 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -681,6 +681,12 @@ and the stylus would go dead; the fallback keeps it usable.) receiver sends MUST be followed by a matching `"up"` (or by disconnect — senders MUST release a still-held key on session loss regardless). `{"type":"keyboard","action":"down","usage":80,"modifiers":["shift"]}` + A `"down"` MAY carry `repeat` (bool, additive, no `pv` bump): `true` + marks the receiver's own key repeat for a key it already sent down and + has not released. A sender SHOULD post it as an auto-repeat key-down + only while that key is held and MUST ignore it otherwise; a sender that + doesn't know the field treats it as the duplicate down it already + ignores. * `"modifierDown"` / `"modifierUp"` (pv 6) — a synthetic modifier's held lifecycle: `modifier` is one of `"command"`, `"option"`, `"control"`, or `"shift"`. Every down MUST be followed by an up; senders MUST also release @@ -696,7 +702,13 @@ unrecognized modifier names. `usage` values are USB HID Usage Tables keyboard-page (0x07) usage numbers. Senders MUST validate `usage` (an integral value in `0...65535`) and MUST ignore an out-of-range, non-integral, or unrecognized value rather than -treat it as fatal. The keys the official apps exchange today: +treat it as fatal. The official Mac Receiver sends any key of a physical +keyboard: letters, digits and punctuation (4–56), F1–F12 (58–69) and +F13–F20 (104–111), Insert/Help (73), Home (74), Page Up (75), End (77), +Page Down (78), the keypad (83–99, 103, 133), the ISO key (100) and the JIS +keys (135, 137, 144, 145). The usage names a physical key, so the +sender's own keyboard layout decides the character. The special keys the +iPhone and iPad apps send: | Usage | Key | |---|---| @@ -1471,3 +1483,4 @@ This file is versioned by git; the authoritative change log is | 2026-09-27 | `pv` 22: optional reliable-stream QUIC transport (section 2.4) with authenticated `transports`/`qv` capability | | 2026-09-30 | Documented optional HEVC video (section 5.4, `pv` 19) and the `pv` 15/16/18 rows; no wire change. The sender now also falls back to H.264 when an HEVC encoder produces no frames or a new `hello` drops `"hevc"` | | 2026-09-26 | Hardening, no `pv` change: session admission and input grants are per connection (section 6.9); the same `hello` identity/current-pin check on USB (section 2.2); the Remote Access knock declares its intent | +| 2026-10-25 | Additive, no `pv` change: `keyboard` `"down"` may carry `repeat`; the Mac Receiver sends the full physical-keyboard usage range | diff --git a/README.md b/README.md index e68035406..4cb126ae3 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ later). Both are free. | **Connections** | USB · local Wi-Fi · Remote Access · Auto-Reconnect · Wake & Connect | | **Audio** | Mac system audio on the receiver · Mac keeps playing locally · A/V sync and Resync | | **Security** | Trusted pairing with a matching code · TLS 1.3 · pinned peer identity | -| **Receivers** | iPhone and iPad · another Mac with the display-only Mac Receiver | +| **Receivers** | iPhone and iPad · another Mac with the Mac Receiver, whose keyboard and trackpad can control the sending Mac | ## Why MeowDisplay? @@ -312,8 +312,9 @@ Either side can start a session, and the Mac sender always streams. When connection approval is required, an explicit Connect waits for acceptance; background reconnects never prompt. -Receivers include **iPhone**, **iPad**, and the **display-only Mac Receiver**. -The Mac Receiver does not forward its keyboard or trackpad to the other Mac. +Receivers include **iPhone**, **iPad**, and the **Mac Receiver**. Once the +sending Mac allows control, the receiving Mac's own keyboard and trackpad drive +it while the video window is in front; press ⌃⌥⎋ to pause. More on [Devices](https://meowdisplay.app/features.html#devices). ## Connections diff --git a/RECEIVER_INPUT.md b/RECEIVER_INPUT.md new file mode 100644 index 000000000..b0c936bc2 --- /dev/null +++ b/RECEIVER_INPUT.md @@ -0,0 +1,130 @@ +# Mac Receiver input + +A Mac running the Mac Receiver is a display **and** a control surface +([issue #12](https://github.com/raiseCatError/meowdisplay/issues/12)). Once the +sending Mac allows it, the receiving Mac's own trackpad, mouse and keyboard +drive the sending Mac while the video window is in front, in Mirror and in +Extend. Drag a window onto the receiving MacBook's extended desktop and use +that MacBook's trackpad and keyboard on it. + +## Using it + +1. Connect as usual. Every session starts **View Only**. +2. Click **Request Control** in the video window's titlebar. In full screen, + move the pointer to the top of the screen to show the titlebar. The + sending Mac asks its owner, or allows it straight away if this device is + set to **Always Allow Requests**. +3. While the titlebar says **Controlling**, the pointer, clicks, scrolling, + pinch/rotate and keyboard go to the sending Mac. +4. To get this Mac back: + - **⌃⌥⎋ (Control-Option-Escape)** pauses control. Click the picture to + resume; that click only resumes and isn't sent. + - Switching to another app or window (Command-Tab, a click outside the + window) stops forwarding until the video window is in front again. + - **Release** in the titlebar gives control back to the sending Mac. A + new request is needed to control it again. + +The sending Mac can revoke control at any time. Its own Allow Input switch, +per-device policy and request prompt all work exactly as for an iPhone or +iPad. + +## What is forwarded + +| Input | Sent as | Notes | +| --- | --- | --- | +| Pointer movement | `pointer` `move` (absolute, normalized) | Over the picture only, or anywhere during a drag (clamped to the picture's edge). | +| Click, double/triple click | `pointer` `down`/`up`, `clickCount` | Left and right (two-finger click, Control-click via the Control modifier). | +| Drag | `down`, moves, `up` | Released correctly even when the button goes up outside the window. | +| Scroll | `scroll` (video pixels) | Trackpad precise deltas and momentum; wheel lines ≈ 10 pt. Direction follows **this** Mac's natural-scrolling setting. | +| Pinch, rotate | `nativeAppGesture` | Full began/changed/ended lifecycle; cancelled if focus is lost. | +| Keys | `keyboard` `down`/`up` with HID usage | Every key of a full keyboard, F-keys, navigation and keypad. The **sending** Mac's keyboard layout and input method decide the character, as with a keyboard plugged into it. | +| Key repeat | `keyboard` `down` + `repeat: true` | This Mac's own repeat rate. Needs a sender from this branch; an older sender simply doesn't repeat. | +| Shift, Control, Option, Command | `keyboard` `modifierDown`/`modifierUp` | Applied to keys, clicks and scrolls on the sending Mac. | +| Caps Lock | `capsLock` in each key's `modifiers` | Not toggled on the sending Mac. | + +**Not forwarded**: +- **Middle and other mouse buttons**: the protocol names left and right only. +- **Fn and Globe-key shortcuts**: macOS turns Fn+arrow into Home/End/Page keys before the app sees it, and those are forwarded. +- **Three- and four-finger swipes, Mission Control, Launchpad and other trackpad system gestures**: macOS keeps these on this Mac, by design. +- **Force Touch pressure**, and **text composed by this Mac's input method**: the sending Mac's own input method composes instead. + +## What stays on this Mac + +Input is never captured globally. Only events macOS delivers to the video +window while it is in front are looked at (an app-local event monitor scoped +to that window), so: + +- System shortcuts macOS handles before any app — **Command-Tab, Command-Option-Escape (Force Quit), Command-Space, Mission Control, screenshots, Control-Command-Q (Lock Screen)** — always act on this Mac. +- **⌃⌥⎋** is reserved for pausing and is never sent. +- While controlling, app shortcuts such as Command-Q, Command-W or Command-, go + to the **sending** Mac (it's what typing into that window means). Pause + first (⌃⌥⎋) or switch away to use them on this Mac. +- A modifier still held from before forwarding started (the Command of the + Command-Tab that brought the window forward) is never sent, so it can't + turn the next key into a shortcut on the sending Mac. + +## Never stuck + +The receiver remembers every key, modifier, button and gesture it sent down. +Pausing, focus loss, Release, revocation by the sending Mac and closing the +window all send the matching releases, then `keyboard` `cancel` as a safety +net. A disconnect resets locally: the sending Mac releases everything on +session loss itself (PROTOCOL.md), and a reconnect starts View Only with +nothing held. The sending Mac ignores input the moment its grant ends, +whatever a receiver sends. + +## Pointer feel + +While controlling, the streamed cursor is drawn at this Mac's pointer +immediately instead of after a network round trip and a video frame. The +sending Mac's reported position takes over 0.25 s after local movement stops, +so the two can't disagree for long (for example when the sending Mac's own +trackpad moves the cursor). + +## Code + +| Piece | Where | +| --- | --- | +| Rules: geometry, key map, forwarding state machine, cursor echo | `MacReceiver/ReceiverInputForwarding.swift` (AppKit-free, unit tested) | +| AppKit events, window focus, sending, titlebar indicator, hint | `MacReceiver/ReceiverInputController.swift` | +| Video view event overrides | `ReceiverVideoView` in `MacReceiver/MacReceiver.swift` | +| Sender: extra HID usages, key repeat, scroll remainder | `Mac/InputRouting.swift`, `Mac/InputInjector.swift` | + +No new wire message exists for any of this. The only addition is the optional +`repeat` field (PROTOCOL.md section 6.1). + +## Automated coverage + +`MacTests/ReceiverInputForwardingTests.swift` drives the forwarder with +synthetic event sequences: +- **Geometry**: exact fit, letterbox, pillarbox, resize, mixed-resolution full screen, clamped drags. +- **Key map**: a full round trip through the sender's injection table. +- **Pointer**: view only by default; request, decline and requests-disabled; old senders; hover; click, drag and release outside; double and right click; letterbox clicks. +- **Focus and pause**: focus loss mid-drag and mid-pinch; the pause shortcut and click-to-resume; Release. +- **Keyboard**: modifiers and shortcuts; Command held from Command-Tab; key repeat; Caps Lock. +- **Session**: revocation mid-typing; disconnect and reconnect with no stale state. +- **Sender side**: scroll remainder and repeat. +- **Cost**: a measured per-move decision cost. + +These prove the receiver's rules and the sender's pure helpers. Nothing here +posts events into the live system. They do not prove how a physical +trackpad or keyboard feels, real latency, or the sending Mac's injected +events in real apps. + +## Physical test checklist + +Needs two Macs on the same network, paired, both built from this branch: +the sender (for example a MacBook Air) and the receiver (an older MacBook). + +1. **View Only.** Connect: the titlebar says View Only; moving, clicking and typing in the window do nothing on the sender. +2. **Request.** Request Control → the sender prompts → Allow: the titlebar says Controlling and the hint names ⌃⌥⎋. +3. **Pointer, Extend.** Drag a window from the sender onto the receiver's extended desktop. With the receiver's trackpad: move, click, double-click a word, right-click (two fingers and Control-click), drag the window by its title bar and drop it with the button released outside the receiver window. +4. **Pointer, Mirror.** Same in Mirror; check the cursor lands exactly under the receiver's pointer at all four corners, windowed and full screen. +5. **Scroll and gestures.** Slow and fast two-finger scrolls (including momentum) in Safari and Finder; the direction matches the receiver's natural-scrolling setting. Pinch-zoom a photo in Preview. +6. **Keyboard.** Type a sentence with Shift; Command-C/Command-V; hold Delete (repeats); arrows, Home/End, F-keys, Escape; Caps Lock. A non-US layout on the sender types its own characters. +7. **Local shortcuts.** While controlling: Command-Tab, Command-Space, Mission Control, Command-Shift-4 and Command-Option-Escape act on the receiver. +8. **Pause and resume.** ⌃⌥⎋ → Paused; typing stays local; click the picture → Controlling again, without a click reaching the sender. +9. **No stuck keys.** Hold Shift (or a mouse button mid-drag), then Command-Tab away, then check on the sender that nothing is held: typing there is lower-case and no drag continues. Repeat with revoking control on the sender, and with unplugging the network mid-drag. +10. **Release and revoke.** Release → View Only, nothing reaches the sender. Request again, then revoke from the sender's device settings: input stops immediately. +11. **Reconnect.** Quit and relaunch the receiver: it starts View Only. +12. **iPhone/iPad unchanged.** With the same sender, an iPhone's Direct Touch, trackpad mode, keyboard, modifiers and scrolling behave as before. Slow two-finger scrolls should be no worse (the sender now carries the scroll remainder instead of rounding each event). diff --git a/SETUP.md b/SETUP.md index 3a80fdc4c..0b75864d9 100644 --- a/SETUP.md +++ b/SETUP.md @@ -186,7 +186,9 @@ Settings. **Always Allow Requests** skips the Mac prompt but still requires that request each session. See [Allow Input & control requests](README.md#allow-input--control-requests). Receivers can still view the Mac with input off. The standalone Mac Receiver -is display-only; its keyboard and trackpad are not forwarded. +uses the same request: once the sending Mac allows it, the receiving Mac's own +keyboard and trackpad control it while the video window is in front. See +[RECEIVER_INPUT.md](RECEIVER_INPUT.md). ### Connecting and reconnecting diff --git a/Shared/StreamReceiver.swift b/Shared/StreamReceiver.swift index 62876818e..364817dd5 100644 --- a/Shared/StreamReceiver.swift +++ b/Shared/StreamReceiver.swift @@ -3324,9 +3324,19 @@ final class StreamReceiver: ObservableObject { /// A hardware key going down, for keys the Mac must hold (arrows, /// modified shortcuts). `modifiers` are named protocol modifiers, not /// raw UIKit flags. + /// + /// `isRepeat`: the receiving Mac's own key repeat for a key already sent + /// down (additive `repeat` field; an older Mac ignores the duplicate + /// down, as it always did). @MainActor func sendKeyboardDown(usage: Int, modifiers: [String]) { + sendKeyboardDown(usage: usage, modifiers: modifiers, isRepeat: false) + } + + @MainActor func sendKeyboardDown(usage: Int, modifiers: [String], isRepeat: Bool) { guard displayState == .running, macSupportsKeyboardWire else { return } - sendUIControl(["type": "keyboard", "action": "down", "usage": usage, "modifiers": modifiers]) + var message: [String: Any] = ["type": "keyboard", "action": "down", "usage": usage, "modifiers": modifiers] + if isRepeat { message["repeat"] = true } + sendUIControl(message) } /// The matching release for `sendKeyboardDown`. diff --git a/project.yml b/project.yml index 3ee9d8291..f6c2927f3 100644 --- a/project.yml +++ b/project.yml @@ -179,6 +179,7 @@ targets: - Mac/SmartTouchTargetClassifier.swift - MacReceiver/ReceiverWindowLifecycle.swift - MacReceiver/ReceiverSettingsCategory.swift + - MacReceiver/ReceiverInputForwarding.swift - Shared/ReceiverGesture.swift - Mac/Log.swift - Mac/LogPolicies.swift