From 14c2e06f367f15f2f0e86be06298978602271ac5 Mon Sep 17 00:00:00 2001 From: Parya Rastegar Date: Tue, 27 Oct 2026 04:42:48 +0100 Subject: [PATCH] Pinch and rotate from the trackpad A trackpad pinch or rotation becomes two fingers on either side of the pointer: they move apart or together with the magnification and turn with the rotation. macOS sends magnify and rotate as separate events, often interleaved in one gesture, so the fingers stay down until both have ended. They use scrcpy's generic and virtual finger pointers (-2 and -3), the ones its own pinch uses, so Android always sees touchscreen fingers. Hover and scroll events wait while the fingers are down. --- apple/AppHost/AndroidDisplayView.swift | 93 +++++++++++++++++++++++-- apple/Shared/Scrcpy/ScrcpySession.swift | 3 + docs/ARCHITECTURE.md | 3 +- site/index.html | 2 +- 4 files changed, 94 insertions(+), 7 deletions(-) diff --git a/apple/AppHost/AndroidDisplayView.swift b/apple/AppHost/AndroidDisplayView.swift index 536fe4e..16811c7 100644 --- a/apple/AppHost/AndroidDisplayView.swift +++ b/apple/AppHost/AndroidDisplayView.swift @@ -3,7 +3,8 @@ // // Mouse: left button = finger touch (scrcpy maps the "mouse" pointer with only the // primary button to a touchscreen event), right button = mouse secondary click, -// trackpad and wheel = scroll, hover = mouse hover. +// trackpad and wheel = scroll, hover = mouse hover, trackpad pinch and rotation = +// two fingers around the pointer. // Keyboard: text goes as text, everything else as keycodes (AndroidKeys). // // Licensed under the GNU General Public License v3 or later. @@ -21,6 +22,7 @@ final class AndroidDisplayView: NSView { let displayLayer = AVSampleBufferDisplayLayer() private var trackingArea: NSTrackingArea? private var buttons: UInt32 = 0 + private var fingers: TwoFingers? private var resizeWork: DispatchWorkItem? override init(frame: NSRect) { @@ -101,7 +103,11 @@ final class AndroidDisplayView: NSView { /// Map a point in the view to Android pixels, honouring the aspect-fit letterbox. private func androidPoint(_ event: NSEvent) -> (Int, Int)? { - let point = convert(event.locationInWindow, from: nil) + androidPoint(inWindow: event.locationInWindow) + } + + private func androidPoint(inWindow location: NSPoint) -> (Int, Int)? { + let point = convert(location, from: nil) let video = AVMakeRect(aspectRatio: frameSize, insideRect: bounds) guard video.width > 0, video.height > 0 else { return nil } let x = (point.x - video.minX) / video.width * frameSize.width @@ -142,12 +148,12 @@ final class AndroidDisplayView: NSView { } override func mouseMoved(with event: NSEvent) { - guard buttons == 0, let (x, y) = androidPoint(event) else { return } + guard buttons == 0, fingers == nil, let (x, y) = androidPoint(event) else { return } session?.injectTouch(.hoverMove, x: x, y: y, pressure: 0) } override func scrollWheel(with event: NSEvent) { - guard let (x, y) = androidPoint(event) else { return } + guard fingers == nil, let (x, y) = androidPoint(event) else { return } // Trackpads report precise point deltas; wheels report lines. let divisor: CGFloat = event.hasPreciseScrollingDeltas ? 30 : 1 let h = Float(event.scrollingDeltaX / divisor) @@ -156,6 +162,85 @@ final class AndroidDisplayView: NSView { session?.injectScroll(x: x, y: y, hScroll: h, vScroll: v) } + // MARK: Trackpad gestures + + /// A pinch or rotation on the trackpad, as two fingers on either side of the + /// point where it started. macOS sends magnify and rotate as separate events, + /// often interleaved in one gesture, so the fingers stay down until both end. + private struct TwoFingers { + enum Gesture { case magnify, rotate } + /// Android pixels: about a fingertip apart. + static let closest: CGFloat = 48 + + var center: CGPoint // Android pixels + var span: CGFloat // distance between the fingers, Android pixels + var angle: CGFloat = 0 // radians, clockwise on the screen + var active: Set = [] + + /// Both fingers' positions, kept inside the frame. + func points(in size: CGSize) -> [(x: Int, y: Int)] { + let dx = cos(angle) * span / 2, dy = sin(angle) * span / 2 + return [CGPoint(x: center.x - dx, y: center.y - dy), CGPoint(x: center.x + dx, y: center.y + dy)].map { + (Int(max(0, min(size.width - 1, $0.x))), Int(max(0, min(size.height - 1, $0.y)))) + } + } + } + + override func magnify(with event: NSEvent) { + let factor = 1 + event.magnification + twoFingerGesture(.magnify, phase: event.phase, location: event.locationInWindow) { fingers in + // Never closer than a fingertip, never wider than the display. + let diagonal = hypot(frameSize.width, frameSize.height) + fingers.span = max(TwoFingers.closest, min(diagonal, fingers.span * factor)) + } + } + + override func rotate(with event: NSEvent) { + // Degrees, counterclockwise; Android's y axis points down. + let radians = -CGFloat(event.rotation) * .pi / 180 + twoFingerGesture(.rotate, phase: event.phase, location: event.locationInWindow) { fingers in + fingers.angle += radians + } + } + + private func twoFingerGesture(_ gesture: TwoFingers.Gesture, phase: NSEvent.Phase, location: NSPoint, + update: (inout TwoFingers) -> Void) { + if phase.contains(.began) { + if fingers == nil { + // A click in progress keeps the mouse pointer; it can't also be a finger. + guard buttons == 0, let (x, y) = androidPoint(inWindow: location) else { return } + // About a thumb and finger apart on a phone. + let span = min(frameSize.width, frameSize.height) / 4 + fingers = TwoFingers(center: CGPoint(x: x, y: y), span: max(TwoFingers.closest, span)) + sendFingers(.down) + } + fingers?.active.insert(gesture) + } + guard fingers?.active.contains(gesture) == true else { return } + if phase.contains(.changed) { + update(&fingers!) + sendFingers(.move) + } + if phase.contains(.ended) || phase.contains(.cancelled) { + fingers?.active.remove(gesture) + if fingers?.active.isEmpty == true { + sendFingers(.up) + fingers = nil + } + } + } + + /// Down in order, up in reverse, so Android sees one finger and then two. + private func sendFingers(_ action: ScrcpySession.TouchAction) { + guard let fingers else { return } + let points = fingers.points(in: frameSize) + var touches = [(ScrcpySession.pointerFinger, points[0]), (ScrcpySession.pointerVirtualFinger, points[1])] + if action == .up { touches.reverse() } + for (pointer, point) in touches { + session?.injectTouch(action, x: point.x, y: point.y, pointer: pointer) + } + } + // MARK: Keyboard override func keyDown(with event: NSEvent) { diff --git a/apple/Shared/Scrcpy/ScrcpySession.swift b/apple/Shared/Scrcpy/ScrcpySession.swift index cb67bf0..e0fe115 100644 --- a/apple/Shared/Scrcpy/ScrcpySession.swift +++ b/apple/Shared/Scrcpy/ScrcpySession.swift @@ -257,6 +257,9 @@ final class ScrcpySession: @unchecked Sendable { enum KeyAction: UInt8 { case down = 0, up = 1 } static let pointerMouse: UInt64 = 0xFFFF_FFFF_FFFF_FFFF // -1 + /// scrcpy's own pinch uses these two: always touchscreen fingers, never a mouse. + static let pointerFinger: UInt64 = 0xFFFF_FFFF_FFFF_FFFE // -2 + static let pointerVirtualFinger: UInt64 = 0xFFFF_FFFF_FFFF_FFFD // -3 static let buttonPrimary: UInt32 = 1 static let buttonSecondary: UInt32 = 2 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6c02f66..8fa84f9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -44,7 +44,7 @@ new_display=/ flex_display=true vd_system_decorations=false display_ime_policy=hide video_codec=h264 audio=false tunnel_forward=true ``` -It then opens `localabstract:scrcpy_` twice through ADB (video, then control; no port forwarding), switches the new virtual display to fullscreen with `cmd window set-display-windowing-mode` (the image declares `android.hardware.type.pc`, so Android would otherwise open the app as a small freeform window) and sends `START_APP `. Video packets become `CMSampleBuffer`s (`H264SampleBuilder`) for an `AVSampleBufferDisplayLayer`, which decodes in hardware. Mouse buttons become touches, the wheel becomes scroll events, printable keys become text and the rest become Android keycodes. Resizing the window sends `RESIZE_DISPLAY`, so the Android display (and the app's layout) follows the window. `dpi = 160 × backing scale × App size`, so one Android dp is one Mac point at the default size. +It then opens `localabstract:scrcpy_` twice through ADB (video, then control; no port forwarding), switches the new virtual display to fullscreen with `cmd window set-display-windowing-mode` (the image declares `android.hardware.type.pc`, so Android would otherwise open the app as a small freeform window) and sends `START_APP `. Video packets become `CMSampleBuffer`s (`H264SampleBuilder`) for an `AVSampleBufferDisplayLayer`, which decodes in hardware. Mouse buttons become touches, the wheel becomes scroll events, a trackpad pinch or rotation becomes two fingers around the pointer (scrcpy's generic and virtual finger pointers, the ones its own pinch uses), printable keys become text and the rest become Android keycodes. Resizing the window sends `RESIZE_DISPLAY`, so the Android display (and the app's layout) follows the window. `dpi = 160 × backing scale × App size`, so one Android dp is one Mac point at the default size. Sound doesn't go through scrcpy: Android plays through the VM's `intel-hda` device, so all apps share one audio path. @@ -109,7 +109,6 @@ The Mac reads them with `content read --uri content://io.github.harkhowalia1.msa ## Next steps - GPU acceleration: virtio-gpu-gl with virglrenderer and ANGLE (as UTM does) instead of software rendering. -- Multi-touch (pinch, rotate) from the trackpad. - Camera, location and shared folders. - Close the Mac window when the Android app finishes its last activity. - Faster *As needed* starts with a saved VM snapshot. diff --git a/site/index.html b/site/index.html index 3ba0fa3..ebd7c1f 100644 --- a/site/index.html +++ b/site/index.html @@ -410,7 +410,7 @@

What works today.

Automatic storage clean-upWorksDownloads, leftovers, logs, Android's unused space. Notifications, clipboard, soundWorksMicrophone off unless you allow it. 3D graphics accelerationSoftwareAndroid draws on the CPU for now; 3D games run slower than they could. - Multi-touch gestures (pinch, rotate)PlannedClicks, drags and scrolling work. + Multi-touch gestures (pinch, rotate)WorksPinch and rotate on the trackpad, as two fingers around the pointer. Google Play and Google servicesNoGoogle doesn't license them for this. Intel Macs, iPhone, iPadNoNeeds Apple silicon and macOS.