Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 89 additions & 4 deletions apple/AppHost/AndroidDisplayView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
//
// Mouse: left button = finger touch (scrcpy maps the "mouse" pointer with only the
// primary button to a touchscreen event), right button = mouse secondary click,
// trackpad and wheel = scroll, hover = mouse hover.
// trackpad and wheel = scroll, hover = mouse hover, trackpad pinch and rotation =
// two fingers around the pointer.
// Keyboard: text goes as text, everything else as keycodes (AndroidKeys).
//
// Licensed under the GNU General Public License v3 or later.
Expand All @@ -21,6 +22,7 @@ final class AndroidDisplayView: NSView {
let displayLayer = AVSampleBufferDisplayLayer()
private var trackingArea: NSTrackingArea?
private var buttons: UInt32 = 0
private var fingers: TwoFingers?
private var resizeWork: DispatchWorkItem?

override init(frame: NSRect) {
Expand Down Expand Up @@ -101,7 +103,11 @@ final class AndroidDisplayView: NSView {

/// Map a point in the view to Android pixels, honouring the aspect-fit letterbox.
private func androidPoint(_ event: NSEvent) -> (Int, Int)? {
let point = convert(event.locationInWindow, from: nil)
androidPoint(inWindow: event.locationInWindow)
}

private func androidPoint(inWindow location: NSPoint) -> (Int, Int)? {
let point = convert(location, from: nil)
let video = AVMakeRect(aspectRatio: frameSize, insideRect: bounds)
guard video.width > 0, video.height > 0 else { return nil }
let x = (point.x - video.minX) / video.width * frameSize.width
Expand Down Expand Up @@ -142,12 +148,12 @@ final class AndroidDisplayView: NSView {
}

override func mouseMoved(with event: NSEvent) {
guard buttons == 0, let (x, y) = androidPoint(event) else { return }
guard buttons == 0, fingers == nil, let (x, y) = androidPoint(event) else { return }
session?.injectTouch(.hoverMove, x: x, y: y, pressure: 0)
}

override func scrollWheel(with event: NSEvent) {
guard let (x, y) = androidPoint(event) else { return }
guard fingers == nil, let (x, y) = androidPoint(event) else { return }
// Trackpads report precise point deltas; wheels report lines.
let divisor: CGFloat = event.hasPreciseScrollingDeltas ? 30 : 1
let h = Float(event.scrollingDeltaX / divisor)
Expand All @@ -156,6 +162,85 @@ final class AndroidDisplayView: NSView {
session?.injectScroll(x: x, y: y, hScroll: h, vScroll: v)
}

// MARK: Trackpad gestures

/// A pinch or rotation on the trackpad, as two fingers on either side of the
/// point where it started. macOS sends magnify and rotate as separate events,
/// often interleaved in one gesture, so the fingers stay down until both end.
private struct TwoFingers {
enum Gesture { case magnify, rotate }
/// Android pixels: about a fingertip apart.
static let closest: CGFloat = 48

var center: CGPoint // Android pixels
var span: CGFloat // distance between the fingers, Android pixels
var angle: CGFloat = 0 // radians, clockwise on the screen
var active: Set<Gesture> = []

/// Both fingers' positions, kept inside the frame.
func points(in size: CGSize) -> [(x: Int, y: Int)] {
let dx = cos(angle) * span / 2, dy = sin(angle) * span / 2
return [CGPoint(x: center.x - dx, y: center.y - dy), CGPoint(x: center.x + dx, y: center.y + dy)].map {
(Int(max(0, min(size.width - 1, $0.x))), Int(max(0, min(size.height - 1, $0.y))))
}
}
}

override func magnify(with event: NSEvent) {
let factor = 1 + event.magnification
twoFingerGesture(.magnify, phase: event.phase, location: event.locationInWindow) { fingers in
// Never closer than a fingertip, never wider than the display.
let diagonal = hypot(frameSize.width, frameSize.height)
fingers.span = max(TwoFingers.closest, min(diagonal, fingers.span * factor))
}
}

override func rotate(with event: NSEvent) {
// Degrees, counterclockwise; Android's y axis points down.
let radians = -CGFloat(event.rotation) * .pi / 180
twoFingerGesture(.rotate, phase: event.phase, location: event.locationInWindow) { fingers in
fingers.angle += radians
}
}

private func twoFingerGesture(_ gesture: TwoFingers.Gesture, phase: NSEvent.Phase, location: NSPoint,
update: (inout TwoFingers) -> Void) {
if phase.contains(.began) {
if fingers == nil {
// A click in progress keeps the mouse pointer; it can't also be a finger.
guard buttons == 0, let (x, y) = androidPoint(inWindow: location) else { return }
// About a thumb and finger apart on a phone.
let span = min(frameSize.width, frameSize.height) / 4
fingers = TwoFingers(center: CGPoint(x: x, y: y), span: max(TwoFingers.closest, span))
sendFingers(.down)
}
fingers?.active.insert(gesture)
}
guard fingers?.active.contains(gesture) == true else { return }
if phase.contains(.changed) {
update(&fingers!)
sendFingers(.move)
}
if phase.contains(.ended) || phase.contains(.cancelled) {
fingers?.active.remove(gesture)
if fingers?.active.isEmpty == true {
sendFingers(.up)
fingers = nil
}
}
}

/// Down in order, up in reverse, so Android sees one finger and then two.
private func sendFingers(_ action: ScrcpySession.TouchAction) {
guard let fingers else { return }
let points = fingers.points(in: frameSize)
var touches = [(ScrcpySession.pointerFinger, points[0]), (ScrcpySession.pointerVirtualFinger, points[1])]
if action == .up { touches.reverse() }
for (pointer, point) in touches {
session?.injectTouch(action, x: point.x, y: point.y, pointer: pointer)
}
}

// MARK: Keyboard

override func keyDown(with event: NSEvent) {
Expand Down
3 changes: 3 additions & 0 deletions apple/Shared/Scrcpy/ScrcpySession.swift
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,9 @@ final class ScrcpySession: @unchecked Sendable {
enum KeyAction: UInt8 { case down = 0, up = 1 }

static let pointerMouse: UInt64 = 0xFFFF_FFFF_FFFF_FFFF // -1
/// scrcpy's own pinch uses these two: always touchscreen fingers, never a mouse.
static let pointerFinger: UInt64 = 0xFFFF_FFFF_FFFF_FFFE // -2
static let pointerVirtualFinger: UInt64 = 0xFFFF_FFFF_FFFF_FFFD // -3
static let buttonPrimary: UInt32 = 1
static let buttonSecondary: UInt32 = 2

Expand Down
3 changes: 1 addition & 2 deletions docs/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ new_display=<window pixels>/<dpi> flex_display=true vd_system_decorations=false
display_ime_policy=hide video_codec=h264 audio=false tunnel_forward=true
```

It then opens `localabstract:scrcpy_<scid>` twice through ADB (video, then control; no port forwarding), switches the new virtual display to fullscreen with `cmd window set-display-windowing-mode` (the image declares `android.hardware.type.pc`, so Android would otherwise open the app as a small freeform window) and sends `START_APP <package>`. Video packets become `CMSampleBuffer`s (`H264SampleBuilder`) for an `AVSampleBufferDisplayLayer`, which decodes in hardware. Mouse buttons become touches, the wheel becomes scroll events, printable keys become text and the rest become Android keycodes. Resizing the window sends `RESIZE_DISPLAY`, so the Android display (and the app's layout) follows the window. `dpi = 160 × backing scale × App size`, so one Android dp is one Mac point at the default size.
It then opens `localabstract:scrcpy_<scid>` twice through ADB (video, then control; no port forwarding), switches the new virtual display to fullscreen with `cmd window set-display-windowing-mode` (the image declares `android.hardware.type.pc`, so Android would otherwise open the app as a small freeform window) and sends `START_APP <package>`. Video packets become `CMSampleBuffer`s (`H264SampleBuilder`) for an `AVSampleBufferDisplayLayer`, which decodes in hardware. Mouse buttons become touches, the wheel becomes scroll events, a trackpad pinch or rotation becomes two fingers around the pointer (scrcpy's generic and virtual finger pointers, the ones its own pinch uses), printable keys become text and the rest become Android keycodes. Resizing the window sends `RESIZE_DISPLAY`, so the Android display (and the app's layout) follows the window. `dpi = 160 × backing scale × App size`, so one Android dp is one Mac point at the default size.

Sound doesn't go through scrcpy: Android plays through the VM's `intel-hda` device, so all apps share one audio path.

Expand Down Expand Up @@ -109,7 +109,6 @@ The Mac reads them with `content read --uri content://io.github.harkhowalia1.msa
## Next steps

- GPU acceleration: virtio-gpu-gl with virglrenderer and ANGLE (as UTM does) instead of software rendering.
- Multi-touch (pinch, rotate) from the trackpad.
- Camera, location and shared folders.
- Close the Mac window when the Android app finishes its last activity.
- Faster *As needed* starts with a saved VM snapshot.
Expand Down
2 changes: 1 addition & 1 deletion site/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -410,7 +410,7 @@ <h2>What works today.</h2>
<tr><td>Automatic storage clean-up</td><td><span class="pill yes">Works</span></td><td>Downloads, leftovers, logs, Android's unused space.</td></tr>
<tr><td>Notifications, clipboard, sound</td><td><span class="pill yes">Works</span></td><td>Microphone off unless you allow it.</td></tr>
<tr><td>3D graphics acceleration</td><td><span class="pill part">Software</span></td><td>Android draws on the CPU for now; 3D games run slower than they could.</td></tr>
<tr><td>Multi-touch gestures (pinch, rotate)</td><td><span class="pill part">Planned</span></td><td>Clicks, drags and scrolling work.</td></tr>
<tr><td>Multi-touch gestures (pinch, rotate)</td><td><span class="pill yes">Works</span></td><td>Pinch and rotate on the trackpad, as two fingers around the pointer.</td></tr>
<tr><td>Google Play and Google services</td><td><span class="pill no">No</span></td><td>Google doesn't license them for this.</td></tr>
<tr><td>Intel Macs, iPhone, iPad</td><td><span class="pill no">No</span></td><td>Needs Apple silicon and macOS.</td></tr>
</tbody>
Expand Down