diff --git a/android/bridge/AndroidManifest.xml b/android/bridge/AndroidManifest.xml
index f89d459..69d9096 100644
--- a/android/bridge/AndroidManifest.xml
+++ b/android/bridge/AndroidManifest.xml
@@ -10,6 +10,9 @@
notifications.json active notifications, from NotificationRelayService
Only the shell user (which holds DUMP) can read it, not other Android apps.
+ MacInputMethod is Android's keyboard while it runs on the Mac: the Mac's
+ windows send it what is typed, in any script (see MacInputMethod.java).
+
Licensed under the GNU General Public License v3 or later.
-->
+
+
+
+
+
+
+
diff --git a/android/bridge/build.sh b/android/bridge/build.sh
index 2d397e8..f7610a2 100755
--- a/android/bridge/build.sh
+++ b/android/bridge/build.sh
@@ -27,8 +27,9 @@ WORK="${ROOT}/build/bridge"
rm -rf "${WORK}"
mkdir -p "${WORK}/classes" "${WORK}/dex" "$(dirname "${OUT}")"
+"${BUILD_TOOLS}/aapt2" compile --dir "${HERE}/res" -o "${WORK}/res.zip"
"${BUILD_TOOLS}/aapt2" link -o "${WORK}/base.apk" -I "${PLATFORM}" \
- --manifest "${HERE}/AndroidManifest.xml" \
+ --manifest "${HERE}/AndroidManifest.xml" "${WORK}/res.zip" \
--min-sdk-version 30 --target-sdk-version 35 \
--version-code "${VERSION}" --version-name "${VERSION}"
diff --git a/android/bridge/res/xml/method.xml b/android/bridge/res/xml/method.xml
new file mode 100644
index 0000000..05f2d67
--- /dev/null
+++ b/android/bridge/res/xml/method.xml
@@ -0,0 +1,3 @@
+
+
+
diff --git a/android/bridge/src/io/github/harkhowalia1/msa/bridge/MacInputMethod.java b/android/bridge/src/io/github/harkhowalia1/msa/bridge/MacInputMethod.java
new file mode 100644
index 0000000..35871a3
--- /dev/null
+++ b/android/bridge/src/io/github/harkhowalia1/msa/bridge/MacInputMethod.java
@@ -0,0 +1,273 @@
+// Mac Android Subsystem - the keyboard Android uses while it runs on the Mac.
+//
+// It has no keys on screen. Each Mac window connects to the abstract socket
+// "msa_ime" over ADB and sends the text the Mac's keyboard and input methods
+// produce, so text in any script reaches the app (scrcpy's text injection only
+// carries characters Android's virtual key map has, so Persian, Russian or
+// Chinese never arrived). Composing text from a Mac input method (Pinyin, Kana,
+// dead keys) shows in the app underlined, as with a phone keyboard. Editing keys
+// go the same way while an editor has input, so they stay in order with the text.
+//
+// The Mac selects it with
+// ime enable io.github.harkhowalia1.msa.bridge/.MacInputMethod
+// ime set io.github.harkhowalia1.msa.bridge/.MacInputMethod
+// Only the shell user (ADB) and root may connect; other apps can't type with it.
+//
+// Messages, big-endian. Mac to Android:
+// 1 hello displayId:i32 the window's virtual display
+// 2 commit length:u32 UTF-8 text typed
+// 3 compose length:u32 UTF-8 the composing text so far
+// 4 finish keep the composing text as typed
+// 5 key action:u8 keycode:i32 metaState:i32 repeat:i32
+// Android to Mac:
+// 1 editing displayId:i32 active:u8 an editor on that display got or lost input
+//
+// Licensed under the GNU General Public License v3 or later.
+
+package io.github.harkhowalia1.msa.bridge;
+
+import android.content.res.Configuration;
+import android.inputmethodservice.InputMethodService;
+import android.net.LocalServerSocket;
+import android.net.LocalSocket;
+import android.os.Handler;
+import android.os.Looper;
+import android.os.Process;
+import android.os.SystemClock;
+import android.text.InputType;
+import android.util.Log;
+import android.view.Display;
+import android.view.InputDevice;
+import android.view.KeyCharacterMap;
+import android.view.KeyEvent;
+import android.view.inputmethod.EditorInfo;
+import android.view.inputmethod.InputConnection;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+public final class MacInputMethod extends InputMethodService {
+ private static final String TAG = "MacInputMethod";
+ private static final String SOCKET = "msa_ime";
+ private static final int MAX_TEXT = 256 * 1024;
+
+ // The socket and the Mac's connections belong to the process, not to one instance:
+ // Android destroys and makes the service again (when input moves to another
+ // display, for one), and the windows stay connected through that.
+ private static final CopyOnWriteArrayList clients = new CopyOnWriteArrayList<>();
+ private static final Handler main = new Handler(Looper.getMainLooper());
+ private static Thread server;
+ /** The running instance, null between two. Main thread only. */
+ private static MacInputMethod current;
+ /** The display of the editor that has input, or -1 when none does. Main thread only. */
+ private static int editingDisplay = Display.INVALID_DISPLAY;
+
+ @Override
+ public void onCreate() {
+ super.onCreate();
+ current = this;
+ if (server == null || !server.isAlive()) {
+ server = new Thread(MacInputMethod::serve, "msa-ime");
+ server.setDaemon(true);
+ server.start();
+ }
+ }
+
+ @Override
+ public void onDestroy() {
+ if (current == this) {
+ current = null;
+ setEditingDisplay(Display.INVALID_DISPLAY);
+ }
+ super.onDestroy();
+ }
+
+ // Nothing to show: the keys are on the Mac.
+ @Override
+ public boolean onEvaluateInputViewShown() {
+ return false;
+ }
+
+ @Override
+ public boolean onEvaluateFullscreenMode() {
+ return false;
+ }
+
+ @Override
+ public void onStartInput(EditorInfo attribute, boolean restarting) {
+ super.onStartInput(attribute, restarting);
+ // A view that takes no text (a game's surface, a terminal) wants raw key events.
+ boolean text = attribute != null && attribute.inputType != InputType.TYPE_NULL;
+ setEditingDisplay(text ? currentDisplay() : Display.INVALID_DISPLAY);
+ }
+
+ @Override
+ public void onFinishInput() {
+ super.onFinishInput();
+ setEditingDisplay(Display.INVALID_DISPLAY);
+ }
+
+ // The IME's window follows the editor to its display, and this service's context with it.
+ @Override
+ public void onConfigurationChanged(Configuration newConfig) {
+ super.onConfigurationChanged(newConfig);
+ if (editingDisplay != Display.INVALID_DISPLAY) setEditingDisplay(currentDisplay());
+ }
+
+ private int currentDisplay() {
+ Display display = getDisplay();
+ return display != null ? display.getDisplayId() : Display.INVALID_DISPLAY;
+ }
+
+ private static void setEditingDisplay(int display) {
+ if (display == editingDisplay) return;
+ int previous = editingDisplay;
+ editingDisplay = display;
+ for (Client client : clients) {
+ if (previous != Display.INVALID_DISPLAY) client.sendEditing(previous, false);
+ if (display != Display.INVALID_DISPLAY) client.sendEditing(display, true);
+ }
+ }
+
+ /** The editor's connection, if it is on the window's display. */
+ private static InputConnection connectionFor(int display) {
+ MacInputMethod service = current;
+ return service != null && display == editingDisplay ? service.getCurrentInputConnection() : null;
+ }
+
+ private static void serve() {
+ try (LocalServerSocket listener = new LocalServerSocket(SOCKET)) {
+ while (true) {
+ LocalSocket socket = listener.accept();
+ int uid = socket.getPeerCredentials().getUid();
+ if (uid != Process.SHELL_UID && uid != Process.ROOT_UID) {
+ socket.close();
+ continue;
+ }
+ Client client;
+ try {
+ client = new Client(socket);
+ } catch (IOException e) {
+ socket.close();
+ continue;
+ }
+ clients.add(client);
+ Thread thread = new Thread(client::run, "msa-ime-client");
+ thread.setDaemon(true);
+ thread.start();
+ }
+ } catch (IOException e) {
+ Log.w(TAG, "socket closed", e);
+ }
+ }
+
+ private static final class Client {
+ private final LocalSocket socket;
+ private final DataOutputStream out;
+ /** The window's display, from its hello. Main thread only. */
+ private int display = Display.INVALID_DISPLAY;
+
+ Client(LocalSocket socket) throws IOException {
+ this.socket = socket;
+ out = new DataOutputStream(socket.getOutputStream());
+ }
+
+ void run() {
+ try (DataInputStream in = new DataInputStream(socket.getInputStream())) {
+ while (true) {
+ int type = in.readUnsignedByte();
+ switch (type) {
+ case 1:
+ int id = in.readInt();
+ main.post(() -> {
+ display = id;
+ if (editingDisplay == id) sendEditing(id, true);
+ });
+ break;
+ case 2: {
+ String text = readText(in);
+ main.post(() -> {
+ InputConnection ic = connectionFor(display);
+ if (ic != null) ic.commitText(text, 1);
+ });
+ break;
+ }
+ case 3: {
+ String text = readText(in);
+ main.post(() -> {
+ InputConnection ic = connectionFor(display);
+ if (ic != null) ic.setComposingText(text, 1);
+ });
+ break;
+ }
+ case 4:
+ main.post(() -> {
+ InputConnection ic = connectionFor(display);
+ if (ic != null) ic.finishComposingText();
+ });
+ break;
+ case 5: {
+ int action = in.readUnsignedByte();
+ int keycode = in.readInt();
+ int metaState = in.readInt();
+ int repeat = in.readInt();
+ main.post(() -> {
+ InputConnection ic = connectionFor(display);
+ if (ic != null) ic.sendKeyEvent(keyEvent(action, keycode, metaState, repeat));
+ });
+ break;
+ }
+ default:
+ throw new IOException("unknown message " + type);
+ }
+ }
+ } catch (EOFException ignored) {
+ // The window closed.
+ } catch (IOException e) {
+ Log.w(TAG, "client", e);
+ } finally {
+ close();
+ }
+ }
+
+ void sendEditing(int id, boolean active) {
+ if (display != Display.INVALID_DISPLAY && display != id) return;
+ try {
+ synchronized (this) {
+ out.writeByte(1);
+ out.writeInt(id);
+ out.writeByte(active ? 1 : 0);
+ out.flush();
+ }
+ } catch (IOException e) {
+ close();
+ }
+ }
+
+ void close() {
+ clients.remove(this);
+ try {
+ socket.close();
+ } catch (IOException ignored) {
+ }
+ }
+ }
+
+ private static String readText(DataInputStream in) throws IOException {
+ int length = in.readInt();
+ if (length < 0 || length > MAX_TEXT) throw new IOException("text of " + length + " bytes");
+ byte[] bytes = new byte[length];
+ in.readFully(bytes);
+ return new String(bytes, StandardCharsets.UTF_8);
+ }
+
+ private static KeyEvent keyEvent(int action, int keycode, int metaState, int repeat) {
+ long now = SystemClock.uptimeMillis();
+ return new KeyEvent(now, now, action, keycode, repeat, metaState, KeyCharacterMap.VIRTUAL_KEYBOARD, 0,
+ KeyEvent.FLAG_SOFT_KEYBOARD | KeyEvent.FLAG_KEEP_TOUCH_MODE, InputDevice.SOURCE_KEYBOARD);
+ }
+}
diff --git a/apple/AppHost/AndroidDisplayView.swift b/apple/AppHost/AndroidDisplayView.swift
index 536fe4e..e5c6062 100644
--- a/apple/AppHost/AndroidDisplayView.swift
+++ b/apple/AppHost/AndroidDisplayView.swift
@@ -4,7 +4,10 @@
// Mouse: left button = finger touch (scrcpy maps the "mouse" pointer with only the
// primary button to a touchscreen event), right button = mouse secondary click,
// trackpad and wheel = scroll, hover = mouse hover.
-// Keyboard: text goes as text, everything else as keycodes (AndroidKeys).
+// Keyboard: the Mac's input methods (any layout, dead keys, Pinyin, Kana) give text
+// and composing text through NSTextInputClient; it goes to the bridge's keyboard
+// while an editor has input (MacKeyboard), as scrcpy text otherwise. Everything
+// else goes as keycodes (AndroidKeys).
//
// Licensed under the GNU General Public License v3 or later.
@@ -13,6 +16,8 @@ import AVFoundation
final class AndroidDisplayView: NSView {
weak var session: ScrcpySession?
+ /// The bridge's keyboard, when Android uses it.
+ var keyboard: MacKeyboard?
/// Size of the Android frame in pixels; input coordinates are scaled to it.
var frameSize = CGSize(width: 1, height: 1)
/// Called when the view's pixel size settles after a resize.
@@ -21,6 +26,13 @@ final class AndroidDisplayView: NSView {
let displayLayer = AVSampleBufferDisplayLayer()
private var trackingArea: NSTrackingArea?
private var buttons: UInt32 = 0
+ /// Keys whose down went to Android as keycodes, and whether through the keyboard.
+ private var keysDown: [UInt16: Bool] = [:]
+ /// The key interpretKeyEvents is working on, for doCommand(by:).
+ private var keyEvent: NSEvent?
+ private var markedText = ""
+ /// Where the last click landed (view coordinates): the input method's candidates go there.
+ private var lastClick: NSPoint?
private var resizeWork: DispatchWorkItem?
override init(frame: NSRect) {
@@ -113,6 +125,7 @@ final class AndroidDisplayView: NSView {
override func mouseDown(with event: NSEvent) {
window?.makeFirstResponder(self)
+ lastClick = convert(event.locationInWindow, from: nil)
guard let (x, y) = androidPoint(event) else { return }
buttons |= ScrcpySession.buttonPrimary
session?.injectTouch(.down, x: x, y: y, actionButton: ScrcpySession.buttonPrimary, buttons: buttons)
@@ -159,19 +172,43 @@ final class AndroidDisplayView: NSView {
// MARK: Keyboard
override func keyDown(with event: NSEvent) {
- if let text = AndroidKeys.text(for: event) {
- session?.injectText(text)
+ let flags = event.modifierFlags.intersection(.deviceIndependentFlagsMask)
+ // Shortcuts, and arrows or function keys outside a composition, are keys, never text.
+ if flags.contains(.control) || flags.contains(.command) || (flags.contains(.function) && markedText.isEmpty) {
+ sendKey(event, .down)
return
}
- guard let code = AndroidKeys.keycodes[Int(event.keyCode)] else { return }
- session?.injectKey(.down, keycode: code, repeat: event.isARepeat ? 1 : 0,
- metaState: AndroidKeys.metaState(event.modifierFlags))
+ // Text and editing commands come back through NSTextInputClient below.
+ keyEvent = event
+ interpretKeyEvents([event])
+ keyEvent = nil
}
override func keyUp(with event: NSEvent) {
- if AndroidKeys.text(for: event) != nil { return }
+ guard let throughKeyboard = keysDown.removeValue(forKey: event.keyCode) else { return }
+ sendKey(event, .up, throughKeyboard: throughKeyboard)
+ }
+
+ /// A key as an Android keycode: through the bridge's keyboard while an editor has input,
+ /// so it stays in order with the text, otherwise through scrcpy. A key goes up the way it went down.
+ private func sendKey(_ event: NSEvent, _ action: ScrcpySession.KeyAction, throughKeyboard: Bool? = nil) {
guard let code = AndroidKeys.keycodes[Int(event.keyCode)] else { return }
- session?.injectKey(.up, keycode: code, metaState: AndroidKeys.metaState(event.modifierFlags))
+ let meta = AndroidKeys.metaState(event.modifierFlags)
+ let count: Int32 = action == .down && event.isARepeat ? 1 : 0
+ let viaKeyboard = throughKeyboard ?? (keyboard?.isEditing == true)
+ if action == .down { keysDown[event.keyCode] = viaKeyboard }
+ if viaKeyboard, let keyboard {
+ keyboard.key(action, keycode: code, metaState: meta, repeat: count)
+ } else {
+ session?.injectKey(action, keycode: code, repeat: count, metaState: meta)
+ }
+ }
+
+ override func resignFirstResponder() -> Bool {
+ // Keep what is being composed, as typed.
+ inputContext?.discardMarkedText()
+ unmarkText()
+ return super.resignFirstResponder()
}
/// Control-key shortcuts reach keyDown; Command shortcuts are menu items (Edit menu).
@@ -184,3 +221,58 @@ final class AndroidDisplayView: NSView {
return super.performKeyEquivalent(with: event)
}
}
+
+// MARK: Text input
+
+extension AndroidDisplayView: NSTextInputClient {
+ func insertText(_ string: Any, replacementRange: NSRange) {
+ let text = (string as? NSAttributedString)?.string ?? string as? String ?? ""
+ markedText = ""
+ guard !text.isEmpty else { return }
+ if let keyboard, keyboard.isEditing {
+ keyboard.commit(text) // replaces the composing text
+ } else {
+ session?.injectText(text)
+ }
+ }
+
+ /// Return, Delete, Tab, Escape and the like: the key itself.
+ override func doCommand(by selector: Selector) {
+ if let keyEvent { sendKey(keyEvent, .down) }
+ }
+
+ func setMarkedText(_ string: Any, selectedRange: NSRange, replacementRange: NSRange) {
+ markedText = (string as? NSAttributedString)?.string ?? string as? String ?? ""
+ if let keyboard, keyboard.isEditing { keyboard.compose(markedText) }
+ }
+
+ func unmarkText() {
+ guard !markedText.isEmpty else { return }
+ markedText = ""
+ if let keyboard, keyboard.isEditing { keyboard.finishComposing() }
+ }
+
+ func selectedRange() -> NSRange {
+ NSRange(location: (markedText as NSString).length, length: 0)
+ }
+
+ func markedRange() -> NSRange {
+ markedText.isEmpty ? NSRange(location: NSNotFound, length: 0)
+ : NSRange(location: 0, length: (markedText as NSString).length)
+ }
+
+ func hasMarkedText() -> Bool { !markedText.isEmpty }
+
+ func attributedSubstring(forProposedRange range: NSRange, actualRange: NSRangePointer?) -> NSAttributedString? { nil }
+
+ func validAttributesForMarkedText() -> [NSAttributedString.Key] { [] }
+
+ /// Android doesn't tell us where its caret is; the last click is usually next to it.
+ func firstRect(forCharacterRange range: NSRange, actualRange: NSRangePointer?) -> NSRect {
+ let point = lastClick ?? NSPoint(x: bounds.midX, y: bounds.midY)
+ let rect = NSRect(x: point.x, y: point.y - 10, width: 1, height: 20)
+ return window?.convertToScreen(convert(rect, to: nil)) ?? .zero
+ }
+
+ func characterIndex(for point: NSPoint) -> Int { NSNotFound }
+}
diff --git a/apple/AppHost/AndroidKeys.swift b/apple/AppHost/AndroidKeys.swift
index 9a899d7..7873f42 100644
--- a/apple/AppHost/AndroidKeys.swift
+++ b/apple/AppHost/AndroidKeys.swift
@@ -1,7 +1,7 @@
// Mac Android Subsystem - Mac keyboard to Android key events.
//
-// Printable characters are sent as text (scrcpy's INJECT_TEXT), so every Mac
-// keyboard layout and dead key works. Everything else (arrows, Return, Delete,
+// Text comes from AppKit's text input (AndroidDisplayView), so every Mac keyboard
+// layout, dead key and input method works. Everything else (arrows, Return, Delete,
// shortcuts with Control) is sent as an Android keycode with its meta state.
//
// Licensed under the GNU General Public License v3 or later.
@@ -65,16 +65,4 @@ enum AndroidKeys {
if flags.contains(.capsLock) { meta |= metaCapsLock }
return meta
}
-
- /// Text to type for a key press, or nil when it should go as a keycode.
- static func text(for event: NSEvent) -> String? {
- let flags = event.modifierFlags.intersection(.deviceIndependentFlagsMask)
- if flags.contains(.control) || flags.contains(.command) || flags.contains(.function) { return nil }
- guard let characters = event.characters, !characters.isEmpty else { return nil }
- // Control characters (Return, Tab, Delete, arrows in the private-use range) go as keycodes.
- let printable = characters.unicodeScalars.allSatisfy { scalar in
- !CharacterSet.controlCharacters.contains(scalar) && !(0xF700...0xF8FF).contains(scalar.value)
- }
- return printable ? characters : nil
- }
}
diff --git a/apple/AppHost/AndroidWindowController.swift b/apple/AppHost/AndroidWindowController.swift
index 56cfb72..ca7a534 100644
--- a/apple/AppHost/AndroidWindowController.swift
+++ b/apple/AppHost/AndroidWindowController.swift
@@ -20,6 +20,7 @@ final class AndroidWindowController: NSWindowController, NSWindowDelegate {
private let overlay = SplashOverlay()
private var connection: ADBConnection?
private var session: ScrcpySession?
+ private var keyboard: MacKeyboard?
private let builder = H264SampleBuilder()
private var lastSentSize = (0, 0)
private var isClosing = false
@@ -114,6 +115,7 @@ final class AndroidWindowController: NSWindowController, NSWindowDelegate {
height: max(height, 320), dpi: dpi)
configuration.showSoftKeyboard = UserDefaults(suiteName: AppConfig.subsystemBundleID)?
.bool(forKey: "showSoftKeyboard") ?? false
+ configuration.macKeyboard = MacKeyboard.select(connection, onScreenKeyboard: configuration.showSoftKeyboard)
// The performance level sets how smooth (and how costly) the stream is.
configuration.maxFPS = min(level.maxFPS, max(screenFPS, 30))
configuration.bitRate = level.videoBitRate
@@ -173,8 +175,13 @@ final class AndroidWindowController: NSWindowController, NSWindowDelegate {
Task.detached(priority: .userInitiated) { [weak self] in
do {
try session.start()
+ // Without it typing still works, as scrcpy text (characters Android's key map has).
+ let keyboard = session.usesMacKeyboard
+ ? session.displayID.map { MacKeyboard(connection: connection, display: $0) } : nil
await MainActor.run { [weak self] in
- guard let self else { return }
+ guard let self else { keyboard?.close(); return }
+ self.keyboard = keyboard
+ self.displayView.keyboard = keyboard
self.lastSentSize = self.displayView.pixelSize
self.window?.makeFirstResponder(self.displayView)
self.gamepads.session = session
@@ -191,6 +198,9 @@ final class AndroidWindowController: NSWindowController, NSWindowDelegate {
guard !isClosing else { return }
orientationWatch?.cancel()
gamepads.session = nil
+ displayView.keyboard = nil
+ keyboard?.close()
+ keyboard = nil
session = nil
connection?.close()
connection = nil
@@ -225,6 +235,7 @@ final class AndroidWindowController: NSWindowController, NSWindowDelegate {
gamepads.deactivate()
isClosing = true
orientationWatch?.cancel()
+ keyboard?.close()
session?.stop()
connection?.close()
NSApp.terminate(nil)
diff --git a/apple/AppHost/MacKeyboard.swift b/apple/AppHost/MacKeyboard.swift
new file mode 100644
index 0000000..aa0b9b9
--- /dev/null
+++ b/apple/AppHost/MacKeyboard.swift
@@ -0,0 +1,136 @@
+// Mac Android Subsystem - typing through the bridge's keyboard (MacInputMethod.java).
+//
+// scrcpy's text injection only carries characters Android's virtual key map has,
+// so Persian, Russian or Chinese never arrived. The bridge's input method commits
+// text itself. While an editor on this window's display has input, text, composing
+// text and editing keys go to it over the abstract socket "msa_ime"; the rest
+// (keys for games and lists, text with no editor) still goes through scrcpy.
+//
+// Licensed under the GNU General Public License v3 or later.
+
+import Foundation
+
+final class MacKeyboard: @unchecked Sendable {
+ static var inputMethod: String { "\(AppConfig.bridgePackage)/.MacInputMethod" }
+
+ /// Make the bridge's keyboard Android's input method, or, when Android's own on-screen
+ /// keyboard is wanted, give that back. True when the bridge's keyboard is the input
+ /// method afterwards (a bridge older than version 2 has none).
+ static func select(_ connection: ADBConnection, onScreenKeyboard: Bool) -> Bool {
+ let current = (try? connection.execString("settings get secure default_input_method", timeout: 5))?
+ .trimmingCharacters(in: .whitespacesAndNewlines)
+ if onScreenKeyboard {
+ if current == inputMethod { _ = try? connection.exec("ime reset", timeout: 5) }
+ return false
+ }
+ if current == inputMethod { return true }
+ _ = try? connection.exec("ime enable \(inputMethod)", timeout: 5)
+ let output = (try? connection.execString("ime set \(inputMethod)", timeout: 5)) ?? ""
+ return output.contains("selected")
+ }
+
+ private weak var connection: ADBConnection?
+ private let display: Int
+ private let queue = DispatchQueue(label: "msa.keyboard")
+ private let lock = NSLock()
+ private var stream: ADBStream?
+ private var editing = false
+ private var closed = false
+
+ /// True while an editor on this window's display has input.
+ var isEditing: Bool {
+ lock.lock(); defer { lock.unlock() }
+ return editing
+ }
+
+ /// Android starts the input method once an app needs it, and may restart it, so the
+ /// link keeps reconnecting until the window closes.
+ init(connection: ADBConnection, display: Int) {
+ self.connection = connection
+ self.display = display
+ let thread = Thread { [weak self] in
+ var delay: TimeInterval = 0.25
+ while let self, !self.isClosed, let connection = self.connection {
+ if let stream = try? connection.openStream("localabstract:msa_ime", timeout: 5) {
+ self.run(stream)
+ delay = 0.25
+ }
+ Thread.sleep(forTimeInterval: delay)
+ delay = min(delay * 2, 4)
+ }
+ }
+ thread.name = "msa keyboard"
+ thread.start()
+ }
+
+ func close() {
+ lock.lock()
+ closed = true
+ let current = stream
+ lock.unlock()
+ current?.close()
+ }
+
+ private var isClosed: Bool {
+ lock.lock(); defer { lock.unlock() }
+ return closed
+ }
+
+ /// Say which display this window is, then follow "editing" messages
+ /// (displayId:i32 active:u8) until the stream ends.
+ private func run(_ stream: ADBStream) {
+ lock.lock()
+ if closed { lock.unlock(); stream.close(); return }
+ self.stream = stream
+ lock.unlock()
+ var hello = ScrcpySession.Message(1)
+ hello.i32(Int32(display))
+ send(hello)
+ while let type = try? stream.readExactly(1)[0], type == 1,
+ let body = try? stream.readExactly(5) {
+ guard Int(Int32(bitPattern: body.be32(0))) == display else { continue }
+ lock.lock()
+ editing = body[body.startIndex + 4] != 0
+ lock.unlock()
+ }
+ stream.close()
+ lock.lock()
+ self.stream = nil
+ editing = false
+ lock.unlock()
+ }
+
+ func commit(_ text: String) {
+ var m = ScrcpySession.Message(2)
+ m.string(text, lengthBytes: 4)
+ send(m)
+ }
+
+ /// The composing text so far (shown underlined); commit replaces it.
+ func compose(_ text: String) {
+ var m = ScrcpySession.Message(3)
+ m.string(text, lengthBytes: 4)
+ send(m)
+ }
+
+ /// Keep the composing text as typed.
+ func finishComposing() {
+ send(ScrcpySession.Message(4))
+ }
+
+ func key(_ action: ScrcpySession.KeyAction, keycode: Int32, metaState: Int32, repeat count: Int32) {
+ var m = ScrcpySession.Message(5)
+ m.u8(action.rawValue); m.i32(keycode); m.i32(metaState); m.i32(count)
+ send(m)
+ }
+
+ private func send(_ message: ScrcpySession.Message) {
+ lock.lock()
+ let current = stream
+ lock.unlock()
+ guard let current else { return }
+ queue.async {
+ try? current.write(message.data)
+ }
+ }
+}
diff --git a/apple/Config/Base.xcconfig b/apple/Config/Base.xcconfig
index abfe478..78a1bcc 100644
--- a/apple/Config/Base.xcconfig
+++ b/apple/Config/Base.xcconfig
@@ -14,7 +14,7 @@ ADB_PORT = 58526
// Must match build/payload/scrcpy-server.jar (core/scripts/fetch-scrcpy-server.sh).
SCRCPY_VERSION = 5.0.1
// Version code of android/bridge (android/bridge/build.sh reads it from here).
-MSA_BRIDGE_VERSION = 1
+MSA_BRIDGE_VERSION = 2
MSA_BRIDGE_PACKAGE = io.github.harkhowalia1.msa.bridge
REPOSITORY = harkhowalia-1/MacSubsystemForAndroid
WEBSITE_URL = https:/$()/harkhowalia-1.github.io/MacSubsystemForAndroid
diff --git a/apple/Shared/AppConfig.swift b/apple/Shared/AppConfig.swift
index 99f9dbd..1907e42 100644
--- a/apple/Shared/AppConfig.swift
+++ b/apple/Shared/AppConfig.swift
@@ -30,6 +30,8 @@ enum AppConfig {
static let websiteURL = URL(string: info("MSAWebsiteURL") ?? "https://github.com")!
/// ADB port on 127.0.0.1 (forwarded by QEMU).
static let adbPort: UInt16 = UInt16(info("MSAADBPort") ?? "") ?? 58526
+ /// Package of the bridge app inside Android.
+ static let bridgePackage: String = info("MSABridgePackage") ?? "io.github.harkhowalia1.msa.bridge"
/// scrcpy server version bundled in Resources/scrcpy-server.jar. Client and server must match.
static let scrcpyVersion: String = info("MSAScrcpyVersion") ?? "5.0.1"
/// Bundle identifier prefix of generated launchers: .
diff --git a/apple/Shared/Scrcpy/ScrcpySession.swift b/apple/Shared/Scrcpy/ScrcpySession.swift
index cb67bf0..11ee1cb 100644
--- a/apple/Shared/Scrcpy/ScrcpySession.swift
+++ b/apple/Shared/Scrcpy/ScrcpySession.swift
@@ -30,6 +30,8 @@ final class ScrcpySession: @unchecked Sendable {
var maxFPS: Int = 60
var bitRate: Int = 12_000_000
var showSoftKeyboard = false
+ /// The bridge's keyboard is Android's input method (MacKeyboard).
+ var macKeyboard = false
var serverPath = "/data/local/tmp/msa-scrcpy-server.jar"
}
@@ -62,6 +64,9 @@ final class ScrcpySession: @unchecked Sendable {
private var serverLog = ""
private(set) var frameSize: (width: Int, height: Int) = (0, 0)
+ var usesMacKeyboard: Bool { configuration.macKeyboard }
+ /// The virtual display's id, once the server has made it.
+ var displayID: Int? { virtualDisplayID() }
init(connection: ADBConnection, configuration: Configuration) {
self.connection = connection
@@ -89,7 +94,10 @@ final class ScrcpySession: @unchecked Sendable {
"cleanup=true",
"power_on=true",
]
- arguments.append("display_ime_policy=\(configuration.showSoftKeyboard ? "local" : "hide")")
+ // The input method works on the window's display: Android's on-screen keyboard when
+ // asked for, or the bridge's keyboard, which shows nothing. Otherwise it stays away.
+ let imeOnDisplay = configuration.showSoftKeyboard || configuration.macKeyboard
+ arguments.append("display_ime_policy=\(imeOnDisplay ? "local" : "hide")")
let command = "CLASSPATH=\(configuration.serverPath) app_process / com.genymobile.scrcpy.Server "
+ "\(AppConfig.scrcpyVersion) " + arguments.joined(separator: " ")
diff --git a/apple/Subsystem/Model/AndroidSetup.swift b/apple/Subsystem/Model/AndroidSetup.swift
index a6622dc..60656f9 100644
--- a/apple/Subsystem/Model/AndroidSetup.swift
+++ b/apple/Subsystem/Model/AndroidSetup.swift
@@ -13,8 +13,7 @@
import Foundation
enum AndroidBridge {
- static let package = Bundle.main.object(forInfoDictionaryKey: "MSABridgePackage") as? String
- ?? "io.github.harkhowalia1.msa.bridge"
+ static var package: String { AppConfig.bridgePackage }
static var authority: String { package }
static var notificationListener: String { "\(package)/\(package).NotificationRelayService" }
static var bundledAPK: URL? { Bundle.main.url(forResource: "bridge", withExtension: "apk") }
diff --git a/apple/Subsystem/Model/LauncherGenerator.swift b/apple/Subsystem/Model/LauncherGenerator.swift
index 00c1ce0..58b5de4 100644
--- a/apple/Subsystem/Model/LauncherGenerator.swift
+++ b/apple/Subsystem/Model/LauncherGenerator.swift
@@ -97,7 +97,7 @@ enum LauncherGenerator {
]
// The window host reads the Subsystem's settings from its own Info.plist.
for key in ["MSADisplayName", "MSAShortName", "MSALegacySupportFolders", "MSASubsystemBundleID", "MSAURLScheme", "MSASupportFolder",
- "MSALauncherFolder", "MSAScrcpyVersion", "MSAADBPort", "MSARepository", "MSAWebsiteURL"] {
+ "MSALauncherFolder", "MSAScrcpyVersion", "MSAADBPort", "MSARepository", "MSAWebsiteURL", "MSABridgePackage"] {
if let value = Bundle.main.object(forInfoDictionaryKey: key) { info[key] = value }
}
let plist = try PropertyListSerialization.data(fromPropertyList: info, format: .xml, options: 0)
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 6c02f66..c6133e9 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -41,10 +41,12 @@
```
new_display=/ flex_display=true vd_system_decorations=false
-display_ime_policy=hide video_codec=h264 audio=false tunnel_forward=true
+display_ime_policy=local video_codec=h264 audio=false tunnel_forward=true
```
-It then opens `localabstract:scrcpy_` twice through ADB (video, then control; no port forwarding), switches the new virtual display to fullscreen with `cmd window set-display-windowing-mode` (the image declares `android.hardware.type.pc`, so Android would otherwise open the app as a small freeform window) and sends `START_APP `. Video packets become `CMSampleBuffer`s (`H264SampleBuilder`) for an `AVSampleBufferDisplayLayer`, which decodes in hardware. Mouse buttons become touches, the wheel becomes scroll events, printable keys become text and the rest become Android keycodes. Resizing the window sends `RESIZE_DISPLAY`, so the Android display (and the app's layout) follows the window. `dpi = 160 × backing scale × App size`, so one Android dp is one Mac point at the default size.
+(`display_ime_policy=hide` when neither the bridge's keyboard nor Android's on-screen keyboard is in use.)
+
+It then opens `localabstract:scrcpy_` twice through ADB (video, then control; no port forwarding), switches the new virtual display to fullscreen with `cmd window set-display-windowing-mode` (the image declares `android.hardware.type.pc`, so Android would otherwise open the app as a small freeform window) and sends `START_APP `. Video packets become `CMSampleBuffer`s (`H264SampleBuilder`) for an `AVSampleBufferDisplayLayer`, which decodes in hardware. Mouse buttons become touches and the wheel becomes scroll events. Typing goes through AppKit's text input (`NSTextInputClient`), so every layout, dead key and input method (Pinyin, Kana) works: while an editor on the window's display has input, text, composing text and editing keys go to the bridge's keyboard (`MacKeyboard`, see below), otherwise text goes as scrcpy text and the rest as Android keycodes. Resizing the window sends `RESIZE_DISPLAY`, so the Android display (and the app's layout) follows the window. `dpi = 160 × backing scale × App size`, so one Android dp is one Mac point at the default size.
Sound doesn't go through scrcpy: Android plays through the VM's `intel-hda` device, so all apps share one audio path.
@@ -96,6 +98,8 @@ Modes map to levels as follows. Standard → Standard and Performance → Perfor
The Mac reads them with `content read --uri content://io.github.harkhowalia1.msa.bridge/`.
+`MacInputMethod` is Android's input method (each app window selects it with `ime set`, unless *Show Android's on-screen keyboard* is on). It shows nothing. scrcpy's text injection only carries characters Android's virtual key map has, so text in Persian, Russian or Chinese never arrived; the input method commits it with `commitText`, and shows a Mac input method's composing text underlined with `setComposingText`. Each window connects to its abstract socket `msa_ime` through ADB (only the shell user and root may), says which display it is, and hears when an editor on that display gains or loses input; while one has it, the window sends text, composing text and editing keys there, so they stay in order. The socket belongs to the bridge's process, since Android destroys and recreates the service, and windows reconnect when the process restarts.
+
## Files
```