Merge pull request #80 from IchenDEV/agent/focus-overlay-on-active-display

fix: place input overlay on focused display

Author: IchenDEV

Sources/App/VoicePipeline+Status.swift

@@ -6,6 +6,7 @@ extension VoicePipeline {
     func showOverlay() {
         overlay.show(
             appState: appState,
+            targetApp: recordingTargetApp,
             onCancel: { [weak self] in
                 self?.cancel()
             },

Sources/UI/OverlayPanel.swift

@@ -1,17 +1,21 @@
 import AppKit
+import ApplicationServices
 import SwiftUI
 
 final class OverlayPanel {
     private var window: NSPanel?
     private var hostingView: NSHostingView<AnyView>?
+    private var targetDisplayID: CGDirectDisplayID?
 
     @MainActor
     func show(
         appState: AppState,
+        targetApp: NSRunningApplication?,
         onCancel: @escaping () -> Void,
         onConfirm: @escaping () -> Void
     ) {
         let initialLayout = OverlayLayout(appState: appState)
+        targetDisplayID = OverlayTargetScreenResolver.displayID(for: targetApp)
 
         if window == nil {
             let panel = NSPanel(
@@ -61,6 +65,7 @@ final class OverlayPanel {
         window?.close()
         window = nil
         hostingView = nil
+        targetDisplayID = nil
     }
 
     @MainActor
@@ -78,15 +83,116 @@ final class OverlayPanel {
 
     @MainActor
     private func frame(for size: CGSize, window: NSWindow) -> NSRect {
-        let screen = window.screen ?? NSScreen.main
+        let screen = OverlayTargetScreenResolver.screen(with: targetDisplayID)
+            ?? NSScreen.main
+            ?? window.screen
         let visibleFrame = screen?.visibleFrame ?? NSRect(x: 0, y: 0, width: size.width, height: size.height)
         return OverlayPanelPlacement.frame(for: size, in: visibleFrame)
     }
 }
 
+@MainActor
+enum OverlayTargetScreenResolver {
+    static func displayID(for app: NSRunningApplication?) -> CGDirectDisplayID? {
+        guard let windowFrame = focusedWindowFrame(for: app) else {
+            return NSScreen.main.flatMap(displayID(for:))
+        }
+
+        let screens = NSScreen.screens
+        // AX window geometry and CG display bounds share the same global coordinate space.
+        let displayFrames = screens.map { screen in
+            displayID(for: screen).map(CGDisplayBounds) ?? .null
+        }
+        guard let index = OverlayPanelPlacement.targetDisplayIndex(
+            for: windowFrame,
+            in: displayFrames
+        ) else {
+            return NSScreen.main.flatMap(displayID(for:))
+        }
+        return displayID(for: screens[index])
+    }
+
+    static func screen(with displayID: CGDirectDisplayID?) -> NSScreen? {
+        guard let displayID else { return nil }
+        return NSScreen.screens.first { self.displayID(for: $0) == displayID }
+    }
+
+    private static func displayID(for screen: NSScreen) -> CGDirectDisplayID? {
+        let key = NSDeviceDescriptionKey("NSScreenNumber")
+        guard let number = screen.deviceDescription[key] as? NSNumber else { return nil }
+        return CGDirectDisplayID(number.uint32Value)
+    }
+
+    private static func focusedWindowFrame(for app: NSRunningApplication?) -> CGRect? {
+        guard let app, AXIsProcessTrusted() else { return nil }
+        let application = AXUIElementCreateApplication(app.processIdentifier)
+
+        for attribute in [kAXFocusedWindowAttribute, kAXMainWindowAttribute] {
+            guard let frame = windowFrame(from: application, attribute: attribute as CFString) else {
+                continue
+            }
+            return frame
+        }
+        return nil
+    }
+
+    private static func windowFrame(from application: AXUIElement, attribute: CFString) -> CGRect? {
+        var windowValue: CFTypeRef?
+        guard AXUIElementCopyAttributeValue(application, attribute, &windowValue) == .success,
+              let rawWindow = windowValue,
+              CFGetTypeID(rawWindow) == AXUIElementGetTypeID() else {
+            return nil
+        }
+
+        let window = rawWindow as! AXUIElement
+        var positionValue: CFTypeRef?
+        var sizeValue: CFTypeRef?
+        guard AXUIElementCopyAttributeValue(
+            window,
+            kAXPositionAttribute as CFString,
+            &positionValue
+        ) == .success,
+              AXUIElementCopyAttributeValue(
+                window,
+                kAXSizeAttribute as CFString,
+                &sizeValue
+              ) == .success,
+              let rawPosition = positionValue,
+              let rawSize = sizeValue,
+              CFGetTypeID(rawPosition) == AXValueGetTypeID(),
+              CFGetTypeID(rawSize) == AXValueGetTypeID() else {
+            return nil
+        }
+
+        var position = CGPoint.zero
+        var size = CGSize.zero
+        guard AXValueGetValue(rawPosition as! AXValue, .cgPoint, &position),
+              AXValueGetValue(rawSize as! AXValue, .cgSize, &size),
+              position.x.isFinite,
+              position.y.isFinite,
+              size.width.isFinite,
+              size.height.isFinite,
+              size.width > 0,
+              size.height > 0 else {
+            return nil
+        }
+        return CGRect(origin: position, size: size)
+    }
+}
+
 enum OverlayPanelPlacement {
     static let bottomMargin: CGFloat = 24
 
+    static func targetDisplayIndex(for windowFrame: CGRect, in displayFrames: [CGRect]) -> Int? {
+        displayFrames.indices
+            .map { index in
+                (index: index, overlap: overlapArea(windowFrame, displayFrames[index]))
+            }
+            .filter { $0.overlap > 0 }
+            .max { lhs, rhs in lhs.overlap < rhs.overlap }?
+            .index
+    }
+
     static func frame(for size: CGSize, in visibleFrame: CGRect) -> CGRect {
         CGRect(
             x: visibleFrame.midX - size.width / 2,
@@ -95,4 +201,10 @@ enum OverlayPanelPlacement {
             height: size.height
         )
     }
+
+    private static func overlapArea(_ lhs: CGRect, _ rhs: CGRect) -> CGFloat {
+        let intersection = lhs.intersection(rhs)
+        guard !intersection.isNull else { return 0 }
+        return intersection.width * intersection.height
+    }
 }

Tests/OpenTypeTests/OverlayLayoutTests.swift

@@ -59,6 +59,48 @@ final class OverlayLayoutTests: XCTestCase {
         XCTAssertEqual(frame.minY, visibleFrame.minY + 24)
     }
 
+    func testOverlayTargetsDisplayContainingFocusedWindow() {
+        let displays = [
+            CGRect(x: 0, y: 0, width: 1_512, height: 982),
+            CGRect(x: -2_560, y: -323, width: 2_560, height: 1_440),
+        ]
+        let focusedWindow = CGRect(x: -2_200, y: -200, width: 1_000, height: 800)
+
+        let index = OverlayPanelPlacement.targetDisplayIndex(
+            for: focusedWindow,
+            in: displays
+        )
+
+        XCTAssertEqual(index, 1)
+    }
+
+    func testOverlayTargetsDisplayWithMostOfFocusedWindow() {
+        let displays = [
+            CGRect(x: -1_920, y: 0, width: 1_920, height: 1_080),
+            CGRect(x: 0, y: 0, width: 1_512, height: 982),
+        ]
+        let focusedWindow = CGRect(x: -300, y: 100, width: 1_000, height: 700)
+
+        let index = OverlayPanelPlacement.targetDisplayIndex(
+            for: focusedWindow,
+            in: displays
+        )
+
+        XCTAssertEqual(index, 1)
+    }
+
+    func testOverlayHasNoTargetForWindowOutsideConnectedDisplays() {
+        let displays = [CGRect(x: 0, y: 0, width: 1_512, height: 982)]
+        let focusedWindow = CGRect(x: 4_000, y: 100, width: 800, height: 600)
+
+        let index = OverlayPanelPlacement.targetDisplayIndex(
+            for: focusedWindow,
+            in: displays
+        )
+
+        XCTAssertNil(index)
+    }
+
     func testCancellingRecordingResetsStateWithoutProcessing() {
         let appState = AppState()
         appState.phase = .recording