diff --git a/MacTests/DemoDesktopTests.swift b/MacTests/DemoDesktopTests.swift index 4034d5918..c32b73267 100644 --- a/MacTests/DemoDesktopTests.swift +++ b/MacTests/DemoDesktopTests.swift @@ -364,3 +364,36 @@ final class DemoDesktopTests: XCTestCase { XCTAssertEqual(second.extendShape, .standard) } } + +/// The demo hands the control overlay the same keyboard geometry as a live +/// session: the area above a docked keyboard, not the keyboard itself, or +/// the keyboard bar is laid out behind the keyboard. +final class DemoKeyboardGeometryTests: XCTestCase { + private let screen = CGRect(x: 0, y: 0, width: 402, height: 874) + + func testDockedKeyboardYieldsTheAreaAboveIt() { + let keyboard = CGRect(x: 0, y: 538, width: 402, height: 336) + let visible = DemoDisplayGeometry.keyboardVisibleRect(keyboardFrame: keyboard, canvas: screen, + screenBounds: screen) + XCTAssertEqual(visible, CGRect(x: 0, y: 0, width: 402, height: 538)) + } + + func testVisibleRectIsInTheCanvasOwnCoordinates() { + let canvas = CGRect(x: 0, y: 0, width: 1180, height: 760) + let screen = CGRect(x: 0, y: 0, width: 1180, height: 820) + let keyboard = CGRect(x: 0, y: 500, width: 1180, height: 320) + let visible = DemoDisplayGeometry.keyboardVisibleRect(keyboardFrame: keyboard, canvas: canvas, + screenBounds: screen) + XCTAssertEqual(visible, CGRect(x: 0, y: 0, width: 1180, height: 500)) + } + + func testHiddenOrFloatingKeyboardYieldsNothing() { + XCTAssertNil(DemoDisplayGeometry.keyboardVisibleRect(keyboardFrame: nil, canvas: screen, screenBounds: screen)) + let offscreen = CGRect(x: 0, y: 874, width: 402, height: 336) + XCTAssertNil(DemoDisplayGeometry.keyboardVisibleRect(keyboardFrame: offscreen, canvas: screen, + screenBounds: screen)) + let floating = CGRect(x: 60, y: 300, width: 320, height: 250) + XCTAssertNil(DemoDisplayGeometry.keyboardVisibleRect(keyboardFrame: floating, canvas: screen, + screenBounds: screen)) + } +} diff --git a/MacTests/PadControlLayoutTests.swift b/MacTests/PadControlLayoutTests.swift index 2414733dd..7b6651a78 100644 --- a/MacTests/PadControlLayoutTests.swift +++ b/MacTests/PadControlLayoutTests.swift @@ -700,7 +700,7 @@ final class PadControlLayoutTests: XCTestCase { func testTemplateEnablesTwoHandAssistWithIdleFunctionActions() { let layout = CustomControlLayout.radialTemplate(name: "Test", corner: .bottomLeading) XCTAssertTrue(layout.twoHandAssist) - XCTAssertEqual(layout.assistActionIDs, ["undo", "redo", "zoom-in", "zoom-out"]) + XCTAssertEqual(layout.assistActionIDs, ["undo", "redo", "zoom-in", "zoom-out", "return"]) let canonicalIDs = Set(FunctionTrayProfile.canonical().items.map(\.id)) XCTAssertTrue(Set(layout.assistActionIDs).isSubset(of: canonicalIDs), "idle actions reuse existing Function Tray items") @@ -776,3 +776,64 @@ final class PadControlLayoutTests: XCTestCase { return (ReceiverControlPreferencesRepository(defaults: defaults), defaults, suite) } } + +/// A fresh iPad starts on the radial Custom layout around a bottom-right +/// Settings anchor; Strip and Overlay stay one setting away. Nothing a +/// user already saved changes, and the iPhone is unaffected. +final class PadDefaultLayoutTests: XCTestCase { + private func emptyDefaults() -> UserDefaults { + let suite = "PadDefaultLayoutTests.\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: suite)! + defaults.removePersistentDomain(forName: suite) + return defaults + } + + func testFreshPadStartsOnTheRadialCustomLayout() { + let preferences = ReceiverControlPreferences.initial(isPad: true) + XCTAssertEqual(preferences.padControlLayout, .custom) + let layout = try? XCTUnwrap(preferences.activeCustomLayout) + XCTAssertEqual(layout?.landscape, .radialTemplate(corner: .bottomTrailing, portrait: false)) + XCTAssertEqual(layout?.portrait, .radialTemplate(corner: .bottomTrailing, portrait: true)) + XCTAssertEqual(preferences.customLayouts.count, 1) + } + + func testFreshPhoneKeepsTheStandardDefaults() { + XCTAssertEqual(ReceiverControlPreferences.initial(isPad: false), ReceiverControlPreferences()) + } + + func testRepositoryUsesTheInitialPreferencesOnlyWhenNothingIsSaved() { + let defaults = emptyDefaults() + let repository = ReceiverControlPreferencesRepository(defaults: defaults) + XCTAssertEqual(repository.load(initial: .initial(isPad: true)).padControlLayout, .custom) + + var saved = ReceiverControlPreferences() + saved.padControlLayout = .strip + repository.save(saved) + XCTAssertEqual(repository.load(initial: .initial(isPad: true)).padControlLayout, .strip) + XCTAssertTrue(repository.load(initial: .initial(isPad: true)).customLayouts.isEmpty) + } +} + +/// The radial layout's idle Two-Hand Assist arc offers Return beside Undo, +/// Redo and Zoom, without crowding the arc. +final class RadialReturnTests: XCTestCase { + func testRadialTemplateOffersReturnInTwoHandAssist() { + XCTAssertEqual(CustomControlLayout.defaultAssistActionIDs, ["undo", "redo", "zoom-in", "zoom-out", "return"]) + let layout = CustomControlLayout.radialTemplate(name: "Test") + XCTAssertTrue(layout.assistActionIDs.contains("return")) + } + + func testFiveAssistActionsDoNotOverlap() { + let area = CGRect(x: 0, y: 0, width: 1180, height: 820) + let diameter: CGFloat = 52 + let points = TwoHandAssist.helperPoints(count: 5, clusterCentroid: CGPoint(x: 0.9, y: 0.9), area: area, + itemDiameter: diameter, spacing: 8) + XCTAssertEqual(points.count, 5) + for (i, a) in points.enumerated() { + XCTAssertTrue(area.insetBy(dx: diameter / 2 - 0.5, dy: diameter / 2 - 0.5).contains(a)) + for b in points[(i + 1)...] { + XCTAssertGreaterThanOrEqual(hypot(a.x - b.x, a.y - b.y), diameter - 0.5) + } + } + } +} diff --git a/MacTests/ReceiverControlsTests.swift b/MacTests/ReceiverControlsTests.swift index df29cdc1f..6ea107052 100644 --- a/MacTests/ReceiverControlsTests.swift +++ b/MacTests/ReceiverControlsTests.swift @@ -383,6 +383,49 @@ final class ReceiverControlsTests: XCTestCase { XCTAssertTrue(visible.contains(layout.paletteFrame)) } + /// Nine 40pt chips need 396pt: wider than a 390pt (or 375pt) iPhone in + /// portrait, and taller than it in landscape, so the end chips ran off + /// the screen and out of the tray's touch area. + func testTrayChipsShrinkJustEnoughToFitTheScreen() { + XCTAssertEqual(ControlTrayGeometry.fittedChipDiameter(count: 9, length: 440), 40) + for length: CGFloat in [402, 390, 375] { + let chip = ControlTrayGeometry.fittedChipDiameter(count: 9, length: length) + XCTAssertLessThan(chip, 40) + let tray = 9 * chip + 8 * 4 + 4 + XCTAssertLessThanOrEqual(tray, length - 24, "\(length)pt screen") + } + XCTAssertEqual(ControlTrayGeometry.fittedChipDiameter(count: 4, length: 390), 40) + XCTAssertEqual(ControlTrayGeometry.fittedChipDiameter(count: 20, length: 390), 30) + } + + /// Avoid Notch lifts the portrait tray off the home indicator; the + /// palette must move with it rather than stay where the tray used to + /// be (it covered half the tray on every Face ID iPhone). + func testAvoidNotchCarriesPortraitPaletteWithTheTray() { + let container = CGRect(x: 0, y: 0, width: 402, height: 874) + let layout = ControlTrayGeometry.layout( + container: container, + safeInsets: ControlSafeInsets(top: 62, leading: 0, bottom: 34, trailing: 0), + keyboardVisibleRect: nil, portrait: true, side: .trailing, + traySize: CGSize(width: 396, height: 44), paletteSize: CGSize(width: 300, height: 42), + avoidNotch: true) + XCTAssertLessThanOrEqual(layout.trayFrame.maxY, container.maxY - 34) + XCTAssertFalse(layout.trayFrame.intersects(layout.paletteFrame)) + XCTAssertEqual(layout.trayFrame.minY - layout.paletteFrame.maxY, 10, accuracy: 0.001) + } + + func testAvoidNotchCarriesLandscapePaletteWithTheTray() { + let layout = ControlTrayGeometry.layout( + container: CGRect(x: 0, y: 0, width: 874, height: 402), + safeInsets: ControlSafeInsets(top: 0, leading: 62, bottom: 21, trailing: 62), + keyboardVisibleRect: nil, portrait: false, side: .leading, + traySize: CGSize(width: 44, height: 300), paletteSize: CGSize(width: 42, height: 260), + avoidNotch: true, notchSide: .leading) + XCTAssertGreaterThanOrEqual(layout.trayFrame.minX, 62) + XCTAssertFalse(layout.trayFrame.intersects(layout.paletteFrame)) + XCTAssertEqual(layout.paletteFrame.minX - layout.trayFrame.maxX, 10, accuracy: 0.001) + } + func testLandscapeCornerFallsBackInsideKeyboardVisibleBounds() { let visible = CGRect(x: 0, y: 0, width: 700, height: 190) let layout = ControlTrayGeometry.layout( @@ -415,18 +458,262 @@ final class ReceiverControlsTests: XCTestCase { // MARK: - Function Tray - func testDefaultFunctionTrayContainsZoomAndEditActionsInOrder() { + func testDefaultFunctionTrayContainsZoomEditAndReturnActionsInOrder() { let profile = FunctionTrayProfile.canonical() + XCTAssertEqual(profile.visibleItems.map(\.id), ["zoom-in", "zoom-out", "undo", "redo", "return"]) + XCTAssertEqual(profile.visibleItems.map(\.title), ["Zoom In", "Zoom Out", "Undo", "Redo", "Return"]) + } + + /// Return gets its own cluster right after Undo/Redo: near the editing + /// actions, but with a gap so a reach for Redo doesn't submit. + func testDefaultFunctionTrayRendersReturnAsItsOwnGroupAfterEditing() { + let groups = FunctionTrayProfile.canonical().visibleGroups + XCTAssertEqual(groups.map { $0.map(\.id) }, [["zoom-in", "zoom-out"], ["undo", "redo"], ["return"]]) + } + + /// One activation is one ordinary Return press: usage 40, no modifiers + /// of its own, so a latched ⌘ or ⇧ (held on the Mac) still applies. + func testReturnIsOnePlainReturnPress() throws { + let item = try XCTUnwrap(FunctionTrayProfile.canonical().visibleItems.first { $0.id == "return" }) + XCTAssertEqual(ControlActionPlan.operations(for: item.action), [.press(usage: 40, modifiers: [])]) + XCTAssertTrue(item.action.isValid) + XCTAssertTrue(FunctionTrayProfile.isBuiltIn("return")) + } + + /// On the Mac, usage 40 is the Return key, and a modifier latched on the + /// receiver (held via modifierDown) joins the press the way + /// `InputInjector.handleKeyboardPress` combines them: ⌘↵, ⇧↵. + func testReturnPressPicksUpLatchedModifiersOnTheMac() throws { + let usage = try XCTUnwrap(HIDKeyUsage.parse(40)) + XCTAssertEqual(usage, .returnOrEnter) + XCTAssertEqual(usage.keyCode, 36) + var held = HeldModifierTracker() + XCTAssertEqual(KeyModifier.flags(named: []).union(held.flags), []) + _ = held.down(.command) + XCTAssertEqual(KeyModifier.flags(named: []).union(held.flags), .maskCommand) + _ = held.up(.command) + _ = held.down(.shift) + XCTAssertEqual(KeyModifier.flags(named: []).union(held.flags), .maskShift) + } + + /// A profile saved before Return existed keeps its tray exactly as it + /// was; Return arrives hidden, ready to switch on in the editor. + func testSavedProfileGainsReturnHiddenWithoutChangingItsLayout() { + var saved = FunctionTrayProfile.canonical() + saved.items.removeAll { $0.id == "return" } + saved.moveItems(from: IndexSet(integer: 3), to: 0) + saved.items[1].isVisible = false + let refreshed = saved.resolvingCanonicalMetadata() + XCTAssertEqual(Array(refreshed.items.prefix(saved.items.count)), saved.items) + let added = refreshed.items.filter { !saved.items.map(\.id).contains($0.id) } + XCTAssertEqual(added.map(\.id), ["return"]) + XCTAssertEqual(added.first?.isVisible, false) + XCTAssertEqual(refreshed.visibleItems.map(\.id), saved.visibleItems.map(\.id)) + } + + // MARK: Return upgrade from v1.20.0 + + /// One Function Tray profile's items as v1.20.0 (schema 18) stored them: + /// every slot started with this tray, and saving any receiver setting + /// wrote it out even when the tray itself was never touched. + private static let v1_20FunctionTrayItems = """ + [ + {"item": {"id": "zoom-in", "title": "Zoom In", "displayKey": "+", "systemImage": "plus.magnifyingglass", + "action": {"keyboardShortcut": {"_0": {"usage": 46, "modifiers": {"modifiers": ["command"]}}}}}, + "isVisible": true, "group": 0}, + {"item": {"id": "zoom-out", "title": "Zoom Out", "displayKey": "−", "systemImage": "minus.magnifyingglass", + "action": {"keyboardShortcut": {"_0": {"usage": 45, "modifiers": {"modifiers": ["command"]}}}}}, + "isVisible": true, "group": 0}, + {"item": {"id": "undo", "title": "Undo", "displayKey": "Z", "systemImage": "arrow.uturn.backward", + "action": {"keyboardShortcut": {"_0": {"usage": 29, "modifiers": {"modifiers": ["command"]}}}}}, + "isVisible": true, "group": 1}, + {"item": {"id": "redo", "title": "Redo", "displayKey": "Z", "systemImage": "arrow.uturn.forward", + "action": {"keyboardShortcut": {"_0": {"usage": 29, + "modifiers": {"modifiers": ["shift", "command"]}}}}}, + "isVisible": true, "group": 1}, + {"item": {"id": "menu-bar", "title": "Menu Bar", "displayKey": "Menu Bar", "systemImage": "menubar.rectangle", + "action": {"receiverGesture": {"_0": "menuBar"}}}, + "isVisible": false, "group": 2}, + {"item": {"id": "launchpad", "title": "Launchpad", "displayKey": "Launchpad", "systemImage": "square.grid.3x3", + "action": {"receiverGesture": {"_0": "launchpad"}}}, + "isVisible": false, "group": 2}, + {"item": {"id": "show-desktop", "title": "Show Desktop", "displayKey": "Show Desktop", + "systemImage": "rectangle.dashed", "action": {"receiverGesture": {"_0": "showDesktop"}}}, + "isVisible": false, "group": 2}, + {"item": {"id": "control-center", "title": "Control Center", "displayKey": "Control Center", + "systemImage": "switch.2", "action": {"receiverGesture": {"_0": "controlCenter"}}}, + "isVisible": false, "group": 2} + ] + """ + + private func v1_20FunctionTrayProfile(_ slot: ControlProfileSlot) throws -> FunctionTrayProfile { + let json = #"{"slot": "\#(slot.rawValue)", "items": \#(Self.v1_20FunctionTrayItems)}"# + return try JSONDecoder().decode(FunctionTrayProfile.self, from: Data(json.utf8)) + } + + /// A schema-18 save holding `trays` (v1.20.0's tray in every slot when + /// nil) with some unrelated receiver settings changed. + private func saveV1_20Preferences(_ defaults: UserDefaults, trays: [FunctionTrayProfile]? = nil) throws { + var old = ReceiverControlPreferences() + old.version = 18 + old.hapticsEnabled = false + old.inputMode = .trackpad + old.activeFunctionTrayProfile = .profile1 + old.functionTrayPosition = .sameSide + old.functionTrayProfiles = try trays ?? ControlProfileSlot.allCases.map { try v1_20FunctionTrayProfile($0) } + defaults.set(try JSONEncoder().encode(old), forKey: ReceiverControlPreferencesRepository.defaultsKey) + } + + private func migrationDefaults() throws -> (UserDefaults, ReceiverControlPreferencesRepository) { + let suite = "ReceiverControlsReturnMigrationTests.\(UUID().uuidString)" + let defaults = try XCTUnwrap(UserDefaults(suiteName: suite)) + addTeardownBlock { UserDefaults(suiteName: suite)?.removePersistentDomain(forName: suite) } + return (defaults, ReceiverControlPreferencesRepository(defaults: defaults)) + } + + /// The fixture is v1.20.0's starting tray: Zoom and Undo/Redo showing, + /// the four system actions hidden, and no Return. + func testV1_20FixtureIsThatReleasesDefaultTray() throws { + let profile = try v1_20FunctionTrayProfile(.default) + XCTAssertEqual(profile.items.map(\.id), ["zoom-in", "zoom-out", "undo", "redo", + "menu-bar", "launchpad", "show-desktop", "control-center"]) XCTAssertEqual(profile.visibleItems.map(\.id), ["zoom-in", "zoom-out", "undo", "redo"]) - XCTAssertEqual(profile.visibleItems.map(\.title), ["Zoom In", "Zoom Out", "Undo", "Redo"]) + XCTAssertEqual(profile.items.map(\.group), [0, 0, 1, 1, 2, 2, 2, 2]) + var today = FunctionTrayProfile.canonical() + today.items.removeAll { $0.id == "return" } + XCTAssertEqual(profile, today) + } + + func testFreshInstallShowsReturnInEverySlot() throws { + let (_, repository) = try migrationDefaults() + for initial in [ReceiverControlPreferences.initial(isPad: false), .initial(isPad: true)] { + let loaded = repository.load(initial: initial) + for slot in ControlProfileSlot.allCases { + let profile = loaded.functionTrayProfile(for: slot) + XCTAssertEqual(profile, .canonical(slot: slot), "\(slot)") + XCTAssertEqual(profile.visibleItems.map(\.id), ["zoom-in", "zoom-out", "undo", "redo", "return"]) + } + } } - func testDefaultFunctionTrayRendersAsTwoDistinctGroups() { - let profile = FunctionTrayProfile.canonical() - let groups = profile.visibleGroups - XCTAssertEqual(groups.count, 2) - XCTAssertEqual(groups[0].map(\.id), ["zoom-in", "zoom-out"]) - XCTAssertEqual(groups[1].map(\.id), ["undo", "redo"]) + /// An untouched v1.20.0 tray, saved only because other settings + /// changed, upgrades to today's default in all three slots, and those + /// other settings stay as they were. + func testUntouchedV1_20TraysUpgradeToShowReturnInEverySlot() throws { + let (defaults, repository) = try migrationDefaults() + try saveV1_20Preferences(defaults) + + let loaded = repository.load() + for slot in ControlProfileSlot.allCases { + XCTAssertEqual(loaded.functionTrayProfiles.first { $0.slot == slot }, .canonical(slot: slot), "\(slot)") + XCTAssertEqual(loaded.functionTrayProfile(for: slot).visibleGroups.map { $0.map(\.id) }, + [["zoom-in", "zoom-out"], ["undo", "redo"], ["return"]], "\(slot)") + } + XCTAssertFalse(loaded.hapticsEnabled) + XCTAssertEqual(loaded.inputMode, .trackpad) + XCTAssertEqual(loaded.activeFunctionTrayProfile, .profile1) + XCTAssertEqual(loaded.functionTrayPosition, .sameSide) + XCTAssertEqual(loaded.version, ReceiverControlPreferences.schemaVersion) + } + + /// Each kind of customization keeps the user's tray exactly and only + /// adds Return hidden, ready in the editor. Untouched slots beside it + /// still upgrade on their own. + func testCustomizedV1_20TraysKeepTheirLayoutAndGainReturnHidden() throws { + let custom = ShortcutItem(id: "custom-save", title: "Save", displayKey: "S", usage: 22, + modifiers: ModifierChord([.command])) + let customizations: [(String, (inout FunctionTrayProfile) -> Void)] = [ + ("order", { $0.moveItems(from: IndexSet(integer: 3), to: 0) }), + ("hidden", { $0.removeItem(id: "zoom-out") }), + ("shown", { $0.items[4].isVisible = true }), + ("group", { $0.items[1].group = 1 }), + ("custom action", { XCTAssertTrue($0.addCustomItem(custom)) }), + ("name", { $0.rename(id: "undo", to: "Back") }), + ("display", { $0.setDisplay(id: "redo", .emoji("🔁")) }), + ] + for (name, customize) in customizations { + let (defaults, repository) = try migrationDefaults() + var tray = try v1_20FunctionTrayProfile(.profile2) + customize(&tray) + try saveV1_20Preferences(defaults, trays: [try v1_20FunctionTrayProfile(.default), + try v1_20FunctionTrayProfile(.profile1), tray]) + + let loaded = repository.load() + let migrated = loaded.functionTrayProfile(for: .profile2) + XCTAssertEqual(Array(migrated.items.dropLast()), tray.items, name) + XCTAssertEqual(migrated.items.last?.id, "return", name) + XCTAssertEqual(migrated.items.last?.isVisible, false, name) + XCTAssertEqual(migrated.visibleItems, tray.visibleItems, name) + XCTAssertEqual(loaded.functionTrayProfile(for: .default), .canonical(slot: .default), name) + XCTAssertEqual(loaded.functionTrayProfile(for: .profile1), .canonical(slot: .profile1), name) + } + } + + /// A tray that already has Return keeps the user's choice for it. + func testSavedReturnChoiceIsKept() throws { + var hidden = FunctionTrayProfile.canonical(slot: .default) + hidden.removeItem(id: "return") + var moved = FunctionTrayProfile.canonical(slot: .profile1) + moved.moveItems(from: IndexSet(integer: 4), to: 0) + // A pre-release build appended it hidden to a v1.20.0 tray. + var appended = try v1_20FunctionTrayProfile(.profile2) + let returnItem = try XCTUnwrap(FunctionTrayProfile.canonical().items.first { $0.id == "return" }) + appended.items.append(FunctionTrayItemConfiguration(item: returnItem.item, isVisible: false, + group: returnItem.group)) + for profile in [hidden, moved, appended] { + XCTAssertEqual(profile.resolvingCanonicalMetadata(), profile, "\(profile.slot)") + } + let (defaults, repository) = try migrationDefaults() + try saveV1_20Preferences(defaults, trays: [hidden, moved, appended]) + let loaded = repository.load() + XCTAssertEqual(loaded.functionTrayProfiles, [hidden, moved, appended]) + } + + /// Upgraded once, a tray is the user's: hiding Return sticks across + /// later loads, and loading again without a save upgrades the same way. + func testReturnUpgradeIsIdempotentAndRespectsALaterHide() throws { + let (defaults, repository) = try migrationDefaults() + try saveV1_20Preferences(defaults) + let first = repository.load() + XCTAssertEqual(repository.load(), first) + repository.save(first) + XCTAssertEqual(repository.load(), first) + + var preferences = first + var tray = preferences.functionTrayProfile(for: .profile1) + tray.removeItem(id: "return") + preferences.updateFunctionTrayProfile(tray) + repository.save(preferences) + for _ in 0..<2 { + let reloaded = repository.load() + XCTAssertEqual(reloaded.functionTrayProfile(for: .profile1), tray) + XCTAssertFalse(reloaded.functionTrayProfile(for: .profile1).visibleItems.contains { $0.id == "return" }) + XCTAssertTrue(reloaded.functionTrayProfile(for: .default).visibleItems.contains { $0.id == "return" }) + repository.save(reloaded) + } + } + + /// While typing, Return sits in the keyboard bar beside Escape and Tab. + func testKeyboardBarOffersReturnBeforeHideKeyboard() { + let items = KeyboardBarItem.defaultItems + XCTAssertEqual(Array(items.suffix(4)), [.escape, .tab, .returnKey, .dismissKeyboard]) + XCTAssertEqual(items.filter { $0 == .returnKey }.count, 1) + } + + /// The eight-key bar (same metrics as `keyboardBarBody`) fits a 375pt + /// and 390pt iPhone inside the layout margins, shrinking only as far as + /// it must and never below the 30pt floor. + func testEightKeyKeyboardBarFitsCompactIPhones() { + let count = KeyboardBarItem.defaultItems.count + XCTAssertEqual(count, 8) + let gap: CGFloat = 8, padding: CGFloat = 6 + for (width, expected) in [(CGFloat(375), CGFloat(35)), (390, 37), (402, 38), (440, 38)] { + let item = ControlTrayGeometry.fittedChipDiameter(count: count, length: width, preferred: 38, + gap: gap, slack: padding * 2) + XCTAssertEqual(item, expected, "\(width)pt") + let barWidth = CGFloat(count) * item + CGFloat(count - 1) * gap + padding * 2 + XCTAssertLessThanOrEqual(barWidth, width - 24, "\(width)pt") + } } func testZoomInResolvesToCommandPlus() throws { diff --git a/MacTests/RemoteViewportTests.swift b/MacTests/RemoteViewportTests.swift index 20e1f02b8..7123fadce 100644 --- a/MacTests/RemoteViewportTests.swift +++ b/MacTests/RemoteViewportTests.swift @@ -720,3 +720,304 @@ final class RemoteViewportTests: XCTestCase { XCTAssertEqual(restored.current.scale, transformed.scale, accuracy: 0.0001) } } + +/// Local pinch-to-zoom up to 8×: geometry stays finite, the content under +/// the fingers stays put, touches still land on the right Mac point, and +/// pan, rotation and reset keep working at every magnification. +final class HighMagnificationViewportTests: XCTestCase { + /// The un-zoomed presentation of a 16:10 Mac on a portrait iPhone. + private let portraitBase = RemoteViewportCalculator.normal( + viewBounds: CGRect(x: 0, y: 0, width: 402, height: 874), remoteAspectSize: CGSize(width: 1728, height: 1080)) + private let scales: [CGFloat] = [1, 3, 4, 6, 8] + + private func zoomed(_ state: ManualViewportState, base: RemoteViewportTransform? = nil) -> RemoteViewportTransform { + RemoteViewportCalculator.applyManualZoom(to: base ?? portraitBase, state: state) + } + + private func assertFinite(_ rect: CGRect, file: StaticString = #filePath, line: UInt = #line) { + XCTAssertTrue([rect.minX, rect.minY, rect.width, rect.height].allSatisfy(\.isFinite), file: file, line: line) + XCTAssertGreaterThan(rect.width, 0, file: file, line: line) + XCTAssertGreaterThan(rect.height, 0, file: file, line: line) + } + + /// Whether `point` lies on the displayed (rotated) picture, edges + /// included: the clamp lets an edge travel exactly to the center, which + /// `CGRect.contains` (open at the max edges) would count as off. + private func isOnPicture(_ point: CGPoint, _ transform: RemoteViewportTransform) -> Bool { + let rect = transform.displayedRect + let dx = point.x - rect.midX + let dy = point.y - rect.midY + let c = cos(-transform.rotationRadians) + let s = sin(-transform.rotationRadians) + let tolerance: CGFloat = 1e-6 + return abs(dx * c - dy * s) <= rect.width / 2 + tolerance + && abs(dx * s + dy * c) <= rect.height / 2 + tolerance + } + + /// How far inside the picture's nearest edge the view's center sits for + /// `state` as given, without clamping it: negative means off the picture. + private func edgeGap(_ state: ManualViewportState, rotation: CGFloat, base: CGRect) -> CGFloat { + let c = cos(-rotation) + let s = sin(-rotation) + let localX = -state.panX * c + state.panY * s + let localY = -state.panX * s - state.panY * c + return min(base.width * state.scale / 2 - abs(localX), base.height * state.scale / 2 - abs(localY)) + } + + /// The clamp at any angle, not only quarter turns: the view's center + /// always ends up on the picture, a pan already on it is kept exactly, + /// a pan that was off it lands on the picture's edge (give or take the + /// 0.01pt rest snap), and clamping twice changes nothing. The grid + /// includes near-zero components, where the rest snap meets the edge. + func testClampKeepsTheCenterOnThePictureAtArbitraryAngles() { + let base = portraitBase.displayedRect + let center = CGPoint(x: base.midX, y: base.midY) + let fractions: [CGFloat] = [-1.5, -1.05, -0.7, -0.45, -0.2, 0, 0.2, 0.45, 0.7, 1.05, 1.5] + let degreesList = Array(stride(from: -180.0, through: 180.0, by: 7.5)) + [0.1, -1, 44.9, 89.9] + for degrees in degreesList { + let rotation = CGFloat(degrees) * .pi / 180 + for scale in scales { + let reach = base.width * scale + let offsets = fractions.map { $0 * reach } + [-0.004, 0.004] + for panX in offsets { + for panY in offsets { + let state = ManualViewportState(scale: scale, panX: panX, panY: panY, rotationRadians: rotation) + let clamped = state.clamped(against: base) + let transform = zoomed(clamped) + XCTAssertTrue(isOnPicture(center, transform), "\(degrees)° \(scale)× \(state)") + XCTAssertEqual(clamped.clamped(against: base), clamped, "\(degrees)° \(scale)×") + let before = edgeGap(state, rotation: clamped.rotationRadians, base: base) + if before >= 0, abs(state.panX) >= 0.01, abs(state.panY) >= 0.01 { + XCTAssertEqual(clamped.panX, state.panX, "\(degrees)° \(scale)×") + XCTAssertEqual(clamped.panY, state.panY, "\(degrees)° \(scale)×") + } else if before < 0 { + // Clamped onto the edge, not pulled further in. + let after = edgeGap(clamped, rotation: clamped.rotationRadians, base: base) + XCTAssertGreaterThanOrEqual(after, -1e-6, "\(degrees)° \(scale)× \(state)") + XCTAssertLessThanOrEqual(after, 0.01, "\(degrees)° \(scale)× \(state)") + } + } + } + } + } + } + + /// The pre-existing failure the rotated-frame clamp fixes: tilted 30°, + /// zoomed and dragged far along a diagonal, the bounding-box limits let + /// the whole picture leave the screen. + func testTiltedDiagonalExtremePanNoLongerLosesThePicture() { + let base = portraitBase.displayedRect + let center = CGPoint(x: base.midX, y: base.midY) + for scale in scales { + let state = ManualViewportState(scale: scale, panX: 1e6, panY: 1e6, rotationRadians: .pi / 6) + .clamped(against: base) + XCTAssertTrue(isOnPicture(center, zoomed(state)), "\(scale)×") + } + } + + func testMaximumManualZoomIsEightTimesAndClampsAbove() { + XCTAssertEqual(ManualViewportState.maxScale, 8) + let over = ManualViewportState(scale: 50).clamped(against: portraitBase.displayedRect) + XCTAssertEqual(over.scale, 8) + XCTAssertEqual(ManualViewportState(scale: 0.2).clamped(against: portraitBase.displayedRect).scale, 1) + XCTAssertEqual(ManualViewportState(scale: .nan).clamped(against: portraitBase.displayedRect).scale, 1) + XCTAssertEqual(ManualViewportState(scale: .infinity).clamped(against: portraitBase.displayedRect).scale, 1) + } + + /// The Separate Zoom While Typing cap is untouched. + func testZoomWhileTypingKeepsItsOwnLimit() { + XCTAssertEqual(RemoteViewportCalculator.defaultMaxZoom, 1.5) + } + + func testPinchKeepsTheContentUnderTheMidpointAtEveryScale() throws { + let base = portraitBase.displayedRect + let midpoint = CGPoint(x: base.minX + base.width * 0.73, y: base.minY + base.height * 0.31) + let before = try XCTUnwrap(portraitBase.remotePoint(forView: midpoint)) + for scale in scales { + let state = ManualViewportState.pinching(from: .identity, initialBase: base, initialMidpoint: midpoint, + currentMidpoint: midpoint, scaleRatio: scale) + XCTAssertEqual(state.scale, scale, accuracy: 1e-9) + let transform = zoomed(state) + assertFinite(transform.displayedRect) + XCTAssertEqual(transform.displayedRect.width, base.width * scale, accuracy: 1e-6) + let after = try XCTUnwrap(transform.remotePoint(forView: midpoint)) + XCTAssertEqual(after.x, before.x, accuracy: 1e-9, "\(scale)×") + XCTAssertEqual(after.y, before.y, accuracy: 1e-9, "\(scale)×") + } + } + + /// A slow two-finger spread from 1× to 8× and back moves smoothly: no + /// jumps, no NaN, and the anchored content never drifts. + func testContinuousPinchToEightAndBackIsSmoothAndAnchored() throws { + let base = portraitBase.displayedRect + let midpoint = CGPoint(x: base.midX + 40, y: base.midY - 25) + let anchor = try XCTUnwrap(portraitBase.remotePoint(forView: midpoint)) + var previous: CGFloat = 1 + let ratios = (0...200).map { step -> CGFloat in + let t = CGFloat(step) / 100 + return t <= 1 ? pow(8, t) : pow(8, 2 - t) + } + for ratio in ratios { + let state = ManualViewportState.pinching(from: .identity, initialBase: base, initialMidpoint: midpoint, + currentMidpoint: midpoint, scaleRatio: ratio) + XCTAssertLessThan(abs(state.scale - previous), 0.17, "jump at ratio \(ratio)") + previous = state.scale + let mapped = try XCTUnwrap(zoomed(state).remotePoint(forView: midpoint)) + XCTAssertEqual(mapped.x, anchor.x, accuracy: 1e-9) + XCTAssertEqual(mapped.y, anchor.y, accuracy: 1e-9) + } + XCTAssertEqual(previous, 1, accuracy: 1e-9) + } + + /// Zoomed further than 8× in one gesture, the pinch stops at 8× without + /// throwing the content away from the fingers. + func testPinchingPastTheCapStaysAtEightAndOnScreen() { + let base = portraitBase.displayedRect + let midpoint = CGPoint(x: base.midX, y: base.midY) + let state = ManualViewportState.pinching(from: ManualViewportState(scale: 6), initialBase: base, + initialMidpoint: midpoint, currentMidpoint: midpoint, scaleRatio: 40) + XCTAssertEqual(state.scale, 8) + XCTAssertTrue(zoomed(state).containsViewPoint(midpoint)) + } + + /// However far the user drags, an edge of the Mac can only reach the + /// middle of the view: the content can't be lost off screen. + func testExtremePanIsClampedSoTheMacStaysReachable() throws { + let base = portraitBase.displayedRect + let center = CGPoint(x: base.midX, y: base.midY) + for scale in scales { + for (dx, dy) in [(1e9, 1e9), (-1e9, 1e9), (1e9, -1e9), (-1e9, -1e9), (CGFloat.infinity, .nan)] { + let state = ManualViewportState(scale: scale, panX: dx, panY: dy).clamped(against: base) + XCTAssertTrue(state.panX.isFinite && state.panY.isFinite) + let transform = zoomed(state) + assertFinite(transform.displayedRect) + XCTAssertTrue(isOnPicture(center, transform), "\(scale)× pan (\(dx), \(dy))") + XCTAssertNotNil(transform.remotePoint(forView: center)) + } + } + } + + /// A pan that arrives through the gesture is bounded the same way. + func testExtremeDragAtEightTimesIsBounded() { + let base = portraitBase.displayedRect + let start = CGPoint(x: base.midX, y: base.midY) + let state = ManualViewportState.pinching(from: ManualViewportState(scale: 8), initialBase: base, + initialMidpoint: start, + currentMidpoint: CGPoint(x: start.x + 50_000, y: start.y - 50_000), + scaleRatio: 1) + let limits = try? XCTUnwrap(state.panLimits(against: base)) + XCTAssertEqual(abs(state.panX), limits?.x ?? -1, accuracy: 1e-6) + XCTAssertEqual(abs(state.panY), limits?.y ?? -1, accuracy: 1e-6) + XCTAssertTrue(isOnPicture(start, zoomed(state))) + } + + /// Reset returns to exactly the normal presentation from any zoom, and + /// the restore toggle brings the same 8× view back. + func testResetAndRestoreAtEveryScale() { + let base = portraitBase.displayedRect + for scale in scales.dropFirst() { + let zoomedState = ManualViewportState(scale: scale, panX: 30, panY: -12, rotationRadians: .pi / 7) + .clamped(against: base) + let reset = ViewportResetRestorePolicy.toggled(current: zoomedState, memory: nil, base: base) + XCTAssertEqual(reset?.current, .identity) + XCTAssertEqual(zoomed(reset?.current ?? zoomedState), portraitBase) + let restored = ViewportResetRestorePolicy.toggled(current: .identity, memory: reset?.memory, base: base) + XCTAssertEqual(restored?.current, zoomedState) + } + } + + /// Touch → Mac → view round trip at high zoom, with and without + /// rotation: a visible Mac control tapped at 8× receives the tap at + /// that exact point. + func testPointerMappingIsExactAtHighMagnificationAndRotation() throws { + let base = portraitBase.displayedRect + let view = CGRect(x: 0, y: 0, width: 402, height: 874) + for scale in scales { + for rotation: CGFloat in [0, .pi / 2, -.pi / 3, .pi] { + let state = ManualViewportState(scale: scale, panX: -base.width * 0.9, panY: base.height * 0.4, + rotationRadians: rotation).clamped(against: base) + let transform = zoomed(state) + for fx in stride(from: 0.05, through: 0.95, by: 0.15) { + for fy in stride(from: 0.05, through: 0.95, by: 0.15) { + let touch = CGPoint(x: view.width * fx, y: view.height * fy) + guard transform.containsViewPoint(touch) else { continue } + let mac = try XCTUnwrap(transform.remotePoint(forView: touch)) + let back = transform.viewPoint(forRemote: mac) + XCTAssertEqual(back.x, touch.x, accuracy: 1e-6, "\(scale)× \(rotation)") + XCTAssertEqual(back.y, touch.y, accuracy: 1e-6, "\(scale)× \(rotation)") + } + } + } + } + } + + /// At 8× one point of finger travel moves 1/8 as far across the Mac as + /// at 1×: magnification makes pointing finer, never coarser. + func testOnePointOfTravelIsFinerAtEightTimes() throws { + let base = portraitBase.displayedRect + let center = CGPoint(x: base.midX, y: base.midY) + func step(at scale: CGFloat) throws -> CGFloat { + let transform = zoomed(ManualViewportState(scale: scale)) + let a = try XCTUnwrap(transform.remotePoint(forView: center)) + let b = try XCTUnwrap(transform.remotePoint(forView: CGPoint(x: center.x + 1, y: center.y))) + return b.x - a.x + } + XCTAssertEqual(try step(at: 8), try step(at: 1) / 8, accuracy: 1e-12) + } + + /// Rotation stays available at 8× and snaps exactly like at 1×. + func testRotationAtEightTimesKeepsScaleAndAnchor() throws { + let base = portraitBase.displayedRect + let midpoint = CGPoint(x: base.midX - 30, y: base.midY + 10) + let start = ManualViewportState.pinching(from: .identity, initialBase: base, initialMidpoint: midpoint, + currentMidpoint: midpoint, scaleRatio: 8) + let anchor = try XCTUnwrap(zoomed(start).remotePoint(forView: midpoint)) + let rotated = ManualViewportState.pinching(from: start, initialBase: base, initialMidpoint: midpoint, + currentMidpoint: midpoint, scaleRatio: 1, + rotationDelta: .pi / 2) + XCTAssertEqual(rotated.scale, 8, accuracy: 1e-9) + XCTAssertEqual(rotated.rotationRadians, .pi / 2, accuracy: 1e-9) + let mapped = try XCTUnwrap(zoomed(rotated).remotePoint(forView: midpoint)) + XCTAssertEqual(mapped.x, anchor.x, accuracy: 1e-9) + XCTAssertEqual(mapped.y, anchor.y, accuracy: 1e-9) + } + + /// Rotating the device, resizing an iPad window or opening the keyboard + /// changes the base. An 8× state re-clamps against the new base into + /// valid, finite geometry that still covers the view's center. + func testEightTimesSurvivesOrientationWindowAndKeyboardChanges() { + let mac = CGSize(width: 1728, height: 1080) + let views = [CGRect(x: 0, y: 0, width: 402, height: 874), CGRect(x: 0, y: 0, width: 874, height: 402), + CGRect(x: 0, y: 0, width: 1032, height: 1376), CGRect(x: 0, y: 0, width: 507, height: 688), + CGRect(x: 0, y: 0, width: 320, height: 240)] + let state = ManualViewportState(scale: 8, panX: 5_000, panY: -5_000, rotationRadians: .pi / 4) + for view in views { + let normal = RemoteViewportCalculator.normal(viewBounds: view, remoteAspectSize: mac) + let typing = RemoteViewportCalculator.keyboardOpen( + viewBounds: view, remoteAspectSize: mac, + visibleRect: CGRect(x: 0, y: 0, width: view.width, height: view.height * 0.55), + anchor: CGPoint(x: 0.5, y: 0.8), zoomEnabled: true) + for base in [normal, typing] where base.isValid { + let clamped = state.clamped(against: base.displayedRect) + XCTAssertEqual(clamped.scale, 8) + let transform = RemoteViewportCalculator.applyManualZoom(to: base, state: clamped) + assertFinite(transform.displayedRect) + let center = CGPoint(x: base.displayedRect.midX, y: base.displayedRect.midY) + XCTAssertTrue(isOnPicture(center, transform), "\(view.size)") + } + } + } + + /// Unusable geometry never yields NaN at any scale. + func testDegenerateGeometryNeverProducesNaN() { + let degenerate = RemoteViewportTransform(remoteCrop: CGRect(x: 0, y: 0, width: 1, height: 1), displayedRect: .zero) + for scale in scales { + let transform = RemoteViewportCalculator.applyManualZoom(to: degenerate, state: ManualViewportState(scale: scale)) + XCTAssertNil(transform.remotePoint(forView: .zero)) + let state = ManualViewportState.pinching(from: ManualViewportState(scale: scale), initialBase: .zero, + initialMidpoint: .zero, currentMidpoint: CGPoint(x: 10, y: 10), + scaleRatio: .nan) + XCTAssertEqual(state.scale, scale) + } + } +} diff --git a/MacTests/ThirdQARefinementTests.swift b/MacTests/ThirdQARefinementTests.swift index 184cde6e6..d29560de0 100644 --- a/MacTests/ThirdQARefinementTests.swift +++ b/MacTests/ThirdQARefinementTests.swift @@ -102,7 +102,7 @@ final class ThirdQARefinementTests: XCTestCase { XCTAssertEqual(ReceiverControlPresentation.mode(softwareKeyboardVisible: true), .keyboardBar) XCTAssertEqual(ReceiverControlPresentation.mode(softwareKeyboardVisible: false), .normal) XCTAssertEqual(KeyboardBarItem.defaultItems, [.modifier(.command), .modifier(.option), .modifier(.control), - .modifier(.shift), .escape, .tab, .dismissKeyboard]) + .modifier(.shift), .escape, .tab, .returnKey, .dismissKeyboard]) } func testNormalControlsReturnUnchangedAfterTheKeyboard() { diff --git a/MacTests/ViewportPanFreedomTests.swift b/MacTests/ViewportPanFreedomTests.swift index f905666a0..00ab7fa70 100644 --- a/MacTests/ViewportPanFreedomTests.swift +++ b/MacTests/ViewportPanFreedomTests.swift @@ -81,10 +81,18 @@ final class ViewportPanFreedomTests: XCTestCase { XCTAssertEqual(corners.map(\.x).min() ?? 0, center.x, accuracy: 0.01) XCTAssertEqual(corners.map(\.y).min() ?? 0, center.y, accuracy: 0.01) + // A tilted picture's own right edge stops at the center: the center + // stays on the picture (the rotated bounding box let it slide off). let tilted = ManualViewportState(scale: 1.5, panX: -10_000, rotationRadians: .pi / 6).clamped(against: base) - let tiltedCorners = [CGPoint(x: 0, y: 0), CGPoint(x: 1, y: 0), CGPoint(x: 0, y: 1), CGPoint(x: 1, y: 1)] - .map(displayed(tilted).viewPoint(forRemote:)) - XCTAssertEqual(tiltedCorners.map(\.x).max() ?? 0, center.x, accuracy: 0.01) + let onPicture = try XCTUnwrap(displayed(tilted).remotePoint(forView: center)) + XCTAssertEqual(onPicture.x, 1, accuracy: 0.0001) + let diagonal = ManualViewportState(scale: 3, panX: 10_000, panY: 10_000, rotationRadians: .pi / 4) + .clamped(against: base) + let unrotated = displayed(diagonal) + let local = CGPoint(x: center.x - unrotated.displayedRect.midX, y: center.y - unrotated.displayedRect.midY) + let c = cos(-diagonal.rotationRadians), s = sin(-diagonal.rotationRadians) + XCTAssertLessThanOrEqual(abs(local.x * c - local.y * s), unrotated.displayedRect.width / 2 + 0.001) + XCTAssertLessThanOrEqual(abs(local.x * s + local.y * c), unrotated.displayedRect.height / 2 + 0.001) } func testInvalidGeometryResetsToIdentity() { diff --git a/Shared/ControlProfile.swift b/Shared/ControlProfile.swift index 4e4c935f6..765f206a2 100644 --- a/Shared/ControlProfile.swift +++ b/Shared/ControlProfile.swift @@ -630,7 +630,12 @@ struct FunctionTrayProfile: Codable, Equatable, Identifiable { /// Visibility, ordering, and grouping remain entirely user-owned. A /// canonical item a saved profile predates is appended HIDDEN, so the /// editor can offer it without changing the tray the user already has. + /// + /// A tray still exactly as an earlier release started it was never + /// customized — saving any other receiver setting writes the trays too — + /// so it takes today's default instead, new visible actions included. func resolvingCanonicalMetadata() -> FunctionTrayProfile { + if isEarlierReleaseDefault { return Self.canonical(slot: slot) } let canonical = Self.canonical(slot: slot).items let canonicalByID = Dictionary(uniqueKeysWithValues: canonical.map { ($0.id, $0.item) }) let savedIDs = Set(items.map(\.id)) @@ -647,6 +652,38 @@ struct FunctionTrayProfile: Codable, Equatable, Identifiable { } return FunctionTrayProfile(slot: slot, items: resolved + added) } + + /// Every slot's starting tray in an earlier release, by the user-owned + /// fields only: action ID, visibility and group, in order. Spelled out + /// rather than derived from `canonical`, which keeps moving. + private static let earlierReleaseDefaults: [[EarlierDefaultEntry]] = [ + // v1.20.0, before Return. + [EarlierDefaultEntry("zoom-in", true, 0), EarlierDefaultEntry("zoom-out", true, 0), + EarlierDefaultEntry("undo", true, 1), EarlierDefaultEntry("redo", true, 1), + EarlierDefaultEntry("menu-bar", false, 2), EarlierDefaultEntry("launchpad", false, 2), + EarlierDefaultEntry("show-desktop", false, 2), EarlierDefaultEntry("control-center", false, 2)], + ] + + private struct EarlierDefaultEntry: Equatable { + let id: String + let isVisible: Bool + let group: Int + + init(_ id: String, _ isVisible: Bool, _ group: Int) { + self.id = id + self.isVisible = isVisible + self.group = group + } + } + + /// Unchanged since an earlier release created it: same actions, order, + /// visibility and groups, and no built-in renamed or restyled. A custom + /// action has its own ID, so a tray with one never matches. + private var isEarlierReleaseDefault: Bool { + let entries = items.map { EarlierDefaultEntry($0.id, $0.isVisible, $0.group) } + return Self.earlierReleaseDefaults.contains(entries) + && items.allSatisfy { $0.customTitle == nil && $0.customDisplay == nil } + } } extension FunctionTrayProfile { @@ -715,6 +752,9 @@ extension FunctionTrayProfile { let zoomGroup = 0 let editGroup = 1 let systemGroup = 2 + // Return sits right after the edit cluster (select, paste, then + // submit) but in its own cluster, so a reach for Redo can't submit. + let returnGroup = 3 let items: [(ShortcutItem, Int)] = [ (ShortcutItem(id: "zoom-in", title: "Zoom In", displayKey: "+", usage: 46, modifiers: command, systemImage: "plus.magnifyingglass"), zoomGroup), @@ -724,6 +764,10 @@ extension FunctionTrayProfile { systemImage: "arrow.uturn.backward"), editGroup), (ShortcutItem(id: "redo", title: "Redo", displayKey: "Z", usage: 29, modifiers: commandShift, systemImage: "arrow.uturn.forward"), editGroup), + // HID Return (40) with no modifiers of its own: a latched ⌘ or ⇧ + // held on the Mac still applies (⌘↵, ⇧↵). + (ShortcutItem(id: "return", title: "Return", displayKey: "↵", usage: 40, modifiers: ModifierChord(), + systemImage: "return"), returnGroup), ] // Opt-in (hidden by default): the only routes to Launchpad and Show // Desktop besides the 4/5-finger pinch/spread, which can collide @@ -1080,11 +1124,13 @@ struct ReceiverControlPreferencesRepository { init(defaults: UserDefaults = .standard) { self.defaults = defaults } - func load() -> ReceiverControlPreferences { + /// `initial` is what a device starts with while nothing usable is saved + /// (see `ReceiverControlPreferences.initial(isPad:)`). + func load(initial: ReceiverControlPreferences = ReceiverControlPreferences()) -> ReceiverControlPreferences { guard let data = defaults.data(forKey: Self.defaultsKey), var value = try? JSONDecoder().decode(ReceiverControlPreferences.self, from: data), value.version <= ReceiverControlPreferences.schemaVersion else { - return ReceiverControlPreferences() + return initial } if value.version < 2 { for index in value.profiles.indices @@ -1214,7 +1260,8 @@ struct ReceiverControlPreferencesRepository { } // Old Function Tray profiles predate `ShortcutItem.systemImage`. // Resolve current canonical metadata by ID without rewriting the - // user's visibility, order, or group choices. + // user's visibility, order, or group choices; a tray still at an + // earlier release's default takes today's (v1.20.0 → Return shown). value.functionTrayProfiles = value.functionTrayProfiles.map { $0.resolvingCanonicalMetadata() } diff --git a/Shared/ControlTrayGeometry.swift b/Shared/ControlTrayGeometry.swift index 1be8c59d6..5a5a2d288 100644 --- a/Shared/ControlTrayGeometry.swift +++ b/Shared/ControlTrayGeometry.swift @@ -388,8 +388,14 @@ enum ControlTrayGeometry { // Obstacle avoidance is deliberately after normal placement. A // non-intersecting frame must remain bit-for-bit unchanged. - trayFrame = avoidingUnsafeRegion(trayFrame, in: container, safeInsets: safeInsets, enabled: avoidNotch, - portrait: portrait, notchSide: notchSide) + let unsafeTrayFrame = avoidingUnsafeRegion(trayFrame, in: container, safeInsets: safeInsets, + enabled: avoidNotch, portrait: portrait, notchSide: notchSide) + // The palette belongs to the tray: when the tray steps off the home + // indicator or notch, the palette moves with it instead of staying + // where the tray used to be. + paletteFrame = paletteFrame.offsetBy(dx: unsafeTrayFrame.minX - trayFrame.minX, + dy: unsafeTrayFrame.minY - trayFrame.minY) + trayFrame = unsafeTrayFrame paletteFrame = avoidingUnsafeRegion(paletteFrame, in: container, safeInsets: safeInsets, enabled: avoidNotch, portrait: portrait, notchSide: notchSide) let obstacles = reservedObstacles(reservedRegions, avoidNotch: avoidNotch) @@ -692,3 +698,17 @@ extension ControlTrayGeometry { } } } + +extension ControlTrayGeometry { + /// The iPhone Main Tray's chip diameter along a screen edge `length` + /// points long: the standard size, shrunk just enough that every chip + /// and the layout margins fit (nine 40pt chips don't fit a 390pt + /// iPhone), never below `minimum`. + static func fittedChipDiameter(count: Int, length: CGFloat, preferred: CGFloat = 40, + gap: CGFloat = 4, slack: CGFloat = 4, margin: CGFloat = 12, + minimum: CGFloat = 30) -> CGFloat { + guard count > 0 else { return preferred } + let room = length - margin * 2 - slack - CGFloat(count - 1) * gap + return max(minimum, min(preferred, (room / CGFloat(count)).rounded(.down))) + } +} diff --git a/Shared/CustomControlLayout.swift b/Shared/CustomControlLayout.swift index e2ce4de56..1cd526d45 100644 --- a/Shared/CustomControlLayout.swift +++ b/Shared/CustomControlLayout.swift @@ -234,7 +234,7 @@ struct CustomControlArrangement: Codable, Equatable { struct CustomControlLayout: Codable, Equatable, Identifiable { static let maximumCount = 5 - static let defaultAssistActionIDs = ["undo", "redo", "zoom-in", "zoom-out"] + static let defaultAssistActionIDs = ["undo", "redo", "zoom-in", "zoom-out", "return"] var id: String var name: String @@ -545,6 +545,18 @@ struct CornerCluster: Equatable { extension ReceiverControlPreferences { var canCreateCustomLayout: Bool { customLayouts.count < CustomControlLayout.maximumCount } + /// What a device starts with before anything is saved. iPad opens on + /// the radial Custom layout around a bottom-right Settings anchor, with + /// Strip and Overlay one setting away; iPhone keeps the standard + /// defaults. Saved preferences are never replaced by this. + static func initial(isPad: Bool) -> ReceiverControlPreferences { + var preferences = ReceiverControlPreferences() + guard isPad else { return preferences } + preferences.createCustomLayout(corner: .bottomTrailing) + preferences.padControlLayout = .custom + return preferences + } + /// The selected Custom layout, falling back to the first one. var activeCustomLayout: CustomControlLayout? { customLayouts.first { $0.id == activeCustomLayoutID } ?? customLayouts.first diff --git a/Shared/ReceiverInputPrompt.swift b/Shared/ReceiverInputPrompt.swift index da9c746b6..afaf11537 100644 --- a/Shared/ReceiverInputPrompt.swift +++ b/Shared/ReceiverInputPrompt.swift @@ -170,9 +170,11 @@ enum KeyboardBarItem: Hashable { case modifier(ControlModifier) case escape case tab + /// One Return press, beside Escape and Tab for ⌘↵ and ⇧↵. + case returnKey /// Dismisses the software keyboard. case dismissKeyboard static let defaultItems: [KeyboardBarItem] = ControlModifier.allCases.map(KeyboardBarItem.modifier) - + [.escape, .tab, .dismissKeyboard] + + [.escape, .tab, .returnKey, .dismissKeyboard] } diff --git a/Shared/RemoteViewport.swift b/Shared/RemoteViewport.swift index f935bef80..304572c3b 100644 --- a/Shared/RemoteViewport.swift +++ b/Shared/RemoteViewport.swift @@ -434,9 +434,15 @@ struct ManualViewportState: Equatable { /// Manual zoom can never go below 1x (no such thing as "zoomed out" /// of the normal fit) — see `clamped(against:)`. static let minScale: CGFloat = 1 - /// Initial cap on manual magnification (PRODUCT RULE: "approximately - /// 3x unless architecture suggests a better bound" — nothing here does). - static let maxScale: CGFloat = 3 + /// Cap on manual magnification. Zoom is a transform on the content layer + /// (see `VideoView.applyContentGeometry`): the stream resolution, the + /// decoded frame and the Metal drawable are the same at any scale, so + /// zooming adds no network, decode or render-pass work. How iOS + /// composites a scaled or rotated video layer is up to the system and + /// can only be measured on hardware (ux-review/ios-control-surface- + /// performance.md). Unrelated to Zoom While Typing's + /// `RemoteViewportCalculator.defaultMaxZoom`. + static let maxScale: CGFloat = 8 var scale: CGFloat = minScale /// Pan offset, in the host view's local points, applied to the scaled @@ -449,11 +455,16 @@ struct ManualViewportState: Equatable { var isIdentity: Bool { self == .identity } - /// Clamps `scale` to `[minScale, maxScale]` and `pan{X,Y}` to - /// `panLimits(against:)`: translation is independent of zoom (100% can - /// pan too), but any edge of the remote display may travel at most to - /// the viewport's center, never past it — so the Mac can be brought - /// within reach from any direction yet never flung off screen. The + /// Clamps `scale` to `[minScale, maxScale]` and `pan{X,Y}` so that + /// translation is independent of zoom (100% can pan too), but any edge + /// of the remote display may travel at most to the viewport's center, + /// never past it — so the Mac can be brought within reach from any + /// direction yet never flung off screen. The rule holds in the + /// picture's own (rotated) frame: the viewport's center always stays on + /// the picture. Clamping each screen axis to the rotated bounding box + /// instead let a tilted, diagonally panned picture leave the screen + /// entirely, and that gap grows with zoom. At 0°, 90° and 180° the two + /// are identical. The /// viewport's center is `base`'s center: the normal presentation is /// centered in the surface, and the keyboard-open one in the area above /// the keyboard. The single source of truth for this math — both @@ -466,18 +477,52 @@ struct ManualViewportState: Equatable { var copy = self copy.scale = scale.isFinite ? min(max(scale, Self.minScale), Self.maxScale) : Self.minScale copy.rotationRadians = rotationRadians.isFinite ? Self.normalizedAngle(rotationRadians) : 0 - guard let limits = copy.panLimits(against: base) else { + guard copy.panLimits(against: base) != nil else { copy.panX = 0 copy.panY = 0 copy.rotationRadians = 0 return copy } - copy.panX = panX.isFinite ? min(max(panX, -limits.x), limits.x) : 0 - copy.panY = panY.isFinite ? min(max(panY, -limits.y), limits.y) : 0 + let halfWidth = base.width * copy.scale / 2 + let halfHeight = base.height * copy.scale / 2 + let c = cos(copy.rotationRadians) + let s = sin(copy.rotationRadians) + let x = panX.isFinite ? panX : 0 + let y = panY.isFinite ? panY : 0 + // The viewport's center relative to the picture's center, in the + // picture's unrotated frame... + func local(_ x: CGFloat, _ y: CGFloat) -> (x: CGFloat, y: CGFloat) { + (-(x * c + y * s), -(-x * s + y * c)) + } + // ...with a millionth of a point of slack: a pan this clamp itself + // put on the edge maps back a rounding error outside it. + func isOnPicture(_ x: CGFloat, _ y: CGFloat) -> Bool { + let point = local(x, y) + return abs(point.x) <= halfWidth + 1e-6 && abs(point.y) <= halfHeight + 1e-6 + } + var resultX = x + var resultY = y + // Already on the picture: the pan is kept exactly, so re-clamping + // (restore, relayout) never nudges it by rounding. Otherwise the + // nearest point of the picture, turned back into a screen-space pan. + if !isOnPicture(x, y) { + let unclamped = local(x, y) + let localX = min(max(unclamped.x, -halfWidth), halfWidth) + let localY = min(max(unclamped.y, -halfHeight), halfHeight) + resultX = -(localX * c - localY * s) + resultY = -(localX * s + localY * c) + } // Floating-point dust from a pinch that ended back at rest must not - // leave a technically-non-identity state behind. - if abs(copy.panX) < 0.01 { copy.panX = 0 } - if abs(copy.panY) < 0.01 { copy.panY = 0 } + // leave a technically-non-identity state behind — unless, tilted, + // dropping it would step the center off the picture's edge. + let snappedX = abs(resultX) < 0.01 ? 0 : resultX + let snappedY = abs(resultY) < 0.01 ? 0 : resultY + if isOnPicture(snappedX, snappedY) { + resultX = snappedX + resultY = snappedY + } + copy.panX = resultX + copy.panY = resultY return copy } diff --git a/iOS/DemoDesktop.swift b/iOS/DemoDesktop.swift index 74884b7d0..a13731fcc 100644 --- a/iOS/DemoDesktop.swift +++ b/iOS/DemoDesktop.swift @@ -478,6 +478,17 @@ enum DemoDisplayGeometry { } } + /// The part of the receiver surface above a docked software keyboard, + /// in the surface's own coordinates: what a live session hands the + /// control overlay (see `VideoView`'s `keyboardWillChangeFrame`). `nil` + /// while the keyboard is hidden or floating. + static func keyboardVisibleRect(keyboardFrame: CGRect?, canvas: CGRect, screenBounds: CGRect) -> CGRect? { + guard let keyboardFrame, keyboardFrame.minY < screenBounds.maxY - 1, + keyboardFrame.maxY >= screenBounds.maxY - 1 else { return nil } + return CGRect(x: 0, y: 0, width: canvas.width, + height: min(canvas.height, max(0, keyboardFrame.minY - canvas.minY))) + } + /// The point size a real Mac would build this Extend display at, from /// the same `ExtendDisplaySizing` the Mac uses. static func extendPointSize(shape: ExtendDisplayShapePreference, receiverPixels: CGSize) -> (wide: Int, high: Int) { diff --git a/iOS/DemoMode.swift b/iOS/DemoMode.swift index 3e07fa92c..8886f51c7 100644 --- a/iOS/DemoMode.swift +++ b/iOS/DemoMode.swift @@ -79,7 +79,8 @@ struct ExploreDemoCard: View { .buttonStyle(.plain) // As wide as the status card on iPad; the phone column's width elsewhere. .frame(maxWidth: horizontalSizeClass == .regular ? .infinity : 420) - .accessibilityElement(children: .ignore) + // Label the button itself: wrapping it in its own element hid the + // button trait (and the button from UI tests) behind a plain group. .accessibilityLabel(Text("Explore Demo")) .accessibilityHint(Text("Opens a sample session. Nothing connects to another device.")) .accessibilityIdentifier("exploreDemo") diff --git a/iOS/DemoSurface.swift b/iOS/DemoSurface.swift index cf1bd9a7d..15ea15600 100644 --- a/iOS/DemoSurface.swift +++ b/iOS/DemoSurface.swift @@ -97,7 +97,8 @@ struct DemoReceiverScreen: View { keyboardActive: $keyboardActive, showSettings: $showSettings, keyboardAvailable: demo.inputAllowed, - keyboardVisibleRect: keyboardRect, + keyboardVisibleRect: DemoDisplayGeometry.keyboardVisibleRect( + keyboardFrame: keyboardRect, canvas: canvas, screenBounds: UIScreen.main.bounds), containerSize: geo.size, safeInsets: insets, notchSide: nil, @@ -144,7 +145,7 @@ struct DemoReceiverScreen: View { } .onReceive(NotificationCenter.default.publisher(for: UIResponder.keyboardWillChangeFrameNotification)) { note in guard let frame = note.userInfo?[UIResponder.keyboardFrameEndUserInfoKey] as? CGRect else { return } - keyboardRect = frame.minY >= UIScreen.main.bounds.maxY - 1 ? nil : frame + keyboardRect = frame } .onReceive(NotificationCenter.default.publisher(for: UIResponder.keyboardWillHideNotification)) { _ in keyboardRect = nil @@ -272,6 +273,9 @@ struct DemoReceiverScreen: View { onResetView: { withAnimation(.easeOut(duration: 0.25)) { workspace.interaction.resetViewport() } }) + // The banner is one fixed-height row over the desktop; past + // this size its text overflowed and covered Exit Demo. + .dynamicTypeSize(...DynamicTypeSize.xxLarge) if let hint { DemoHintChip(hint: hint) .transition(.opacity.combined(with: .move(edge: .top))) @@ -280,6 +284,8 @@ struct DemoReceiverScreen: View { .transition(.opacity) } } + // Chips over the picture keep scaling, but stop before they cover it. + .dynamicTypeSize(...DynamicTypeSize.xxxLarge) .frame(width: canvas.width) .padding(.top, Self.bandTop(canvas: canvas, insets: insets)) .offset(x: canvas.minX) diff --git a/iOS/ReceiverControls.swift b/iOS/ReceiverControls.swift index ead48fe33..4b138acea 100644 --- a/iOS/ReceiverControls.swift +++ b/iOS/ReceiverControls.swift @@ -57,7 +57,7 @@ final class ReceiverControlStore: ObservableObject { isPad: Bool = UIDevice.current.userInterfaceIdiom == .pad) { self.repository = repository self.isPad = isPad - preferences = repository.load() + preferences = repository.load(initial: .initial(isPad: isPad)) } /// The iPad layout mode actually in effect: Custom needs a layout to @@ -323,6 +323,7 @@ struct ReceiverControlOverlay: View { @State private var interaction = ControlInteractionState() @State private var initiatingMoveView = false + @State private var initiatingReturn = false /// The Wheel palette on screen, for hit-testing its wedges. @State private var wheelHit: WheelHitTarget? /// The modifier whose placement a Custom-layout palette blooms around. @@ -507,7 +508,12 @@ struct ReceiverControlOverlay: View { let portrait = containerSize.height > containerSize.width let collapsed = store.preferences.trayCollapsed || !store.preferences.trayCanBeShown let count = max(controls.count, 1) - let traySize = calculatedTraySize(collapsed: collapsed, portrait: portrait, count: count) + // Shrinks only when the chips wouldn't fit the screen edge (nine + // standard chips are wider than a 390pt iPhone). + let trayItem = ControlTrayGeometry.fittedChipDiameter( + count: count, length: portrait ? containerSize.width : containerSize.height, + preferred: Metrics.trayItem, gap: Metrics.trayGap, slack: Metrics.touchSlack) + let traySize = calculatedTraySize(collapsed: collapsed, portrait: portrait, count: count, item: trayItem) // Auto-hide fades the (always-mounted, see `controls`/`functionGroups`) // tray content out in place rather than removing it — `collapsed` // above is untouched by it, since `store.autoHidden` can only ever @@ -626,9 +632,10 @@ struct ReceiverControlOverlay: View { .background(frameReader("tray:more")) .background(chip(selected: false)) .accessibilityLabel(ControlTrayItem.settings.title) + .trayButtonAccessibility { activateForAccessibility(.settings) } .position(x: layout.trayFrame.midX, y: layout.trayFrame.midY) } else { - controlsRow(axis: layout.axis == .horizontal ? .horizontal : .vertical) + controlsRow(axis: layout.axis == .horizontal ? .horizontal : .vertical, item: trayItem) .frame(width: layout.trayFrame.width, height: layout.trayFrame.height) .position(x: layout.trayFrame.midX, y: layout.trayFrame.midY) .opacity(autoHideOpacity) @@ -752,13 +759,13 @@ struct ReceiverControlOverlay: View { } #endif - private func calculatedTraySize(collapsed: Bool, portrait: Bool, count: Int) -> CGSize { - let thickness = Metrics.trayItem + Metrics.touchSlack + private func calculatedTraySize(collapsed: Bool, portrait: Bool, count: Int, item: CGFloat) -> CGSize { + let thickness = item + Metrics.touchSlack guard !collapsed else { return CGSize(width: Metrics.collapsed + Metrics.touchSlack, height: Metrics.collapsed + Metrics.touchSlack) } - let length = CGFloat(count) * Metrics.trayItem + let length = CGFloat(count) * item + CGFloat(Swift.max(0, count - 1)) * Metrics.trayGap + Metrics.touchSlack return portrait ? CGSize(width: length, height: thickness) @@ -804,16 +811,16 @@ struct ReceiverControlOverlay: View { } @ViewBuilder - private func controlsRow(axis: Axis) -> some View { + private func controlsRow(axis: Axis, item diameter: CGFloat) -> some View { let stack = axis == .horizontal ? AnyLayout(HStackLayout(spacing: Metrics.trayGap)) : AnyLayout(VStackLayout(spacing: Metrics.trayGap)) stack { ForEach(controls) { item in if let modifier = item.modifier { - modifierButton(modifier) + modifierButton(modifier, diameter: diameter) } else { - actionButton(item) + actionButton(item, diameter: diameter) } } } @@ -844,6 +851,17 @@ struct ReceiverControlOverlay: View { .contentShape(Circle()) .onTapGesture { performFunctionAction(item) } .accessibilityLabel(item.title) + .trayButtonAccessibility { performFunctionAction(item) } + } + + /// One plain Return press through the normal keyboard path. Modifiers + /// latched on the tray are held on the Mac, so they still apply. Gated + /// like the Function Tray's one-tap actions; the Mac enforces the grant + /// again on its side. + fileprivate func sendReturn() { + guard store.preferences.allowInput else { return } + receiver.sendKeyboardPress(usage: 40) + haptics.play(.confirmation) } private func performFunctionAction(_ item: ShortcutItem) { @@ -852,33 +870,36 @@ struct ReceiverControlOverlay: View { store.scheduleAutoHide() } - private func modifierButton(_ modifier: ControlModifier) -> some View { + private func modifierButton(_ modifier: ControlModifier, diameter: CGFloat) -> some View { let selected = interaction.latchedModifiers.contains(modifier) || interaction.temporaryModifiers.contains(modifier) return Text(modifier.symbol) - .font(.system(size: 21, weight: .semibold)) + .font(.system(size: (21 * diameter / Metrics.trayItem).rounded(), weight: .semibold)) .foregroundStyle(selected ? Color.black : Color.white) - .frame(width: Metrics.trayItem, height: Metrics.trayItem) + .frame(width: diameter, height: diameter) .background(frameReader("modifier:\(modifier.rawValue)")) .background(chip(selected: selected)) .accessibilityLabel(modifier.title) .accessibilityValue(selected ? "On" : "Off") + .trayButtonAccessibility { activateForAccessibility(ControlTrayItem.item(for: modifier)) } } - private func actionButton(_ item: ControlTrayItem) -> some View { - Group { + private func actionButton(_ item: ControlTrayItem, diameter: CGFloat) -> some View { + let scale = diameter / Metrics.trayItem + return Group { if item == .escape || item == .tab { - Text(item.displayLabel).font(.system(size: 13, weight: .semibold)) + Text(item.displayLabel).font(.system(size: (13 * scale).rounded(), weight: .semibold)) } else { Image(systemName: item.displayLabel) - .font(.system(size: 18, weight: .semibold)) + .font(.system(size: (18 * scale).rounded(), weight: .semibold)) } } .foregroundStyle(.white) - .frame(width: Metrics.trayItem, height: Metrics.trayItem) + .frame(width: diameter, height: diameter) .background(frameReader("tray:\(item.rawValue)")) .background(chip(selected: false)) .accessibilityLabel(item.title) + .trayButtonAccessibility { activateForAccessibility(item) } } private func shortcutPalette(_ actions: [ShortcutItem], portrait: Bool, @@ -925,6 +946,7 @@ struct ReceiverControlOverlay: View { .scaleEffect(selected ? 1.12 : 1) .animation(.snappy(duration: 0.14), value: selected) .accessibilityLabel(action.title) + .trayButtonAccessibility { apply(interaction.finish(with: action)) } } /// Shows the combination the finger is currently on — the live chord, @@ -959,6 +981,8 @@ struct ReceiverControlOverlay: View { initiatingItem = trayItem(at: value.startLocation) initiatingMoveView = initiatingItem == nil && frames["control:moveView"]?.contains(value.startLocation) == true + initiatingReturn = initiatingItem == nil + && frames["control:return"]?.contains(value.startLocation) == true if let modifier = initiatingItem?.modifier { initiatingModifier = modifier paletteAnchor = modifier @@ -1008,6 +1032,10 @@ struct ReceiverControlOverlay: View { } else if let item = initiatingItem, frames["tray:\(item.rawValue)"]?.contains(value.location) == true { perform(item) + } else if initiatingReturn, + frames["control:return"]?.contains(value.location) == true { + // A latched chord stays latched: ⌘ then ↵ is ⌘↵. + sendReturn() } else if initiatingMoveView, frames["control:moveView"]?.contains(value.location) == true { store.setMoveViewActive(!store.moveViewActive) @@ -1018,6 +1046,7 @@ struct ReceiverControlOverlay: View { initiatingModifier = nil initiatingItem = nil initiatingMoveView = false + initiatingReturn = false lastHoveredModifier = nil gestureActive = false // The touch lifted: safe to (re)start counting down again, @@ -1084,6 +1113,20 @@ struct ReceiverControlOverlay: View { } } + /// What a tap on a Main control does, for VoiceOver, Voice Control and + /// Switch Control. The chips are only drawn; taps reach them through + /// `controlGesture`'s hit-testing, which those technologies don't use. + /// A modifier latches, exactly like a quick tap. + fileprivate func activateForAccessibility(_ item: ControlTrayItem) { + store.beginAutoHideActivity() + if let modifier = item.modifier { + apply(interaction.tap(modifier)) + } else { + perform(item) + } + store.scheduleAutoHide(interactionIdle: interaction.phase == .idle) + } + private func apply(_ effects: [ControlInteractionEffect]) { for effect in effects { switch effect { @@ -1110,11 +1153,13 @@ struct ReceiverControlOverlay: View { extension ReceiverControlOverlay { /// The tray items the bar's gesture can start on — see `KeyboardBarItem`. fileprivate var keyboardBarTrayItems: [ControlTrayItem] { - keyboardBarItems.map { item in + keyboardBarItems.compactMap { item in switch item { case .modifier(let modifier): return ControlTrayItem.item(for: modifier) case .escape: return .escape case .tab: return .tab + // Not a tray item; `controlGesture` tracks it by its frame. + case .returnKey: return nil case .dismissKeyboard: return .keyboard } } @@ -1137,9 +1182,12 @@ extension ReceiverControlOverlay { var keyboardBarBody: some View { let keyboardTop = (keyboardVisibleRect?.maxY ?? containerSize.height) + surfaceOrigin.y let items = keyboardBarItems - let item: CGFloat = store.isPad ? (PadControlMetrics.baseItem * CGFloat(PadControlScale.clamped(store.preferences.padControlScale))).rounded() : 38 let gap: CGFloat = 8 let padding: CGFloat = 6 + let preferredItem: CGFloat = store.isPad ? (PadControlMetrics.baseItem * CGFloat(PadControlScale.clamped(store.preferences.padControlScale))).rounded() : 38 + // Eight keys need a little more than a 375pt iPhone offers at 38pt. + let item = ControlTrayGeometry.fittedChipDiameter(count: items.count, length: containerSize.width, + preferred: preferredItem, gap: gap, slack: padding * 2) let barWidth = CGFloat(items.count) * item + CGFloat(items.count - 1) * gap + padding * 2 let bar = CGRect(x: containerSize.width / 2 - barWidth / 2, y: keyboardTop - 8 - item - padding * 2, width: barWidth, height: item + padding * 2) @@ -1217,6 +1265,7 @@ extension ReceiverControlOverlay { .position(x: frame.midX, y: frame.midY) .accessibilityLabel(modifier.title) .accessibilityValue(selected ? "On" : "Off") + .trayButtonAccessibility { activateForAccessibility(ControlTrayItem.item(for: modifier)) } case .escape, .tab: let item: ControlTrayItem = entry == .escape ? .escape : .tab Text(item.displayLabel) @@ -1227,6 +1276,7 @@ extension ReceiverControlOverlay { .background(PadChip(selected: false, onRail: true)) .position(x: frame.midX, y: frame.midY) .accessibilityLabel(item.title) + .trayButtonAccessibility { activateForAccessibility(item) } case .dismissKeyboard: Image(systemName: "keyboard.chevron.compact.down") .font(.system(size: frame.width * 0.38, weight: .semibold)) @@ -1236,6 +1286,17 @@ extension ReceiverControlOverlay { .background(PadChip(selected: false, onRail: true)) .position(x: frame.midX, y: frame.midY) .accessibilityLabel(Text("Hide Keyboard")) + .trayButtonAccessibility { activateForAccessibility(.keyboard) } + case .returnKey: + Image(systemName: "return") + .font(.system(size: frame.width * 0.38, weight: .semibold)) + .foregroundStyle(.white) + .frame(width: frame.width, height: frame.height) + .background(frameReader("control:return")) + .background(PadChip(selected: false, onRail: true)) + .position(x: frame.midX, y: frame.midY) + .accessibilityLabel(Text("Return")) + .trayButtonAccessibility { sendReturn() } } } } @@ -1402,7 +1463,8 @@ extension ReceiverControlOverlay { } if let paletteChord, let wheel { WheelPaletteView(layout: wheel, actions: actions, selectedID: interaction.selectedActionID, - centerLabel: paletteChord.symbols) + centerLabel: paletteChord.symbols, + onActivate: { apply(interaction.finish(with: $0)) }) .transition(.scale(scale: 0.85).combined(with: .opacity)) } else if let paletteChord, let paletteFrame { if !actions.isEmpty { @@ -1563,7 +1625,8 @@ extension ReceiverControlOverlay { } if let paletteChord, let wheel { WheelPaletteView(layout: wheel, actions: actions, selectedID: interaction.selectedActionID, - centerLabel: paletteChord.symbols) + centerLabel: paletteChord.symbols, + onActivate: { apply(interaction.finish(with: $0)) }) .transition(.scale(scale: 0.85).combined(with: .opacity)) } else if let paletteChord { if paletteKeys.isEmpty, let anchorControl { @@ -1753,6 +1816,17 @@ extension ReceiverControlOverlay { .accessibilityElement(children: .ignore) .accessibilityLabel(padAccessibilityLabel(kind)) .accessibilityValue(kind.modifier != nil || kind == .moveView ? (selected ? "On" : "Off") : "") + .trayButtonAccessibility { + switch kind { + case .tray(let item): + activateForAccessibility(item) + case .moveView: + store.setMoveViewActive(!store.moveViewActive) + haptics.play(.selection) + case .function, .shortcut: + break + } + } } private func padAccessibilityLabel(_ kind: CustomControlKind) -> String { @@ -1787,7 +1861,9 @@ extension ReceiverControlOverlay { .position(x: frame.midX, y: frame.minY + (diameter + (hint == nil ? 0 : PadControlMetrics.hintHeight + 2)) / 2) .accessibilityElement(children: .ignore) .accessibilityLabel(item.title) - .accessibilityAddTraits(.isButton) + // `.ignore` drops the face's tap gesture from the element, so the + // action has to be given to it directly. + .trayButtonAccessibility { performFunctionAction(item) } } @ViewBuilder @@ -1835,6 +1911,7 @@ extension ReceiverControlOverlay { .scaleEffect(selected ? 1.05 : 1) .animation(.snappy(duration: 0.14), value: selected) .accessibilityLabel(action.title) + .trayButtonAccessibility { apply(interaction.finish(with: action)) } } private func emptyPaletteLabel(_ chord: ModifierChord) -> some View { @@ -1930,6 +2007,9 @@ struct WheelPaletteView: View { let actions: [ShortcutItem] let selectedID: String? let centerLabel: String + /// Fires a wedge for VoiceOver, Voice Control and Switch Control; touch + /// selection goes through the overlay's gesture. `nil` in the editor. + var onActivate: ((ShortcutItem) -> Void)? var body: some View { ZStack(alignment: .topLeading) { @@ -1949,6 +2029,8 @@ struct WheelPaletteView: View { .scaleEffect(selected ? 1.12 : 1) .position(segment.labelPoint) .accessibilityLabel(action.title) + .accessibilityAddTraits(onActivate == nil ? [] : .isButton) + .accessibilityAction { onActivate?(action) } } Circle() .fill(Color(white: 0.12).opacity(0.92)) @@ -2350,3 +2432,12 @@ struct ShortcutButtonFaceView: View { extension StreamReceiver: ReceiverControlSource { var controlSessionPhase: ReceiverSessionPhase { session.phase } } + +private extension View { + /// Marks a drawn control as a button and gives assistive technologies + /// its tap action directly — see `activateForAccessibility`. + func trayButtonAccessibility(_ action: @escaping () -> Void) -> some View { + accessibilityAddTraits(.isButton) + .accessibilityAction { action() } + } +} diff --git a/iOSUITests/ReceiverControlsTourTests.swift b/iOSUITests/ReceiverControlsTourTests.swift new file mode 100644 index 000000000..5618d6165 --- /dev/null +++ b/iOSUITests/ReceiverControlsTourTests.swift @@ -0,0 +1,200 @@ +import XCTest + +/// Walks the receiver's touch controls in Demo Mode (the real control +/// overlay over a sample desktop) and keeps a screenshot of each state: +/// the trays, a held ⌘ palette, the keyboard bar, input off and video off. +/// Run per device and orientation to review layout; it asserts only what +/// must hold everywhere. +final class ReceiverControlsTourTests: XCTestCase { + private var app: XCUIApplication! + + override func setUp() { + continueAfterFailure = false + } + + @MainActor + func testControlsTourPortrait() { + XCUIDevice.shared.orientation = .portrait + tour(prefix: "portrait") + } + + @MainActor + func testControlsTourLandscape() { + XCUIDevice.shared.orientation = .landscapeLeft + defer { XCUIDevice.shared.orientation = .portrait } + tour(prefix: "landscape") + } + + /// The tray is drawn and driven by one gesture, but each control must + /// still be a real button to VoiceOver, Voice Control and Switch Control. + @MainActor + func testTrayControlsAreAccessibilityButtons() { + XCUIDevice.shared.orientation = .portrait + enterDemo() + for label in ["Command", "Option", "Control", "Shift", "Escape", "Tab", "Toggle Dock", "Settings", "Return"] { + XCTAssertTrue(app.buttons[label].firstMatch.exists, "\(label) is not exposed as a button") + } + app.buttons["Exit Demo"].tap() + } + + /// Local pinch-zoom to the 8× cap, then a twist, over the sample + /// desktop: captured for review, and the controls must stay usable. + @MainActor + func testHighMagnificationTour() { + XCUIDevice.shared.orientation = .portrait + enterDemo() + let desktop = element("Sample Mac desktop") + XCTAssertTrue(desktop.waitForExistence(timeout: 3)) + for _ in 0..<3 { + desktop.pinch(withScale: 2.2, velocity: 2) + sleep(1) + } + snapshot("zoom-01-eight-times") + desktop.rotate(.pi / 7, withVelocity: 1) + sleep(1) + snapshot("zoom-02-eight-times-rotated") + // One finger pans only under Move View (the iPad radial layout's). + let moveView = element("Move View") + if moveView.exists, moveView.isHittable { + moveView.tap() + for _ in 0..<4 { desktop.swipeLeft(velocity: .fast); desktop.swipeUp(velocity: .fast) } + sleep(1) + snapshot("zoom-03-extreme-pan") + moveView.tap() + } + XCTAssertTrue(element("Settings").exists, "Settings stays reachable while zoomed") + app.buttons["Exit Demo"].tap() + } + + /// Return reaches the (sample) Mac from the Function Tray with the + /// keyboard closed, and from the keyboard bar with it open. + @MainActor + func testReturnSendsAPressWithKeyboardClosedAndOpen() { + XCUIDevice.shared.orientation = .portrait + enterDemo() + let sent = NSPredicate(format: "label BEGINSWITH 'Return' OR label == 'New line'") + let trayReturn = app.buttons["Return"].firstMatch + XCTAssertTrue(trayReturn.waitForExistence(timeout: 3)) + revealControls(trayReturn) + trayReturn.tap() + XCTAssertTrue(app.staticTexts.matching(sent).firstMatch.waitForExistence(timeout: 3), + "Function Tray Return sent nothing") + + let keyboard = element("Keyboard") + if keyboard.exists { + revealControls(keyboard) + keyboard.tap() + sleep(2) + if app.buttons["Continue"].exists { app.buttons["Continue"].tap(); sleep(1) } + // Not the software keyboard's own return key. + let keys = app.keyboards.firstMatch.frame + let barReturn = app.buttons.matching(identifier: "Return").allElementsBoundByIndex + .first { !keys.intersects($0.frame) } + XCTAssertNotNil(barReturn, "keyboard bar has no Return") + barReturn?.tap() + XCTAssertTrue(app.staticTexts["New line"].firstMatch.waitForExistence(timeout: 3), + "keyboard bar Return sent nothing") + let hide = element("Hide Keyboard") + if hide.waitForExistence(timeout: 3) { hide.tap() } + } + app.buttons["Exit Demo"].tap() + } + + @MainActor + private func tour(prefix: String) { + enterDemo() + snapshot("\(prefix)-01-surface") + + // Hold ⌘ until its palette opens, capture, then slide off and lift. + let command = element("Command") + XCTAssertTrue(command.waitForExistence(timeout: 3)) + revealControls(command) + let start = command.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)) + start.press(forDuration: 1.2) + // A press that became a hold leaves nothing latched. + XCTAssertEqual(command.value as? String, "Off") + + // Tap ⌘ to latch it: the palette stays up for browsing. + command.tap() + XCTAssertEqual(command.value as? String, "On") + sleep(1) + snapshot("\(prefix)-02-command-latched-palette") + command.tap() + XCTAssertEqual(command.value as? String, "Off") + + // Keyboard bar. + let keyboard = element("Keyboard") + if keyboard.exists { + revealControls(keyboard) + keyboard.tap() + sleep(2) + // A fresh simulator shows the one-time slide-to-type intro. + if app.buttons["Continue"].exists { app.buttons["Continue"].tap(); sleep(1) } + snapshot("\(prefix)-03-keyboard-bar") + let hide = element("Hide Keyboard") + if hide.waitForExistence(timeout: 3) { hide.tap() } + sleep(1) + } + + // Input off: the trays give way to the gear. + let input = element("Allow Input (Demo)") + XCTAssertTrue(input.exists) + input.tap() + sleep(1) + element("Sample Mac desktop").coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)).tap() + sleep(1) + snapshot("\(prefix)-04-input-off") + XCTAssertTrue(element("Settings").exists, "Settings stays reachable with input off") + input.tap() + sleep(1) + + app.buttons["Exit Demo"].tap() + XCTAssertTrue(app.descendants(matching: .any)["exploreDemo"].firstMatch.waitForExistence(timeout: 5)) + } + + // MARK: Helpers + + @MainActor + private func enterDemo() { + app = XCUIApplication() + app.launch() + sleep(2) + if app.buttons["Close"].waitForExistence(timeout: 3) { app.buttons["Close"].tap() } + sleep(1) + let explore = app.descendants(matching: .any)["exploreDemo"].firstMatch + XCTAssertTrue(explore.waitForExistence(timeout: 5)) + // Explore Demo can sit below the fold in iPhone landscape. + for _ in 0..<4 where !explore.isHittable { + app.scrollViews.firstMatch.swipeUp(velocity: .slow) + } + sleep(1) + explore.tap() + XCTAssertTrue(app.buttons["Exit Demo"].waitForExistence(timeout: 5)) + sleep(1) + } + + private func element(_ label: String) -> XCUIElement { + app.descendants(matching: .any)[label].firstMatch + } + + /// Auto-hide fades the trays after a while; a touch on the desktop + /// brings them back. + @MainActor + private func revealControls(_ control: XCUIElement) { + guard !control.isHittable else { return } + element("Sample Mac desktop").coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.02)).tap() + sleep(1) + } + + @MainActor + private func snapshot(_ name: String) { + let attachment = XCTAttachment(screenshot: app.screenshot()) + attachment.name = name + attachment.lifetime = .keepAlways + add(attachment) + // Landscape XCUITest captures come out rotated; the tour script + // takes a `simctl io screenshot` when it sees this marker. + print("TOURSHOT \(name)") + sleep(3) + } +} diff --git a/ux-review/ios-control-surface-performance.md b/ux-review/ios-control-surface-performance.md new file mode 100644 index 000000000..98418fcd0 --- /dev/null +++ b/ux-review/ios-control-surface-performance.md @@ -0,0 +1,110 @@ +# Performance audit: receiver controls, 8× zoom and the streaming path + +Companion to [`ios-control-surface.md`](ios-control-surface.md). Released +v1.20.0 is reported as responsive in daily use, so the bar here is evidence: +nothing in the streaming path changes in this branch, and anything worth +doing is listed for a separate performance PR. + +No physical iPhone, iPad or second Mac was available. Everything below is +either measured on this Mac, read from the code, or marked as needing +hardware. + +## What already exists + +| Tool | What it gives | Where | +|---|---|---| +| Performance overlay (`PerfOverlay`) | Live fps, Mbps, inter-frame intervals and a bar graph, stalls (frames >50 ms late), e2e P50/P95, Mac encode P50, control RTT, cursor updates/s and drops, Mac encoder/network drops and send-queue depth, capture fps, input P50/P95 (touch sent → CGEvent injected), and on the Metal path decode P50 and photon P50/P95 (capture → on glass). | `Shared/PerfOverlay.swift`, `PerfStats` in `Shared/StreamReceiver.swift` | +| Decode/present counters | Decode durations and display-layer flushes, lock-backed and off the actor path. | `Shared/ReceiverVideoTelemetry.swift` | +| DEBUG pipeline log | `receiverPipeline:` frames decoded/presented per second. | `StreamReceiver` (DEBUG only) | +| Renderer A/B switch | `metalRenderer` user default picks the Metal path (photon metrics) over `AVSampleBufferDisplayLayer` (the default). | `iOS/MetalVideoRenderer.swift` | +| Unit tests | 1,875 macOS-hosted tests of the pure geometry, gesture, input and protocol code. | `MacTests/` | + +Not present: `os_signpost` intervals, XCTest performance metrics, and any +automated latency benchmark. The overlay is the only end-to-end measurement, +and it needs a live session. + +## Confirmed measurements (this Mac) + +MacBook Air, Swift `-O`, three runs each, compiled against the branch's +`Shared/RemoteViewport.swift`: + +| Operation | Cost per call | +|---|---| +| Pinch update: `ManualViewportState.pinching` + `applyManualZoom` + `remotePoint` at 1× | ~125 ns | +| Same at 8× | ~124 ns | +| Same at 8×, rotated 30° | ~145 ns | +| `VideoView.layoutSubviews`' diagnostic line: build string + `NSLog` + date format (main thread) | ~9 µs | +| That line's file append (`Log`'s background queue) | ~2.7 µs | + +Local gesture math is roughly 1/60,000 of a 120 Hz frame and doesn't depend +on magnification. It isn't a bottleneck at any zoom. + +## Code-level findings + +- **Zoom doesn't touch the stream.** `VideoView.applyContentGeometry` sets the + content layer's bounds to the *unzoomed* rect and applies scale and rotation + as a layer transform. Stream resolution, bitrate, decoder output and the + Metal drawable (`drawableSize` = decoded pixel-buffer size) are identical at + 1× and 8×. No buffer is allocated or resized when zooming. +- **What zoom does change** is compositing. At 1× the video covers the + aspect-fit rect; a 16:10 Mac fills a 402 × 874 pt iPhone from about 1.4× in + landscape and 3.5× in portrait, and when rotated it's sampled at an angle. Compositor work is bounded by the screen's + pixels, not by the 8× layer size, because off-screen parts are clipped + (`clipsToBounds`). +- **The open question is the video plane.** The code records an earlier A/B: + `AVSampleBufferDisplayLayer` reached glass faster than the Metal path + because iOS gives it a dedicated compositor plane. Whether iOS keeps that + plane for a scaled or rotated layer isn't documented. If it doesn't, zooming + could add compositor work and possibly latency. That's a **hypothesis**: + it can't be measured in the simulator, whose GPU path is the Mac's. +- **Per-frame logging during gestures.** Each pinch, rotate or Move View update + calls `setNeedsLayout()` + `layoutIfNeeded()`. `layoutSubviews` logs a + "layout:" line whenever the displayed size changes, which is every update of + a pinch. At 120 Hz that's ~0.1 % of the main thread's 8.3 ms frame budget + (if device cost is similar to this Mac's): small, but it also fills the + 1 MB diagnostics log with noise during a long pinch (about 90 KB per 10 s). The line exists for rotation diagnostics and + only needs bounds and video size. +- **The send path is already bounded and tuned.** VideoToolbox runs RealTime, + without frame reordering, with a capped frame delay and speed over quality. + At most 2 encodes and 3 sends are in flight. A busy encoder or full TCP queue + drops the capture rather than queueing it (counted as Mac enc/net drops). + ScreenCaptureKit's queue depth is 8. +- **Telemetry memory is bounded.** Interval and latency rings cap at ~120 + samples, and decode durations are cleared every 5 s report. `removeFirst()` + on a 120-element array is negligible. +- **Input dispatch** is a direct JSON control message per event. On the Mac, + `handleKeyboardPress` posts down and up under one lock, merged with held + modifiers. Nothing in this branch adds work to it: Return reuses the existing + press path. + +## Needs hardware + +| Question | How to measure | +|---|---| +| Does zoom or rotation cost latency on the default video layer? | Overlay e2e P50/P95 and stalls at 1×, 8× and 8× rotated, 2 minutes each, same scene. Then repeat with `metalRenderer` on for photon P50/P95. | +| Does zoom or rotation cost GPU or frame pacing? | Instruments, Animation Hitches + Core Animation, at 1×/8×/rotated, during a pinch and at rest. | +| Sustained CPU, memory and thermals | 30-minute session with Xcode's gauges and Thermal State, 1× vs 8×. | +| Input latency | Overlay input P50/P95 while typing and dragging, USB vs Wi-Fi. | + +## Worth trying in a separate performance PR + +1. **Log layout only when bounds or video size change**, not on every + displayed-size change. Small, local, no behaviour change. Measured at ~9 µs + per gesture frame on this Mac, so the main gain is a diagnostics log that + stays useful; low priority. +2. **Add `os_signpost` intervals** around decode, present, input send and + inject. That makes the questions above answerable in Instruments without + new UI. +3. **Run the zoom A/B above.** If the video plane is lost under transform, + consider dropping rotation (not scale) from the default path's transform + above some threshold, and only with numbers in hand. + +## Not worth the risk without evidence + +- Retuning encoder properties, in-flight limits or the capture queue depth. + They're deliberate, and earlier measurements are recorded in the code. +- Changing `maximumDrawableCount` or switching the default renderer. The code + records that 2 drawables measured worse and that `AVSampleBufferDisplayLayer` + beats Metal on iOS. +- Moving viewport math off the main thread. It costs ~0.13 µs. +- Replacing the telemetry arrays with ring buffers. diff --git a/ux-review/ios-control-surface.md b/ux-review/ios-control-surface.md new file mode 100644 index 000000000..6ab1c22e3 --- /dev/null +++ b/ux-review/ios-control-surface.md @@ -0,0 +1,271 @@ +# iPhone and iPad control surface: UX review + +Branch `feature/ios-control-surface-ux`, based on `cbbad7f` (the `v1.20.0` +App Store build, `release/app-store-rc`). It is independent of the Mac PRs +#24, #25 and #26. + +## Method + +- **Where:** every screen below is Demo Mode, which draws a sample desktop + under the real `ReceiverControlOverlay`. The trays, palettes, keyboard bar, + input prompt and Settings are the shipping code paths; only the Mac is + simulated. +- **Simulators (iOS 27.1):** iPhone 17 (402 × 874 pt), iPhone 17e + (390 × 844 pt), iPad mini (A17 Pro), iPad Air 11-inch (M4) and iPad Pro + 13-inch (M5), each in portrait and landscape. +- **Follow-up round (iOS 26.5):** iPhone SE 3rd generation (375 × 667 pt), + iPhone 17e, iPhone 17 and iPad Pro 13-inch (M5), one simulator at a time. +- **Automation:** `iOSUITests/ReceiverControlsTourTests` walks the surface: + the trays, a held and latched ⌘ palette, the keyboard bar and Input off. The + tour script takes a `simctl io screenshot` at each step, because XCUITest + captures come out rotated in landscape. +- **Not verified here:** how touch feels, Apple Pencil accuracy, network + latency, and end-to-end streaming with a real Mac. The hardware checklist at + the end covers these. + +## Feature inventory + +| Area | What exists | Assessment | +|---|---|---| +| Direct Touch / Trackpad | One-finger pointer model in Settings › Input (segmented). Trackpad sensitivity slider. Per-axis scroll inversion. | Works. Clear explanations in Settings. | +| Precise Control | A second finger on an anchored first finger switches to relative fine-tuning; clicks land at the pointer. Works in both modes, with no setting to turn on. | Works; covered by `PointerGestureEngine` tests. You can only discover it from the README or the website. | +| Smart Touch (Experimental) | Direct Touch only. One-finger scroll; hold a title bar, then drag, to move the window. Optional build-up haptic. | Behind a clear Experimental badge. | +| Multi-finger gestures | Two-finger scroll and right-click. Three-finger swipes (Mission Control, App Exposé, Spaces). Three-finger tap for Spotlight. Four/five-finger pinch and spread. Pinch/rotate can target the Mac or the local view, with Snap Rotation and per-app gesture commands. | Mature. The "Prefer MeowDisplay Gestures" toggle defers the system edge swipes. | +| Main Control Tray | ⌘ ⌥ ⌃ ⇧, esc, tab, Dock, Keyboard, Settings. Tap latches, hold opens a palette, slide across modifiers to build a chord. Profiles, Auto-hide, Collapse, landscape side, Avoid Notch. | Strong model. **Bugs:** see findings 1, 2 and 3. | +| Function Tray | One-tap actions in groups (Zoom, Undo/Redo and Return by default), custom shortcuts and sequences, separate profiles. | Works. | +| Keyboard | Tray button opens the software keyboard. A flat accessory bar (modifiers, esc, tab, Return, hide) replaces the trays while typing. Hardware keyboard passes through. | Live sessions work. **Demo Mode bug:** finding 4. | +| iPad layouts | Strip (solid edge rails), Overlay (floating), Custom (user-built, radial corner template, Two-Hand Assist, Wheel or Keys palette), Control Size, Control Hints. | Rich. **Shipped default was Strip:** finding 6. | +| Apple Pencil | Pen-tablet events with pressure and tilt, hover, palm rejection while the tip is down, tap/double-tap clicks. | Code path reviewed. Needs hardware to verify. | +| Picture in Picture | View-only PiP of the live Mac. Toggle in Settings › Display. | Needs a live session; not exercised in Demo Mode. | +| Video Off | Turns the receiver into a trackpad and control panel: trays, keyboard and input mode stay live, and a trackpad preview replaces the picture. | This is already the "Control Surface" mode. See the assessment below. | +| Input permission | The Mac is authoritative. Settings › Input shows the confirmed state with Request/Turn Off. A blocked touch shows a short-lived on-surface **Request Input** prompt that rate-limits its requests. The gear stays reachable with input off. | Clear and safe. | +| Rotation / iPad resizing | Tray geometry is pure and unit-tested (safe areas, notch side, iOS 27.1 reserved regions and folds, keyboard). | Solid. Two placement bugs found (findings 1 and 2). | + +## Findings + +Ranked by impact within each round. Findings 1 to 7 come from the first +round; 8 and 9 (Return and 8× zoom) were added in a follow-up. All are fixed +on this branch. + +### 1. The ⌘ palette covered half the iPhone tray (every Face ID iPhone) + +With Avoid Notch on (the default), the tray steps up off the home indicator +in portrait, or in from the notch in landscape. The palette was avoided +separately and stayed where the tray used to be, so the shortcut keys sat on +top of ⌥ ⌃ ⇧ esc tab and Dock. The live receiver and Demo Mode share this +geometry. + +Fix: carry the palette by the tray's offset, as the reserved-region pass +already did (`ControlTrayGeometry.layout`). Unit tests cover portrait and +landscape. + +| Before (iPhone 17) | After | +|---|---| +| ![](ios-control-surface/before-iphone17-portrait-palette.png) | ![](ios-control-surface/after-iphone17-portrait-palette.png) | + +### 2. The default iPhone tray ran off 390 pt and 402 pt screens + +Nine 40 pt chips need 396 pt. On iPhone 17e (390 pt) ⌘ and Settings were cut +off at the edges in portrait, and at the top and bottom in landscape. The +geometry clamps the tray's touch frame to the screen minus margins, so the +outer halves of those chips also didn't respond to touches. + +Fix: `ControlTrayGeometry.fittedChipDiameter` shrinks the chips just enough +to fit (36 pt on a 390 pt phone, 38 pt on a 402 pt phone, never below 30 pt). +Trays that already fit are unchanged. + +| Before (iPhone 17e) | After | +|---|---| +| ![](ios-control-surface/before-iphone17e-portrait-palette.png) | ![](ios-control-surface/after-iphone17e-portrait-palette.png) | +| ![](ios-control-surface/before-iphone17e-landscape.png) | ![](ios-control-surface/after-iphone17e-landscape.png) | + +### 3. Tray controls weren't buttons to VoiceOver, Voice Control or Switch Control + +The tray chips are drawn views. Taps reach them through one hit-testing +gesture, so assistive technologies saw labelled text with no button trait and +no action. The Main Tray, keyboard bar, palette keys and iPad Main controls +now carry the button trait and their tap action: a modifier latches, a +palette key fires its shortcut. `testTrayControlsAreAccessibilityButtons` +failed on the shipped build ("Command is not exposed as a button") and passes +now. + +### 4. Demo Mode hid the keyboard bar behind the keyboard + +Demo Mode passed the keyboard's own frame where the overlay expects the +visible area above it. The modifier/esc/tab bar was laid out behind the +keyboard, and the trays vanished while typing, on iPhone and iPad alike. Demo +Mode is what App Review and first-time users try, so this mattered. Live +sessions were not affected. + +| Before | After | +|---|---| +| ![](ios-control-surface/before-iphone17-keyboard.png) | ![](ios-control-surface/after-iphone17-keyboard.png) | + +### 5. The Explore Demo card hid its button + +The card wrapped its `Button` in a separate accessibility element. VoiceOver +got a plain group, and on iOS 27.1 the existing App Review smoke tests could +no longer find `app.buttons["Explore Demo"]`. The button now carries the label +itself. + +### 6. iPad started on Strip, not the radial layout + +A fresh iPad install (and Demo Mode) opened on Strip. The radial bottom-right +layout only existed once a Custom layout had been created. A fresh iPad now +starts on the radial Custom layout, with Strip and Overlay still in Settings › +Controls › Control Layout. Saved preferences are never replaced, and iPhone +is unchanged. + +| Before: Strip (iPad Air 11) | After: radial Custom (iPad Pro 13) | After: windowed (iPad Air 11) | +|---|---|---| +| ![](ios-control-surface/before-ipadair11-palette.png) | ![](ios-control-surface/after-ipadpro13-palette.png) | ![](ios-control-surface/after-ipadair11-windowed.png) | + +The radial layout follows the window when iPadOS runs the app windowed (third +image). Two-Hand Assist mirrors the modifiers on the left, and ⌘'s palette +fans out around the cluster. + +### 7. Demo Mode's banner broke at accessibility text sizes + +At the largest text size the fixed-height Demo banner's text overflowed and +covered **Exit Demo**, and the status chip wrapped as "Al- low In- put" over +the desktop. The banner is now capped at `xxLarge` and the chips at +`xxxLarge`. Settings and the home screen still scale fully. + +| Before (iPhone 17, largest accessibility size) | After | +|---|---| +| ![](ios-control-surface/before-iphone17-ax5.png) | ![](ios-control-surface/after-iphone17-ax5.png) | + +### 8. There was no Return key on the touch controls + +Submitting a form, a chat message or a terminal command needed the software +keyboard, even with ⌘ latched for ⌘↵. Return is now in three places, all +sending one ordinary Return press (HID usage 40) through the existing +keyboard path: + +- **Function Tray:** a built-in Return action in its own group after + Undo/Redo, so a reach for Redo can't submit. Visible on fresh installs. + Saved Function Tray profiles keep their layout exactly; Return is added + hidden, ready to switch on in the editor. +- **Keyboard bar:** between tab and Hide Keyboard. With eight keys the bar + fits by shrinking: 35 pt keys on a 375 pt iPhone, 37 pt on 390 pt, 38 pt + (unchanged) from 402 pt up, never below 30 pt. +- **iPad radial layout:** in Two-Hand Assist's idle arc for new radial + layouts. Saved Custom layouts keep their own arc. + +Return has no modifiers of its own. A modifier latched on the tray is held +on the Mac, which merges it into the press, so ⌘ then ↵ is ⌘↵ and ⇧ then ↵ +is ⇧↵. The press is a single down/up pair under one lock on the Mac, so it +can't leave a key held. It's sent only while input is allowed, and the Mac +enforces the grant again. + +The Function Tray chips (iPhone and iPad) and the iPad Wheel palette's +wedges now perform their action for VoiceOver, Voice Control and Switch +Control, instead of only being labelled. On iPad, the chip's +`.accessibilityElement(children: .ignore)` had dropped its tap action. A +wedge's action ends the palette through `ControlInteractionState.finish(with:)`, +the same path a touch selection takes. That's checked in source only. A UI +test confirms the Function Tray's Return chip is exposed as a button, but no +automated test covers Wheel wedges, and XCTest can't trigger accessibility +actions. Activation with real assistive technology is in the hardware +checklist. + +| Function Tray, iPhone 17e (390 pt) | Keyboard bar, iPhone SE (375 pt) | Radial layout, iPad Pro 13 | +|---|---|---| +| ![](ios-control-surface/return-iphone17e-tray.png) | ![](ios-control-surface/return-iphonese-keyboard.png) | ![](ios-control-surface/return-ipadpro13-radial.png) | + +### 9. Rotated video could be panned off screen; zoom stopped at 3× + +Manual viewport zoom now goes to **8×** (it was 3×). Zoom While Typing keeps +its own 1.5× limit, and App Gesture Commands are unchanged. Zoom is a layer +transform, so the stream resolution, decoding and networking don't change +(see the [performance audit](ios-control-surface-performance.md) for what it +does change). + +Raising the cap exposed a pre-existing bug. Pan limits came from the rotated +picture's bounding box, so a tilted picture dragged along a diagonal could +leave the screen entirely, and the gap grew with zoom. The clamp now works in +the picture's own rotated frame. The point under the view's center is the +nearest point on the picture, so the picture can't be lost at any angle or +zoom. At 0°, 90° and 180° the result is identical to before. +Re-clamping an in-range state (restore, relayout) leaves it exactly as it was. + +| 8× with Move View dragged to the edge, iPad Pro 13 | +|---| +| ![](ios-control-surface/zoom-ipadpro13-eight-times-pan.png) | + +### Noted, not changed + +- **Precise Control is invisible in the app.** One footnote under Input Mode + would help. The feature works; it just isn't discoverable. +- **Keyboard bar gap:** on iPhone the bar sits roughly 60 pt above the keyboard + in the simulator. Check on hardware before changing anything. +- **Fixed-size chip glyphs:** chips over live video ignore Dynamic Type by + design. Settings, the input prompt and the interruption overlay use system + text styles. +- **`DemoModeSmokeTests` can't walk Settings on a 375 pt iPhone in + landscape. This predates the branch: it's a test scrolling issue, not an + app bug.** + - **Cause:** on the iPhone SE (3rd generation, iOS 26.5) the landscape + Settings sheet shows three rows above its pinned Search field and Demo + footer. The Gestures row sits under them, so XCTest reports it as not + hittable. The test's single `swipeUp()` then flings the list to the end + (Permissions … Developer), the row is recycled, and the lookup fails. + The row scrolls into view normally by hand. + - **Baseline comparison:** the shipped base (`cbbad7f`) and this branch + behave identically with the same test on the same simulator: portrait + 3/3 pass and landscape 0/3 on each, with the same list state at failure. + For that comparison both builds located Explore Demo by its identifier. + On the base the unchanged test can't find the Explore Demo button at all + (finding 5). + - **Test unchanged:** two ad-hoc scrolling changes didn't hold in both + orientations and were reverted. A fix needs a scrolling helper verified + on every device size. The test passes on all larger devices. + +## Control Surface mode + +There is no need for a new mode. **Video Off** already turns the device into +a trackpad and control panel: trays, Function Tray, keyboard and input mode +stay live, and a trackpad preview replaces the picture. Making it a first-class +"Control Surface" would mostly mean making it easier to find, for example a +Function Tray action that toggles video. That needs no protocol change, but it +is a product decision, so it isn't in this branch. + +## Hardware-only checklist + +- [ ] iPhone with a home indicator, portrait: hold ⌘. The palette sits clear + above the tray, with no overlap, and sliding to a key fires it. +- [ ] Same in landscape with the notch on each side (rotate both ways). +- [ ] 390 pt / 375 pt iPhone: every tray chip is fully on screen, and tapping + the outer edge of ⌘ and Settings works. +- [ ] VoiceOver: swipe through the tray. Each control reads as a button; + double-tap ⌘ latches it (Mac receives ⌘ down), double-tap again + releases; double-tap a palette key sends the shortcut. +- [ ] Voice Control: "Tap Command", "Tap Settings", "Tap Escape". +- [ ] Switch Control: scan to the tray and activate a modifier and Settings. +- [ ] Fresh iPad install: the radial bottom-right layout appears; switching + to Strip/Overlay and back works; an existing install keeps its layout. +- [ ] Live session keyboard bar on iPhone and iPad: position relative to the + keyboard (see the gap note above). +- [ ] Live session at the largest accessibility text size: the on-surface + Request Input prompt and the interruption card stay legible and + tappable (Demo Mode has its own chips, so this couldn't be checked + here). +- [ ] Return from the Function Tray, the keyboard bar and the iPad radial + arc submits once in a Mac text field; ⌘ latched then ↵ sends ⌘↵, ⇧ + then ↵ sends ⇧↵, and nothing stays held afterwards. +- [ ] VoiceOver double-tap, "Tap Return" (Voice Control) and Switch Control + on a Function Tray chip and an iPad Wheel wedge fire the action. +- [ ] Upgrade over v1.20.0 with a customized Function Tray: the tray is + unchanged and Return is listed, hidden, in the editor. +- [ ] Upgrade over v1.20.0 with other settings changed but the Function + Tray untouched: Return shows after Undo/Redo in every profile. +- [ ] Pinch to 8× and back on a live Mac: smooth, anchored under the + fingers, and taps land on the magnified control. Rotate, pan hard in + every direction, rotate the device and open the keyboard while zoomed. +- [ ] Zoom cost on the live stream: the measurements in the + [performance audit](ios-control-surface-performance.md#needs-hardware). +- [ ] Apple Pencil: pressure, tilt, hover and palm rejection on a supported + iPad. +- [ ] Picture in Picture with a live Mac. +- [ ] Real network latency and touch feel for Direct Touch, Trackpad and + Precise Control. diff --git a/ux-review/ios-control-surface/after-ipadair11-windowed.png b/ux-review/ios-control-surface/after-ipadair11-windowed.png new file mode 100644 index 000000000..6eb48fece Binary files /dev/null and b/ux-review/ios-control-surface/after-ipadair11-windowed.png differ diff --git a/ux-review/ios-control-surface/after-ipadpro13-palette.png b/ux-review/ios-control-surface/after-ipadpro13-palette.png new file mode 100644 index 000000000..4a011787e Binary files /dev/null and b/ux-review/ios-control-surface/after-ipadpro13-palette.png differ diff --git a/ux-review/ios-control-surface/after-iphone17-ax5.png b/ux-review/ios-control-surface/after-iphone17-ax5.png new file mode 100644 index 000000000..a76d2930f Binary files /dev/null and b/ux-review/ios-control-surface/after-iphone17-ax5.png differ diff --git a/ux-review/ios-control-surface/after-iphone17-keyboard.png b/ux-review/ios-control-surface/after-iphone17-keyboard.png new file mode 100644 index 000000000..4cdf9762f Binary files /dev/null and b/ux-review/ios-control-surface/after-iphone17-keyboard.png differ diff --git a/ux-review/ios-control-surface/after-iphone17-portrait-palette.png b/ux-review/ios-control-surface/after-iphone17-portrait-palette.png new file mode 100644 index 000000000..f1b3daee8 Binary files /dev/null and b/ux-review/ios-control-surface/after-iphone17-portrait-palette.png differ diff --git a/ux-review/ios-control-surface/after-iphone17e-landscape.png b/ux-review/ios-control-surface/after-iphone17e-landscape.png new file mode 100644 index 000000000..41e955a77 Binary files /dev/null and b/ux-review/ios-control-surface/after-iphone17e-landscape.png differ diff --git a/ux-review/ios-control-surface/after-iphone17e-portrait-palette.png b/ux-review/ios-control-surface/after-iphone17e-portrait-palette.png new file mode 100644 index 000000000..b951085c6 Binary files /dev/null and b/ux-review/ios-control-surface/after-iphone17e-portrait-palette.png differ diff --git a/ux-review/ios-control-surface/before-ipadair11-palette.png b/ux-review/ios-control-surface/before-ipadair11-palette.png new file mode 100644 index 000000000..96fc14057 Binary files /dev/null and b/ux-review/ios-control-surface/before-ipadair11-palette.png differ diff --git a/ux-review/ios-control-surface/before-iphone17-ax5.png b/ux-review/ios-control-surface/before-iphone17-ax5.png new file mode 100644 index 000000000..a9db2216d Binary files /dev/null and b/ux-review/ios-control-surface/before-iphone17-ax5.png differ diff --git a/ux-review/ios-control-surface/before-iphone17-keyboard.png b/ux-review/ios-control-surface/before-iphone17-keyboard.png new file mode 100644 index 000000000..12ef415d6 Binary files /dev/null and b/ux-review/ios-control-surface/before-iphone17-keyboard.png differ diff --git a/ux-review/ios-control-surface/before-iphone17-portrait-palette.png b/ux-review/ios-control-surface/before-iphone17-portrait-palette.png new file mode 100644 index 000000000..7ee0d01a9 Binary files /dev/null and b/ux-review/ios-control-surface/before-iphone17-portrait-palette.png differ diff --git a/ux-review/ios-control-surface/before-iphone17e-landscape.png b/ux-review/ios-control-surface/before-iphone17e-landscape.png new file mode 100644 index 000000000..d755cec7f Binary files /dev/null and b/ux-review/ios-control-surface/before-iphone17e-landscape.png differ diff --git a/ux-review/ios-control-surface/before-iphone17e-portrait-palette.png b/ux-review/ios-control-surface/before-iphone17e-portrait-palette.png new file mode 100644 index 000000000..196b372b3 Binary files /dev/null and b/ux-review/ios-control-surface/before-iphone17e-portrait-palette.png differ diff --git a/ux-review/ios-control-surface/return-ipadpro13-radial.png b/ux-review/ios-control-surface/return-ipadpro13-radial.png new file mode 100644 index 000000000..182e32c8d Binary files /dev/null and b/ux-review/ios-control-surface/return-ipadpro13-radial.png differ diff --git a/ux-review/ios-control-surface/return-iphone17e-tray.png b/ux-review/ios-control-surface/return-iphone17e-tray.png new file mode 100644 index 000000000..d4c5a64ea Binary files /dev/null and b/ux-review/ios-control-surface/return-iphone17e-tray.png differ diff --git a/ux-review/ios-control-surface/return-iphonese-keyboard.png b/ux-review/ios-control-surface/return-iphonese-keyboard.png new file mode 100644 index 000000000..62ce45040 Binary files /dev/null and b/ux-review/ios-control-surface/return-iphonese-keyboard.png differ diff --git a/ux-review/ios-control-surface/zoom-ipadpro13-eight-times-pan.png b/ux-review/ios-control-surface/zoom-ipadpro13-eight-times-pan.png new file mode 100644 index 000000000..9dcd27add Binary files /dev/null and b/ux-review/ios-control-surface/zoom-ipadpro13-eight-times-pan.png differ