diff --git a/AGENTS.md b/AGENTS.md index c280cfbc..a8facf46 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -80,7 +80,7 @@ Private simulator behavior is implemented locally in: The current repo uses the private boot path, private display bridge, and private accessibility translation bridge directly. The browser streams frames from that bridge, injects touch and keyboard events through the same native session layer, inspects accessibility through `AccessibilityPlatformTranslation`, and renders device chrome from `packages/server/native/XCWChromeRenderer.*`. CoreSimulator service contexts resolve the active developer directory from `DEVELOPER_DIR`, then `xcode-select -p`, then `/Applications/Xcode.app/Contents/Developer`. The display bridge prefers direct CoreSimulator screen IOSurface callbacks and activates the SimulatorKit offscreen renderable view only if direct callbacks are unavailable. Accessibility recovery may use simulator launchctl UIKit application state plus hit-tested translations to recover candidate foreground pids; the returned tree must still be rooted at tokenized `AXPTranslator` application objects, because `translationApplicationObjectForPid:` can omit the bridge delegate token after private display lifecycle changes. Full-tree snapshots merge those recovered roots with the private frontmost application translation. Shallow snapshots with `maxDepth <= 2` use the tokenized frontmost application translation directly when it is available, and only run the expensive recovery sweep if frontmost lookup fails, so agent-oriented describe loops avoid launchctl and hit-test recovery overhead. Interactive-only snapshots also prune non-actionable native AX leaves during Objective-C serialization before the Rust-side compacting pass; keep this native pruning conservative so selector taps still retain actionable rows plus their ancestors. When multiple candidate application roots are discovered, serialize all of them in preferred order: non-extension app roots first, then largest translated roots, with `.appex`/PlugIns processes de-prioritized so SpringBoard and Safari app roots stay primary while widgets and WebContent roots remain debuggable. Widget renderer extension roots may report local frames; normalize those roots and children against matching SpringBoard widget placeholder frames before returning the snapshot. -Physical chrome button support uses DeviceKit `chrome.json` input geometry for browser hit targets. Volume, action, mute, Apple Watch digital crown, Watch side button, and Watch left-side button dispatch through `IndigoHIDMessageForHIDArbitrary` with consumer/telephony/vendor HID usage pairs from the device chrome metadata; home, lock, and app-switcher remain on the existing SimulatorKit button paths. Apple Watch Digital Crown rotation dispatches through `IndigoHIDMessageForDigitalCrownEvent` when SimulatorKit exposes it, with `IndigoHIDMessageForScrollEvent(..., target=0x34)` as the fallback. tvOS simulators do not support direct screen touch; browser/API tap maps to Enter, swipe maps to arrow keys, and the native bridge rejects tvOS touch packets before they reach guest `SimulatorHID`. watchOS/tvOS skip dynamic pointer/mouse service warm-up because those guest runtimes abort on unsupported virtual services. Apple TV and Apple Watch simulators are fixed-orientation devices, so client and server rotation paths must not expose or dispatch device rotation for those families. +Physical chrome button support uses DeviceKit `chrome.json` input geometry for browser hit targets. Volume, action, mute, Apple Watch digital crown, Watch side button, and Watch left-side button dispatch through `IndigoHIDMessageForHIDArbitrary` with consumer/telephony/vendor HID usage pairs from the device chrome metadata; home, lock, and app-switcher remain on the existing SimulatorKit button paths. Apple Watch Digital Crown rotation dispatches through `IndigoHIDMessageForDigitalCrownEvent` when SimulatorKit exposes it, with `IndigoHIDMessageForScrollEvent(..., target=0x34)` as the fallback. Browser mouse/trackpad wheel input over the device screen sends the normalized screen point with the scroll delta, moves the SimulatorKit pointer target there, then dispatches native scroll packets through `IndigoHIDMessageForScrollEvent(..., target=0x2)` with a digitizer-target fallback instead of synthesizing touch drags. tvOS simulators do not support direct screen touch; browser/API tap maps to Enter, swipe maps to arrow keys, and the native bridge rejects tvOS touch packets before they reach guest `SimulatorHID`. watchOS/tvOS skip dynamic pointer/mouse service warm-up because those guest runtimes abort on unsupported virtual services. Apple TV and Apple Watch simulators are fixed-orientation devices, so client and server rotation paths must not expose or dispatch device rotation for those families. On macOS/Xcode 27-era CoreSimulator profiles, `mainScreenWidth`, `mainScreenHeight`, and `mainScreenScale` may be absent from `profile.plist`; DeviceKit chrome rendering must read `capabilities.plist` `ScreenDimensionsCapability` or the primary `displays` entry before falling back to the framebuffer mask PDF. If none of those sources produce usable display geometry, the chrome profile must fail instead of returning a tiny synthetic bezel that hides the stream. Two-point multi-touch dispatch prefers the current SimulatorKit/Indigo packet constructor and falls back to SimDeck's manual Indigo packet adapter. On Xcode 26 SimulatorKit, the constructor expects pixel-space points and stable two-finger movement requires sending `LeftMouseDown` for both `began` and `moved`, then `LeftMouseUp` for `ended`/`cancelled`; using `LeftMouseDragged` for multi-touch moves only advances one contact in UIKit. Do not coalesce multi-touch move packets in the WebSocket or WebRTC control paths, because gesture recognizers need the intermediate two-contact samples. WebKit inspection uses the simulator `webinspectord` Unix socket named `com.apple.webinspectord_sim.socket` and WebKit's binary-plist Remote Inspector selectors. It lists only WebKit content that the runtime exposes as inspectable. For app-owned `WKWebView` on iOS 16.4 and newer, the app must set `isInspectable = true`. diff --git a/docs/api/rest.md b/docs/api/rest.md index 5acba0e8..4fdf9f5d 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -174,6 +174,13 @@ Performance query parameters: For normal clients, copy the browser behavior instead of hand-coding a raw decoder. The UI uses the WebRTC offer endpoint for live video. Android emulator IDs use the same WebRTC endpoint; their H.264 frames are produced from the emulator `-share-vid` display surface, not screenshot polling. +The input/control WebSocket accepts JSON control messages with camelCase fields, +including `touch`, `edgeTouch`, `multiTouch`, `key`, `button`, `crown`, +`scroll`, `home`, `appSwitcher`, and rotation controls. Native iOS scroll wheel +input uses `{ "type": "scroll", "deltaX": 0, "deltaY": 24, "x": 0.5, "y": 0.5 }`, +where `x` and `y` are optional normalized screen coordinates from `0.0` to +`1.0`. Touch-like messages use normalized screen coordinates too. + Minimal WebRTC request: ```json diff --git a/packages/client/src/api/controls.ts b/packages/client/src/api/controls.ts index 6e04a7c3..acd40b35 100644 --- a/packages/client/src/api/controls.ts +++ b/packages/client/src/api/controls.ts @@ -9,6 +9,7 @@ import type { LaunchPayload, MultiTouchPayload, OpenUrlPayload, + ScrollPayload, SimulatorMetadata, SimulatorResponse, TouchPayload, @@ -21,6 +22,7 @@ export type ControlMessage = | ({ type: "key" } & KeyPayload) | ({ type: "button" } & ButtonPayload) | ({ type: "crown" } & CrownPayload) + | ({ type: "scroll" } & ScrollPayload) | { type: "dismissKeyboard" } | { type: "toggleSoftwareKeyboard" } | { type: "home" } diff --git a/packages/client/src/api/types.ts b/packages/client/src/api/types.ts index e6da623a..31dbde3d 100644 --- a/packages/client/src/api/types.ts +++ b/packages/client/src/api/types.ts @@ -569,6 +569,13 @@ export interface CrownPayload { delta: number; } +export interface ScrollPayload { + deltaX: number; + deltaY: number; + x?: number; + y?: number; +} + export interface LaunchPayload { bundleId: string; } diff --git a/packages/client/src/app/AppShell.tsx b/packages/client/src/app/AppShell.tsx index 962d0e09..ec218be0 100644 --- a/packages/client/src/app/AppShell.tsx +++ b/packages/client/src/app/AppShell.tsx @@ -44,6 +44,7 @@ import type { } from "../api/types"; import { AccessibilityInspector } from "../features/accessibility/AccessibilityInspector"; import { DevToolsPanel } from "../features/devtools/DevToolsPanel"; +import { normalizedClientCoordinates } from "../features/input/gestureMath"; import { isEditableTarget } from "../features/input/keycodes"; import { useKeyboardInput } from "../features/input/useKeyboardInput"; import { usePointerInput } from "../features/input/usePointerInput"; @@ -71,6 +72,7 @@ import type { Point, Size, TouchIndicator, + TouchPreviewPoint, ViewMode, } from "../features/viewport/types"; import { useViewportLayout } from "../features/viewport/useViewportLayout"; @@ -84,6 +86,7 @@ import { computeChromeBackingRect, computeChromeScreenBorderRadius, computeChromeScreenRect, + mapDisplayedPointToNaturalOrientation, normalizeQuarterTurns, screenAspectRatio, shellSize, @@ -104,6 +107,7 @@ import { HIERARCHY_VISIBLE_STORAGE_KEY, nextAccessibilitySourcePreference, readPersistedUiState, + readStoredAccessibilitySkeletonMode, readStoredAccessibilitySource, readStoredFlag, sanitizeAccessibilitySources, @@ -111,18 +115,32 @@ import { TOUCH_OVERLAY_VISIBLE_STORAGE_KEY, viewportStateForUDID, writePersistedUiState, + writeStoredAccessibilitySkeletonMode, writeStoredFlag, } from "./uiState"; import { isMoveControlMessage } from "./controlMessages"; const ACCESSIBILITY_REFRESH_MS = 1500; +const ACCESSIBILITY_SKELETON_REFRESH_MS = 500; const REACT_NATIVE_ACCESSIBILITY_REFRESH_MS = 500; const FLUTTER_ACCESSIBILITY_REFRESH_MS = 1000; const ACCESSIBILITY_BACKGROUND_REFRESH_MS = 3000; const ANDROID_METADATA_REFRESH_MS = 1000; +const BROWSER_EDGE_GESTURE_WIDTH = 32; +const BROWSER_EDGE_GESTURE_MIN_DELTA = 8; +const SAFARI_SCREEN_GESTURE_INITIAL_SPREAD = 0.28; +const SAFARI_SCREEN_GESTURE_MIN_SCALE = 0.25; +const SAFARI_SCREEN_GESTURE_MAX_SCALE = 4; +const REAL_MULTITOUCH_GESTURE_GRACE_MS = 120; +const NATIVE_SCROLL_DELTA_SCALE = 4; +const NATIVE_SCROLL_MAX_DELTA = 180; const DEFAULT_ACCESSIBILITY_MAX_DEPTH = 10; const LOGICAL_INSPECTOR_MAX_DEPTH = 80; const FLUTTER_INSPECTOR_MAX_DEPTH = 48; + +function clampNumber(value: number, min: number, max: number): number { + return Math.min(Math.max(value, min), max); +} const AUTH_REQUIRED_MESSAGE = "SimDeck API access token is required."; const NOT_CONNECTED_MESSAGE = "Not connected"; const LOCAL_STREAM_DEFAULTS: StreamConfig = { @@ -407,6 +425,20 @@ type AppInstallState = { phase: "dragging" | "installing" | "installed"; }; +type SafariScreenGesturePair = { + first: Point; + second: Point; + firstPreview: TouchPreviewPoint; + secondPreview: TouchPreviewPoint; +}; + +type SafariScreenGestureState = { + blockedByPointerInput: boolean; + lastPair: SafariScreenGesturePair | null; + screenElement: HTMLElement; + startRotationDegrees: number; +}; + type CaptureStatus = { busy: boolean; label: string; @@ -529,6 +561,9 @@ export function AppShell({ useState([]); const [accessibilityPreferredSource, setAccessibilityPreferredSource] = useState(readStoredAccessibilitySource); + const [accessibilitySkeletonMode, setAccessibilitySkeletonMode] = useState( + readStoredAccessibilitySkeletonMode, + ); const [zoomAnimating, setZoomAnimating] = useState(false); const [touchOverlayVisible, setTouchOverlayVisible] = useState(() => readStoredFlag(TOUCH_OVERLAY_VISIBLE_STORAGE_KEY, true), @@ -572,6 +607,12 @@ export function AppShell({ const zoomAnimationTimeoutRef = useRef(0); const touchIndicatorTimeoutRef = useRef(0); const gestureStartZoomRef = useRef(1); + const screenGestureRef = useRef(null); + const recentPointerInputMultiTouchAtRef = useRef(0); + const cancelPointerGestureForScreenGestureRef = useRef<() => void>(() => {}); + const scrollAccumulatorRef = useRef({ x: 0, y: 0 }); + const scrollFrameRef = useRef(0); + const scrollPointRef = useRef({ x: 0.5, y: 0.5 }); const effectiveZoomRef = useRef(initialViewportState.zoom ?? 1); const panRef = useRef(initialViewportState.pan); const applyZoomAtClientPointRef = useRef< @@ -614,6 +655,15 @@ export function AppShell({ setZoomDockElement(node); }, []); + useEffect(() => { + return () => { + if (scrollFrameRef.current !== 0) { + window.cancelAnimationFrame(scrollFrameRef.current); + scrollFrameRef.current = 0; + } + }; + }, []); + const searchNeedle = search.trim().toLowerCase(); const filteredSimulators = simulators.filter((simulator) => { if (!searchNeedle) { @@ -1102,6 +1152,10 @@ export function AppShell({ writeStoredFlag(DEVICE_CHROME_VISIBLE_STORAGE_KEY, deviceChromeVisible); }, [deviceChromeVisible]); + useEffect(() => { + writeStoredAccessibilitySkeletonMode(accessibilitySkeletonMode); + }, [accessibilitySkeletonMode]); + const toggleDevTools = useCallback(() => { setDevToolsVisible((current) => !current); }, []); @@ -1442,16 +1496,23 @@ export function AppShell({ [accessibilityPreferredSource, updateAccessibilityRoots], ); + const accessibilitySkeletonVisible = + accessibilitySource === "native-ax" && + (accessibilitySkeletonMode === "always" || + (accessibilitySkeletonMode === "auto" && hierarchyVisible)); + useEffect(() => { - const refreshMs = hierarchyVisible - ? accessibilityPreferredSource === "react-native" || - accessibilitySource === "react-native" - ? REACT_NATIVE_ACCESSIBILITY_REFRESH_MS - : accessibilityPreferredSource === "flutter" || - accessibilitySource === "flutter" - ? FLUTTER_ACCESSIBILITY_REFRESH_MS - : ACCESSIBILITY_REFRESH_MS - : ACCESSIBILITY_BACKGROUND_REFRESH_MS; + const refreshMs = accessibilitySkeletonVisible + ? ACCESSIBILITY_SKELETON_REFRESH_MS + : hierarchyVisible + ? accessibilityPreferredSource === "react-native" || + accessibilitySource === "react-native" + ? REACT_NATIVE_ACCESSIBILITY_REFRESH_MS + : accessibilityPreferredSource === "flutter" || + accessibilitySource === "flutter" + ? FLUTTER_ACCESSIBILITY_REFRESH_MS + : ACCESSIBILITY_REFRESH_MS + : ACCESSIBILITY_BACKGROUND_REFRESH_MS; let disposed = false; let timeout: number | null = null; const refreshLoop = async () => { @@ -1475,6 +1536,7 @@ export function AppShell({ }, [ accessibilityPreferredSource, accessibilitySource, + accessibilitySkeletonVisible, hierarchyVisible, loadAccessibilityTree, ]); @@ -1792,6 +1854,101 @@ export function AppShell({ }; }, []); + useEffect(() => { + let edgeTouch: { + identifier: number; + startX: number; + startY: number; + } | null = null; + + function touchByIdentifier(touches: TouchList, identifier: number) { + for (let index = 0; index < touches.length; index += 1) { + const touch = touches.item(index); + if (touch?.identifier === identifier) { + return touch; + } + } + return null; + } + + function handleDocumentTouchStart(event: TouchEvent) { + if (event.touches.length !== 1) { + edgeTouch = null; + return; + } + const touch = event.touches.item(0); + if (!touch) { + edgeTouch = null; + return; + } + const viewportWidth = window.visualViewport?.width ?? window.innerWidth; + const fromHorizontalEdge = + touch.clientX <= BROWSER_EDGE_GESTURE_WIDTH || + touch.clientX >= viewportWidth - BROWSER_EDGE_GESTURE_WIDTH; + edgeTouch = fromHorizontalEdge + ? { + identifier: touch.identifier, + startX: touch.clientX, + startY: touch.clientY, + } + : null; + } + + function handleDocumentTouchMove(event: TouchEvent) { + if (!edgeTouch) { + return; + } + const touch = touchByIdentifier(event.touches, edgeTouch.identifier); + if (!touch) { + edgeTouch = null; + return; + } + const deltaX = touch.clientX - edgeTouch.startX; + const deltaY = touch.clientY - edgeTouch.startY; + if ( + Math.abs(deltaX) >= BROWSER_EDGE_GESTURE_MIN_DELTA && + Math.abs(deltaX) > Math.abs(deltaY) + ) { + event.preventDefault(); + } + } + + function clearDocumentEdgeTouch() { + edgeTouch = null; + } + + document.addEventListener("touchstart", handleDocumentTouchStart, { + capture: true, + passive: false, + }); + document.addEventListener("touchmove", handleDocumentTouchMove, { + capture: true, + passive: false, + }); + document.addEventListener("touchend", clearDocumentEdgeTouch, { + capture: true, + passive: true, + }); + document.addEventListener("touchcancel", clearDocumentEdgeTouch, { + capture: true, + passive: true, + }); + return () => { + document.removeEventListener("touchstart", handleDocumentTouchStart, { + capture: true, + }); + document.removeEventListener("touchmove", handleDocumentTouchMove, { + capture: true, + }); + document.removeEventListener("touchend", clearDocumentEdgeTouch, { + capture: true, + }); + document.removeEventListener("touchcancel", clearDocumentEdgeTouch, { + capture: true, + }); + }; + }, []); + useEffect(() => { if (!touchOverlayVisible) { setTouchIndicators([]); @@ -1847,6 +2004,7 @@ export function AppShell({ if (!selectedSimulator) { return; } + recentPointerInputMultiTouchAtRef.current = performance.now(); if (phase === "began" && accessibilitySelectedId) { setAccessibilitySelectedId(""); setAccessibilityHoveredId(null); @@ -1867,6 +2025,8 @@ export function AppShell({ rotationQuarterTurns: viewportRotationQuarterTurns, setPan, }); + cancelPointerGestureForScreenGestureRef.current = + pointerInput.cancelActiveGestureForExternalMultiTouch; const pairingRequired = !remoteStream && @@ -2421,6 +2581,42 @@ export function AppShell({ applyZoomAtClientPointRef.current = applyZoomAtClientPoint; }); + function screenElementForWheel(event: React.WheelEvent) { + const target = event.target; + return target instanceof Element + ? (target.closest(".device-screen") as HTMLElement | null) + : null; + } + + function handleNativeScreenWheel( + event: React.WheelEvent, + deltaX: number, + deltaY: number, + ): boolean { + const screenElement = screenElementForWheel(event); + if (!screenElement || !selectedSimulator?.isBooted) { + return false; + } + if (deltaX === 0 && deltaY === 0) { + return true; + } + + const displayedPoint = normalizedClientCoordinates( + screenElement, + event.clientX, + event.clientY, + { clamp: true }, + ); + const screenPoint = mapDisplayedPointToNaturalOrientation( + displayedPoint ?? { x: 0.5, y: 0.5 }, + viewportRotationQuarterTurns, + ); + setAccessibilitySelectedId(""); + setAccessibilityHoveredId(null); + sendScrollWheel(deltaX, deltaY, screenPoint.x, screenPoint.y); + return true; + } + function handleViewportWheel(event: React.WheelEvent) { if (!selectedSimulator) { return; @@ -2448,6 +2644,10 @@ export function AppShell({ return; } + if (handleNativeScreenWheel(event, deltaX, deltaY)) { + return; + } + setPan( (currentPan) => nextViewportWheelPanState({ @@ -2467,22 +2667,53 @@ export function AppShell({ ); } - function showTouchIndicator(phase: TouchPhase, coords: Point) { + function showTouchIndicator(phase: TouchPhase, coords: TouchPreviewPoint) { showTouchIndicators(phase, [coords]); } - function showTouchIndicators(phase: TouchPhase, coords: Point[]) { + function showTouchIndicators(phase: TouchPhase, coords: TouchPreviewPoint[]) { if (!touchOverlayVisible) { return; } + const canvasElement = outerCanvasRef.current ?? outerCanvasElement; + const canvasRect = canvasElement?.getBoundingClientRect() ?? null; setTouchIndicators( - coords.map((coord, index) => ({ - id: index + 1, - phase, - x: coord.x, - y: coord.y, - })), + coords.map((coord, index) => { + if ( + canvasRect && + Number.isFinite(coord.pageX) && + Number.isFinite(coord.pageY) + ) { + return { + id: index + 1, + phase, + space: "canvas", + x: (coord.pageX ?? 0) - (canvasRect.left + window.scrollX), + y: (coord.pageY ?? 0) - (canvasRect.top + window.scrollY), + }; + } + if ( + canvasRect && + Number.isFinite(coord.clientX) && + Number.isFinite(coord.clientY) + ) { + return { + id: index + 1, + phase, + space: "canvas", + x: (coord.clientX ?? 0) - canvasRect.left, + y: (coord.clientY ?? 0) - canvasRect.top, + }; + } + return { + id: index + 1, + phase, + space: "screen", + x: coord.x, + y: coord.y, + }; + }), ); if (touchIndicatorTimeoutRef.current) { clearTimeout(touchIndicatorTimeoutRef.current); @@ -2505,15 +2736,240 @@ export function AppShell({ type WebKitGestureEvent = Event & { clientX?: number; clientY?: number; + rotation?: number; scale?: number; }; + function finiteGestureNumber(value: number | undefined, fallback: number) { + return typeof value === "number" && Number.isFinite(value) + ? value + : fallback; + } + + function currentBrowserTimestamp() { + return performance.now(); + } + + function clampSafariGestureScale(value: number | undefined) { + return Math.min( + Math.max( + finiteGestureNumber(value, 1), + SAFARI_SCREEN_GESTURE_MIN_SCALE, + ), + SAFARI_SCREEN_GESTURE_MAX_SCALE, + ); + } + + function screenElementForGesture(event: Event): HTMLElement | null { + const target = event.target; + return target instanceof Element + ? (target.closest(".device-screen") as HTMLElement | null) + : null; + } + + function screenGestureAnchor( + screenElement: HTMLElement, + event: Event, + ): Point & { clientX: number; clientY: number } { + const gestureEvent = event as WebKitGestureEvent; + const screenRect = screenElement.getBoundingClientRect(); + let clientX = finiteGestureNumber( + gestureEvent.clientX, + screenRect.left + screenRect.width / 2, + ); + let clientY = finiteGestureNumber( + gestureEvent.clientY, + screenRect.top + screenRect.height / 2, + ); + let displayed = normalizedClientCoordinates( + screenElement, + clientX, + clientY, + { + clamp: false, + }, + ); + if ( + !displayed || + displayed.x < -0.25 || + displayed.x > 1.25 || + displayed.y < -0.25 || + displayed.y > 1.25 + ) { + clientX = screenRect.left + screenRect.width / 2; + clientY = screenRect.top + screenRect.height / 2; + displayed = normalizedClientCoordinates( + screenElement, + clientX, + clientY, + { + clamp: false, + }, + ) ?? { x: 0.5, y: 0.5 }; + } + return { + ...displayed, + clientX, + clientY, + }; + } + + function previewPointFromClient( + screenElement: HTMLElement, + clientX: number, + clientY: number, + ): TouchPreviewPoint | null { + const displayed = normalizedClientCoordinates( + screenElement, + clientX, + clientY, + { + clamp: false, + }, + ); + return displayed + ? { + ...displayed, + clientX, + clientY, + pageX: clientX + window.scrollX, + pageY: clientY + window.scrollY, + } + : null; + } + + function screenGesturePairFromEvent( + screenElement: HTMLElement, + event: Event, + startRotationDegrees: number, + ): SafariScreenGesturePair | null { + const gestureEvent = event as WebKitGestureEvent; + const screenRect = screenElement.getBoundingClientRect(); + const anchor = screenGestureAnchor(screenElement, event); + const minScreenPixels = Math.max( + 1, + Math.min(screenRect.width, screenRect.height), + ); + const scale = clampSafariGestureScale(gestureEvent.scale); + const rotationDegrees = + finiteGestureNumber(gestureEvent.rotation, startRotationDegrees) - + startRotationDegrees; + const radians = (rotationDegrees * Math.PI) / 180; + const halfSpreadPixels = + (minScreenPixels * SAFARI_SCREEN_GESTURE_INITIAL_SPREAD * scale) / 2; + const offsetX = Math.cos(radians) * halfSpreadPixels; + const offsetY = Math.sin(radians) * halfSpreadPixels; + const firstPreview = previewPointFromClient( + screenElement, + anchor.clientX + offsetX, + anchor.clientY + offsetY, + ); + const secondPreview = previewPointFromClient( + screenElement, + anchor.clientX - offsetX, + anchor.clientY - offsetY, + ); + if (!firstPreview || !secondPreview) { + return null; + } + return { + first: mapDisplayedPointToNaturalOrientation( + firstPreview, + viewportRotationQuarterTurns, + ), + second: mapDisplayedPointToNaturalOrientation( + secondPreview, + viewportRotationQuarterTurns, + ), + firstPreview, + secondPreview, + }; + } + + function sendScreenGestureMultiTouch( + phase: TouchPhase, + state: SafariScreenGestureState, + event: Event, + ) { + if (!selectedSimulator?.isBooted) { + return false; + } + const pair = + screenGesturePairFromEvent( + state.screenElement, + event, + state.startRotationDegrees, + ) ?? state.lastPair; + if (!pair) { + return false; + } + state.lastPair = pair; + if (phase === "began" && accessibilitySelectedId) { + setAccessibilitySelectedId(""); + setAccessibilityHoveredId(null); + } + showTouchIndicators(phase, [pair.firstPreview, pair.secondPreview]); + sendControl(selectedSimulator.udid, { + type: "multiTouch", + x1: pair.first.x, + y1: pair.first.y, + x2: pair.second.x, + y2: pair.second.y, + phase, + }); + return true; + } + function handleGestureStart(event: Event) { + const screenElement = screenElementForGesture(event); + if (screenElement) { + event.preventDefault(); + event.stopPropagation(); + const gestureEvent = event as WebKitGestureEvent; + const blockedByPointerInput = + currentBrowserTimestamp() - + recentPointerInputMultiTouchAtRef.current < + REAL_MULTITOUCH_GESTURE_GRACE_MS; + const startRotationDegrees = finiteGestureNumber( + gestureEvent.rotation, + 0, + ); + const pair = screenGesturePairFromEvent( + screenElement, + event, + startRotationDegrees, + ); + const state: SafariScreenGestureState = { + blockedByPointerInput, + lastPair: pair, + screenElement, + startRotationDegrees, + }; + screenGestureRef.current = state; + if (blockedByPointerInput || !pair || !selectedSimulator?.isBooted) { + return; + } + cancelPointerGestureForScreenGestureRef.current(); + sendScreenGestureMultiTouch("began", state, event); + return; + } + + screenGestureRef.current = null; event.preventDefault(); gestureStartZoomRef.current = effectiveZoomRef.current; } function handleGestureChange(event: Event) { + const screenGesture = screenGestureRef.current; + if (screenGesture) { + event.preventDefault(); + event.stopPropagation(); + if (!screenGesture.blockedByPointerInput) { + sendScreenGestureMultiTouch("moved", screenGesture, event); + } + return; + } + event.preventDefault(); const gestureEvent = event as WebKitGestureEvent; const bounds = canvasElement.getBoundingClientRect(); @@ -2524,17 +2980,41 @@ export function AppShell({ ); } + function handleGestureEnd(event: Event) { + const screenGesture = screenGestureRef.current; + if (screenGesture) { + event.preventDefault(); + event.stopPropagation(); + if (!screenGesture.blockedByPointerInput) { + sendScreenGestureMultiTouch("ended", screenGesture, event); + } + screenGestureRef.current = null; + return; + } + + event.preventDefault(); + } + canvasElement.addEventListener("gesturestart", handleGestureStart, { passive: false, }); canvasElement.addEventListener("gesturechange", handleGestureChange, { passive: false, }); + canvasElement.addEventListener("gestureend", handleGestureEnd, { + passive: false, + }); return () => { canvasElement.removeEventListener("gesturestart", handleGestureStart); canvasElement.removeEventListener("gesturechange", handleGestureChange); + canvasElement.removeEventListener("gestureend", handleGestureEnd); }; - }, [outerCanvasElement]); + }, [ + accessibilitySelectedId, + outerCanvasElement, + selectedSimulator, + viewportRotationQuarterTurns, + ]); function promptForURL() { if (!selectedSimulator) { @@ -2630,6 +3110,95 @@ export function AppShell({ } } + function flushScrollWheel() { + scrollFrameRef.current = 0; + const accumulated = scrollAccumulatorRef.current; + scrollAccumulatorRef.current = { x: 0, y: 0 }; + if (accumulated.x === 0 && accumulated.y === 0) { + return; + } + + const deltaX = clampNumber( + accumulated.x, + -NATIVE_SCROLL_MAX_DELTA, + NATIVE_SCROLL_MAX_DELTA, + ); + const deltaY = clampNumber( + accumulated.y, + -NATIVE_SCROLL_MAX_DELTA, + NATIVE_SCROLL_MAX_DELTA, + ); + const remainingX = accumulated.x - deltaX; + const remainingY = accumulated.y - deltaY; + if (remainingX !== 0 || remainingY !== 0) { + scrollAccumulatorRef.current = { x: remainingX, y: remainingY }; + scrollFrameRef.current = window.requestAnimationFrame(flushScrollWheel); + } + + sendScrollWheelNow( + deltaX, + deltaY, + scrollPointRef.current.x, + scrollPointRef.current.y, + ); + } + + function sendScrollWheel( + deltaX: number, + deltaY: number, + x: number, + y: number, + ) { + if ( + !Number.isFinite(deltaX) || + !Number.isFinite(deltaY) || + (deltaX === 0 && deltaY === 0) + ) { + return; + } + + scrollPointRef.current = { + x: Number.isFinite(x) ? clampNumber(x, 0, 1) : 0.5, + y: Number.isFinite(y) ? clampNumber(y, 0, 1) : 0.5, + }; + scrollAccumulatorRef.current = { + x: scrollAccumulatorRef.current.x + deltaX * NATIVE_SCROLL_DELTA_SCALE, + y: scrollAccumulatorRef.current.y + deltaY * NATIVE_SCROLL_DELTA_SCALE, + }; + if (scrollFrameRef.current === 0) { + scrollFrameRef.current = window.requestAnimationFrame(flushScrollWheel); + } + } + + function sendScrollWheelNow( + deltaX: number, + deltaY: number, + x: number, + y: number, + ) { + if ( + !selectedSimulator || + !Number.isFinite(deltaX) || + !Number.isFinite(deltaY) || + !Number.isFinite(x) || + !Number.isFinite(y) || + (deltaX === 0 && deltaY === 0) + ) { + return; + } + if ( + !sendControl(selectedSimulator.udid, { + type: "scroll", + deltaX, + deltaY, + x: clampNumber(x, 0, 1), + y: clampNumber(y, 0, 1), + }) + ) { + setLocalError("Simulator control stream disconnected."); + } + } + function prepareSimulatorInput() { setMenuOpen(false); setAccessibilitySelectedId(""); @@ -2890,6 +3459,7 @@ export function AppShell({ type="file" /> setMenuOpen(false)} @@ -3012,6 +3582,7 @@ export function AppShell({ onToggleTouchOverlay={() => setTouchOverlayVisible((current) => !current) } + onAccessibilitySkeletonModeChange={setAccessibilitySkeletonMode} recordingActive={screenRecording?.phase === "recording"} recordingStopping={screenRecording?.phase === "stopping"} remoteStream={remoteStream} @@ -3083,6 +3654,7 @@ export function AppShell({ accessibilityPickerActive={accessibilityPickerActive} accessibilityRoots={accessibilityRoots} accessibilitySelectedId={accessibilitySelectedId} + accessibilitySkeletonVisible={accessibilitySkeletonVisible} chromeLoaded={chromeLoaded} chromeProfile={viewportChromeProfile} chromeRequired={chromeRequired} @@ -3120,6 +3692,14 @@ export function AppShell({ onAppInstallDragOver={handleAppInstallDragOver} onAppInstallDrop={handleAppInstallDrop} onChromeButtonEvent={sendHardwareButtonEvent} + onBottomBezelPointerCancel={pointerInput.handleBottomBezelPointerCancel} + onBottomBezelPointerDown={pointerInput.handleBottomBezelPointerDown} + onBottomBezelPointerMove={pointerInput.handleBottomBezelPointerMove} + onBottomBezelPointerUp={pointerInput.handleBottomBezelPointerUp} + onBottomBezelTouchCancel={pointerInput.handleBottomBezelTouchCancel} + onBottomBezelTouchEnd={pointerInput.handleBottomBezelTouchEnd} + onBottomBezelTouchMove={pointerInput.handleBottomBezelTouchMove} + onBottomBezelTouchStart={pointerInput.handleBottomBezelTouchStart} onPanPointerMove={pointerInput.handlePanPointerMove} onPanPointerUp={pointerInput.handlePanPointerUp} onPickerHover={setAccessibilityHoveredId} @@ -3133,6 +3713,10 @@ export function AppShell({ onScreenPointerDown={pointerInput.handleScreenPointerDown} onScreenPointerMove={pointerInput.handleScreenPointerMove} onScreenPointerUp={pointerInput.handleScreenPointerUp} + onScreenTouchCancel={pointerInput.handleScreenTouchCancel} + onScreenTouchEnd={pointerInput.handleScreenTouchEnd} + onScreenTouchMove={pointerInput.handleScreenTouchMove} + onScreenTouchStart={pointerInput.handleScreenTouchStart} onStartPanning={pointerInput.startPanning} onViewportWheel={handleViewportWheel} onZoomActual={() => diff --git a/packages/client/src/app/controlMessages.test.ts b/packages/client/src/app/controlMessages.test.ts index e77c1b87..0bf41e6a 100644 --- a/packages/client/src/app/controlMessages.test.ts +++ b/packages/client/src/app/controlMessages.test.ts @@ -42,6 +42,9 @@ describe("controlMessages", () => { phase: "ended", }), ).toBe(false); + expect( + isMoveControlMessage({ type: "scroll", deltaX: 0, deltaY: 42 }), + ).toBe(false); expect(isMoveControlMessage({ type: "home" })).toBe(false); }); }); diff --git a/packages/client/src/app/uiState.test.ts b/packages/client/src/app/uiState.test.ts index c0dae8b4..b69c5119 100644 --- a/packages/client/src/app/uiState.test.ts +++ b/packages/client/src/app/uiState.test.ts @@ -4,6 +4,7 @@ import { DEFAULT_VIEWPORT_STATE, nextAccessibilitySourcePreference, preferredRicherAccessibilitySource, + readStoredAccessibilitySkeletonMode, readStoredFlag, sanitizeAccessibilitySources, sanitizePersistedUiState, @@ -240,4 +241,8 @@ describe("uiState", () => { it("uses the supplied stored-flag default outside the browser", () => { expect(readStoredFlag("missing-flag", true)).toBe(true); }); + + it("defaults accessibility skeleton frames to auto outside the browser", () => { + expect(readStoredAccessibilitySkeletonMode()).toBe("auto"); + }); }); diff --git a/packages/client/src/app/uiState.ts b/packages/client/src/app/uiState.ts index 084f9300..6069c192 100644 --- a/packages/client/src/app/uiState.ts +++ b/packages/client/src/app/uiState.ts @@ -2,6 +2,10 @@ import type { AccessibilitySource, AccessibilitySourcePreference, } from "../api/types"; +import { + isAccessibilitySkeletonMode, + type AccessibilitySkeletonMode, +} from "../features/accessibility/skeletonMode"; import type { Point, ViewMode } from "../features/viewport/types"; export interface PersistedViewportState { @@ -29,6 +33,8 @@ export const WEBKIT_INSPECTOR_VISIBLE_STORAGE_KEY = export const CHROME_DEVTOOLS_VISIBLE_STORAGE_KEY = "xcw-chrome-devtools-visible"; export const ACCESSIBILITY_SOURCE_STORAGE_KEY = "xcw-hierarchy-source"; +export const ACCESSIBILITY_SKELETON_MODE_STORAGE_KEY = + "xcw-accessibility-skeleton-mode"; export const TOUCH_OVERLAY_VISIBLE_STORAGE_KEY = "xcw-touch-overlay-visible"; export const DEVICE_CHROME_VISIBLE_STORAGE_KEY = "xcw-device-chrome-visible"; @@ -111,6 +117,27 @@ export function readStoredAccessibilitySource(): AccessibilitySourcePreference { return source === "auto" || isAccessibilitySource(source) ? source : "auto"; } +export function readStoredAccessibilitySkeletonMode(): AccessibilitySkeletonMode { + if (typeof window === "undefined") { + return "auto"; + } + + const mode = window.localStorage.getItem( + ACCESSIBILITY_SKELETON_MODE_STORAGE_KEY, + ); + return isAccessibilitySkeletonMode(mode) ? mode : "auto"; +} + +export function writeStoredAccessibilitySkeletonMode( + mode: AccessibilitySkeletonMode, +): void { + if (typeof window === "undefined") { + return; + } + + window.localStorage.setItem(ACCESSIBILITY_SKELETON_MODE_STORAGE_KEY, mode); +} + export function sanitizeAccessibilitySources( value: unknown, ): AccessibilitySource[] { diff --git a/packages/client/src/features/accessibility/AccessibilityOverlay.test.ts b/packages/client/src/features/accessibility/AccessibilityOverlay.test.ts index c9880849..08b071ac 100644 --- a/packages/client/src/features/accessibility/AccessibilityOverlay.test.ts +++ b/packages/client/src/features/accessibility/AccessibilityOverlay.test.ts @@ -103,4 +103,30 @@ describe("AccessibilityOverlay", () => { expect(markup).not.toContain("; disabled"); expect(markup).not.toContain(" title="); }); + + it("draws label-free skeleton frames when requested", () => { + const markup = renderToStaticMarkup( + createElement(AccessibilityOverlay, { + hoveredId: null, + roots: [ + { + frame: { height: 844, width: 390, x: 0, y: 0 }, + role: "application", + children: [ + { + AXLabel: "Continue", + frame: { height: 48, width: 180, x: 105, y: 720 }, + type: "Button", + }, + ], + }, + ], + selectedId: "", + skeletonVisible: true, + }), + ); + + expect(markup).toContain("accessibility-rect skeleton"); + expect(markup).not.toContain("Continue"); + }); }); diff --git a/packages/client/src/features/accessibility/AccessibilityOverlay.tsx b/packages/client/src/features/accessibility/AccessibilityOverlay.tsx index 8e1adcae..9c8e4d6e 100644 --- a/packages/client/src/features/accessibility/AccessibilityOverlay.tsx +++ b/packages/client/src/features/accessibility/AccessibilityOverlay.tsx @@ -17,12 +17,14 @@ interface AccessibilityOverlayProps { hoveredId: string | null; roots: AccessibilityNode[]; selectedId: string; + skeletonVisible?: boolean; } export function AccessibilityOverlay({ hoveredId, roots, selectedId, + skeletonVisible = false, }: AccessibilityOverlayProps) { const rootFrame = accessibilityRootFrame(roots); const tree = buildAccessibilityTree(roots); @@ -63,6 +65,16 @@ export function AccessibilityOverlay({ ))}
diff --git a/packages/client/src/features/toolbar/Toolbar.tsx b/packages/client/src/features/toolbar/Toolbar.tsx index 685325ff..7e8819f3 100644 --- a/packages/client/src/features/toolbar/Toolbar.tsx +++ b/packages/client/src/features/toolbar/Toolbar.tsx @@ -12,6 +12,7 @@ import { import { useEffect, useState, type RefObject } from "react"; import type { SimulatorMetadata } from "../../api/types"; +import type { AccessibilitySkeletonMode } from "../accessibility/skeletonMode"; import type { StreamConfig, StreamEncoder, @@ -24,6 +25,7 @@ import { SimulatorMenu } from "../simulators/SimulatorMenu"; import { SimulatorPickerMenu } from "../simulators/SimulatorPickerMenu"; interface ToolbarProps { + accessibilitySkeletonMode: AccessibilitySkeletonMode; debugVisible: boolean; deviceChromeAvailable: boolean; deviceChromeVisible: boolean; @@ -48,6 +50,7 @@ interface ToolbarProps { onOpenUrlPrompt: () => void; onRotateLeft: () => void; onRotateRight: () => void; + onAccessibilitySkeletonModeChange: (mode: AccessibilitySkeletonMode) => void; onShutdown: () => void; onStreamEncoderChange: (encoder: StreamEncoder) => void; onStreamFpsChange: (fps: StreamFps) => void; @@ -85,6 +88,7 @@ interface ToolbarProps { } export function Toolbar({ + accessibilitySkeletonMode, captureBusy, closeSimulatorMenu, closeMenu, @@ -114,6 +118,7 @@ export function Toolbar({ onOpenUrlPrompt, onRotateLeft, onRotateRight, + onAccessibilitySkeletonModeChange, onShutdown, onStreamEncoderChange, onStreamFpsChange, @@ -195,6 +200,7 @@ export function Toolbar({ onOpenBundlePrompt={onOpenBundlePrompt} onOpenUrlPrompt={onOpenUrlPrompt} onRotateRight={onRotateRight} + onAccessibilitySkeletonModeChange={onAccessibilitySkeletonModeChange} onShutdown={onShutdown} onStreamEncoderChange={onStreamEncoderChange} onStreamFpsChange={onStreamFpsChange} @@ -216,6 +222,7 @@ export function Toolbar({ canInstallApp={canInstallApp} streamConfig={streamConfig} streamTransport={streamTransport} + accessibilitySkeletonMode={accessibilitySkeletonMode} deviceChromeAvailable={deviceChromeAvailable} deviceChromeVisible={deviceChromeVisible} touchOverlayVisible={touchOverlayVisible} diff --git a/packages/client/src/features/viewport/DeviceChrome.tsx b/packages/client/src/features/viewport/DeviceChrome.tsx index f9c13212..770db811 100644 --- a/packages/client/src/features/viewport/DeviceChrome.tsx +++ b/packages/client/src/features/viewport/DeviceChrome.tsx @@ -15,6 +15,7 @@ interface DeviceChromeProps { accessibilityPickerActive: boolean; accessibilityRoots: AccessibilityNode[]; accessibilitySelectedId: string; + accessibilitySkeletonVisible: boolean; chromeProfile: ChromeProfile | null; chromeButtonsRenderedInChrome: boolean; chromeScreenBackingStyle: CSSProperties | null; @@ -31,6 +32,14 @@ interface DeviceChromeProps { usagePage?: number, usage?: number, ) => void; + onBottomBezelPointerCancel: (event: React.PointerEvent) => void; + onBottomBezelPointerDown: (event: React.PointerEvent) => void; + onBottomBezelPointerMove: (event: React.PointerEvent) => void; + onBottomBezelPointerUp: (event: React.PointerEvent) => void; + onBottomBezelTouchCancel: (event: React.TouchEvent) => void; + onBottomBezelTouchEnd: (event: React.TouchEvent) => void; + onBottomBezelTouchMove: (event: React.TouchEvent) => void; + onBottomBezelTouchStart: (event: React.TouchEvent) => void; onPanPointerCancel: (event: React.PointerEvent) => void; onPanPointerMove: (event: React.PointerEvent) => void; onPanPointerUp: () => void; @@ -41,6 +50,10 @@ interface DeviceChromeProps { onScreenPointerDown: (event: React.PointerEvent) => void; onScreenPointerMove: (event: React.PointerEvent) => void; onScreenPointerUp: (event: React.PointerEvent) => void; + onScreenTouchCancel: (event: React.TouchEvent) => void; + onScreenTouchEnd: (event: React.TouchEvent) => void; + onScreenTouchMove: (event: React.TouchEvent) => void; + onScreenTouchStart: (event: React.TouchEvent) => void; onStartPanning: (event: React.PointerEvent) => void; rotationQuarterTurns: number; screenAspect: string; @@ -62,6 +75,7 @@ export function DeviceChrome({ accessibilityPickerActive, accessibilityRoots, accessibilitySelectedId, + accessibilitySkeletonVisible, chromeProfile, chromeButtonsRenderedInChrome, chromeScreenBackingStyle, @@ -73,6 +87,14 @@ export function DeviceChrome({ isLoadingStream, isStreamError, onChromeButtonEvent, + onBottomBezelPointerCancel, + onBottomBezelPointerDown, + onBottomBezelPointerMove, + onBottomBezelPointerUp, + onBottomBezelTouchCancel, + onBottomBezelTouchEnd, + onBottomBezelTouchMove, + onBottomBezelTouchStart, onPanPointerCancel, onPanPointerMove, onPanPointerUp, @@ -83,6 +105,10 @@ export function DeviceChrome({ onScreenPointerDown, onScreenPointerMove, onScreenPointerUp, + onScreenTouchCancel, + onScreenTouchEnd, + onScreenTouchMove, + onScreenTouchStart, onStartPanning, rotationQuarterTurns, screenAspect, @@ -136,11 +162,24 @@ export function DeviceChrome({ onEvent={onChromeButtonEvent} renderImages={!chromeButtonsRenderedInChrome} /> + ) => void; + onPointerDown: (event: React.PointerEvent) => void; + onPointerMove: (event: React.PointerEvent) => void; + onPointerUp: (event: React.PointerEvent) => void; + onSimulatorInteraction: () => void; + onTouchCancel: (event: React.TouchEvent) => void; + onTouchEnd: (event: React.TouchEvent) => void; + onTouchMove: (event: React.TouchEvent) => void; + onTouchStart: (event: React.TouchEvent) => void; +}) { + const style = screenEdgeCatcherStyle(chromeScreenStyle); + if (!style) { + return null; + } + return ( +