From bf90be3a1e27161bca7941e91d5dd8a37c7d450c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 20 Jun 2026 01:35:14 -0400 Subject: [PATCH] fix: disable unsafe native scroll packets --- docs/api/rest.md | 9 +- packages/client/src/app/AppShell.tsx | 185 +++++++++++++++++++--- packages/server/src/api/routes.rs | 28 ++-- packages/server/src/native/bridge.rs | 29 ---- packages/server/src/native/ffi.rs | 8 - packages/server/src/simulators/session.rs | 10 -- packages/server/src/transport/webrtc.rs | 2 +- 7 files changed, 185 insertions(+), 86 deletions(-) diff --git a/docs/api/rest.md b/docs/api/rest.md index 4fdf9f5d..a531eb7c 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -176,10 +176,11 @@ For normal clients, copy the browser behavior instead of hand-coding a raw decod The input/control WebSocket accepts JSON control messages with camelCase fields, including `touch`, `edgeTouch`, `multiTouch`, `key`, `button`, `crown`, -`scroll`, `home`, `appSwitcher`, and rotation controls. Native iOS scroll wheel -input uses `{ "type": "scroll", "deltaX": 0, "deltaY": 24, "x": 0.5, "y": 0.5 }`, -where `x` and `y` are optional normalized screen coordinates from `0.0` to -`1.0`. Touch-like messages use normalized screen coordinates too. +`home`, `appSwitcher`, and rotation controls. Use short `touch` drag sequences +for simulator scrolling. The legacy raw `scroll` control is parsed for +compatibility but rejected because SimulatorKit scroll packets can destabilize +iOS runtimes. Touch-like messages use normalized screen coordinates from `0.0` +to `1.0`. Minimal WebRTC request: diff --git a/packages/client/src/app/AppShell.tsx b/packages/client/src/app/AppShell.tsx index ec218be0..456411ae 100644 --- a/packages/client/src/app/AppShell.tsx +++ b/packages/client/src/app/AppShell.tsx @@ -132,8 +132,10 @@ const SAFARI_SCREEN_GESTURE_INITIAL_SPREAD = 0.28; const SAFARI_SCREEN_GESTURE_MIN_SCALE = 0.25; const SAFARI_SCREEN_GESTURE_MAX_SCALE = 4; const REAL_MULTITOUCH_GESTURE_GRACE_MS = 120; -const NATIVE_SCROLL_DELTA_SCALE = 4; -const NATIVE_SCROLL_MAX_DELTA = 180; +const WHEEL_TOUCH_SCROLL_DELTA_SCALE = 0.0012; +const WHEEL_TOUCH_SCROLL_MAX_STEP = 0.18; +const WHEEL_TOUCH_SCROLL_EDGE_INSET = 0.08; +const WHEEL_TOUCH_SCROLL_IDLE_MS = 90; const DEFAULT_ACCESSIBILITY_MAX_DEPTH = 10; const LOGICAL_INSPECTOR_MAX_DEPTH = 80; const FLUTTER_INSPECTOR_MAX_DEPTH = 48; @@ -176,6 +178,11 @@ interface StreamQualityResponse { videoCodec?: string; } +interface WheelTouchScrollState { + udid: string; + current: Point; +} + function buildChromeUrl( udid: string, stamp: string, @@ -613,6 +620,8 @@ export function AppShell({ const scrollAccumulatorRef = useRef({ x: 0, y: 0 }); const scrollFrameRef = useRef(0); const scrollPointRef = useRef({ x: 0.5, y: 0.5 }); + const wheelTouchScrollRef = useRef(null); + const wheelTouchScrollEndTimeoutRef = useRef(0); const effectiveZoomRef = useRef(initialViewportState.zoom ?? 1); const panRef = useRef(initialViewportState.pan); const applyZoomAtClientPointRef = useRef< @@ -661,6 +670,11 @@ export function AppShell({ window.cancelAnimationFrame(scrollFrameRef.current); scrollFrameRef.current = 0; } + if (wheelTouchScrollEndTimeoutRef.current !== 0) { + window.clearTimeout(wheelTouchScrollEndTimeoutRef.current); + wheelTouchScrollEndTimeoutRef.current = 0; + } + endWheelTouchScroll("cancelled"); }; }, []); @@ -2613,7 +2627,7 @@ export function AppShell({ ); setAccessibilitySelectedId(""); setAccessibilityHoveredId(null); - sendScrollWheel(deltaX, deltaY, screenPoint.x, screenPoint.y); + sendWheelTouchScroll(deltaX, deltaY, screenPoint.x, screenPoint.y); return true; } @@ -3110,6 +3124,97 @@ export function AppShell({ } } + function clampWheelTouchPoint(point: Point): Point { + return { + x: clampNumber( + point.x, + WHEEL_TOUCH_SCROLL_EDGE_INSET, + 1 - WHEEL_TOUCH_SCROLL_EDGE_INSET, + ), + y: clampNumber( + point.y, + WHEEL_TOUCH_SCROLL_EDGE_INSET, + 1 - WHEEL_TOUCH_SCROLL_EDGE_INSET, + ), + }; + } + + function wheelTouchStartPoint(anchor: Point, step: Point): Point { + const clampedAnchor = clampWheelTouchPoint(anchor); + const maxX = 1 - WHEEL_TOUCH_SCROLL_EDGE_INSET; + const maxY = 1 - WHEEL_TOUCH_SCROLL_EDGE_INSET; + let x = clampedAnchor.x; + let y = clampedAnchor.y; + + if (step.x > 0) { + x = Math.min(x, maxX - Math.abs(step.x)); + } else if (step.x < 0) { + x = Math.max(x, WHEEL_TOUCH_SCROLL_EDGE_INSET + Math.abs(step.x)); + } + + if (step.y > 0) { + y = Math.min(y, maxY - Math.abs(step.y)); + } else if (step.y < 0) { + y = Math.max(y, WHEEL_TOUCH_SCROLL_EDGE_INSET + Math.abs(step.y)); + } + + return clampWheelTouchPoint({ x, y }); + } + + function wheelDeltaToTouchStep(deltaX: number, deltaY: number): Point { + return { + x: clampNumber( + -deltaX * WHEEL_TOUCH_SCROLL_DELTA_SCALE, + -WHEEL_TOUCH_SCROLL_MAX_STEP, + WHEEL_TOUCH_SCROLL_MAX_STEP, + ), + y: clampNumber( + -deltaY * WHEEL_TOUCH_SCROLL_DELTA_SCALE, + -WHEEL_TOUCH_SCROLL_MAX_STEP, + WHEEL_TOUCH_SCROLL_MAX_STEP, + ), + }; + } + + function endWheelTouchScroll(phase: "ended" | "cancelled" = "ended") { + if (wheelTouchScrollEndTimeoutRef.current !== 0) { + window.clearTimeout(wheelTouchScrollEndTimeoutRef.current); + wheelTouchScrollEndTimeoutRef.current = 0; + } + + const active = wheelTouchScrollRef.current; + wheelTouchScrollRef.current = null; + if (!active) { + return; + } + + sendControl(active.udid, { + type: "touch", + ...active.current, + phase, + }); + } + + function scheduleWheelTouchScrollEnd() { + if (wheelTouchScrollEndTimeoutRef.current !== 0) { + window.clearTimeout(wheelTouchScrollEndTimeoutRef.current); + } + wheelTouchScrollEndTimeoutRef.current = window.setTimeout(() => { + wheelTouchScrollEndTimeoutRef.current = 0; + endWheelTouchScroll("ended"); + }, WHEEL_TOUCH_SCROLL_IDLE_MS); + } + + function beginWheelTouchScroll(udid: string, anchor: Point, step: Point) { + const start = wheelTouchStartPoint(anchor, step); + wheelTouchScrollRef.current = { udid, current: start }; + sendControl(udid, { + type: "touch", + ...start, + phase: "began", + }); + } + function flushScrollWheel() { scrollFrameRef.current = 0; const accumulated = scrollAccumulatorRef.current; @@ -3120,13 +3225,13 @@ export function AppShell({ const deltaX = clampNumber( accumulated.x, - -NATIVE_SCROLL_MAX_DELTA, - NATIVE_SCROLL_MAX_DELTA, + -WHEEL_TOUCH_SCROLL_MAX_STEP / WHEEL_TOUCH_SCROLL_DELTA_SCALE, + WHEEL_TOUCH_SCROLL_MAX_STEP / WHEEL_TOUCH_SCROLL_DELTA_SCALE, ); const deltaY = clampNumber( accumulated.y, - -NATIVE_SCROLL_MAX_DELTA, - NATIVE_SCROLL_MAX_DELTA, + -WHEEL_TOUCH_SCROLL_MAX_STEP / WHEEL_TOUCH_SCROLL_DELTA_SCALE, + WHEEL_TOUCH_SCROLL_MAX_STEP / WHEEL_TOUCH_SCROLL_DELTA_SCALE, ); const remainingX = accumulated.x - deltaX; const remainingY = accumulated.y - deltaY; @@ -3135,7 +3240,7 @@ export function AppShell({ scrollFrameRef.current = window.requestAnimationFrame(flushScrollWheel); } - sendScrollWheelNow( + sendWheelTouchScrollNow( deltaX, deltaY, scrollPointRef.current.x, @@ -3143,7 +3248,7 @@ export function AppShell({ ); } - function sendScrollWheel( + function sendWheelTouchScroll( deltaX: number, deltaY: number, x: number, @@ -3162,15 +3267,15 @@ export function AppShell({ y: Number.isFinite(y) ? clampNumber(y, 0, 1) : 0.5, }; scrollAccumulatorRef.current = { - x: scrollAccumulatorRef.current.x + deltaX * NATIVE_SCROLL_DELTA_SCALE, - y: scrollAccumulatorRef.current.y + deltaY * NATIVE_SCROLL_DELTA_SCALE, + x: scrollAccumulatorRef.current.x + deltaX, + y: scrollAccumulatorRef.current.y + deltaY, }; if (scrollFrameRef.current === 0) { scrollFrameRef.current = window.requestAnimationFrame(flushScrollWheel); } } - function sendScrollWheelNow( + function sendWheelTouchScrollNow( deltaX: number, deltaY: number, x: number, @@ -3186,17 +3291,61 @@ export function AppShell({ ) { return; } + + const udid = selectedSimulator.udid; + const anchor = { + x: clampNumber(x, 0, 1), + y: clampNumber(y, 0, 1), + }; + const step = wheelDeltaToTouchStep(deltaX, deltaY); + if (step.x === 0 && step.y === 0) { + return; + } + + if (wheelTouchScrollRef.current?.udid !== udid) { + endWheelTouchScroll("cancelled"); + } + + if (!wheelTouchScrollRef.current) { + beginWheelTouchScroll(udid, anchor, step); + } + + let active = wheelTouchScrollRef.current; + if (!active) { + return; + } + + let next = clampWheelTouchPoint({ + x: active.current.x + step.x, + y: active.current.y + step.y, + }); if ( - !sendControl(selectedSimulator.udid, { - type: "scroll", - deltaX, - deltaY, - x: clampNumber(x, 0, 1), - y: clampNumber(y, 0, 1), + Math.hypot(next.x - active.current.x, next.y - active.current.y) < 0.001 + ) { + endWheelTouchScroll("ended"); + beginWheelTouchScroll(udid, anchor, step); + active = wheelTouchScrollRef.current; + if (!active) { + return; + } + next = clampWheelTouchPoint({ + x: active.current.x + step.x, + y: active.current.y + step.y, + }); + } + + active.current = next; + if ( + !sendControl(udid, { + type: "touch", + ...next, + phase: "moved", }) ) { setLocalError("Simulator control stream disconnected."); + return; } + scheduleWheelTouchScrollEnd(); } function prepareSimulatorInput() { diff --git a/packages/server/src/api/routes.rs b/packages/server/src/api/routes.rs index 156ab73e..22ad2306 100644 --- a/packages/server/src/api/routes.rs +++ b/packages/server/src/api/routes.rs @@ -561,10 +561,14 @@ pub(crate) enum ControlMessage { delta: f64, }, Scroll { - delta_x: f64, - delta_y: f64, - x: Option, - y: Option, + #[serde(rename = "deltaX")] + _delta_x: f64, + #[serde(rename = "deltaY")] + _delta_y: f64, + #[serde(rename = "x")] + _x: Option, + #[serde(rename = "y")] + _y: Option, }, DismissKeyboard, ToggleSoftwareKeyboard, @@ -2805,7 +2809,7 @@ async fn run_android_control_message( "Digital Crown rotation is only available for Apple Watch simulators.", )), ControlMessage::Scroll { .. } => Err(AppError::bad_request( - "Native scroll wheel input is only available for iOS simulators.", + "Native scroll wheel input is disabled because SimulatorKit scroll packets can destabilize iOS runtimes. Send touch drag input instead.", )), ControlMessage::ToggleAppearance => android.toggle_appearance(&udid), ControlMessage::Touch { .. } @@ -3071,17 +3075,9 @@ pub(crate) async fn run_control_message( } } ControlMessage::Crown { delta } => session.rotate_crown(delta), - ControlMessage::Scroll { - delta_x, - delta_y, - x, - y, - } => { - if !delta_x.is_finite() || !delta_y.is_finite() { - return Err(AppError::bad_request("Scroll deltas must be finite.")); - } - session.send_scroll(delta_x, delta_y, x.unwrap_or(0.5), y.unwrap_or(0.5)) - } + ControlMessage::Scroll { .. } => Err(AppError::bad_request( + "Native scroll wheel input is disabled because SimulatorKit scroll packets can destabilize iOS runtimes. Send touch drag input instead.", + )), ControlMessage::DismissKeyboard => session.send_key(41, 0), ControlMessage::ToggleSoftwareKeyboard => session.press_button("software-keyboard", 0), ControlMessage::Home => session.press_home(), diff --git a/packages/server/src/native/bridge.rs b/packages/server/src/native/bridge.rs index d4371608..22039229 100644 --- a/packages/server/src/native/bridge.rs +++ b/packages/server/src/native/bridge.rs @@ -1238,35 +1238,6 @@ impl NativeSession { } } - pub fn send_scroll( - &self, - delta_x: f64, - delta_y: f64, - normalized_x: f64, - normalized_y: f64, - ) -> Result<(), AppError> { - if !delta_x.is_finite() || !delta_y.is_finite() { - return Err(AppError::bad_request("Scroll deltas must be finite.")); - } - if !normalized_x.is_finite() || !normalized_y.is_finite() { - return Err(AppError::bad_request("Scroll coordinates must be finite.")); - } - unsafe { - let mut error = ptr::null_mut(); - bool_result( - ffi::xcw_native_session_send_scroll( - self.handle, - delta_x, - delta_y, - normalized_x, - normalized_y, - &mut error, - ), - error, - ) - } - } - pub fn open_app_switcher(&self) -> Result<(), AppError> { unsafe { let mut error = ptr::null_mut(); diff --git a/packages/server/src/native/ffi.rs b/packages/server/src/native/ffi.rs index 41486253..4fe9e724 100644 --- a/packages/server/src/native/ffi.rs +++ b/packages/server/src/native/ffi.rs @@ -321,14 +321,6 @@ unsafe extern "C" { delta: f64, error_message: *mut *mut c_char, ) -> bool; - pub fn xcw_native_session_send_scroll( - handle: *mut c_void, - delta_x: f64, - delta_y: f64, - normalized_x: f64, - normalized_y: f64, - error_message: *mut *mut c_char, - ) -> bool; pub fn xcw_native_session_open_app_switcher( handle: *mut c_void, error_message: *mut *mut c_char, diff --git a/packages/server/src/simulators/session.rs b/packages/server/src/simulators/session.rs index c81c59c1..9b424ed4 100644 --- a/packages/server/src/simulators/session.rs +++ b/packages/server/src/simulators/session.rs @@ -380,16 +380,6 @@ impl SimulatorSession { self.inner.native.rotate_crown(delta) } - pub fn send_scroll(&self, delta_x: f64, delta_y: f64, x: f64, y: f64) -> Result<(), AppError> { - self.mark_activity(); - if self.is_tvos() { - return Err(AppError::bad_request( - "tvOS simulators do not support direct scroll wheel input. Use Enter and arrow keys instead.", - )); - } - self.inner.native.send_scroll(delta_x, delta_y, x, y) - } - pub fn open_app_switcher(&self) -> Result<(), AppError> { self.mark_activity(); self.inner.native.open_app_switcher() diff --git a/packages/server/src/transport/webrtc.rs b/packages/server/src/transport/webrtc.rs index 607f02a6..1a311cce 100644 --- a/packages/server/src/transport/webrtc.rs +++ b/packages/server/src/transport/webrtc.rs @@ -828,7 +828,7 @@ async fn run_android_webrtc_control_message( "Digital Crown rotation is only available for Apple Watch simulators.", )), ControlMessage::Scroll { .. } => Err(AppError::bad_request( - "Native scroll wheel input is only available for iOS simulators.", + "Native scroll wheel input is disabled because SimulatorKit scroll packets can destabilize iOS runtimes. Send touch drag input instead.", )), ControlMessage::ToggleAppearance => state.android.toggle_appearance(&udid), ControlMessage::Touch { .. }