Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions docs/api/rest.md
Original file line number Diff line number Diff line change
Expand Up @@ -176,10 +176,11 @@ For normal clients, copy the browser behavior instead of hand-coding a raw decod

The input/control WebSocket accepts JSON control messages with camelCase fields,
including `touch`, `edgeTouch`, `multiTouch`, `key`, `button`, `crown`,
`scroll`, `home`, `appSwitcher`, and rotation controls. Native iOS scroll wheel
input uses `{ "type": "scroll", "deltaX": 0, "deltaY": 24, "x": 0.5, "y": 0.5 }`,
where `x` and `y` are optional normalized screen coordinates from `0.0` to
`1.0`. Touch-like messages use normalized screen coordinates too.
`home`, `appSwitcher`, and rotation controls. Use short `touch` drag sequences
for simulator scrolling. The legacy raw `scroll` control is parsed for
compatibility but rejected because SimulatorKit scroll packets can destabilize
iOS runtimes. Touch-like messages use normalized screen coordinates from `0.0`
to `1.0`.

Minimal WebRTC request:

Expand Down
185 changes: 167 additions & 18 deletions packages/client/src/app/AppShell.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,10 @@ const SAFARI_SCREEN_GESTURE_INITIAL_SPREAD = 0.28;
const SAFARI_SCREEN_GESTURE_MIN_SCALE = 0.25;
const SAFARI_SCREEN_GESTURE_MAX_SCALE = 4;
const REAL_MULTITOUCH_GESTURE_GRACE_MS = 120;
const NATIVE_SCROLL_DELTA_SCALE = 4;
const NATIVE_SCROLL_MAX_DELTA = 180;
const WHEEL_TOUCH_SCROLL_DELTA_SCALE = 0.0012;
const WHEEL_TOUCH_SCROLL_MAX_STEP = 0.18;
const WHEEL_TOUCH_SCROLL_EDGE_INSET = 0.08;
const WHEEL_TOUCH_SCROLL_IDLE_MS = 90;
const DEFAULT_ACCESSIBILITY_MAX_DEPTH = 10;
const LOGICAL_INSPECTOR_MAX_DEPTH = 80;
const FLUTTER_INSPECTOR_MAX_DEPTH = 48;
Expand Down Expand Up @@ -176,6 +178,11 @@ interface StreamQualityResponse {
videoCodec?: string;
}

interface WheelTouchScrollState {
udid: string;
current: Point;
}

function buildChromeUrl(
udid: string,
stamp: string,
Expand Down Expand Up @@ -613,6 +620,8 @@ export function AppShell({
const scrollAccumulatorRef = useRef<Point>({ x: 0, y: 0 });
const scrollFrameRef = useRef<number>(0);
const scrollPointRef = useRef<Point>({ x: 0.5, y: 0.5 });
const wheelTouchScrollRef = useRef<WheelTouchScrollState | null>(null);
const wheelTouchScrollEndTimeoutRef = useRef<number>(0);
const effectiveZoomRef = useRef(initialViewportState.zoom ?? 1);
const panRef = useRef<Point>(initialViewportState.pan);
const applyZoomAtClientPointRef = useRef<
Expand Down Expand Up @@ -661,6 +670,11 @@ export function AppShell({
window.cancelAnimationFrame(scrollFrameRef.current);
scrollFrameRef.current = 0;
}
if (wheelTouchScrollEndTimeoutRef.current !== 0) {
window.clearTimeout(wheelTouchScrollEndTimeoutRef.current);
wheelTouchScrollEndTimeoutRef.current = 0;
}
endWheelTouchScroll("cancelled");
};
}, []);

Expand Down Expand Up @@ -2613,7 +2627,7 @@ export function AppShell({
);
setAccessibilitySelectedId("");
setAccessibilityHoveredId(null);
sendScrollWheel(deltaX, deltaY, screenPoint.x, screenPoint.y);
sendWheelTouchScroll(deltaX, deltaY, screenPoint.x, screenPoint.y);
return true;
}

Expand Down Expand Up @@ -3110,6 +3124,97 @@ export function AppShell({
}
}

function clampWheelTouchPoint(point: Point): Point {
return {
x: clampNumber(
point.x,
WHEEL_TOUCH_SCROLL_EDGE_INSET,
1 - WHEEL_TOUCH_SCROLL_EDGE_INSET,
),
y: clampNumber(
point.y,
WHEEL_TOUCH_SCROLL_EDGE_INSET,
1 - WHEEL_TOUCH_SCROLL_EDGE_INSET,
),
};
}

function wheelTouchStartPoint(anchor: Point, step: Point): Point {
const clampedAnchor = clampWheelTouchPoint(anchor);
const maxX = 1 - WHEEL_TOUCH_SCROLL_EDGE_INSET;
const maxY = 1 - WHEEL_TOUCH_SCROLL_EDGE_INSET;
let x = clampedAnchor.x;
let y = clampedAnchor.y;

if (step.x > 0) {
x = Math.min(x, maxX - Math.abs(step.x));
} else if (step.x < 0) {
x = Math.max(x, WHEEL_TOUCH_SCROLL_EDGE_INSET + Math.abs(step.x));
}

if (step.y > 0) {
y = Math.min(y, maxY - Math.abs(step.y));
} else if (step.y < 0) {
y = Math.max(y, WHEEL_TOUCH_SCROLL_EDGE_INSET + Math.abs(step.y));
}

return clampWheelTouchPoint({ x, y });
}

function wheelDeltaToTouchStep(deltaX: number, deltaY: number): Point {
return {
x: clampNumber(
-deltaX * WHEEL_TOUCH_SCROLL_DELTA_SCALE,
-WHEEL_TOUCH_SCROLL_MAX_STEP,
WHEEL_TOUCH_SCROLL_MAX_STEP,
),
y: clampNumber(
-deltaY * WHEEL_TOUCH_SCROLL_DELTA_SCALE,
-WHEEL_TOUCH_SCROLL_MAX_STEP,
WHEEL_TOUCH_SCROLL_MAX_STEP,
),
};
}

function endWheelTouchScroll(phase: "ended" | "cancelled" = "ended") {
if (wheelTouchScrollEndTimeoutRef.current !== 0) {
window.clearTimeout(wheelTouchScrollEndTimeoutRef.current);
wheelTouchScrollEndTimeoutRef.current = 0;
}

const active = wheelTouchScrollRef.current;
wheelTouchScrollRef.current = null;
if (!active) {
return;
}

sendControl(active.udid, {
type: "touch",
...active.current,
phase,
});
}

function scheduleWheelTouchScrollEnd() {
if (wheelTouchScrollEndTimeoutRef.current !== 0) {
window.clearTimeout(wheelTouchScrollEndTimeoutRef.current);
}
wheelTouchScrollEndTimeoutRef.current = window.setTimeout(() => {
wheelTouchScrollEndTimeoutRef.current = 0;
endWheelTouchScroll("ended");
}, WHEEL_TOUCH_SCROLL_IDLE_MS);
}

function beginWheelTouchScroll(udid: string, anchor: Point, step: Point) {
const start = wheelTouchStartPoint(anchor, step);
wheelTouchScrollRef.current = { udid, current: start };
sendControl(udid, {
type: "touch",
...start,
phase: "began",
});
}

function flushScrollWheel() {
scrollFrameRef.current = 0;
const accumulated = scrollAccumulatorRef.current;
Expand All @@ -3120,13 +3225,13 @@ export function AppShell({

const deltaX = clampNumber(
accumulated.x,
-NATIVE_SCROLL_MAX_DELTA,
NATIVE_SCROLL_MAX_DELTA,
-WHEEL_TOUCH_SCROLL_MAX_STEP / WHEEL_TOUCH_SCROLL_DELTA_SCALE,
WHEEL_TOUCH_SCROLL_MAX_STEP / WHEEL_TOUCH_SCROLL_DELTA_SCALE,
);
const deltaY = clampNumber(
accumulated.y,
-NATIVE_SCROLL_MAX_DELTA,
NATIVE_SCROLL_MAX_DELTA,
-WHEEL_TOUCH_SCROLL_MAX_STEP / WHEEL_TOUCH_SCROLL_DELTA_SCALE,
WHEEL_TOUCH_SCROLL_MAX_STEP / WHEEL_TOUCH_SCROLL_DELTA_SCALE,
);
const remainingX = accumulated.x - deltaX;
const remainingY = accumulated.y - deltaY;
Expand All @@ -3135,15 +3240,15 @@ export function AppShell({
scrollFrameRef.current = window.requestAnimationFrame(flushScrollWheel);
}

sendScrollWheelNow(
sendWheelTouchScrollNow(
deltaX,
deltaY,
scrollPointRef.current.x,
scrollPointRef.current.y,
);
}

function sendScrollWheel(
function sendWheelTouchScroll(
deltaX: number,
deltaY: number,
x: number,
Expand All @@ -3162,15 +3267,15 @@ export function AppShell({
y: Number.isFinite(y) ? clampNumber(y, 0, 1) : 0.5,
};
scrollAccumulatorRef.current = {
x: scrollAccumulatorRef.current.x + deltaX * NATIVE_SCROLL_DELTA_SCALE,
y: scrollAccumulatorRef.current.y + deltaY * NATIVE_SCROLL_DELTA_SCALE,
x: scrollAccumulatorRef.current.x + deltaX,
y: scrollAccumulatorRef.current.y + deltaY,
};
if (scrollFrameRef.current === 0) {
scrollFrameRef.current = window.requestAnimationFrame(flushScrollWheel);
}
}

function sendScrollWheelNow(
function sendWheelTouchScrollNow(
deltaX: number,
deltaY: number,
x: number,
Expand All @@ -3186,17 +3291,61 @@ export function AppShell({
) {
return;
}

const udid = selectedSimulator.udid;
const anchor = {
x: clampNumber(x, 0, 1),
y: clampNumber(y, 0, 1),
};
const step = wheelDeltaToTouchStep(deltaX, deltaY);
if (step.x === 0 && step.y === 0) {
return;
}

if (wheelTouchScrollRef.current?.udid !== udid) {
endWheelTouchScroll("cancelled");
}

if (!wheelTouchScrollRef.current) {
beginWheelTouchScroll(udid, anchor, step);
}

let active = wheelTouchScrollRef.current;
if (!active) {
return;
}

let next = clampWheelTouchPoint({
x: active.current.x + step.x,
y: active.current.y + step.y,
});
if (
!sendControl(selectedSimulator.udid, {
type: "scroll",
deltaX,
deltaY,
x: clampNumber(x, 0, 1),
y: clampNumber(y, 0, 1),
Math.hypot(next.x - active.current.x, next.y - active.current.y) < 0.001
) {
endWheelTouchScroll("ended");
beginWheelTouchScroll(udid, anchor, step);
active = wheelTouchScrollRef.current;
if (!active) {
return;
}
next = clampWheelTouchPoint({
x: active.current.x + step.x,
y: active.current.y + step.y,
});
}

active.current = next;
if (
!sendControl(udid, {
type: "touch",
...next,
phase: "moved",
})
) {
setLocalError("Simulator control stream disconnected.");
return;
}
scheduleWheelTouchScrollEnd();
}

function prepareSimulatorInput() {
Expand Down
28 changes: 12 additions & 16 deletions packages/server/src/api/routes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -561,10 +561,14 @@ pub(crate) enum ControlMessage {
delta: f64,
},
Scroll {
delta_x: f64,
delta_y: f64,
x: Option<f64>,
y: Option<f64>,
#[serde(rename = "deltaX")]
_delta_x: f64,
#[serde(rename = "deltaY")]
_delta_y: f64,
#[serde(rename = "x")]
_x: Option<f64>,
#[serde(rename = "y")]
_y: Option<f64>,
},
DismissKeyboard,
ToggleSoftwareKeyboard,
Expand Down Expand Up @@ -2805,7 +2809,7 @@ async fn run_android_control_message(
"Digital Crown rotation is only available for Apple Watch simulators.",
)),
ControlMessage::Scroll { .. } => Err(AppError::bad_request(
"Native scroll wheel input is only available for iOS simulators.",
"Native scroll wheel input is disabled because SimulatorKit scroll packets can destabilize iOS runtimes. Send touch drag input instead.",
)),
ControlMessage::ToggleAppearance => android.toggle_appearance(&udid),
ControlMessage::Touch { .. }
Expand Down Expand Up @@ -3071,17 +3075,9 @@ pub(crate) async fn run_control_message(
}
}
ControlMessage::Crown { delta } => session.rotate_crown(delta),
ControlMessage::Scroll {
delta_x,
delta_y,
x,
y,
} => {
if !delta_x.is_finite() || !delta_y.is_finite() {
return Err(AppError::bad_request("Scroll deltas must be finite."));
}
session.send_scroll(delta_x, delta_y, x.unwrap_or(0.5), y.unwrap_or(0.5))
}
ControlMessage::Scroll { .. } => Err(AppError::bad_request(
"Native scroll wheel input is disabled because SimulatorKit scroll packets can destabilize iOS runtimes. Send touch drag input instead.",
)),
ControlMessage::DismissKeyboard => session.send_key(41, 0),
ControlMessage::ToggleSoftwareKeyboard => session.press_button("software-keyboard", 0),
ControlMessage::Home => session.press_home(),
Expand Down
29 changes: 0 additions & 29 deletions packages/server/src/native/bridge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1238,35 +1238,6 @@ impl NativeSession {
}
}

pub fn send_scroll(
&self,
delta_x: f64,
delta_y: f64,
normalized_x: f64,
normalized_y: f64,
) -> Result<(), AppError> {
if !delta_x.is_finite() || !delta_y.is_finite() {
return Err(AppError::bad_request("Scroll deltas must be finite."));
}
if !normalized_x.is_finite() || !normalized_y.is_finite() {
return Err(AppError::bad_request("Scroll coordinates must be finite."));
}
unsafe {
let mut error = ptr::null_mut();
bool_result(
ffi::xcw_native_session_send_scroll(
self.handle,
delta_x,
delta_y,
normalized_x,
normalized_y,
&mut error,
),
error,
)
}
}

pub fn open_app_switcher(&self) -> Result<(), AppError> {
unsafe {
let mut error = ptr::null_mut();
Expand Down
8 changes: 0 additions & 8 deletions packages/server/src/native/ffi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -321,14 +321,6 @@ unsafe extern "C" {
delta: f64,
error_message: *mut *mut c_char,
) -> bool;
pub fn xcw_native_session_send_scroll(
handle: *mut c_void,
delta_x: f64,
delta_y: f64,
normalized_x: f64,
normalized_y: f64,
error_message: *mut *mut c_char,
) -> bool;
pub fn xcw_native_session_open_app_switcher(
handle: *mut c_void,
error_message: *mut *mut c_char,
Expand Down
Loading
Loading