From 19030b3a590ebe8edfe908f63168e84a7b79865c Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Wed, 23 Sep 2026 12:04:11 +0100 Subject: [PATCH 1/5] Add switch-scanned mouse panel --- src-tauri/src/lib.rs | 1 + src-tauri/src/point_scan.rs | 21 +- src-tauri/src/point_scan_runtime.rs | 63 +++++- src-tauri/src/point_workflow.rs | 294 +++++++++++++++++++++++++++- src-tauri/src/scan_executor.rs | 80 +++++++- src-tauri/src/scan_menu.rs | 5 +- src-tauri/src/scan_mouse.rs | 276 ++++++++++++++++++++++++++ src-tauri/src/scan_preferences.rs | 5 + src-tauri/src/scan_tile.rs | 4 +- src-tauri/src/scanning.rs | 8 +- src-tauri/src/scanning_runtime.rs | 16 +- src-tauri/src/switch_gestures.rs | 13 +- src-tauri/src/switches.rs | 4 +- src/scanning/preferences.ts | 9 +- src/scanning/useScanning.ts | 2 +- src/scanning/useSwitches.ts | 1 + src/settings/ScannerPreferences.tsx | 12 +- src/settings/ScanningSection.tsx | 3 + 18 files changed, 787 insertions(+), 30 deletions(-) create mode 100644 src-tauri/src/scan_mouse.rs diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 90f5f26f..8f357745 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -28,6 +28,7 @@ mod scan_host; mod scan_items; mod scan_keyboard; mod scan_menu; +mod scan_mouse; mod scan_preferences; mod scan_tile; mod scan_tree; diff --git a/src-tauri/src/point_scan.rs b/src-tauri/src/point_scan.rs index 9a119e61..612ba17c 100644 --- a/src-tauri/src/point_scan.rs +++ b/src-tauri/src/point_scan.rs @@ -60,6 +60,7 @@ impl Config { crate::scan_preferences::Area::Point, crate::scan_preferences::Area::Menu, crate::scan_preferences::Area::Keyboard, + crate::scan_preferences::Area::Mouse, ] .into_iter() .all(|area| self.resolved(area).automatic), @@ -73,18 +74,27 @@ impl Config { &self, area: crate::scan_preferences::Area, ) -> crate::scan_preferences::Resolved { - self.scan_preferences.resolve( + let mut resolved = self.scan_preferences.resolve( area, self.automatic, self.block_interval_ms, self.scanner_color, - ) + ); + if area == crate::scan_preferences::Area::Mouse + && self.scan_preferences.mouse.automatic.is_none() + { + resolved.automatic = self + .resolved(crate::scan_preferences::Area::Keyboard) + .automatic; + } + resolved } pub fn point(&self) -> PointSettings { PointSettings { scan: self.resolved(crate::scan_preferences::Area::Point), menu_scan: self.resolved(crate::scan_preferences::Area::Menu), keyboard_scan: self.resolved(crate::scan_preferences::Area::Keyboard), + mouse_scan: self.resolved(crate::scan_preferences::Area::Mouse), word_prediction: self.word_prediction, keyboard_wait_after_typing: self.keyboard_wait_after_typing, scanner_color: self.resolved(crate::scan_preferences::Area::Point).color, @@ -108,6 +118,7 @@ pub struct PointSettings { pub scan: crate::scan_preferences::Resolved, pub menu_scan: crate::scan_preferences::Resolved, pub keyboard_scan: crate::scan_preferences::Resolved, + pub mouse_scan: crate::scan_preferences::Resolved, pub word_prediction: bool, pub keyboard_wait_after_typing: bool, pub scanner_color: crate::scanning::ScannerColor, @@ -269,7 +280,11 @@ impl Engine { self.step(TICK_MS); } Action::Reverse => self.direction = -self.direction, - Action::Pause | Action::Stop | Action::Cancel | Action::OpenKeyboard => {} + Action::Pause + | Action::Stop + | Action::Cancel + | Action::OpenKeyboard + | Action::OpenMouse => {} } None } diff --git a/src-tauri/src/point_scan_runtime.rs b/src-tauri/src/point_scan_runtime.rs index 812e3607..2bd24fce 100644 --- a/src-tauri/src/point_scan_runtime.rs +++ b/src-tauri/src/point_scan_runtime.rs @@ -72,10 +72,42 @@ impl Adapter for PointScan { crate::prediction::record(stroke, result.is_ok(), scope); return result.map(|()| None); } - Request::OpenKeyboard => { + Request::OpenKeyboard | Request::OpenMouse => { crate::prediction::stop(); crate::scan_executor::activate(request) } + Request::MouseDrag => { + let (position, _) = display_navigation::displays(app).map_err(|e| e.message)?; + return crate::scan_executor::toggle_mouse_drag(( + position.0.round() as i32, + position.1.round() as i32, + )) + .map(|()| None); + } + Request::MouseSpeed(direction) => { + let model = app.state::(); + let old = model.snapshot().settings.pointer_scale_percent; + let next = (i16::from(old) + i16::from(direction) * 5).clamp(5, 225) as u8; + return model.apply_pointer_scale_percent(next).map(|()| None); + } + Request::MouseMonitor(dx, dy) => { + let (cursor, displays) = + display_navigation::displays(app).map_err(|e| e.message)?; + let source = display_navigation::current_display(cursor, &displays) + .ok_or("No scanning display is available.")?; + let direction = match (dx, dy) { + (-1, 0) => "left", + (1, 0) => "right", + (0, -1) => "up", + (0, 1) => "down", + _ => return Err("Invalid monitor direction.".into()), + }; + let target = display_navigation::target_center(source, &displays, direction) + .map_err(|e| e.message)?; + crate::scan_executor::move_to(target)?; + let (_, environment) = new_engine(app, app.state::().view().config)?; + return Ok(Some(environment)); + } Request::Setting(setting) => scanning_runtime::update_point_setting(app, setting), Request::Display(next) => scanning_runtime::restart_point_on_display(app, next), request => crate::scan_executor::activate(request), @@ -88,7 +120,29 @@ impl Adapter for PointScan { technique: Option<&mut Workflow>, ) -> Result { if let (Some(environment), Some(technique)) = (environment, technique) { - if technique.keyboard_open() { + if technique.mouse_open() { + let (cursor, displays) = + display_navigation::displays(app).map_err(|e| e.message)?; + let current = display_navigation::current_display(cursor, &displays) + .ok_or("No scanning display is available.")? + .clone(); + let rect = Rect { + x: current.x.into(), + y: current.y.into(), + width: current.width.into(), + height: current.height.into(), + }; + let area = crate::scan_host::work_area(rect)?; + environment.display = current; + environment.keyboard_area = area; + environment.foreground = crate::scan_host::foreground()?; + technique.set_mouse_area(area, displays.len()); + let settings = app.state::().snapshot().settings; + technique.set_mouse_settings( + settings.pointer_scale_percent, + settings.mouse_repeat_acceleration_duration_ms, + ); + } else if technique.keyboard_open() { crate::point_scan_ready(app)?; let foreground = crate::scan_host::foreground()?; if environment.foreground != foreground { @@ -102,7 +156,10 @@ impl Adapter for PointScan { Ok(true) } fn preserve_visuals(request: &Request) -> bool { - matches!(request, Request::Keyboard(_) | Request::Prediction { .. }) + matches!( + request, + Request::Keyboard(_) | Request::Prediction { .. } | Request::MouseMove { .. } + ) } fn deferred(request: &Request) -> bool { matches!(request, Request::Prediction { .. }) diff --git a/src-tauri/src/point_workflow.rs b/src-tauri/src/point_workflow.rs index 456c0192..0a58a328 100644 --- a/src-tauri/src/point_workflow.rs +++ b/src-tauri/src/point_workflow.rs @@ -13,7 +13,23 @@ pub enum Request { index: usize, }, OpenKeyboard, + OpenMouse, + CloseMouse, Keyboard(crate::scan_keyboard::Stroke), + MouseMove { + dx: i32, + dy: i32, + }, + MouseClick { + right: bool, + count: u8, + }, + MouseScroll { + dy: i32, + }, + MouseDrag, + MouseSpeed(i8), + MouseMonitor(i8, i8), Click { point: Point, right: bool, @@ -61,6 +77,9 @@ pub enum WorkflowPhase { Keyboard, KeyboardSuspended, KeyboardOpening, + Mouse, + MouseSuspended, + MouseMoving, Menu, MenuSuspended, AutoSelecting, @@ -72,6 +91,8 @@ pub enum WorkflowPhase { enum Stage { Keyboard, KeyboardOpening, + Mouse, + MouseMoving, Idle, Countdown, Point, @@ -82,6 +103,13 @@ enum Stage { pub struct Workflow { keyboard: crate::scan_keyboard::Keyboard, keyboard_area: Rect, + mouse: crate::scan_mouse::MousePanel, + mouse_area: Rect, + return_to_mouse: bool, + move_direction: (i8, i8), + move_elapsed: u64, + move_residual: (f64, f64), + mouse_acceleration_ms: u32, point: Engine, stage: Stage, menu: Menu, @@ -96,6 +124,7 @@ impl Workflow { pub fn new(config: PointSettings, screen: Rect, scale: f64) -> Result { let menu_options = config.menu_scan; let keyboard_options = config.keyboard_scan; + let mouse_options = config.mouse_scan; Ok(Self { keyboard: crate::scan_keyboard::Keyboard::configured( cfg!(target_os = "macos"), @@ -103,6 +132,13 @@ impl Workflow { ) .with_wait_after_typing(config.keyboard_wait_after_typing), keyboard_area: screen, + mouse: crate::scan_mouse::MousePanel::new(mouse_options, 1, 100), + mouse_area: screen, + return_to_mouse: false, + move_direction: (0, 0), + move_elapsed: 0, + move_residual: (0.0, 0.0), + mouse_acceleration_ms: 1000, point: Engine::new(config, screen, scale)?, stage: Stage::Idle, menu: Menu::configured(Kind::Actions, menu_options), @@ -117,6 +153,7 @@ impl Workflow { fn active_preferences(&self) -> crate::scan_preferences::Resolved { match self.stage { Stage::Keyboard | Stage::KeyboardOpening => self.point.config.keyboard_scan, + Stage::Mouse | Stage::MouseMoving => self.point.config.mouse_scan, Stage::Menu => self.point.config.menu_scan, _ => self.point.config.scan, } @@ -139,6 +176,12 @@ impl Workflow { self.point.config.keyboard_scan, ) .with_wait_after_typing(self.point.config.keyboard_wait_after_typing); + self.mouse = crate::scan_mouse::MousePanel::new( + self.point.config.mouse_scan, + 1, + self.mouse.speed_percent, + ); + self.return_to_mouse = false; self.parent_menu.clear(); self.stage = Stage::Point; self.point.start(); @@ -156,6 +199,17 @@ impl Workflow { pub fn keyboard_open(&self) -> bool { matches!(self.stage, Stage::Keyboard | Stage::KeyboardOpening) } + pub fn mouse_open(&self) -> bool { + matches!(self.stage, Stage::Mouse | Stage::MouseMoving) + } + pub fn set_mouse_area(&mut self, area: Rect, displays: usize) { + self.mouse_area = area; + self.mouse.set_displays(displays); + } + pub fn set_mouse_settings(&mut self, speed: u8, acceleration_ms: u32) { + self.mouse.speed_percent = speed; + self.mouse_acceleration_ms = acceleration_ms; + } pub fn foreground_changed(&mut self) { self.keyboard.reset_context(); } @@ -178,9 +232,86 @@ impl Workflow { self.open(Kind::Actions); } } + fn open_mouse(&mut self) -> Option { + self.stage = Stage::Mouse; + self.mouse = crate::scan_mouse::MousePanel::new( + self.point.config.mouse_scan, + 1, + self.mouse.speed_percent, + ); + self.pending = None; + self.parent_menu.clear(); + self.elapsed = 0; + self.error = None; + self.return_to_mouse = false; + Some(Request::OpenMouse) + } + fn mouse_key(&mut self, key: crate::scan_mouse::Key) -> Option { + use crate::scan_mouse::Key; + self.mouse.choose(key); + match key { + Key::Move(dx, dy) => { + self.move_direction = (dx, dy); + self.move_elapsed = 0; + self.move_residual = (0.0, 0.0); + self.stage = Stage::MouseMoving; + None + } + Key::Click => Some(Request::MouseClick { + right: false, + count: 1, + }), + Key::RightClick => Some(Request::MouseClick { + right: true, + count: 1, + }), + Key::DoubleClick => Some(Request::MouseClick { + right: false, + count: 2, + }), + Key::Drag => { + self.mouse.dragging = !self.mouse.dragging; + Some(Request::MouseDrag) + } + Key::Scroll(direction) => Some(Request::MouseScroll { + dy: i32::from(direction) * 5, + }), + Key::Speed(direction) => Some(Request::MouseSpeed(direction)), + Key::Monitor(dx, dy) => Some(Request::MouseMonitor(dx, dy)), + Key::Keyboard => { + self.return_to_mouse = true; + self.mouse.dragging = false; + self.stage = Stage::KeyboardOpening; + self.keyboard = crate::scan_keyboard::Keyboard::configured( + cfg!(target_os = "macos"), + self.point.config.keyboard_scan, + ) + .with_wait_after_typing(self.point.config.keyboard_wait_after_typing); + Some(Request::OpenKeyboard) + } + Key::Close => { + self.reset(); + Some(Request::CloseMouse) + } + Key::More | Key::Movement | Key::Dock => None, + } + } + fn keyboard_closed(&mut self) { + if self.return_to_mouse { + self.return_to_mouse = false; + self.stage = Stage::Mouse; + self.mouse.restart(); + } else { + self.start(); + } + } fn selected(&mut self, item: Item) -> Option { match item { Item::Keyboard => { + self.return_to_mouse = self.mouse_open(); + if self.return_to_mouse { + self.mouse.dragging = false; + } self.stage = Stage::KeyboardOpening; self.keyboard = crate::scan_keyboard::Keyboard::configured( cfg!(target_os = "macos"), @@ -194,6 +325,7 @@ impl Workflow { return Some(Request::OpenKeyboard); } Item::KeyboardKey => {} + Item::MousePanel => return self.open_mouse(), Item::More | Item::Group(_) => { let kind = if let Item::Group(kind) = item { kind @@ -283,6 +415,14 @@ impl Technique for Workflow { self.keyboard.failed(); return; } + if self.mouse_open() { + self.stage = Stage::Mouse; + self.mouse.dragging = false; + self.mouse.failed(); + self.error = None; + let _ = message; + return; + } self.pending = None; self.stage = Stage::Menu; self.menu = Menu::configured(Kind::Actions, self.point.config.menu_scan); @@ -308,6 +448,14 @@ impl Technique for Workflow { self.point.config.keyboard_scan, ) .with_wait_after_typing(self.point.config.keyboard_wait_after_typing); + self.mouse = crate::scan_mouse::MousePanel::new( + self.point.config.mouse_scan, + 1, + self.mouse.speed_percent, + ); + self.return_to_mouse = false; + self.move_direction = (0, 0); + self.move_residual = (0.0, 0.0); self.stage = Stage::Idle; self.parent_menu.clear(); self.pending = None; @@ -326,6 +474,16 @@ impl Technique for Workflow { fn automatic(&self, _fallback: bool) -> bool { self.active_preferences().automatic } + fn switch_pressed(&mut self) -> bool { + if self.stage != Stage::MouseMoving { + return false; + } + self.stage = Stage::Mouse; + self.move_direction = (0, 0); + self.move_residual = (0.0, 0.0); + self.mouse.restart(); + true + } fn auto_selecting(&self) -> bool { self.stage == Stage::Countdown } @@ -333,6 +491,12 @@ impl Technique for Workflow { self.stage != Stage::Executing } fn handle(&mut self, action: Action) -> Option { + if action == Action::OpenMouse { + if self.mouse_open() { + return None; + } + return self.open_mouse(); + } if action == Action::OpenKeyboard { if self.keyboard_open() { return None; @@ -347,10 +511,16 @@ impl Technique for Workflow { Some(crate::scan_keyboard::Output::Prediction { token, index }) => { return Some(Request::Prediction { token, index }) } - Some(crate::scan_keyboard::Output::Close) => self.start(), + Some(crate::scan_keyboard::Output::Close) => self.keyboard_closed(), None => {} }, Stage::KeyboardOpening => {} + Stage::Mouse => { + if let Some(key) = self.mouse.handle(action) { + return self.mouse_key(key); + } + } + Stage::MouseMoving => {} Stage::Point | Stage::Destination => { if let Some(point) = self.point.handle(action) { if self.stage == Stage::Destination { @@ -394,6 +564,9 @@ impl Technique for Workflow { Stage::Keyboard => self .keyboard .advance(ms, self.point.config.keyboard_scan.interval_ms), + Stage::Mouse => self + .mouse + .advance(ms, self.point.config.mouse_scan.interval_ms), Stage::Point | Stage::Destination => { self.point.advance(ms); if self.point.exhausted() { @@ -410,7 +583,33 @@ impl Technique for Workflow { } } fn update(&mut self, ms: u64, context: UpdateContext) { - if self.stage == Stage::Countdown { + if self.stage == Stage::MouseMoving { + let before = self.move_elapsed; + self.move_elapsed = self.move_elapsed.saturating_add(ms); + let acceleration = u64::from(self.mouse_acceleration_ms); + let ramp = if acceleration == 0 { + 1.0 + } else { + (self.move_elapsed.min(acceleration) as f64 / acceleration as f64) + .mul_add(0.75, 0.25) + }; + let seconds = self.move_elapsed.saturating_sub(before) as f64 / 1000.0; + let speed = 240.0 * f64::from(self.mouse.speed_percent) / 100.0 * ramp; + let diagonal = if self.move_direction.0 != 0 && self.move_direction.1 != 0 { + std::f64::consts::FRAC_1_SQRT_2 + } else { + 1.0 + }; + self.move_residual.0 += f64::from(self.move_direction.0) * speed * seconds * diagonal; + self.move_residual.1 += f64::from(self.move_direction.1) * speed * seconds * diagonal; + let dx = self.move_residual.0.trunc() as i32; + let dy = self.move_residual.1.trunc() as i32; + self.move_residual.0 -= f64::from(dx); + self.move_residual.1 -= f64::from(dy); + if dx != 0 || dy != 0 { + self.pending = Some(Request::MouseMove { dx, dy }); + } + } else if self.stage == Stage::Countdown { if !context.paused && !context.switch_held { self.elapsed = self .elapsed @@ -446,6 +645,12 @@ impl Technique for Workflow { fn phase(&self) -> Phase { match self.stage { Stage::KeyboardOpening => Phase::Workflow(WorkflowPhase::KeyboardOpening), + Stage::Mouse => Phase::Workflow(if self.mouse.suspended() { + WorkflowPhase::MouseSuspended + } else { + WorkflowPhase::Mouse + }), + Stage::MouseMoving => Phase::Workflow(WorkflowPhase::MouseMoving), Stage::Keyboard => Phase::Workflow(if self.keyboard.suspended() { WorkflowPhase::KeyboardSuspended } else { @@ -472,6 +677,22 @@ impl Technique for Workflow { self.point.units_per_logical_pixel, self.point.config.scanner_color, ), + Stage::Mouse | Stage::MouseMoving => { + let mut frame = self.mouse.frame( + self.mouse_area, + self.point.units_per_logical_pixel, + self.point.config.mouse_scan.color, + ); + if self.stage == Stage::MouseMoving { + if let Some(label) = frame.label.as_mut() { + label.text = "Moving pointer · Press any switch to stop".into(); + } + for tile in &mut frame.tiles { + tile.selected = false; + } + } + frame + } Stage::Countdown => { let scale = self.point.units_per_logical_pixel; let screen = self.point.screen; @@ -564,6 +785,73 @@ impl Technique for Workflow { mod tests { use super::*; use crate::{point_scan::Config, scanning::Session}; + + #[test] + fn mouse_motion_stops_on_press_and_returns_to_first_row() { + let screen = Rect { + x: 0.0, + y: 0.0, + width: 1280.0, + height: 720.0, + }; + let mut session = Session::new( + Workflow::new(Config::default().point(), screen, 1.0).unwrap(), + true, + ); + assert_eq!(session.action(Action::OpenMouse), Some(Request::OpenMouse)); + assert_eq!( + session.technique.phase(), + Phase::Workflow(WorkflowPhase::Mouse) + ); + session + .technique + .mouse_key(crate::scan_mouse::Key::Move(1, 0)); + session.tick(250, false); + assert!( + matches!(session.take_selection(), Some(Request::MouseMove { dx, dy: 0 }) if dx > 0) + ); + assert!(session.technique.switch_pressed()); + assert!(!session.technique.switch_pressed()); + assert_eq!( + session.technique.phase(), + Phase::Workflow(WorkflowPhase::Mouse) + ); + assert!(session + .technique + .mouse + .frame(screen, 1.0, Default::default()) + .tiles + .iter() + .skip(1) + .any(|tile| tile.selected)); + } + + #[test] + fn mouse_keyboard_return_and_drag_state() { + let screen = Rect { + x: 0.0, + y: 0.0, + width: 1280.0, + height: 720.0, + }; + let mut workflow = Workflow::new(Config::default().point(), screen, 1.0).unwrap(); + workflow.open_mouse(); + assert_eq!( + workflow.mouse_key(crate::scan_mouse::Key::Drag), + Some(Request::MouseDrag) + ); + assert!(workflow.mouse.dragging); + assert_eq!( + workflow.mouse_key(crate::scan_mouse::Key::Keyboard), + Some(Request::OpenKeyboard) + ); + assert!(!workflow.mouse.dragging); + workflow.execution_succeeded(); + assert_eq!(workflow.phase(), Phase::Workflow(WorkflowPhase::Keyboard)); + assert!(workflow.return_to_mouse); + workflow.keyboard_closed(); + assert_eq!(workflow.phase(), Phase::Workflow(WorkflowPhase::Mouse)); + } fn session(automatic: bool) -> Session { Session::new( Workflow::new( @@ -1029,7 +1317,7 @@ mod tests { for (column, right, count) in [(0, false, 1), (1, true, 1), (2, false, 2)] { let mut s = session(false); open(&mut s); - assert_eq!(s.frame().tiles.len(), 10); + assert_eq!(s.frame().tiles.len(), 11); assert_eq!( choose(&mut s, 0, column), Some(Request::Click { diff --git a/src-tauri/src/scan_executor.rs b/src-tauri/src/scan_executor.rs index e4916da7..1a4e017b 100644 --- a/src-tauri/src/scan_executor.rs +++ b/src-tauri/src/scan_executor.rs @@ -16,7 +16,32 @@ pub fn execute( } match request { Request::Prediction { .. } => Err("Prediction requires the scan controller.".into()), - Request::OpenKeyboard => input.release_all(), + Request::OpenKeyboard | Request::OpenMouse | Request::CloseMouse => input.release_all(), + Request::MouseMove { dx, dy } => input.move_pointer_pixels(dx, dy).map(|_| ()), + Request::MouseClick { right, count } => { + if input.has_active_drag() { + return Err("End the drag before clicking.".into()); + } + input.click_pointer( + if right { + MouseButton::Right + } else { + MouseButton::Left + }, + count, + ) + } + Request::MouseScroll { dy } => input.execute_repeat_scroll(0, dy).map(|_| ()), + Request::MouseDrag => { + if input.has_active_drag() { + input.release_all() + } else { + Err("Mouse drag requires the scan controller.".into()) + } + } + Request::MouseSpeed(_) | Request::MouseMonitor(..) => { + Err("Mouse action requires the scan controller.".into()) + } Request::Keyboard(stroke) => { if input.has_active_drag() { return Err("End the active drag before typing.".into()); @@ -103,6 +128,33 @@ pub fn activate(request: Request) -> Result<(), String> { result }) } +pub fn toggle_mouse_drag(point: (i32, i32)) -> Result<(), String> { + if !crate::scan_host::modifiers_released() { + return Err("Release modifiers before dragging.".into()); + } + INPUT.with(|slot| { + let mut slot = slot.borrow_mut(); + if slot.is_none() { + *slot = Some(DesktopInput::new( + enigo::Enigo::new(&crate::input::injection_settings()) + .map_err(|_| "Scan input could not be initialized.")?, + )); + } + let input = slot.as_mut().unwrap(); + if input.has_active_switch_session() { + return Err("End switch forwarding before dragging.".into()); + } + let result = if input.has_active_drag() { + input.release_all() + } else { + input.start_scan_drag(point) + }; + if result.is_err() { + let _ = input.release_all(); + } + result + }) +} pub fn prediction_text(text: &str) -> Result<(), String> { if text.chars().count() > 64 || !crate::scan_host::modifiers_released() { return Err("Prediction input is unavailable.".into()); @@ -273,8 +325,9 @@ mod tests { self.events.push(format!("text {text}")); Ok(()) } - fn move_pointer(&mut self, _: i32, _: i32) -> Result<(), String> { - panic!("No relative movement") + fn move_pointer(&mut self, dx: i32, dy: i32) -> Result<(), String> { + self.events.push(format!("relative {dx} {dy}")); + Ok(()) } fn move_pointer_absolute(&mut self, x: i32, y: i32) -> Result<(), String> { self.events.push(format!("move {x} {y}")); @@ -362,6 +415,27 @@ mod tests { assert!(!input.has_active_drag()); } #[test] + fn mouse_actions_use_fake_input_and_close_releases_drag() { + let mut input = DesktopInput::new(Fake::default()); + execute(&mut input, Request::MouseMove { dx: 4, dy: -2 }, true).unwrap(); + execute( + &mut input, + Request::MouseClick { + right: false, + count: 2, + }, + true, + ) + .unwrap(); + execute(&mut input, Request::MouseScroll { dy: 5 }, true).unwrap(); + execute(&mut input, Request::DragStart((10, 20)), true).unwrap(); + execute(&mut input, Request::CloseMouse, true).unwrap(); + assert!(!input.has_active_drag()); + assert!(input.injector.events.contains(&"relative 4 -2".into())); + assert!(input.injector.events.contains(&"scroll 0 5".into())); + assert!(input.injector.events.contains(&"button Left false".into())); + } + #[test] fn keyboard_text_chords_and_current_focus_use_fake_input() { use crate::scan_keyboard::{Key, Stroke}; let mut input = DesktopInput::new(Fake::default()); diff --git a/src-tauri/src/scan_menu.rs b/src-tauri/src/scan_menu.rs index 33e88aab..bde47754 100644 --- a/src-tauri/src/scan_menu.rs +++ b/src-tauri/src/scan_menu.rs @@ -6,6 +6,7 @@ use crate::{ #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Item { Keyboard, + MousePanel, KeyboardKey, More, Group(Kind), @@ -34,6 +35,7 @@ impl Item { pub fn label(self) -> &'static str { match self { Self::Keyboard => "Keyboard", + Self::MousePanel => "Mouse", Self::KeyboardKey => "Key", Self::More => "More", Self::Group(kind) => kind.label(), @@ -434,7 +436,7 @@ impl Kind { return vec![ vec![LeftClick, RightClick, DoubleClick], vec![Scroll, Drag, More], - vec![Keyboard], + vec![Keyboard, MousePanel], vec![NewPoint, Cancel], ] } @@ -444,6 +446,7 @@ impl Kind { } Self::More => vec![ Keyboard, + MousePanel, Group(Self::Mouse), Group(Self::Editing), Group(Self::Windows), diff --git a/src-tauri/src/scan_mouse.rs b/src-tauri/src/scan_mouse.rs new file mode 100644 index 00000000..338d9bef --- /dev/null +++ b/src-tauri/src/scan_mouse.rs @@ -0,0 +1,276 @@ +//! Switch-owned mouse panel. Native input stays in the scanning adapter. +use crate::{ + scan_items::{ItemScanner, Policy}, + scan_menu::Item, + scan_preferences::Resolved, + scanning::{Action, Frame, FrameLabel, FrameTile, Rect, ScannerColor, TileRole, TileStyle}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Key { + Move(i8, i8), + Click, + RightClick, + DoubleClick, + Drag, + Scroll(i8), + Speed(i8), + Monitor(i8, i8), + More, + Movement, + Keyboard, + Dock, + Close, +} + +pub struct MousePanel { + pub more: bool, + pub top: bool, + pub dragging: bool, + pub speed_percent: u8, + pub error: bool, + displays: usize, + rows: Vec>, + scan: ItemScanner, +} + +impl MousePanel { + pub fn new(options: Resolved, displays: usize, speed_percent: u8) -> Self { + let rows = Self::rows(false, displays); + let scan = ItemScanner::configured_rows(&rows, Policy::KEYBOARD, options); + Self { + more: false, + top: false, + dragging: false, + speed_percent, + error: false, + displays, + rows, + scan, + } + } + fn rows(more: bool, displays: usize) -> Vec> { + use Key::*; + if !more { + vec![ + vec![Move(-1, -1), Move(0, -1), Move(1, -1)], + vec![Move(-1, 0), Click, Move(1, 0)], + vec![Move(-1, 1), Move(0, 1), Move(1, 1)], + vec![RightClick, DoubleClick, Drag], + vec![More, Keyboard, Dock, Close], + ] + } else { + let mut rows = vec![vec![Scroll(1), Scroll(-1)], vec![Speed(-1), Speed(1)]]; + if displays > 1 { + rows.push(vec![ + Monitor(-1, 0), + Monitor(0, -1), + Monitor(0, 1), + Monitor(1, 0), + ]); + } + rows.push(vec![Movement, Keyboard, Dock, Close]); + rows + } + } + pub fn choose(&mut self, key: Key) { + match key { + Key::More | Key::Movement => { + self.more = key == Key::More; + self.rows = Self::rows(self.more, self.displays); + self.scan = + ItemScanner::configured_rows(&self.rows, Policy::KEYBOARD, self.scan.options); + } + Key::Dock => { + self.top = !self.top; + self.scan.restart(); + } + _ => self.scan.restart(), + } + } + pub fn handle(&mut self, action: Action) -> Option { + if self.scan.suspended { + if action == Action::Select { + self.error = false; + self.scan.restart(); + } + return None; + } + self.scan.handle(action) + } + pub fn advance(&mut self, ms: u64, period: u64) { + self.scan.advance(ms, period); + } + pub fn restart(&mut self) { + self.scan.restart(); + } + pub fn suspended(&self) -> bool { + self.scan.suspended + } + pub fn failed(&mut self) { + self.error = true; + self.scan.suspended = true; + } + pub fn set_displays(&mut self, displays: usize) { + if self.displays != displays { + self.displays = displays; + self.rows = Self::rows(self.more, displays); + self.scan = + ItemScanner::configured_rows(&self.rows, Policy::KEYBOARD, self.scan.options); + } + } + fn label(&self, key: Key) -> String { + use Key::*; + match key { + Move(-1, -1) => "↖".into(), + Move(0, -1) => "↑".into(), + Move(1, -1) => "↗".into(), + Move(-1, 0) => "←".into(), + Move(1, 0) => "→".into(), + Move(-1, 1) => "↙".into(), + Move(0, 1) => "↓".into(), + Move(1, 1) => "↘".into(), + Move(..) => "Move".into(), + Click => "Left click".into(), + RightClick => "Right click".into(), + DoubleClick => "Double click".into(), + Drag => if self.dragging { + "End drag" + } else { + "Start drag" + } + .into(), + Scroll(1) => "Scroll up".into(), + Scroll(_) => "Scroll down".into(), + Speed(-1) => format!("Slower · {}%", self.speed_percent), + Speed(_) => format!("Faster · {}%", self.speed_percent), + Monitor(-1, 0) => "Monitor left".into(), + Monitor(0, -1) => "Monitor up".into(), + Monitor(0, 1) => "Monitor down".into(), + Monitor(1, 0) => "Monitor right".into(), + Monitor(..) => "Monitor".into(), + More => "More controls".into(), + Movement => "Movement".into(), + Keyboard => "Keyboard".into(), + Dock => if self.top { "Dock bottom" } else { "Dock top" }.into(), + Close => "Close mouse".into(), + } + } + pub fn frame(&self, screen: Rect, units: f64, color: ScannerColor) -> Frame { + let columns = self.rows.iter().map(Vec::len).max().unwrap_or(1) as f64; + let width = (columns * 160.0 + 32.0) + .min(screen.width / units.max(0.1) - 8.0) + .max(1.0) + * units; + let height = (self.rows.len() as f64 * 98.0 + 52.0) + .min(screen.height / units.max(0.1) * 0.65) + .max(1.0) + * units; + let x = screen.x + (screen.width - width) / 2.0; + let y = if self.top { + screen.y + } else { + screen.y + screen.height - height + }; + let scale = units + .min(width / (columns * 160.0 + 32.0)) + .min(height / (self.rows.len() as f64 * 98.0 + 52.0)); + let mut frame = Frame::default(); + frame.tiles.push(FrameTile::panel_background( + Rect { + x, + y, + width, + height, + }, + scale, + color, + )); + let (active_row, active_column) = self.scan.position(&self.rows); + let row_height = (height - 52.0 * scale) / self.rows.len() as f64; + for (r, row) in self.rows.iter().enumerate() { + let cell_width = (width - 24.0 * scale) / row.len() as f64; + for (c, key) in row.iter().enumerate() { + frame.tiles.push(FrameTile { + thickness: self.scan.options.thickness, + color, + text: self.label(*key), + rect: Rect { + x: x + 12.0 * scale + c as f64 * cell_width, + y: y + 44.0 * scale + r as f64 * row_height, + width: (cell_width - 6.0 * scale).max(1.0), + height: (row_height - 6.0 * scale).max(1.0), + }, + scale, + icon: Item::KeyboardKey, + style: Some(TileStyle { + role: TileRole::Utility, + active: *key == Key::Drag && self.dragging, + row_scan: self.scan.row_scan(), + }), + selected: !self.scan.suspended + && !self.scan.nav.escaping() + && active_row == r + && active_column.is_none_or(|column| column == c), + }); + } + } + frame.label = Some(FrameLabel { + text: if self.error { + "Mouse action failed · Select to resume".into() + } else if self.more { + "Mouse · More controls".into() + } else { + "Mouse · Movement".into() + }, + rect: Rect { + x: x + 12.0 * scale, + y: y + 4.0 * scale, + width: width - 24.0 * scale, + height: 38.0 * scale, + }, + scale: scale * 0.8, + hud: None, + }); + frame + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pages_and_monitor_controls_follow_available_displays() { + let mut panel = MousePanel::new(Resolved::default(), 1, 100); + assert_eq!(panel.rows[1][1], Key::Click); + assert_eq!(panel.rows[0].len(), 3); + panel.choose(Key::More); + assert!(!panel + .rows + .iter() + .flatten() + .any(|key| matches!(key, Key::Monitor(..)))); + panel.set_displays(2); + assert_eq!(panel.rows[2].len(), 4); + panel.choose(Key::Movement); + assert_eq!(panel.rows[1][1], Key::Click); + } + + #[test] + fn panel_renders_with_selected_tiles_inside_the_screen() { + let panel = MousePanel::new(Resolved::default(), 2, 100); + let screen = Rect { + x: -1280.0, + y: 0.0, + width: 1280.0, + height: 720.0, + }; + let frame = panel.frame(screen, 1.0, ScannerColor::default()); + assert!(frame.tiles.iter().any(|tile| tile.selected)); + assert!(frame + .tiles + .iter() + .all(|tile| tile.rect.x >= screen.x && tile.rect.y >= screen.y)); + } +} diff --git a/src-tauri/src/scan_preferences.rs b/src-tauri/src/scan_preferences.rs index 45b1d369..20cea2b8 100644 --- a/src-tauri/src/scan_preferences.rs +++ b/src-tauri/src/scan_preferences.rs @@ -37,6 +37,7 @@ pub enum Area { Point, Menu, Keyboard, + Mouse, } #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase", default)] @@ -59,6 +60,7 @@ pub struct Preferences { pub point: Overrides, pub menu: Overrides, pub keyboard: Overrides, + pub mouse: Overrides, } impl Default for Preferences { fn default() -> Self { @@ -70,6 +72,7 @@ impl Default for Preferences { point: Default::default(), menu: Default::default(), keyboard: Default::default(), + mouse: Default::default(), } } } @@ -108,6 +111,7 @@ impl Preferences { Area::Point => self.point, Area::Menu => self.menu, Area::Keyboard => self.keyboard, + Area::Mouse => self.mouse, }; Resolved { automatic: local.automatic.unwrap_or(automatic), @@ -145,6 +149,7 @@ pub fn deserialize_preferences<'de, D: serde::Deserializer<'de>>( &mut preferences.point, &mut preferences.menu, &mut preferences.keyboard, + &mut preferences.mouse, ] { local.interval_ms = local.interval_ms.filter(|v| (100..=10000).contains(v)); local.pass_limit = local.pass_limit.filter(|v| valid_passes(*v)); diff --git a/src-tauri/src/scan_tile.rs b/src-tauri/src/scan_tile.rs index a4a7a281..4358efb7 100644 --- a/src-tauri/src/scan_tile.rs +++ b/src-tauri/src/scan_tile.rs @@ -181,7 +181,8 @@ pub fn bitmap(tile: &FrameTile) -> Result { lines(&[(62., 46.), (106., 90.)]); lines(&[(106., 46.), (62., 90.)]); } - Keyboard | More | Group(_) | Command(_) | Setting(_) | Display(_) | Pause | Reverse => { + Keyboard | MousePanel | More | Group(_) | Command(_) | Setting(_) | Display(_) | Pause + | Reverse => { artwork(&mut path, tile.icon); } DragHere => { @@ -366,6 +367,7 @@ fn artwork(path: &mut PathBuilder, item: Item) { } line(path, &[(66., 81.), (102., 81.)]); } + Item::MousePanel => mouse(path, 65.), Item::More => { for x in [60., 84., 108.] { path.push_circle(x, 68., 5.); diff --git a/src-tauri/src/scanning.rs b/src-tauri/src/scanning.rs index 778b7e08..3318ee07 100644 --- a/src-tauri/src/scanning.rs +++ b/src-tauri/src/scanning.rs @@ -15,11 +15,13 @@ pub enum Action { Stop, Cancel, OpenKeyboard, + OpenMouse, } impl Action { pub fn label(self) -> &'static str { match self { Self::OpenKeyboard => "Open keyboard", + Self::OpenMouse => "Open mouse", Self::Select => "Select", Self::Next => "Next", Self::Back => "Previous", @@ -317,6 +319,10 @@ pub trait Technique { fn auto_selecting(&self) -> bool { false } + /// Consumes the next physical press while a technique is moving the pointer. + fn switch_pressed(&mut self) -> bool { + false + } fn execution_failed(&mut self, _message: String) {} fn execution_succeeded(&mut self) {} fn start(&mut self); @@ -380,7 +386,7 @@ impl Session { self.reset(); return None; } - if action == Action::OpenKeyboard { + if matches!(action, Action::OpenKeyboard | Action::OpenMouse) { let selection = self.technique.handle(action); if selection.is_some() { self.active = true; diff --git a/src-tauri/src/scanning_runtime.rs b/src-tauri/src/scanning_runtime.rs index c2041730..a0909123 100644 --- a/src-tauri/src/scanning_runtime.rs +++ b/src-tauri/src/scanning_runtime.rs @@ -411,7 +411,7 @@ fn switch(app: &AppHandle, action: Action, input_generation: u64, re hide_prompt(); let result = (|| -> Result<(), String> { let mut d = c.data.lock().unwrap_or_else(|p| p.into_inner()); - if action == Action::OpenKeyboard + if matches!(action, Action::OpenKeyboard | Action::OpenMouse) || (d.engine.as_ref().is_none_or(|e| !e.active()) && action == Action::Select) { let (engine, display) = A::create(app, d.config.clone())?; @@ -684,6 +684,13 @@ fn tick(app: &AppHandle) { match edge { crate::remote_scan::Edge::Reset => unreachable!(), crate::remote_scan::Edge::Down(id) => { + if d.engine + .as_mut() + .is_some_and(|engine| engine.technique.switch_pressed()) + { + d.pressed.cancel(); + continue; + } if !d.pressed.held() { d.remote_hold_started = Some(now_ms); } @@ -755,6 +762,13 @@ fn tick(app: &AppHandle) { continue; } if action == crate::switch_input::Action::Pressed { + if d.engine + .as_mut() + .is_some_and(|engine| engine.technique.switch_pressed()) + { + d.pressed.cancel(); + continue; + } let settings = d.switches.clone(); let countdown = d .engine diff --git a/src-tauri/src/switch_gestures.rs b/src-tauri/src/switch_gestures.rs index bd7c7879..a2cb6300 100644 --- a/src-tauri/src/switch_gestures.rs +++ b/src-tauri/src/switch_gestures.rs @@ -34,9 +34,14 @@ impl Gestures { let mut binding = binding.clone(); if countdown && !binding.hold_actions.contains(&Action::OpenKeyboard) + && !binding.hold_actions.contains(&Action::OpenMouse) && !matches!( binding.press_action, - Action::Stop | Action::Pause | Action::Cancel | Action::OpenKeyboard + Action::Stop + | Action::Pause + | Action::Cancel + | Action::OpenKeyboard + | Action::OpenMouse ) { binding.press_action = Action::Select; @@ -108,7 +113,7 @@ mod tests { } } #[test] - fn countdown_preserves_direct_and_held_keyboard_actions() { + fn countdown_preserves_direct_and_held_panel_actions() { let mut settings = Settings { bindings: vec![Binding { id: "keyboard".into(), @@ -132,6 +137,10 @@ mod tests { gestures.released("keyboard", 2100), Some(Action::OpenKeyboard) ); + settings.bindings[0].press_action = Action::OpenMouse; + settings.bindings[0].hold_actions.clear(); + gestures.pressed_for_scan("keyboard", 3000, &settings, true); + assert_eq!(gestures.released("keyboard", 3100), Some(Action::OpenMouse)); } #[test] fn countdown_press_overrides_normal_actions_and_holds_once() { diff --git a/src-tauri/src/switches.rs b/src-tauri/src/switches.rs index 126526bd..fe621d94 100644 --- a/src-tauri/src/switches.rs +++ b/src-tauri/src/switches.rs @@ -122,7 +122,7 @@ impl Settings { mod tests { use super::*; #[test] - fn version_one_assignments_round_trip_with_additive_keyboard_action() { + fn version_one_assignments_round_trip_with_additive_keyboard_and_mouse_actions() { let legacy = serde_json::json!({ "schemaVersion": 1, "holdIntervalMs": 1000, "bindings": [{ "id": "one", "name": "One", "key": "Space", @@ -132,9 +132,11 @@ mod tests { settings.validate_actions(false).unwrap(); assert_eq!(serde_json::to_value(&settings).unwrap(), legacy); settings.bindings[0].hold_actions.push(Action::OpenKeyboard); + settings.bindings[0].hold_actions.push(Action::OpenMouse); settings.validate_actions(false).unwrap(); let updated = serde_json::to_value(&settings).unwrap(); assert_eq!(updated["bindings"][0]["holdActions"][5], "openKeyboard"); + assert_eq!(updated["bindings"][0]["holdActions"][6], "openMouse"); assert_eq!( serde_json::from_value::(updated).unwrap(), settings diff --git a/src/scanning/preferences.ts b/src/scanning/preferences.ts index f8c85d6b..5212a596 100644 --- a/src/scanning/preferences.ts +++ b/src/scanning/preferences.ts @@ -1,5 +1,5 @@ import type { PointScanConfig, ScannerColor } from './useScanning'; -export type ScanArea = 'point' | 'menu' | 'keyboard'; +export type ScanArea = 'point' | 'menu' | 'keyboard' | 'mouse'; export type ScanOptions = { automatic: boolean; intervalMs: number; @@ -12,7 +12,7 @@ export type ScanOptions = { export type ScanPreferences = Pick & Record>; export const defaultScanPreferences: ScanPreferences = { direction: 'forward', passLimit: 3, pattern: 'grouped', thickness: 'standard', - point: {}, menu: {}, keyboard: {}, + point: {}, menu: {}, keyboard: {}, mouse: {}, }; export function sharedOptions(config: PointScanConfig): ScanOptions { const settings = config.scanPreferences ?? defaultScanPreferences; @@ -21,7 +21,8 @@ export function sharedOptions(config: PointScanConfig): ScanOptions { } export function areaOptions(config: PointScanConfig, area: ScanArea): ScanOptions { const shared = sharedOptions(config); - const overrides = (config.scanPreferences ?? defaultScanPreferences)[area]; - return { ...shared, ...Object.fromEntries(Object.entries(overrides).filter(([, value]) => value != null)), + const overrides = (config.scanPreferences ?? defaultScanPreferences)[area] ?? {}; + return { ...shared, ...(area === 'mouse' && overrides.automatic == null ? { automatic: areaOptions(config, 'keyboard').automatic } : {}), + ...Object.fromEntries(Object.entries(overrides).filter(([, value]) => value != null)), ...(area === 'point' ? { pattern: 'grouped' as const } : {}) }; } diff --git a/src/scanning/useScanning.ts b/src/scanning/useScanning.ts index 2644ef18..e725d633 100644 --- a/src/scanning/useScanning.ts +++ b/src/scanning/useScanning.ts @@ -23,7 +23,7 @@ export type PointScanState = { remote?: boolean; config: PointScanConfig; enabled: boolean; - phase: "autoSelecting" | "idle" | "row" | "rowEscape" | "cell" | "x" | "y" | "menu" | "menuSuspended" | "dragDestination" | "dragConfirmation" | "executing" | "keyboard" | "keyboardSuspended" | "keyboardOpening"; + phase: "autoSelecting" | "idle" | "row" | "rowEscape" | "cell" | "x" | "y" | "menu" | "menuSuspended" | "dragDestination" | "dragConfirmation" | "executing" | "keyboard" | "keyboardSuspended" | "keyboardOpening" | "mouse" | "mouseSuspended" | "mouseMoving"; paused: boolean; message: string; supported: boolean; diff --git a/src/scanning/useSwitches.ts b/src/scanning/useSwitches.ts index 8b99a0bc..b8229e07 100644 --- a/src/scanning/useSwitches.ts +++ b/src/scanning/useSwitches.ts @@ -4,6 +4,7 @@ import { listen } from "@tauri-apps/api/event"; export const actions = { select: "Select", openKeyboard: "Open keyboard", + openMouse: "Open mouse", next: "Next", back: "Previous", reverse: "Reverse direction", diff --git a/src/settings/ScannerPreferences.tsx b/src/settings/ScannerPreferences.tsx index 18260d5f..e727c8bf 100644 --- a/src/settings/ScannerPreferences.tsx +++ b/src/settings/ScannerPreferences.tsx @@ -33,14 +33,14 @@ export function ScannerPreferences({ controller }: { controller: ScanningControl changedView.current = true; setArea(next); }; - const names = { point: 'Point scanning', menu: 'Menus', keyboard: 'Keyboard' }; + const names = { point: 'Point scanning', menu: 'Menus', keyboard: 'Keyboard', mouse: 'Mouse' }; const disabled = !state?.supported; const settings = config.scanPreferences ?? defaultScanPreferences; const shared = sharedOptions(config); const effective = area === 'shared' ? shared : areaOptions(config, area); const change = (key: K, value: ScanOptions[K] | undefined) => { if (area !== 'shared') { - const overrides = { ...settings[area], [key]: value }; + const overrides = { ...(settings[area] ?? {}), [key]: value }; update('scanPreferences', { ...settings, [area]: overrides }); } else if (key === 'automatic') update('automatic', value as boolean); else if (key === 'intervalMs') update('blockIntervalMs', value as number); @@ -48,7 +48,7 @@ export function ScannerPreferences({ controller }: { controller: ScanningControl else update('scanPreferences', { ...settings, [key]: value }); }; const field = (key: keyof ScanOptions, label: string, children: (locked: boolean) => ReactNode) => { - const custom = area !== 'shared' && settings[area][key] != null; + const custom = area !== 'shared' && settings[area]?.[key] != null; return
{area !== 'shared' &&
{custom ? 'Custom' : 'Default'} {custom && } @@ -157,9 +157,9 @@ export function ScannerPreferences({ controller }: { controller: ScanningControl {area === 'shared' &&

Customise an area

Give an area different settings, or keep using your defaults.

-
{(['point', 'menu', 'keyboard'] as const).map(key => { +
{(['point', 'menu', 'keyboard', 'mouse'] as const).map(key => { const options = areaOptions(config, key); - const count = Object.entries(settings[key]).filter(([name, value]) => value != null && !(key === 'point' && name === 'pattern')).length; + const count = Object.entries(settings[key] ?? {}).filter(([name, value]) => value != null && !(key === 'point' && name === 'pattern')).length; return
} - {area !== 'shared' && } + {area !== 'shared' && }

Saved automatically. Press Select to scan again.

; } diff --git a/src/settings/ScanningSection.tsx b/src/settings/ScanningSection.tsx index 740bcb67..caff6df8 100644 --- a/src/settings/ScanningSection.tsx +++ b/src/settings/ScanningSection.tsx @@ -7,6 +7,9 @@ const phases = { keyboard: "Choose a keyboard row, then a key", keyboardSuspended: "Select to resume the keyboard", keyboardOpening: "Opening the keyboard", + mouse: "Choose a mouse control", + mouseSuspended: "Select to resume the mouse", + mouseMoving: "Moving pointer. Press any switch to stop", idle: "Ready to begin", autoSelecting: "Waiting to click. Press a switch for the action menu", menu: "Choose an action at the selected point", From 0ca7947b60375d9ee6cd8a62c3f4247fbde77fec Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Wed, 23 Sep 2026 12:06:42 +0100 Subject: [PATCH 2/5] Refresh pointer speed state and explain mouse scanning --- src-tauri/src/point_scan_runtime.rs | 4 +++- src/settings/ScanningSection.tsx | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src-tauri/src/point_scan_runtime.rs b/src-tauri/src/point_scan_runtime.rs index 2bd24fce..e2390c05 100644 --- a/src-tauri/src/point_scan_runtime.rs +++ b/src-tauri/src/point_scan_runtime.rs @@ -88,7 +88,9 @@ impl Adapter for PointScan { let model = app.state::(); let old = model.snapshot().settings.pointer_scale_percent; let next = (i16::from(old) + i16::from(direction) * 5).clamp(5, 225) as u8; - return model.apply_pointer_scale_percent(next).map(|()| None); + model.apply_pointer_scale_percent(next)?; + crate::state::emit_state(app, &model.shared); + return Ok(None); } Request::MouseMonitor(dx, dy) => { let (cursor, displays) = diff --git a/src/settings/ScanningSection.tsx b/src/settings/ScanningSection.tsx index caff6df8..210c6f56 100644 --- a/src/settings/ScanningSection.tsx +++ b/src/settings/ScanningSection.tsx @@ -37,6 +37,7 @@ export function ScanningSection({ controller }: { controller: ScanningController
How scanning works

Focus the application you want to use, then press Select to scan the display under the pointer. Choose a point, then use the action menu for clicks, scrolling and dragging.

+

Choose Mouse from the action menu, or assign Open mouse to a switch. Select a direction to move continuously; press any switch to stop and scan the mouse panel again. The panel also offers clicks, dragging, scrolling, speed, monitor navigation and Keyboard.

Assign switch actions in the Switches page. Actions run on release; holding a switch freezes movement. Manual scanning needs Select, Next and Previous. After clicking or reaching the pass limit, use Select to start again.

Assigned keys stay reserved while Switchify runs. Escape resets the scan. Mobile connections pause local scanning until they end.

On Windows, assigned keys remain switches with Shift, Ctrl, Alt or Windows held, even in the background. Other keys and Switchify-generated shortcuts still work normally.

From 5c9af92aab0f6607587e1dcc3fa68fa93ee3287d Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Wed, 23 Sep 2026 12:10:30 +0100 Subject: [PATCH 3/5] Preserve macOS pointer bounds and publish movement stop --- src-tauri/src/point_scan_runtime.rs | 27 +++++++++++++++++++++++++-- src-tauri/src/point_workflow.rs | 8 +++++++- src-tauri/src/scan_executor.rs | 8 ++++++++ src-tauri/src/scanning_runtime.rs | 6 ++++++ 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src-tauri/src/point_scan_runtime.rs b/src-tauri/src/point_scan_runtime.rs index e2390c05..85203e38 100644 --- a/src-tauri/src/point_scan_runtime.rs +++ b/src-tauri/src/point_scan_runtime.rs @@ -84,6 +84,17 @@ impl Adapter for PointScan { )) .map(|()| None); } + Request::MouseMove { dx, dy } if cfg!(target_os = "macos") => { + let (cursor, displays) = + display_navigation::displays(app).map_err(|e| e.message)?; + let target = display_navigation::clamped_pointer_target(cursor, dx, dy, &displays) + .ok_or("No active display could be resolved.")?; + return crate::scan_executor::activate(Request::MouseMoveAbsolute { + x: target.0, + y: target.1, + }) + .map(|()| None); + } Request::MouseSpeed(direction) => { let model = app.state::(); let old = model.snapshot().settings.pointer_scale_percent; @@ -138,7 +149,16 @@ impl Adapter for PointScan { environment.display = current; environment.keyboard_area = area; environment.foreground = crate::scan_host::foreground()?; - technique.set_mouse_area(area, displays.len()); + technique.set_mouse_area( + area, + rect, + if cfg!(target_os = "windows") { + environment.display.scale_factor + } else { + 1.0 + }, + displays.len(), + ); let settings = app.state::().snapshot().settings; technique.set_mouse_settings( settings.pointer_scale_percent, @@ -160,7 +180,10 @@ impl Adapter for PointScan { fn preserve_visuals(request: &Request) -> bool { matches!( request, - Request::Keyboard(_) | Request::Prediction { .. } | Request::MouseMove { .. } + Request::Keyboard(_) + | Request::Prediction { .. } + | Request::MouseMove { .. } + | Request::MouseMoveAbsolute { .. } ) } fn deferred(request: &Request) -> bool { diff --git a/src-tauri/src/point_workflow.rs b/src-tauri/src/point_workflow.rs index 0a58a328..cf93890b 100644 --- a/src-tauri/src/point_workflow.rs +++ b/src-tauri/src/point_workflow.rs @@ -20,6 +20,10 @@ pub enum Request { dx: i32, dy: i32, }, + MouseMoveAbsolute { + x: i32, + y: i32, + }, MouseClick { right: bool, count: u8, @@ -202,8 +206,10 @@ impl Workflow { pub fn mouse_open(&self) -> bool { matches!(self.stage, Stage::Mouse | Stage::MouseMoving) } - pub fn set_mouse_area(&mut self, area: Rect, displays: usize) { + pub fn set_mouse_area(&mut self, area: Rect, screen: Rect, scale: f64, displays: usize) { self.mouse_area = area; + self.point.screen = screen; + self.point.units_per_logical_pixel = scale; self.mouse.set_displays(displays); } pub fn set_mouse_settings(&mut self, speed: u8, acceleration_ms: u32) { diff --git a/src-tauri/src/scan_executor.rs b/src-tauri/src/scan_executor.rs index 1a4e017b..ec05df4f 100644 --- a/src-tauri/src/scan_executor.rs +++ b/src-tauri/src/scan_executor.rs @@ -18,6 +18,7 @@ pub fn execute( Request::Prediction { .. } => Err("Prediction requires the scan controller.".into()), Request::OpenKeyboard | Request::OpenMouse | Request::CloseMouse => input.release_all(), Request::MouseMove { dx, dy } => input.move_pointer_pixels(dx, dy).map(|_| ()), + Request::MouseMoveAbsolute { x, y } => input.move_pointer_pixels_absolute(x, y).map(|_| ()), Request::MouseClick { right, count } => { if input.has_active_drag() { return Err("End the drag before clicking.".into()); @@ -418,6 +419,12 @@ mod tests { fn mouse_actions_use_fake_input_and_close_releases_drag() { let mut input = DesktopInput::new(Fake::default()); execute(&mut input, Request::MouseMove { dx: 4, dy: -2 }, true).unwrap(); + execute( + &mut input, + Request::MouseMoveAbsolute { x: 40, y: 50 }, + true, + ) + .unwrap(); execute( &mut input, Request::MouseClick { @@ -432,6 +439,7 @@ mod tests { execute(&mut input, Request::CloseMouse, true).unwrap(); assert!(!input.has_active_drag()); assert!(input.injector.events.contains(&"relative 4 -2".into())); + assert!(input.injector.events.contains(&"move 40 50".into())); assert!(input.injector.events.contains(&"scroll 0 5".into())); assert!(input.injector.events.contains(&"button Left false".into())); } diff --git a/src-tauri/src/scanning_runtime.rs b/src-tauri/src/scanning_runtime.rs index a0909123..f72d147c 100644 --- a/src-tauri/src/scanning_runtime.rs +++ b/src-tauri/src/scanning_runtime.rs @@ -689,6 +689,9 @@ fn tick(app: &AppHandle) { .is_some_and(|engine| engine.technique.switch_pressed()) { d.pressed.cancel(); + d.last_tick = Instant::now(); + drop(d); + publish::(app); continue; } if !d.pressed.held() { @@ -767,6 +770,9 @@ fn tick(app: &AppHandle) { .is_some_and(|engine| engine.technique.switch_pressed()) { d.pressed.cancel(); + d.last_tick = Instant::now(); + drop(d); + publish::(app); continue; } let settings = d.switches.clone(); From 779c1f93f4b933bacad9fa5d027d9eace5e55688 Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Wed, 23 Sep 2026 12:28:19 +0100 Subject: [PATCH 4/5] test: wait for switch practice focus effect --- src/scanning/SwitchPractice.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanning/SwitchPractice.test.tsx b/src/scanning/SwitchPractice.test.tsx index 5983ec95..a7ca5e59 100644 --- a/src/scanning/SwitchPractice.test.tsx +++ b/src/scanning/SwitchPractice.test.tsx @@ -26,7 +26,7 @@ describe("safe switch practice", () => { render(); fireEvent.click(screen.getByRole("button", { name: "Test switches" })); const dialog = await screen.findByRole("dialog", { name: "Safe switch practice" }); expect(invoke).toHaveBeenCalledWith("begin_switch_practice", { remote: false }); - expect(dialog).toHaveFocus(); + await waitFor(() => expect(dialog).toHaveFocus()); expect(screen.queryByRole("button", { name: "Start another test" })).not.toBeInTheDocument(); fireEvent.click(screen.getByRole("button", { name: "Exit practice" })); await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); From 7477e2fc7b8fbc947b4387cbc0fb7c88a45dd303 Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Wed, 23 Sep 2026 13:39:06 +0100 Subject: [PATCH 5/5] Match scanned mouse with remote repeat and pointer ring --- src-tauri/src/mouse_repeat.rs | 89 ++++++++++++ src-tauri/src/overlay.rs | 208 ++++++++++++++++++++++++++-- src-tauri/src/point_scan_runtime.rs | 21 +++ src-tauri/src/point_workflow.rs | 193 ++++++++++++++++++++------ src-tauri/src/scanning_runtime.rs | 49 ++++++- src/settings/ScanningSection.tsx | 2 +- 6 files changed, 505 insertions(+), 57 deletions(-) diff --git a/src-tauri/src/mouse_repeat.rs b/src-tauri/src/mouse_repeat.rs index bb0aae0d..67f445e0 100644 --- a/src-tauri/src/mouse_repeat.rs +++ b/src-tauri/src/mouse_repeat.rs @@ -147,6 +147,57 @@ struct MoveRepeatState { initial_emitted: bool, } +/// Drives a scanned direction with the same movement integral as a remote +/// repeat. Scanner ticks are slower than remote ticks, so split their elapsed +/// time into remote-sized slices before asking the shared motion state. +pub struct ScanMoveRepeat { + direction: (i32, i32), + now: Instant, + movement: MoveRepeatState, +} + +impl ScanMoveRepeat { + pub fn new(dx: i32, dy: i32, now: Instant) -> Self { + Self { + direction: (dx, dy), + now, + movement: MoveRepeatState::new(now), + } + } + + pub fn initial_move(&mut self) -> (i32, i32) { + self.movement + .initial_delta(self.direction.0, self.direction.1) + } + + pub fn advance( + &mut self, + elapsed_ms: u64, + move_interval_ms: u32, + pointer_scale_percent: u8, + acceleration_duration_ms: u32, + ) -> (i32, i32) { + let mut remaining = elapsed_ms.min(64); + let mut result = (0, 0); + while remaining > 0 { + let step = remaining.min(MOVE_TICK_INTERVAL_MS); + self.now += std::time::Duration::from_millis(step); + let delta = self.movement.advance( + self.now, + self.direction.0, + self.direction.1, + move_interval_ms, + pointer_scale_percent, + acceleration_duration_ms, + ); + result.0 += delta.0; + result.1 += delta.1; + remaining -= step; + } + result + } +} + impl MoveRepeatState { fn new(now: Instant) -> Self { Self { @@ -603,6 +654,44 @@ mod tests { ); } + #[test] + fn scanning_motion_matches_remote_repeat_ticks() { + for direction in [(12, 0), (-12, 12), (0, -12)] { + for acceleration in [0, 1000] { + let now = Instant::now(); + let mut scan = ScanMoveRepeat::new(direction.0, direction.1, now); + let mut remote = MouseRepeatController::default(); + let active = remote.start( + "remote".into(), + RepeatCommand::Move { + dx: direction.0, + dy: direction.1, + }, + acceleration, + now, + ); + assert_eq!( + scan.initial_move(), + remote.initial_move("remote", active.generation).unwrap() + ); + let mut time = now; + for _ in 0..40 { + let scanned = scan.advance(32, 250, 100, acceleration); + let mut expected = (0, 0); + for _ in 0..4 { + time += Duration::from_millis(8); + let delta = remote + .advance_move("remote", active.generation, time, 250, 100) + .unwrap(); + expected.0 += delta.0; + expected.1 += delta.1; + } + assert_eq!(scanned, expected); + } + } + } + } + #[test] fn parses_only_bounded_move_and_scroll_commands() { assert_eq!( diff --git a/src-tauri/src/overlay.rs b/src-tauri/src/overlay.rs index 0b67d5b5..955e1f70 100644 --- a/src-tauri/src/overlay.rs +++ b/src-tauri/src/overlay.rs @@ -77,6 +77,7 @@ struct Visibility { epoch: u64, suppressed: bool, hidden: bool, + mouse: bool, } impl VisibilityGate { pub(crate) fn present( @@ -85,7 +86,7 @@ impl VisibilityGate { render: impl FnOnce() -> Result<(), String>, ) -> Result<(), String> { let state = self.0.lock().unwrap_or_else(|p| p.into_inner()); - if state.epoch == epoch && !state.suppressed { + if state.epoch == epoch && (!state.suppressed || state.mouse) { render() } else { Ok(()) @@ -120,6 +121,7 @@ impl CursorOverlay { state.epoch += 1; state.suppressed = true; state.hidden = false; + state.mouse = false; self.sender .send(Command::Suppress(state.epoch)) .map_err(|_| "Cursor overlay is unavailable.".to_string())?; @@ -131,6 +133,32 @@ impl CursorOverlay { state.epoch == epoch && state.suppressed && state.hidden } + /// Feedback from the current scanning mouse is the only feedback allowed + /// to render while the scan owns the overlay. + pub(crate) fn show_scan_mouse( + &self, + epoch: u64, + feedback: PointerFeedback, + settings: AppSettings, + ) -> Result<(), String> { + let mut state = self.gate.0.lock().unwrap_or_else(|p| p.into_inner()); + if state.epoch != epoch || !state.suppressed || !state.hidden { + return Err("Mouse overlay scan token is no longer active.".into()); + } + state.mouse = true; + self.sender + .send(Command::ScanMouse(epoch, feedback, settings)) + .map_err(|_| "Cursor overlay is unavailable.".to_string()) + } + + pub(crate) fn end_scan_mouse(&self, epoch: u64) { + let mut state = self.gate.0.lock().unwrap_or_else(|p| p.into_inner()); + if state.epoch == epoch && state.suppressed && state.mouse { + state.mouse = false; + let _ = self.sender.send(Command::EndScanMouse(epoch)); + } + } + /// Called only after all scanner windows have been hidden. pub(crate) fn release_scan(&self, epoch: u64) { let mut state = self.gate.0.lock().unwrap_or_else(|p| p.into_inner()); @@ -140,6 +168,7 @@ impl CursorOverlay { state.epoch += 1; state.suppressed = false; state.hidden = false; + state.mouse = false; let _ = self.sender.send(Command::Resume(state.epoch)); } @@ -197,6 +226,8 @@ pub(crate) enum Command { Scoped(u64, Box), Suppress(u64), Resume(u64), + ScanMouse(u64, PointerFeedback, AppSettings), + EndScanMouse(u64), Show(PointerFeedback, AppSettings), MarkControlActive(AppSettings), ApplySettings(AppSettings), @@ -234,6 +265,7 @@ pub(crate) struct OverlayEngine { next_follow: Instant, visible: bool, typing_suppressed: bool, + scan_mouse: bool, } impl OverlayEngine { @@ -266,12 +298,42 @@ impl OverlayEngine { next_follow: now, visible: false, typing_suppressed: false, + scan_mouse: false, + } + } + + fn show_scan_mouse( + &mut self, + feedback: PointerFeedback, + mut settings: AppSettings, + now: Instant, + ) -> Update { + let dragging = self.scan_mouse && self.drag_active; + self.scan_mouse = true; + settings.cursor_overlay_enabled = true; + settings.cursor_overlay_visibility = "whileControlling".into(); + let update = self.handle(Command::Show(feedback, settings), now); + if dragging && matches!(feedback, PointerFeedback::Scroll { .. }) { + self.drag_active = true; + } + if matches!(feedback, PointerFeedback::RepeatMove { .. }) { + self.deadline = None; } + update + } + + fn end_scan_mouse(&mut self, now: Instant) -> Update { + self.scan_mouse = false; + self.handle(Command::EndSession, now) } pub(crate) fn handle(&mut self, command: Command, now: Instant) -> Update { match command { - Command::Scoped(_, _) | Command::Suppress(_) | Command::Resume(_) => { + Command::Scoped(_, _) + | Command::Suppress(_) + | Command::Resume(_) + | Command::ScanMouse(..) + | Command::EndScanMouse(_) => { unreachable!("handled by overlay worker") } Command::Show(feedback, settings) => { @@ -281,7 +343,10 @@ impl OverlayEngine { return self.hide(); } self.control_active = true; - self.drag_active = matches!(feedback, PointerFeedback::Drag); + self.drag_active = matches!( + feedback, + PointerFeedback::Drag | PointerFeedback::RepeatMove { dragging: true, .. } + ); self.deadline = deadline_for(feedback, self.persistent(), now); self.feedback = Some(feedback); self.next_follow = now + FOLLOW_INTERVAL; @@ -826,13 +891,14 @@ pub(crate) fn run_loop( break; } let command = receiver.recv_timeout(Duration::from_millis(25)); - let (epoch, suppressed) = { + let (epoch, suppressed, mouse) = { let state = gate.0.lock().unwrap_or_else(|p| p.into_inner()); - (state.epoch, state.suppressed) + (state.epoch, state.suppressed, state.mouse) }; let update = match command { Ok(Command::Suppress(token)) => { engine.clear_scan_feedback(); + engine.scan_mouse = false; hide(Some(token)); continue; } @@ -840,22 +906,44 @@ pub(crate) fn run_loop( if token != epoch || suppressed { continue; } - engine.resume_after_scan(Instant::now()) + if engine.scan_mouse { + engine.end_scan_mouse(Instant::now()) + } else { + engine.resume_after_scan(Instant::now()) + } } Ok(Command::Scoped(token, command)) => { - let update = engine.handle(*command, Instant::now()); - if suppressed || token != epoch { + if suppressed && mouse { + Update::None + } else if suppressed || token != epoch { + let _ = engine.handle(*command, Instant::now()); engine.clear_scan_feedback(); Update::None } else { - update + engine.handle(*command, Instant::now()) + } + } + Ok(Command::ScanMouse(token, feedback, settings)) => { + if token == epoch && suppressed && mouse { + engine.show_scan_mouse(feedback, settings, Instant::now()) + } else { + Update::None + } + } + Ok(Command::EndScanMouse(token)) => { + if (token == epoch && suppressed) + || (token.checked_add(1) == Some(epoch) && !suppressed) + { + engine.end_scan_mouse(Instant::now()) + } else { + Update::None } } Ok(command) => engine.handle(command, Instant::now()), Err(mpsc::RecvTimeoutError::Timeout) => engine.tick(Instant::now()), Err(mpsc::RecvTimeoutError::Disconnected) => Update::Shutdown, }; - let update = if suppressed && matches!(update, Update::Render(_)) { + let update = if suppressed && !mouse && matches!(update, Update::Render(_)) { engine.clear_scan_feedback(); Update::None } else { @@ -935,6 +1023,106 @@ mod tests { assert!(!visible.get()); } + #[test] + fn only_the_current_mouse_scan_can_render_through_suppression() { + let (overlay, _receiver) = test_overlay(); + let first = overlay.suppress_for_scan().unwrap(); + overlay.gate.hide(Some(first), || {}); + assert!(overlay + .show_scan_mouse(first, PointerFeedback::Move, AppSettings::default()) + .is_ok()); + let visible = std::cell::Cell::new(false); + overlay + .gate + .present(first, || { + visible.set(true); + Ok(()) + }) + .unwrap(); + assert!(visible.get()); + overlay.end_scan_mouse(first); + visible.set(false); + overlay + .gate + .present(first, || { + visible.set(true); + Ok(()) + }) + .unwrap(); + assert!(!visible.get()); + overlay.release_scan(first); + let second = overlay.suppress_for_scan().unwrap(); + overlay.gate.hide(Some(second), || {}); + assert!(overlay + .show_scan_mouse(first, PointerFeedback::Move, AppSettings::default()) + .is_err()); + assert!(overlay + .show_scan_mouse(second, PointerFeedback::Move, AppSettings::default()) + .is_ok()); + } + + #[test] + fn mouse_scan_forces_persistent_remote_ring_and_cleans_up() { + let now = Instant::now(); + let mut engine = OverlayEngine::new(now); + let settings = AppSettings { + cursor_overlay_enabled: false, + cursor_overlay_visibility: "onInput".into(), + cursor_overlay_size: "large".into(), + cursor_overlay_color: "blue".into(), + cursor_crosshairs: true, + ..Default::default() + }; + let Update::Render(frame) = + engine.show_scan_mouse(PointerFeedback::Move, settings.clone(), now) + else { + panic!("mouse ring did not appear"); + }; + assert_eq!(frame.logical_size, 176); + assert_eq!(frame.color, color_for("blue")); + assert!(frame.crosshairs); + assert!(matches!( + engine.tick(now + DEFAULT_DURATION), + Update::Render(_) + )); + engine.show_scan_mouse( + PointerFeedback::RepeatMove { + accelerated: true, + dragging: true, + }, + settings.clone(), + now, + ); + assert_eq!(engine.deadline, None); + engine.show_scan_mouse(PointerFeedback::Drag, settings, now); + assert!(matches!(engine.end_scan_mouse(now), Update::Hide)); + assert!(!engine.visible); + assert!(!engine.control_active); + } + + #[test] + fn queued_mouse_close_cannot_restore_ring_after_scan_release() { + let (overlay, receiver) = test_overlay(); + overlay.mark_control_active(AppSettings::default()); + let token = overlay.suppress_for_scan().unwrap(); + overlay.gate.hide(Some(token), || {}); + overlay + .show_scan_mouse(token, PointerFeedback::Move, AppSettings::default()) + .unwrap(); + overlay.end_scan_mouse(token); + overlay.release_scan(token); + let gate = overlay.gate.clone(); + let hide_gate = gate.clone(); + drop(overlay); + run_loop( + |_, _| panic!("mouse ring was restored after closing"), + |epoch| hide_gate.hide(epoch, || {}), + || true, + receiver, + gate, + ); + } + #[test] fn queued_feedback_is_discarded_and_only_persistent_cursor_is_restored() { for visibility in ["onInput", "whileControlling"] { diff --git a/src-tauri/src/point_scan_runtime.rs b/src-tauri/src/point_scan_runtime.rs index 85203e38..04b07b7c 100644 --- a/src-tauri/src/point_scan_runtime.rs +++ b/src-tauri/src/point_scan_runtime.rs @@ -41,6 +41,25 @@ impl Adapter for PointScan { type Environment = Environment; const EVENT: &'static str = "point-scan-changed"; const FILE: &'static str = "point-scan.json"; + fn cursor_feedback(technique: &Workflow) -> Option { + technique.mouse_feedback() + } + fn cursor_action_feedback(request: &Request) -> Option { + use crate::input::PointerFeedback; + use crate::protocol::MouseButton; + match *request { + Request::MouseClick { right, count } => Some(PointerFeedback::Click { + button: if right { + MouseButton::Right + } else { + MouseButton::Left + }, + count, + }), + Request::MouseScroll { dy } => Some(PointerFeedback::Scroll { dx: 0, dy }), + _ => None, + } + } fn validate(config: &Config) -> Result<(), String> { config.validate() } @@ -163,6 +182,8 @@ impl Adapter for PointScan { technique.set_mouse_settings( settings.pointer_scale_percent, settings.mouse_repeat_acceleration_duration_ms, + settings.move_repeat_interval_ms, + settings.mouse_repeat_enabled, ); } else if technique.keyboard_open() { crate::point_scan_ready(app)?; diff --git a/src-tauri/src/point_workflow.rs b/src-tauri/src/point_workflow.rs index cf93890b..ccf74aaa 100644 --- a/src-tauri/src/point_workflow.rs +++ b/src-tauri/src/point_workflow.rs @@ -110,9 +110,9 @@ pub struct Workflow { mouse: crate::scan_mouse::MousePanel, mouse_area: Rect, return_to_mouse: bool, - move_direction: (i8, i8), - move_elapsed: u64, - move_residual: (f64, f64), + move_repeat: Option, + mouse_repeat_enabled: bool, + mouse_move_interval_ms: u32, mouse_acceleration_ms: u32, point: Engine, stage: Stage, @@ -139,9 +139,9 @@ impl Workflow { mouse: crate::scan_mouse::MousePanel::new(mouse_options, 1, 100), mouse_area: screen, return_to_mouse: false, - move_direction: (0, 0), - move_elapsed: 0, - move_residual: (0.0, 0.0), + move_repeat: None, + mouse_repeat_enabled: true, + mouse_move_interval_ms: 250, mouse_acceleration_ms: 1000, point: Engine::new(config, screen, scale)?, stage: Stage::Idle, @@ -186,6 +186,7 @@ impl Workflow { self.mouse.speed_percent, ); self.return_to_mouse = false; + self.move_repeat = None; self.parent_menu.clear(); self.stage = Stage::Point; self.point.start(); @@ -212,9 +213,41 @@ impl Workflow { self.point.units_per_logical_pixel = scale; self.mouse.set_displays(displays); } - pub fn set_mouse_settings(&mut self, speed: u8, acceleration_ms: u32) { + pub fn set_mouse_settings( + &mut self, + speed: u8, + acceleration_ms: u32, + move_interval_ms: u32, + repeat_enabled: bool, + ) { self.mouse.speed_percent = speed; self.mouse_acceleration_ms = acceleration_ms; + self.mouse_move_interval_ms = move_interval_ms; + self.mouse_repeat_enabled = repeat_enabled; + if !repeat_enabled && self.stage == Stage::MouseMoving { + self.stage = Stage::Mouse; + self.move_repeat = None; + self.pending = None; + self.mouse.restart(); + } + } + pub fn mouse_feedback(&self) -> Option { + use crate::input::PointerFeedback; + if self.mouse.error { + return None; + } + match self.stage { + Stage::MouseMoving => Some(PointerFeedback::RepeatMove { + accelerated: self.mouse_acceleration_ms > 0, + dragging: self.mouse.dragging, + }), + Stage::Mouse => Some(if self.mouse.dragging { + PointerFeedback::Drag + } else { + PointerFeedback::Move + }), + _ => None, + } } pub fn foreground_changed(&mut self) { self.keyboard.reset_context(); @@ -240,6 +273,7 @@ impl Workflow { } fn open_mouse(&mut self) -> Option { self.stage = Stage::Mouse; + self.move_repeat = None; self.mouse = crate::scan_mouse::MousePanel::new( self.point.config.mouse_scan, 1, @@ -257,11 +291,27 @@ impl Workflow { self.mouse.choose(key); match key { Key::Move(dx, dy) => { - self.move_direction = (dx, dy); - self.move_elapsed = 0; - self.move_residual = (0.0, 0.0); - self.stage = Stage::MouseMoving; - None + let delta = (i32::from(dx) * 12, i32::from(dy) * 12); + if self.mouse_repeat_enabled { + let mut repeat = crate::mouse_repeat::ScanMoveRepeat::new( + delta.0, + delta.1, + std::time::Instant::now(), + ); + let initial = repeat.initial_move(); + self.move_repeat = Some(repeat); + self.stage = Stage::MouseMoving; + Some(Request::MouseMove { + dx: initial.0, + dy: initial.1, + }) + } else { + let scale = f64::from(self.mouse.speed_percent) / 100.0; + Some(Request::MouseMove { + dx: (f64::from(delta.0) * scale).round() as i32, + dy: (f64::from(delta.1) * scale).round() as i32, + }) + } } Key::Click => Some(Request::MouseClick { right: false, @@ -423,6 +473,7 @@ impl Technique for Workflow { } if self.mouse_open() { self.stage = Stage::Mouse; + self.move_repeat = None; self.mouse.dragging = false; self.mouse.failed(); self.error = None; @@ -460,8 +511,7 @@ impl Technique for Workflow { self.mouse.speed_percent, ); self.return_to_mouse = false; - self.move_direction = (0, 0); - self.move_residual = (0.0, 0.0); + self.move_repeat = None; self.stage = Stage::Idle; self.parent_menu.clear(); self.pending = None; @@ -485,8 +535,8 @@ impl Technique for Workflow { return false; } self.stage = Stage::Mouse; - self.move_direction = (0, 0); - self.move_residual = (0.0, 0.0); + self.move_repeat = None; + self.pending = None; self.mouse.restart(); true } @@ -590,28 +640,14 @@ impl Technique for Workflow { } fn update(&mut self, ms: u64, context: UpdateContext) { if self.stage == Stage::MouseMoving { - let before = self.move_elapsed; - self.move_elapsed = self.move_elapsed.saturating_add(ms); - let acceleration = u64::from(self.mouse_acceleration_ms); - let ramp = if acceleration == 0 { - 1.0 - } else { - (self.move_elapsed.min(acceleration) as f64 / acceleration as f64) - .mul_add(0.75, 0.25) - }; - let seconds = self.move_elapsed.saturating_sub(before) as f64 / 1000.0; - let speed = 240.0 * f64::from(self.mouse.speed_percent) / 100.0 * ramp; - let diagonal = if self.move_direction.0 != 0 && self.move_direction.1 != 0 { - std::f64::consts::FRAC_1_SQRT_2 - } else { - 1.0 - }; - self.move_residual.0 += f64::from(self.move_direction.0) * speed * seconds * diagonal; - self.move_residual.1 += f64::from(self.move_direction.1) * speed * seconds * diagonal; - let dx = self.move_residual.0.trunc() as i32; - let dy = self.move_residual.1.trunc() as i32; - self.move_residual.0 -= f64::from(dx); - self.move_residual.1 -= f64::from(dy); + let (dx, dy) = self.move_repeat.as_mut().map_or((0, 0), |repeat| { + repeat.advance( + ms, + self.mouse_move_interval_ms, + self.mouse.speed_percent, + self.mouse_acceleration_ms, + ) + }); if dx != 0 || dy != 0 { self.pending = Some(Request::MouseMove { dx, dy }); } @@ -809,13 +845,22 @@ mod tests { session.technique.phase(), Phase::Workflow(WorkflowPhase::Mouse) ); - session - .technique - .mouse_key(crate::scan_mouse::Key::Move(1, 0)); - session.tick(250, false); - assert!( - matches!(session.take_selection(), Some(Request::MouseMove { dx, dy: 0 }) if dx > 0) + assert_eq!( + session + .technique + .mouse_key(crate::scan_mouse::Key::Move(1, 0)), + Some(Request::MouseMove { dx: 1, dy: 0 }) ); + let mut moved = false; + for _ in 0..20 { + session.tick(33, false); + if matches!(session.take_selection(), Some(Request::MouseMove { dx, dy: 0 }) if dx > 0) + { + moved = true; + break; + } + } + assert!(moved); assert!(session.technique.switch_pressed()); assert!(!session.technique.switch_pressed()); assert_eq!( @@ -832,6 +877,38 @@ mod tests { .any(|tile| tile.selected)); } + #[test] + fn mouse_without_repeat_moves_once_and_keeps_scanning() { + let screen = Rect { + x: 0.0, + y: 0.0, + width: 1280.0, + height: 720.0, + }; + let mut workflow = Workflow::new(Config::default().point(), screen, 1.0).unwrap(); + workflow.open_mouse(); + workflow.set_mouse_settings(50, 1000, 250, false); + assert_eq!( + workflow.mouse_key(crate::scan_mouse::Key::Move(-1, 1)), + Some(Request::MouseMove { dx: -6, dy: 6 }) + ); + assert_eq!(workflow.phase(), Phase::Workflow(WorkflowPhase::Mouse)); + assert_eq!( + workflow.mouse_feedback(), + Some(crate::input::PointerFeedback::Move) + ); + workflow.update( + 1000, + UpdateContext { + movement_enabled: true, + paused: false, + switch_held: false, + }, + ); + assert!(workflow.take_selection().is_none()); + assert!(!workflow.switch_pressed()); + } + #[test] fn mouse_keyboard_return_and_drag_state() { let screen = Rect { @@ -847,16 +924,44 @@ mod tests { Some(Request::MouseDrag) ); assert!(workflow.mouse.dragging); + assert_eq!( + workflow.mouse_feedback(), + Some(crate::input::PointerFeedback::Drag) + ); assert_eq!( workflow.mouse_key(crate::scan_mouse::Key::Keyboard), Some(Request::OpenKeyboard) ); assert!(!workflow.mouse.dragging); + assert_eq!(workflow.mouse_feedback(), None); workflow.execution_succeeded(); assert_eq!(workflow.phase(), Phase::Workflow(WorkflowPhase::Keyboard)); assert!(workflow.return_to_mouse); workflow.keyboard_closed(); assert_eq!(workflow.phase(), Phase::Workflow(WorkflowPhase::Mouse)); + assert_eq!( + workflow.mouse_feedback(), + Some(crate::input::PointerFeedback::Move) + ); + } + + #[test] + fn mouse_input_failure_clears_repeat_and_ring_until_resumed() { + let mut workflow = session(false).technique; + workflow.open_mouse(); + workflow.mouse_key(crate::scan_mouse::Key::Move(1, 0)); + assert!(matches!( + workflow.mouse_feedback(), + Some(crate::input::PointerFeedback::RepeatMove { .. }) + )); + workflow.execution_failed("input failed".into()); + assert_eq!(workflow.mouse_feedback(), None); + assert!(!workflow.switch_pressed()); + workflow.handle(Action::Select); + assert_eq!( + workflow.mouse_feedback(), + Some(crate::input::PointerFeedback::Move) + ); } fn session(automatic: bool) -> Session { Session::new( diff --git a/src-tauri/src/scanning_runtime.rs b/src-tauri/src/scanning_runtime.rs index f72d147c..eb0140fd 100644 --- a/src-tauri/src/scanning_runtime.rs +++ b/src-tauri/src/scanning_runtime.rs @@ -16,6 +16,14 @@ pub trait Adapter: Send + Sync + 'static { fn preserve_visuals(_request: &::Selection) -> bool { false } + fn cursor_feedback(_technique: &Self::Technique) -> Option { + None + } + fn cursor_action_feedback( + _request: &::Selection, + ) -> Option { + None + } fn poll(_app: &AppHandle, _technique: &mut Self::Technique, _captured_keys: &[String]) {} fn cleanup(app: &AppHandle) -> Result<(), String>; fn validate(config: &Self::Config) -> Result<(), String>; @@ -70,6 +78,7 @@ pub struct Controller { } struct Data { cursor_suppression: Option<(u64, Instant)>, + cursor_mouse: Option<(crate::input::PointerFeedback, crate::state::AppSettings)>, config: A::Config, engine: Option>, display: Option, @@ -107,6 +116,7 @@ impl Controller { generation: AtomicU64::new(0), data: Mutex::new(Data { cursor_suppression: None, + cursor_mouse: None, config, engine: None, display: None, @@ -221,6 +231,7 @@ fn reset_scanner_for_capture(app: &AppHandle, message: &str, recover app.state::().stop(); } hide_scan_visuals(); + let cursor = app.state::(); let token = c .data .lock() @@ -228,9 +239,13 @@ fn reset_scanner_for_capture(app: &AppHandle, message: &str, recover .cursor_suppression .take(); if let Some((token, _)) = token { - app.state::() - .release_scan(token); + cursor.end_scan_mouse(token); + cursor.release_scan(token); } + c.data + .lock() + .unwrap_or_else(|p| p.into_inner()) + .cursor_mouse = None; publish::(app); } /// Pauses scanning so settings can be saved. Must run on the main thread, as @@ -461,6 +476,7 @@ fn dispatch( hide_scan_visuals(); } let deferred = A::deferred(&request); + let pointer_feedback = A::cursor_action_feedback(&request); match A::activate(app, request) { Ok(environment) => { let mut d = c.data.lock().unwrap_or_else(|p| p.into_inner()); @@ -472,6 +488,16 @@ fn dispatch( engine.technique.execution_succeeded(); } } + let mouse_token = d + .cursor_suppression + .map(|(token, _)| token) + .filter(|_| d.cursor_mouse.is_some()); + drop(d); + if let (Some(token), Some(feedback)) = (mouse_token, pointer_feedback) { + let settings = app.state::().snapshot().settings; + app.state::() + .show_scan_mouse(token, feedback, settings)?; + } } Err(error) => { A::cleanup(app)?; @@ -591,8 +617,10 @@ fn render( if !active { hide_scan_visuals(); if let Some((token, _)) = d.cursor_suppression.take() { + cursor.end_scan_mouse(token); cursor.release_scan(token); } + d.cursor_mouse = None; return Ok(()); } if active { @@ -601,6 +629,7 @@ fn render( None => { let lease = (cursor.suppress_for_scan()?, Instant::now()); d.cursor_suppression = Some(lease); + d.cursor_mouse = None; lease } }; @@ -612,6 +641,22 @@ fn render( return Ok(()); } } + if let Some((token, _)) = d.cursor_suppression { + let feedback = d + .engine + .as_ref() + .and_then(|engine| A::cursor_feedback(&engine.technique)); + if let Some(feedback) = feedback { + let settings = app.state::().snapshot().settings; + let next = (feedback, settings); + if d.cursor_mouse.as_ref() != Some(&next) { + cursor.show_scan_mouse(token, feedback, next.1.clone())?; + d.cursor_mouse = Some(next); + } + } else if d.cursor_mouse.take().is_some() { + cursor.end_scan_mouse(token); + } + } let frame = d .engine .as_ref() diff --git a/src/settings/ScanningSection.tsx b/src/settings/ScanningSection.tsx index 210c6f56..428bde95 100644 --- a/src/settings/ScanningSection.tsx +++ b/src/settings/ScanningSection.tsx @@ -37,7 +37,7 @@ export function ScanningSection({ controller }: { controller: ScanningController
How scanning works

Focus the application you want to use, then press Select to scan the display under the pointer. Choose a point, then use the action menu for clicks, scrolling and dragging.

-

Choose Mouse from the action menu, or assign Open mouse to a switch. Select a direction to move continuously; press any switch to stop and scan the mouse panel again. The panel also offers clicks, dragging, scrolling, speed, monitor navigation and Keyboard.

+

Choose Mouse from the action menu, or assign Open mouse to a switch. The pointer ring stays visible while Mouse is open. Select a direction to move with the saved Remote repeat settings; press any switch to stop repeating and scan the mouse panel again. If repeat is off, each selection moves one step. The panel also offers clicks, dragging, scrolling, speed, monitor navigation and Keyboard.

Assign switch actions in the Switches page. Actions run on release; holding a switch freezes movement. Manual scanning needs Select, Next and Previous. After clicking or reaching the pass limit, use Select to start again.

Assigned keys stay reserved while Switchify runs. Escape resets the scan. Mobile connections pause local scanning until they end.

On Windows, assigned keys remain switches with Shift, Ctrl, Alt or Windows held, even in the background. Other keys and Switchify-generated shortcuts still work normally.