diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 90f5f26f..8f357745 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -28,6 +28,7 @@ mod scan_host; mod scan_items; mod scan_keyboard; mod scan_menu; +mod scan_mouse; mod scan_preferences; mod scan_tile; mod scan_tree; diff --git a/src-tauri/src/mouse_repeat.rs b/src-tauri/src/mouse_repeat.rs index bb0aae0d..67f445e0 100644 --- a/src-tauri/src/mouse_repeat.rs +++ b/src-tauri/src/mouse_repeat.rs @@ -147,6 +147,57 @@ struct MoveRepeatState { initial_emitted: bool, } +/// Drives a scanned direction with the same movement integral as a remote +/// repeat. Scanner ticks are slower than remote ticks, so split their elapsed +/// time into remote-sized slices before asking the shared motion state. +pub struct ScanMoveRepeat { + direction: (i32, i32), + now: Instant, + movement: MoveRepeatState, +} + +impl ScanMoveRepeat { + pub fn new(dx: i32, dy: i32, now: Instant) -> Self { + Self { + direction: (dx, dy), + now, + movement: MoveRepeatState::new(now), + } + } + + pub fn initial_move(&mut self) -> (i32, i32) { + self.movement + .initial_delta(self.direction.0, self.direction.1) + } + + pub fn advance( + &mut self, + elapsed_ms: u64, + move_interval_ms: u32, + pointer_scale_percent: u8, + acceleration_duration_ms: u32, + ) -> (i32, i32) { + let mut remaining = elapsed_ms.min(64); + let mut result = (0, 0); + while remaining > 0 { + let step = remaining.min(MOVE_TICK_INTERVAL_MS); + self.now += std::time::Duration::from_millis(step); + let delta = self.movement.advance( + self.now, + self.direction.0, + self.direction.1, + move_interval_ms, + pointer_scale_percent, + acceleration_duration_ms, + ); + result.0 += delta.0; + result.1 += delta.1; + remaining -= step; + } + result + } +} + impl MoveRepeatState { fn new(now: Instant) -> Self { Self { @@ -603,6 +654,44 @@ mod tests { ); } + #[test] + fn scanning_motion_matches_remote_repeat_ticks() { + for direction in [(12, 0), (-12, 12), (0, -12)] { + for acceleration in [0, 1000] { + let now = Instant::now(); + let mut scan = ScanMoveRepeat::new(direction.0, direction.1, now); + let mut remote = MouseRepeatController::default(); + let active = remote.start( + "remote".into(), + RepeatCommand::Move { + dx: direction.0, + dy: direction.1, + }, + acceleration, + now, + ); + assert_eq!( + scan.initial_move(), + remote.initial_move("remote", active.generation).unwrap() + ); + let mut time = now; + for _ in 0..40 { + let scanned = scan.advance(32, 250, 100, acceleration); + let mut expected = (0, 0); + for _ in 0..4 { + time += Duration::from_millis(8); + let delta = remote + .advance_move("remote", active.generation, time, 250, 100) + .unwrap(); + expected.0 += delta.0; + expected.1 += delta.1; + } + assert_eq!(scanned, expected); + } + } + } + } + #[test] fn parses_only_bounded_move_and_scroll_commands() { assert_eq!( diff --git a/src-tauri/src/overlay.rs b/src-tauri/src/overlay.rs index 0b67d5b5..955e1f70 100644 --- a/src-tauri/src/overlay.rs +++ b/src-tauri/src/overlay.rs @@ -77,6 +77,7 @@ struct Visibility { epoch: u64, suppressed: bool, hidden: bool, + mouse: bool, } impl VisibilityGate { pub(crate) fn present( @@ -85,7 +86,7 @@ impl VisibilityGate { render: impl FnOnce() -> Result<(), String>, ) -> Result<(), String> { let state = self.0.lock().unwrap_or_else(|p| p.into_inner()); - if state.epoch == epoch && !state.suppressed { + if state.epoch == epoch && (!state.suppressed || state.mouse) { render() } else { Ok(()) @@ -120,6 +121,7 @@ impl CursorOverlay { state.epoch += 1; state.suppressed = true; state.hidden = false; + state.mouse = false; self.sender .send(Command::Suppress(state.epoch)) .map_err(|_| "Cursor overlay is unavailable.".to_string())?; @@ -131,6 +133,32 @@ impl CursorOverlay { state.epoch == epoch && state.suppressed && state.hidden } + /// Feedback from the current scanning mouse is the only feedback allowed + /// to render while the scan owns the overlay. + pub(crate) fn show_scan_mouse( + &self, + epoch: u64, + feedback: PointerFeedback, + settings: AppSettings, + ) -> Result<(), String> { + let mut state = self.gate.0.lock().unwrap_or_else(|p| p.into_inner()); + if state.epoch != epoch || !state.suppressed || !state.hidden { + return Err("Mouse overlay scan token is no longer active.".into()); + } + state.mouse = true; + self.sender + .send(Command::ScanMouse(epoch, feedback, settings)) + .map_err(|_| "Cursor overlay is unavailable.".to_string()) + } + + pub(crate) fn end_scan_mouse(&self, epoch: u64) { + let mut state = self.gate.0.lock().unwrap_or_else(|p| p.into_inner()); + if state.epoch == epoch && state.suppressed && state.mouse { + state.mouse = false; + let _ = self.sender.send(Command::EndScanMouse(epoch)); + } + } + /// Called only after all scanner windows have been hidden. pub(crate) fn release_scan(&self, epoch: u64) { let mut state = self.gate.0.lock().unwrap_or_else(|p| p.into_inner()); @@ -140,6 +168,7 @@ impl CursorOverlay { state.epoch += 1; state.suppressed = false; state.hidden = false; + state.mouse = false; let _ = self.sender.send(Command::Resume(state.epoch)); } @@ -197,6 +226,8 @@ pub(crate) enum Command { Scoped(u64, Box), Suppress(u64), Resume(u64), + ScanMouse(u64, PointerFeedback, AppSettings), + EndScanMouse(u64), Show(PointerFeedback, AppSettings), MarkControlActive(AppSettings), ApplySettings(AppSettings), @@ -234,6 +265,7 @@ pub(crate) struct OverlayEngine { next_follow: Instant, visible: bool, typing_suppressed: bool, + scan_mouse: bool, } impl OverlayEngine { @@ -266,12 +298,42 @@ impl OverlayEngine { next_follow: now, visible: false, typing_suppressed: false, + scan_mouse: false, + } + } + + fn show_scan_mouse( + &mut self, + feedback: PointerFeedback, + mut settings: AppSettings, + now: Instant, + ) -> Update { + let dragging = self.scan_mouse && self.drag_active; + self.scan_mouse = true; + settings.cursor_overlay_enabled = true; + settings.cursor_overlay_visibility = "whileControlling".into(); + let update = self.handle(Command::Show(feedback, settings), now); + if dragging && matches!(feedback, PointerFeedback::Scroll { .. }) { + self.drag_active = true; + } + if matches!(feedback, PointerFeedback::RepeatMove { .. }) { + self.deadline = None; } + update + } + + fn end_scan_mouse(&mut self, now: Instant) -> Update { + self.scan_mouse = false; + self.handle(Command::EndSession, now) } pub(crate) fn handle(&mut self, command: Command, now: Instant) -> Update { match command { - Command::Scoped(_, _) | Command::Suppress(_) | Command::Resume(_) => { + Command::Scoped(_, _) + | Command::Suppress(_) + | Command::Resume(_) + | Command::ScanMouse(..) + | Command::EndScanMouse(_) => { unreachable!("handled by overlay worker") } Command::Show(feedback, settings) => { @@ -281,7 +343,10 @@ impl OverlayEngine { return self.hide(); } self.control_active = true; - self.drag_active = matches!(feedback, PointerFeedback::Drag); + self.drag_active = matches!( + feedback, + PointerFeedback::Drag | PointerFeedback::RepeatMove { dragging: true, .. } + ); self.deadline = deadline_for(feedback, self.persistent(), now); self.feedback = Some(feedback); self.next_follow = now + FOLLOW_INTERVAL; @@ -826,13 +891,14 @@ pub(crate) fn run_loop( break; } let command = receiver.recv_timeout(Duration::from_millis(25)); - let (epoch, suppressed) = { + let (epoch, suppressed, mouse) = { let state = gate.0.lock().unwrap_or_else(|p| p.into_inner()); - (state.epoch, state.suppressed) + (state.epoch, state.suppressed, state.mouse) }; let update = match command { Ok(Command::Suppress(token)) => { engine.clear_scan_feedback(); + engine.scan_mouse = false; hide(Some(token)); continue; } @@ -840,22 +906,44 @@ pub(crate) fn run_loop( if token != epoch || suppressed { continue; } - engine.resume_after_scan(Instant::now()) + if engine.scan_mouse { + engine.end_scan_mouse(Instant::now()) + } else { + engine.resume_after_scan(Instant::now()) + } } Ok(Command::Scoped(token, command)) => { - let update = engine.handle(*command, Instant::now()); - if suppressed || token != epoch { + if suppressed && mouse { + Update::None + } else if suppressed || token != epoch { + let _ = engine.handle(*command, Instant::now()); engine.clear_scan_feedback(); Update::None } else { - update + engine.handle(*command, Instant::now()) + } + } + Ok(Command::ScanMouse(token, feedback, settings)) => { + if token == epoch && suppressed && mouse { + engine.show_scan_mouse(feedback, settings, Instant::now()) + } else { + Update::None + } + } + Ok(Command::EndScanMouse(token)) => { + if (token == epoch && suppressed) + || (token.checked_add(1) == Some(epoch) && !suppressed) + { + engine.end_scan_mouse(Instant::now()) + } else { + Update::None } } Ok(command) => engine.handle(command, Instant::now()), Err(mpsc::RecvTimeoutError::Timeout) => engine.tick(Instant::now()), Err(mpsc::RecvTimeoutError::Disconnected) => Update::Shutdown, }; - let update = if suppressed && matches!(update, Update::Render(_)) { + let update = if suppressed && !mouse && matches!(update, Update::Render(_)) { engine.clear_scan_feedback(); Update::None } else { @@ -935,6 +1023,106 @@ mod tests { assert!(!visible.get()); } + #[test] + fn only_the_current_mouse_scan_can_render_through_suppression() { + let (overlay, _receiver) = test_overlay(); + let first = overlay.suppress_for_scan().unwrap(); + overlay.gate.hide(Some(first), || {}); + assert!(overlay + .show_scan_mouse(first, PointerFeedback::Move, AppSettings::default()) + .is_ok()); + let visible = std::cell::Cell::new(false); + overlay + .gate + .present(first, || { + visible.set(true); + Ok(()) + }) + .unwrap(); + assert!(visible.get()); + overlay.end_scan_mouse(first); + visible.set(false); + overlay + .gate + .present(first, || { + visible.set(true); + Ok(()) + }) + .unwrap(); + assert!(!visible.get()); + overlay.release_scan(first); + let second = overlay.suppress_for_scan().unwrap(); + overlay.gate.hide(Some(second), || {}); + assert!(overlay + .show_scan_mouse(first, PointerFeedback::Move, AppSettings::default()) + .is_err()); + assert!(overlay + .show_scan_mouse(second, PointerFeedback::Move, AppSettings::default()) + .is_ok()); + } + + #[test] + fn mouse_scan_forces_persistent_remote_ring_and_cleans_up() { + let now = Instant::now(); + let mut engine = OverlayEngine::new(now); + let settings = AppSettings { + cursor_overlay_enabled: false, + cursor_overlay_visibility: "onInput".into(), + cursor_overlay_size: "large".into(), + cursor_overlay_color: "blue".into(), + cursor_crosshairs: true, + ..Default::default() + }; + let Update::Render(frame) = + engine.show_scan_mouse(PointerFeedback::Move, settings.clone(), now) + else { + panic!("mouse ring did not appear"); + }; + assert_eq!(frame.logical_size, 176); + assert_eq!(frame.color, color_for("blue")); + assert!(frame.crosshairs); + assert!(matches!( + engine.tick(now + DEFAULT_DURATION), + Update::Render(_) + )); + engine.show_scan_mouse( + PointerFeedback::RepeatMove { + accelerated: true, + dragging: true, + }, + settings.clone(), + now, + ); + assert_eq!(engine.deadline, None); + engine.show_scan_mouse(PointerFeedback::Drag, settings, now); + assert!(matches!(engine.end_scan_mouse(now), Update::Hide)); + assert!(!engine.visible); + assert!(!engine.control_active); + } + + #[test] + fn queued_mouse_close_cannot_restore_ring_after_scan_release() { + let (overlay, receiver) = test_overlay(); + overlay.mark_control_active(AppSettings::default()); + let token = overlay.suppress_for_scan().unwrap(); + overlay.gate.hide(Some(token), || {}); + overlay + .show_scan_mouse(token, PointerFeedback::Move, AppSettings::default()) + .unwrap(); + overlay.end_scan_mouse(token); + overlay.release_scan(token); + let gate = overlay.gate.clone(); + let hide_gate = gate.clone(); + drop(overlay); + run_loop( + |_, _| panic!("mouse ring was restored after closing"), + |epoch| hide_gate.hide(epoch, || {}), + || true, + receiver, + gate, + ); + } + #[test] fn queued_feedback_is_discarded_and_only_persistent_cursor_is_restored() { for visibility in ["onInput", "whileControlling"] { diff --git a/src-tauri/src/point_scan.rs b/src-tauri/src/point_scan.rs index 9a119e61..612ba17c 100644 --- a/src-tauri/src/point_scan.rs +++ b/src-tauri/src/point_scan.rs @@ -60,6 +60,7 @@ impl Config { crate::scan_preferences::Area::Point, crate::scan_preferences::Area::Menu, crate::scan_preferences::Area::Keyboard, + crate::scan_preferences::Area::Mouse, ] .into_iter() .all(|area| self.resolved(area).automatic), @@ -73,18 +74,27 @@ impl Config { &self, area: crate::scan_preferences::Area, ) -> crate::scan_preferences::Resolved { - self.scan_preferences.resolve( + let mut resolved = self.scan_preferences.resolve( area, self.automatic, self.block_interval_ms, self.scanner_color, - ) + ); + if area == crate::scan_preferences::Area::Mouse + && self.scan_preferences.mouse.automatic.is_none() + { + resolved.automatic = self + .resolved(crate::scan_preferences::Area::Keyboard) + .automatic; + } + resolved } pub fn point(&self) -> PointSettings { PointSettings { scan: self.resolved(crate::scan_preferences::Area::Point), menu_scan: self.resolved(crate::scan_preferences::Area::Menu), keyboard_scan: self.resolved(crate::scan_preferences::Area::Keyboard), + mouse_scan: self.resolved(crate::scan_preferences::Area::Mouse), word_prediction: self.word_prediction, keyboard_wait_after_typing: self.keyboard_wait_after_typing, scanner_color: self.resolved(crate::scan_preferences::Area::Point).color, @@ -108,6 +118,7 @@ pub struct PointSettings { pub scan: crate::scan_preferences::Resolved, pub menu_scan: crate::scan_preferences::Resolved, pub keyboard_scan: crate::scan_preferences::Resolved, + pub mouse_scan: crate::scan_preferences::Resolved, pub word_prediction: bool, pub keyboard_wait_after_typing: bool, pub scanner_color: crate::scanning::ScannerColor, @@ -269,7 +280,11 @@ impl Engine { self.step(TICK_MS); } Action::Reverse => self.direction = -self.direction, - Action::Pause | Action::Stop | Action::Cancel | Action::OpenKeyboard => {} + Action::Pause + | Action::Stop + | Action::Cancel + | Action::OpenKeyboard + | Action::OpenMouse => {} } None } diff --git a/src-tauri/src/point_scan_runtime.rs b/src-tauri/src/point_scan_runtime.rs index 812e3607..04b07b7c 100644 --- a/src-tauri/src/point_scan_runtime.rs +++ b/src-tauri/src/point_scan_runtime.rs @@ -41,6 +41,25 @@ impl Adapter for PointScan { type Environment = Environment; const EVENT: &'static str = "point-scan-changed"; const FILE: &'static str = "point-scan.json"; + fn cursor_feedback(technique: &Workflow) -> Option { + technique.mouse_feedback() + } + fn cursor_action_feedback(request: &Request) -> Option { + use crate::input::PointerFeedback; + use crate::protocol::MouseButton; + match *request { + Request::MouseClick { right, count } => Some(PointerFeedback::Click { + button: if right { + MouseButton::Right + } else { + MouseButton::Left + }, + count, + }), + Request::MouseScroll { dy } => Some(PointerFeedback::Scroll { dx: 0, dy }), + _ => None, + } + } fn validate(config: &Config) -> Result<(), String> { config.validate() } @@ -72,10 +91,55 @@ impl Adapter for PointScan { crate::prediction::record(stroke, result.is_ok(), scope); return result.map(|()| None); } - Request::OpenKeyboard => { + Request::OpenKeyboard | Request::OpenMouse => { crate::prediction::stop(); crate::scan_executor::activate(request) } + Request::MouseDrag => { + let (position, _) = display_navigation::displays(app).map_err(|e| e.message)?; + return crate::scan_executor::toggle_mouse_drag(( + position.0.round() as i32, + position.1.round() as i32, + )) + .map(|()| None); + } + Request::MouseMove { dx, dy } if cfg!(target_os = "macos") => { + let (cursor, displays) = + display_navigation::displays(app).map_err(|e| e.message)?; + let target = display_navigation::clamped_pointer_target(cursor, dx, dy, &displays) + .ok_or("No active display could be resolved.")?; + return crate::scan_executor::activate(Request::MouseMoveAbsolute { + x: target.0, + y: target.1, + }) + .map(|()| None); + } + Request::MouseSpeed(direction) => { + let model = app.state::(); + let old = model.snapshot().settings.pointer_scale_percent; + let next = (i16::from(old) + i16::from(direction) * 5).clamp(5, 225) as u8; + model.apply_pointer_scale_percent(next)?; + crate::state::emit_state(app, &model.shared); + return Ok(None); + } + Request::MouseMonitor(dx, dy) => { + let (cursor, displays) = + display_navigation::displays(app).map_err(|e| e.message)?; + let source = display_navigation::current_display(cursor, &displays) + .ok_or("No scanning display is available.")?; + let direction = match (dx, dy) { + (-1, 0) => "left", + (1, 0) => "right", + (0, -1) => "up", + (0, 1) => "down", + _ => return Err("Invalid monitor direction.".into()), + }; + let target = display_navigation::target_center(source, &displays, direction) + .map_err(|e| e.message)?; + crate::scan_executor::move_to(target)?; + let (_, environment) = new_engine(app, app.state::().view().config)?; + return Ok(Some(environment)); + } Request::Setting(setting) => scanning_runtime::update_point_setting(app, setting), Request::Display(next) => scanning_runtime::restart_point_on_display(app, next), request => crate::scan_executor::activate(request), @@ -88,7 +152,40 @@ impl Adapter for PointScan { technique: Option<&mut Workflow>, ) -> Result { if let (Some(environment), Some(technique)) = (environment, technique) { - if technique.keyboard_open() { + if technique.mouse_open() { + let (cursor, displays) = + display_navigation::displays(app).map_err(|e| e.message)?; + let current = display_navigation::current_display(cursor, &displays) + .ok_or("No scanning display is available.")? + .clone(); + let rect = Rect { + x: current.x.into(), + y: current.y.into(), + width: current.width.into(), + height: current.height.into(), + }; + let area = crate::scan_host::work_area(rect)?; + environment.display = current; + environment.keyboard_area = area; + environment.foreground = crate::scan_host::foreground()?; + technique.set_mouse_area( + area, + rect, + if cfg!(target_os = "windows") { + environment.display.scale_factor + } else { + 1.0 + }, + displays.len(), + ); + let settings = app.state::().snapshot().settings; + technique.set_mouse_settings( + settings.pointer_scale_percent, + settings.mouse_repeat_acceleration_duration_ms, + settings.move_repeat_interval_ms, + settings.mouse_repeat_enabled, + ); + } else if technique.keyboard_open() { crate::point_scan_ready(app)?; let foreground = crate::scan_host::foreground()?; if environment.foreground != foreground { @@ -102,7 +199,13 @@ impl Adapter for PointScan { Ok(true) } fn preserve_visuals(request: &Request) -> bool { - matches!(request, Request::Keyboard(_) | Request::Prediction { .. }) + matches!( + request, + Request::Keyboard(_) + | Request::Prediction { .. } + | Request::MouseMove { .. } + | Request::MouseMoveAbsolute { .. } + ) } fn deferred(request: &Request) -> bool { matches!(request, Request::Prediction { .. }) diff --git a/src-tauri/src/point_workflow.rs b/src-tauri/src/point_workflow.rs index 456c0192..ccf74aaa 100644 --- a/src-tauri/src/point_workflow.rs +++ b/src-tauri/src/point_workflow.rs @@ -13,7 +13,27 @@ pub enum Request { index: usize, }, OpenKeyboard, + OpenMouse, + CloseMouse, Keyboard(crate::scan_keyboard::Stroke), + MouseMove { + dx: i32, + dy: i32, + }, + MouseMoveAbsolute { + x: i32, + y: i32, + }, + MouseClick { + right: bool, + count: u8, + }, + MouseScroll { + dy: i32, + }, + MouseDrag, + MouseSpeed(i8), + MouseMonitor(i8, i8), Click { point: Point, right: bool, @@ -61,6 +81,9 @@ pub enum WorkflowPhase { Keyboard, KeyboardSuspended, KeyboardOpening, + Mouse, + MouseSuspended, + MouseMoving, Menu, MenuSuspended, AutoSelecting, @@ -72,6 +95,8 @@ pub enum WorkflowPhase { enum Stage { Keyboard, KeyboardOpening, + Mouse, + MouseMoving, Idle, Countdown, Point, @@ -82,6 +107,13 @@ enum Stage { pub struct Workflow { keyboard: crate::scan_keyboard::Keyboard, keyboard_area: Rect, + mouse: crate::scan_mouse::MousePanel, + mouse_area: Rect, + return_to_mouse: bool, + move_repeat: Option, + mouse_repeat_enabled: bool, + mouse_move_interval_ms: u32, + mouse_acceleration_ms: u32, point: Engine, stage: Stage, menu: Menu, @@ -96,6 +128,7 @@ impl Workflow { pub fn new(config: PointSettings, screen: Rect, scale: f64) -> Result { let menu_options = config.menu_scan; let keyboard_options = config.keyboard_scan; + let mouse_options = config.mouse_scan; Ok(Self { keyboard: crate::scan_keyboard::Keyboard::configured( cfg!(target_os = "macos"), @@ -103,6 +136,13 @@ impl Workflow { ) .with_wait_after_typing(config.keyboard_wait_after_typing), keyboard_area: screen, + mouse: crate::scan_mouse::MousePanel::new(mouse_options, 1, 100), + mouse_area: screen, + return_to_mouse: false, + move_repeat: None, + mouse_repeat_enabled: true, + mouse_move_interval_ms: 250, + mouse_acceleration_ms: 1000, point: Engine::new(config, screen, scale)?, stage: Stage::Idle, menu: Menu::configured(Kind::Actions, menu_options), @@ -117,6 +157,7 @@ impl Workflow { fn active_preferences(&self) -> crate::scan_preferences::Resolved { match self.stage { Stage::Keyboard | Stage::KeyboardOpening => self.point.config.keyboard_scan, + Stage::Mouse | Stage::MouseMoving => self.point.config.mouse_scan, Stage::Menu => self.point.config.menu_scan, _ => self.point.config.scan, } @@ -139,6 +180,13 @@ impl Workflow { self.point.config.keyboard_scan, ) .with_wait_after_typing(self.point.config.keyboard_wait_after_typing); + self.mouse = crate::scan_mouse::MousePanel::new( + self.point.config.mouse_scan, + 1, + self.mouse.speed_percent, + ); + self.return_to_mouse = false; + self.move_repeat = None; self.parent_menu.clear(); self.stage = Stage::Point; self.point.start(); @@ -156,6 +204,51 @@ impl Workflow { pub fn keyboard_open(&self) -> bool { matches!(self.stage, Stage::Keyboard | Stage::KeyboardOpening) } + pub fn mouse_open(&self) -> bool { + matches!(self.stage, Stage::Mouse | Stage::MouseMoving) + } + pub fn set_mouse_area(&mut self, area: Rect, screen: Rect, scale: f64, displays: usize) { + self.mouse_area = area; + self.point.screen = screen; + self.point.units_per_logical_pixel = scale; + self.mouse.set_displays(displays); + } + pub fn set_mouse_settings( + &mut self, + speed: u8, + acceleration_ms: u32, + move_interval_ms: u32, + repeat_enabled: bool, + ) { + self.mouse.speed_percent = speed; + self.mouse_acceleration_ms = acceleration_ms; + self.mouse_move_interval_ms = move_interval_ms; + self.mouse_repeat_enabled = repeat_enabled; + if !repeat_enabled && self.stage == Stage::MouseMoving { + self.stage = Stage::Mouse; + self.move_repeat = None; + self.pending = None; + self.mouse.restart(); + } + } + pub fn mouse_feedback(&self) -> Option { + use crate::input::PointerFeedback; + if self.mouse.error { + return None; + } + match self.stage { + Stage::MouseMoving => Some(PointerFeedback::RepeatMove { + accelerated: self.mouse_acceleration_ms > 0, + dragging: self.mouse.dragging, + }), + Stage::Mouse => Some(if self.mouse.dragging { + PointerFeedback::Drag + } else { + PointerFeedback::Move + }), + _ => None, + } + } pub fn foreground_changed(&mut self) { self.keyboard.reset_context(); } @@ -178,9 +271,103 @@ impl Workflow { self.open(Kind::Actions); } } + fn open_mouse(&mut self) -> Option { + self.stage = Stage::Mouse; + self.move_repeat = None; + self.mouse = crate::scan_mouse::MousePanel::new( + self.point.config.mouse_scan, + 1, + self.mouse.speed_percent, + ); + self.pending = None; + self.parent_menu.clear(); + self.elapsed = 0; + self.error = None; + self.return_to_mouse = false; + Some(Request::OpenMouse) + } + fn mouse_key(&mut self, key: crate::scan_mouse::Key) -> Option { + use crate::scan_mouse::Key; + self.mouse.choose(key); + match key { + Key::Move(dx, dy) => { + let delta = (i32::from(dx) * 12, i32::from(dy) * 12); + if self.mouse_repeat_enabled { + let mut repeat = crate::mouse_repeat::ScanMoveRepeat::new( + delta.0, + delta.1, + std::time::Instant::now(), + ); + let initial = repeat.initial_move(); + self.move_repeat = Some(repeat); + self.stage = Stage::MouseMoving; + Some(Request::MouseMove { + dx: initial.0, + dy: initial.1, + }) + } else { + let scale = f64::from(self.mouse.speed_percent) / 100.0; + Some(Request::MouseMove { + dx: (f64::from(delta.0) * scale).round() as i32, + dy: (f64::from(delta.1) * scale).round() as i32, + }) + } + } + Key::Click => Some(Request::MouseClick { + right: false, + count: 1, + }), + Key::RightClick => Some(Request::MouseClick { + right: true, + count: 1, + }), + Key::DoubleClick => Some(Request::MouseClick { + right: false, + count: 2, + }), + Key::Drag => { + self.mouse.dragging = !self.mouse.dragging; + Some(Request::MouseDrag) + } + Key::Scroll(direction) => Some(Request::MouseScroll { + dy: i32::from(direction) * 5, + }), + Key::Speed(direction) => Some(Request::MouseSpeed(direction)), + Key::Monitor(dx, dy) => Some(Request::MouseMonitor(dx, dy)), + Key::Keyboard => { + self.return_to_mouse = true; + self.mouse.dragging = false; + self.stage = Stage::KeyboardOpening; + self.keyboard = crate::scan_keyboard::Keyboard::configured( + cfg!(target_os = "macos"), + self.point.config.keyboard_scan, + ) + .with_wait_after_typing(self.point.config.keyboard_wait_after_typing); + Some(Request::OpenKeyboard) + } + Key::Close => { + self.reset(); + Some(Request::CloseMouse) + } + Key::More | Key::Movement | Key::Dock => None, + } + } + fn keyboard_closed(&mut self) { + if self.return_to_mouse { + self.return_to_mouse = false; + self.stage = Stage::Mouse; + self.mouse.restart(); + } else { + self.start(); + } + } fn selected(&mut self, item: Item) -> Option { match item { Item::Keyboard => { + self.return_to_mouse = self.mouse_open(); + if self.return_to_mouse { + self.mouse.dragging = false; + } self.stage = Stage::KeyboardOpening; self.keyboard = crate::scan_keyboard::Keyboard::configured( cfg!(target_os = "macos"), @@ -194,6 +381,7 @@ impl Workflow { return Some(Request::OpenKeyboard); } Item::KeyboardKey => {} + Item::MousePanel => return self.open_mouse(), Item::More | Item::Group(_) => { let kind = if let Item::Group(kind) = item { kind @@ -283,6 +471,15 @@ impl Technique for Workflow { self.keyboard.failed(); return; } + if self.mouse_open() { + self.stage = Stage::Mouse; + self.move_repeat = None; + self.mouse.dragging = false; + self.mouse.failed(); + self.error = None; + let _ = message; + return; + } self.pending = None; self.stage = Stage::Menu; self.menu = Menu::configured(Kind::Actions, self.point.config.menu_scan); @@ -308,6 +505,13 @@ impl Technique for Workflow { self.point.config.keyboard_scan, ) .with_wait_after_typing(self.point.config.keyboard_wait_after_typing); + self.mouse = crate::scan_mouse::MousePanel::new( + self.point.config.mouse_scan, + 1, + self.mouse.speed_percent, + ); + self.return_to_mouse = false; + self.move_repeat = None; self.stage = Stage::Idle; self.parent_menu.clear(); self.pending = None; @@ -326,6 +530,16 @@ impl Technique for Workflow { fn automatic(&self, _fallback: bool) -> bool { self.active_preferences().automatic } + fn switch_pressed(&mut self) -> bool { + if self.stage != Stage::MouseMoving { + return false; + } + self.stage = Stage::Mouse; + self.move_repeat = None; + self.pending = None; + self.mouse.restart(); + true + } fn auto_selecting(&self) -> bool { self.stage == Stage::Countdown } @@ -333,6 +547,12 @@ impl Technique for Workflow { self.stage != Stage::Executing } fn handle(&mut self, action: Action) -> Option { + if action == Action::OpenMouse { + if self.mouse_open() { + return None; + } + return self.open_mouse(); + } if action == Action::OpenKeyboard { if self.keyboard_open() { return None; @@ -347,10 +567,16 @@ impl Technique for Workflow { Some(crate::scan_keyboard::Output::Prediction { token, index }) => { return Some(Request::Prediction { token, index }) } - Some(crate::scan_keyboard::Output::Close) => self.start(), + Some(crate::scan_keyboard::Output::Close) => self.keyboard_closed(), None => {} }, Stage::KeyboardOpening => {} + Stage::Mouse => { + if let Some(key) = self.mouse.handle(action) { + return self.mouse_key(key); + } + } + Stage::MouseMoving => {} Stage::Point | Stage::Destination => { if let Some(point) = self.point.handle(action) { if self.stage == Stage::Destination { @@ -394,6 +620,9 @@ impl Technique for Workflow { Stage::Keyboard => self .keyboard .advance(ms, self.point.config.keyboard_scan.interval_ms), + Stage::Mouse => self + .mouse + .advance(ms, self.point.config.mouse_scan.interval_ms), Stage::Point | Stage::Destination => { self.point.advance(ms); if self.point.exhausted() { @@ -410,7 +639,19 @@ impl Technique for Workflow { } } fn update(&mut self, ms: u64, context: UpdateContext) { - if self.stage == Stage::Countdown { + if self.stage == Stage::MouseMoving { + let (dx, dy) = self.move_repeat.as_mut().map_or((0, 0), |repeat| { + repeat.advance( + ms, + self.mouse_move_interval_ms, + self.mouse.speed_percent, + self.mouse_acceleration_ms, + ) + }); + if dx != 0 || dy != 0 { + self.pending = Some(Request::MouseMove { dx, dy }); + } + } else if self.stage == Stage::Countdown { if !context.paused && !context.switch_held { self.elapsed = self .elapsed @@ -446,6 +687,12 @@ impl Technique for Workflow { fn phase(&self) -> Phase { match self.stage { Stage::KeyboardOpening => Phase::Workflow(WorkflowPhase::KeyboardOpening), + Stage::Mouse => Phase::Workflow(if self.mouse.suspended() { + WorkflowPhase::MouseSuspended + } else { + WorkflowPhase::Mouse + }), + Stage::MouseMoving => Phase::Workflow(WorkflowPhase::MouseMoving), Stage::Keyboard => Phase::Workflow(if self.keyboard.suspended() { WorkflowPhase::KeyboardSuspended } else { @@ -472,6 +719,22 @@ impl Technique for Workflow { self.point.units_per_logical_pixel, self.point.config.scanner_color, ), + Stage::Mouse | Stage::MouseMoving => { + let mut frame = self.mouse.frame( + self.mouse_area, + self.point.units_per_logical_pixel, + self.point.config.mouse_scan.color, + ); + if self.stage == Stage::MouseMoving { + if let Some(label) = frame.label.as_mut() { + label.text = "Moving pointer · Press any switch to stop".into(); + } + for tile in &mut frame.tiles { + tile.selected = false; + } + } + frame + } Stage::Countdown => { let scale = self.point.units_per_logical_pixel; let screen = self.point.screen; @@ -564,6 +827,142 @@ impl Technique for Workflow { mod tests { use super::*; use crate::{point_scan::Config, scanning::Session}; + + #[test] + fn mouse_motion_stops_on_press_and_returns_to_first_row() { + let screen = Rect { + x: 0.0, + y: 0.0, + width: 1280.0, + height: 720.0, + }; + let mut session = Session::new( + Workflow::new(Config::default().point(), screen, 1.0).unwrap(), + true, + ); + assert_eq!(session.action(Action::OpenMouse), Some(Request::OpenMouse)); + assert_eq!( + session.technique.phase(), + Phase::Workflow(WorkflowPhase::Mouse) + ); + assert_eq!( + session + .technique + .mouse_key(crate::scan_mouse::Key::Move(1, 0)), + Some(Request::MouseMove { dx: 1, dy: 0 }) + ); + let mut moved = false; + for _ in 0..20 { + session.tick(33, false); + if matches!(session.take_selection(), Some(Request::MouseMove { dx, dy: 0 }) if dx > 0) + { + moved = true; + break; + } + } + assert!(moved); + assert!(session.technique.switch_pressed()); + assert!(!session.technique.switch_pressed()); + assert_eq!( + session.technique.phase(), + Phase::Workflow(WorkflowPhase::Mouse) + ); + assert!(session + .technique + .mouse + .frame(screen, 1.0, Default::default()) + .tiles + .iter() + .skip(1) + .any(|tile| tile.selected)); + } + + #[test] + fn mouse_without_repeat_moves_once_and_keeps_scanning() { + let screen = Rect { + x: 0.0, + y: 0.0, + width: 1280.0, + height: 720.0, + }; + let mut workflow = Workflow::new(Config::default().point(), screen, 1.0).unwrap(); + workflow.open_mouse(); + workflow.set_mouse_settings(50, 1000, 250, false); + assert_eq!( + workflow.mouse_key(crate::scan_mouse::Key::Move(-1, 1)), + Some(Request::MouseMove { dx: -6, dy: 6 }) + ); + assert_eq!(workflow.phase(), Phase::Workflow(WorkflowPhase::Mouse)); + assert_eq!( + workflow.mouse_feedback(), + Some(crate::input::PointerFeedback::Move) + ); + workflow.update( + 1000, + UpdateContext { + movement_enabled: true, + paused: false, + switch_held: false, + }, + ); + assert!(workflow.take_selection().is_none()); + assert!(!workflow.switch_pressed()); + } + + #[test] + fn mouse_keyboard_return_and_drag_state() { + let screen = Rect { + x: 0.0, + y: 0.0, + width: 1280.0, + height: 720.0, + }; + let mut workflow = Workflow::new(Config::default().point(), screen, 1.0).unwrap(); + workflow.open_mouse(); + assert_eq!( + workflow.mouse_key(crate::scan_mouse::Key::Drag), + Some(Request::MouseDrag) + ); + assert!(workflow.mouse.dragging); + assert_eq!( + workflow.mouse_feedback(), + Some(crate::input::PointerFeedback::Drag) + ); + assert_eq!( + workflow.mouse_key(crate::scan_mouse::Key::Keyboard), + Some(Request::OpenKeyboard) + ); + assert!(!workflow.mouse.dragging); + assert_eq!(workflow.mouse_feedback(), None); + workflow.execution_succeeded(); + assert_eq!(workflow.phase(), Phase::Workflow(WorkflowPhase::Keyboard)); + assert!(workflow.return_to_mouse); + workflow.keyboard_closed(); + assert_eq!(workflow.phase(), Phase::Workflow(WorkflowPhase::Mouse)); + assert_eq!( + workflow.mouse_feedback(), + Some(crate::input::PointerFeedback::Move) + ); + } + + #[test] + fn mouse_input_failure_clears_repeat_and_ring_until_resumed() { + let mut workflow = session(false).technique; + workflow.open_mouse(); + workflow.mouse_key(crate::scan_mouse::Key::Move(1, 0)); + assert!(matches!( + workflow.mouse_feedback(), + Some(crate::input::PointerFeedback::RepeatMove { .. }) + )); + workflow.execution_failed("input failed".into()); + assert_eq!(workflow.mouse_feedback(), None); + assert!(!workflow.switch_pressed()); + workflow.handle(Action::Select); + assert_eq!( + workflow.mouse_feedback(), + Some(crate::input::PointerFeedback::Move) + ); + } fn session(automatic: bool) -> Session { Session::new( Workflow::new( @@ -1029,7 +1428,7 @@ mod tests { for (column, right, count) in [(0, false, 1), (1, true, 1), (2, false, 2)] { let mut s = session(false); open(&mut s); - assert_eq!(s.frame().tiles.len(), 10); + assert_eq!(s.frame().tiles.len(), 11); assert_eq!( choose(&mut s, 0, column), Some(Request::Click { diff --git a/src-tauri/src/scan_executor.rs b/src-tauri/src/scan_executor.rs index e4916da7..ec05df4f 100644 --- a/src-tauri/src/scan_executor.rs +++ b/src-tauri/src/scan_executor.rs @@ -16,7 +16,33 @@ pub fn execute( } match request { Request::Prediction { .. } => Err("Prediction requires the scan controller.".into()), - Request::OpenKeyboard => input.release_all(), + Request::OpenKeyboard | Request::OpenMouse | Request::CloseMouse => input.release_all(), + Request::MouseMove { dx, dy } => input.move_pointer_pixels(dx, dy).map(|_| ()), + Request::MouseMoveAbsolute { x, y } => input.move_pointer_pixels_absolute(x, y).map(|_| ()), + Request::MouseClick { right, count } => { + if input.has_active_drag() { + return Err("End the drag before clicking.".into()); + } + input.click_pointer( + if right { + MouseButton::Right + } else { + MouseButton::Left + }, + count, + ) + } + Request::MouseScroll { dy } => input.execute_repeat_scroll(0, dy).map(|_| ()), + Request::MouseDrag => { + if input.has_active_drag() { + input.release_all() + } else { + Err("Mouse drag requires the scan controller.".into()) + } + } + Request::MouseSpeed(_) | Request::MouseMonitor(..) => { + Err("Mouse action requires the scan controller.".into()) + } Request::Keyboard(stroke) => { if input.has_active_drag() { return Err("End the active drag before typing.".into()); @@ -103,6 +129,33 @@ pub fn activate(request: Request) -> Result<(), String> { result }) } +pub fn toggle_mouse_drag(point: (i32, i32)) -> Result<(), String> { + if !crate::scan_host::modifiers_released() { + return Err("Release modifiers before dragging.".into()); + } + INPUT.with(|slot| { + let mut slot = slot.borrow_mut(); + if slot.is_none() { + *slot = Some(DesktopInput::new( + enigo::Enigo::new(&crate::input::injection_settings()) + .map_err(|_| "Scan input could not be initialized.")?, + )); + } + let input = slot.as_mut().unwrap(); + if input.has_active_switch_session() { + return Err("End switch forwarding before dragging.".into()); + } + let result = if input.has_active_drag() { + input.release_all() + } else { + input.start_scan_drag(point) + }; + if result.is_err() { + let _ = input.release_all(); + } + result + }) +} pub fn prediction_text(text: &str) -> Result<(), String> { if text.chars().count() > 64 || !crate::scan_host::modifiers_released() { return Err("Prediction input is unavailable.".into()); @@ -273,8 +326,9 @@ mod tests { self.events.push(format!("text {text}")); Ok(()) } - fn move_pointer(&mut self, _: i32, _: i32) -> Result<(), String> { - panic!("No relative movement") + fn move_pointer(&mut self, dx: i32, dy: i32) -> Result<(), String> { + self.events.push(format!("relative {dx} {dy}")); + Ok(()) } fn move_pointer_absolute(&mut self, x: i32, y: i32) -> Result<(), String> { self.events.push(format!("move {x} {y}")); @@ -362,6 +416,34 @@ mod tests { assert!(!input.has_active_drag()); } #[test] + fn mouse_actions_use_fake_input_and_close_releases_drag() { + let mut input = DesktopInput::new(Fake::default()); + execute(&mut input, Request::MouseMove { dx: 4, dy: -2 }, true).unwrap(); + execute( + &mut input, + Request::MouseMoveAbsolute { x: 40, y: 50 }, + true, + ) + .unwrap(); + execute( + &mut input, + Request::MouseClick { + right: false, + count: 2, + }, + true, + ) + .unwrap(); + execute(&mut input, Request::MouseScroll { dy: 5 }, true).unwrap(); + execute(&mut input, Request::DragStart((10, 20)), true).unwrap(); + execute(&mut input, Request::CloseMouse, true).unwrap(); + assert!(!input.has_active_drag()); + assert!(input.injector.events.contains(&"relative 4 -2".into())); + assert!(input.injector.events.contains(&"move 40 50".into())); + assert!(input.injector.events.contains(&"scroll 0 5".into())); + assert!(input.injector.events.contains(&"button Left false".into())); + } + #[test] fn keyboard_text_chords_and_current_focus_use_fake_input() { use crate::scan_keyboard::{Key, Stroke}; let mut input = DesktopInput::new(Fake::default()); diff --git a/src-tauri/src/scan_menu.rs b/src-tauri/src/scan_menu.rs index 33e88aab..bde47754 100644 --- a/src-tauri/src/scan_menu.rs +++ b/src-tauri/src/scan_menu.rs @@ -6,6 +6,7 @@ use crate::{ #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Item { Keyboard, + MousePanel, KeyboardKey, More, Group(Kind), @@ -34,6 +35,7 @@ impl Item { pub fn label(self) -> &'static str { match self { Self::Keyboard => "Keyboard", + Self::MousePanel => "Mouse", Self::KeyboardKey => "Key", Self::More => "More", Self::Group(kind) => kind.label(), @@ -434,7 +436,7 @@ impl Kind { return vec![ vec![LeftClick, RightClick, DoubleClick], vec![Scroll, Drag, More], - vec![Keyboard], + vec![Keyboard, MousePanel], vec![NewPoint, Cancel], ] } @@ -444,6 +446,7 @@ impl Kind { } Self::More => vec![ Keyboard, + MousePanel, Group(Self::Mouse), Group(Self::Editing), Group(Self::Windows), diff --git a/src-tauri/src/scan_mouse.rs b/src-tauri/src/scan_mouse.rs new file mode 100644 index 00000000..338d9bef --- /dev/null +++ b/src-tauri/src/scan_mouse.rs @@ -0,0 +1,276 @@ +//! Switch-owned mouse panel. Native input stays in the scanning adapter. +use crate::{ + scan_items::{ItemScanner, Policy}, + scan_menu::Item, + scan_preferences::Resolved, + scanning::{Action, Frame, FrameLabel, FrameTile, Rect, ScannerColor, TileRole, TileStyle}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Key { + Move(i8, i8), + Click, + RightClick, + DoubleClick, + Drag, + Scroll(i8), + Speed(i8), + Monitor(i8, i8), + More, + Movement, + Keyboard, + Dock, + Close, +} + +pub struct MousePanel { + pub more: bool, + pub top: bool, + pub dragging: bool, + pub speed_percent: u8, + pub error: bool, + displays: usize, + rows: Vec>, + scan: ItemScanner, +} + +impl MousePanel { + pub fn new(options: Resolved, displays: usize, speed_percent: u8) -> Self { + let rows = Self::rows(false, displays); + let scan = ItemScanner::configured_rows(&rows, Policy::KEYBOARD, options); + Self { + more: false, + top: false, + dragging: false, + speed_percent, + error: false, + displays, + rows, + scan, + } + } + fn rows(more: bool, displays: usize) -> Vec> { + use Key::*; + if !more { + vec![ + vec![Move(-1, -1), Move(0, -1), Move(1, -1)], + vec![Move(-1, 0), Click, Move(1, 0)], + vec![Move(-1, 1), Move(0, 1), Move(1, 1)], + vec![RightClick, DoubleClick, Drag], + vec![More, Keyboard, Dock, Close], + ] + } else { + let mut rows = vec![vec![Scroll(1), Scroll(-1)], vec![Speed(-1), Speed(1)]]; + if displays > 1 { + rows.push(vec![ + Monitor(-1, 0), + Monitor(0, -1), + Monitor(0, 1), + Monitor(1, 0), + ]); + } + rows.push(vec![Movement, Keyboard, Dock, Close]); + rows + } + } + pub fn choose(&mut self, key: Key) { + match key { + Key::More | Key::Movement => { + self.more = key == Key::More; + self.rows = Self::rows(self.more, self.displays); + self.scan = + ItemScanner::configured_rows(&self.rows, Policy::KEYBOARD, self.scan.options); + } + Key::Dock => { + self.top = !self.top; + self.scan.restart(); + } + _ => self.scan.restart(), + } + } + pub fn handle(&mut self, action: Action) -> Option { + if self.scan.suspended { + if action == Action::Select { + self.error = false; + self.scan.restart(); + } + return None; + } + self.scan.handle(action) + } + pub fn advance(&mut self, ms: u64, period: u64) { + self.scan.advance(ms, period); + } + pub fn restart(&mut self) { + self.scan.restart(); + } + pub fn suspended(&self) -> bool { + self.scan.suspended + } + pub fn failed(&mut self) { + self.error = true; + self.scan.suspended = true; + } + pub fn set_displays(&mut self, displays: usize) { + if self.displays != displays { + self.displays = displays; + self.rows = Self::rows(self.more, displays); + self.scan = + ItemScanner::configured_rows(&self.rows, Policy::KEYBOARD, self.scan.options); + } + } + fn label(&self, key: Key) -> String { + use Key::*; + match key { + Move(-1, -1) => "↖".into(), + Move(0, -1) => "↑".into(), + Move(1, -1) => "↗".into(), + Move(-1, 0) => "←".into(), + Move(1, 0) => "→".into(), + Move(-1, 1) => "↙".into(), + Move(0, 1) => "↓".into(), + Move(1, 1) => "↘".into(), + Move(..) => "Move".into(), + Click => "Left click".into(), + RightClick => "Right click".into(), + DoubleClick => "Double click".into(), + Drag => if self.dragging { + "End drag" + } else { + "Start drag" + } + .into(), + Scroll(1) => "Scroll up".into(), + Scroll(_) => "Scroll down".into(), + Speed(-1) => format!("Slower · {}%", self.speed_percent), + Speed(_) => format!("Faster · {}%", self.speed_percent), + Monitor(-1, 0) => "Monitor left".into(), + Monitor(0, -1) => "Monitor up".into(), + Monitor(0, 1) => "Monitor down".into(), + Monitor(1, 0) => "Monitor right".into(), + Monitor(..) => "Monitor".into(), + More => "More controls".into(), + Movement => "Movement".into(), + Keyboard => "Keyboard".into(), + Dock => if self.top { "Dock bottom" } else { "Dock top" }.into(), + Close => "Close mouse".into(), + } + } + pub fn frame(&self, screen: Rect, units: f64, color: ScannerColor) -> Frame { + let columns = self.rows.iter().map(Vec::len).max().unwrap_or(1) as f64; + let width = (columns * 160.0 + 32.0) + .min(screen.width / units.max(0.1) - 8.0) + .max(1.0) + * units; + let height = (self.rows.len() as f64 * 98.0 + 52.0) + .min(screen.height / units.max(0.1) * 0.65) + .max(1.0) + * units; + let x = screen.x + (screen.width - width) / 2.0; + let y = if self.top { + screen.y + } else { + screen.y + screen.height - height + }; + let scale = units + .min(width / (columns * 160.0 + 32.0)) + .min(height / (self.rows.len() as f64 * 98.0 + 52.0)); + let mut frame = Frame::default(); + frame.tiles.push(FrameTile::panel_background( + Rect { + x, + y, + width, + height, + }, + scale, + color, + )); + let (active_row, active_column) = self.scan.position(&self.rows); + let row_height = (height - 52.0 * scale) / self.rows.len() as f64; + for (r, row) in self.rows.iter().enumerate() { + let cell_width = (width - 24.0 * scale) / row.len() as f64; + for (c, key) in row.iter().enumerate() { + frame.tiles.push(FrameTile { + thickness: self.scan.options.thickness, + color, + text: self.label(*key), + rect: Rect { + x: x + 12.0 * scale + c as f64 * cell_width, + y: y + 44.0 * scale + r as f64 * row_height, + width: (cell_width - 6.0 * scale).max(1.0), + height: (row_height - 6.0 * scale).max(1.0), + }, + scale, + icon: Item::KeyboardKey, + style: Some(TileStyle { + role: TileRole::Utility, + active: *key == Key::Drag && self.dragging, + row_scan: self.scan.row_scan(), + }), + selected: !self.scan.suspended + && !self.scan.nav.escaping() + && active_row == r + && active_column.is_none_or(|column| column == c), + }); + } + } + frame.label = Some(FrameLabel { + text: if self.error { + "Mouse action failed · Select to resume".into() + } else if self.more { + "Mouse · More controls".into() + } else { + "Mouse · Movement".into() + }, + rect: Rect { + x: x + 12.0 * scale, + y: y + 4.0 * scale, + width: width - 24.0 * scale, + height: 38.0 * scale, + }, + scale: scale * 0.8, + hud: None, + }); + frame + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pages_and_monitor_controls_follow_available_displays() { + let mut panel = MousePanel::new(Resolved::default(), 1, 100); + assert_eq!(panel.rows[1][1], Key::Click); + assert_eq!(panel.rows[0].len(), 3); + panel.choose(Key::More); + assert!(!panel + .rows + .iter() + .flatten() + .any(|key| matches!(key, Key::Monitor(..)))); + panel.set_displays(2); + assert_eq!(panel.rows[2].len(), 4); + panel.choose(Key::Movement); + assert_eq!(panel.rows[1][1], Key::Click); + } + + #[test] + fn panel_renders_with_selected_tiles_inside_the_screen() { + let panel = MousePanel::new(Resolved::default(), 2, 100); + let screen = Rect { + x: -1280.0, + y: 0.0, + width: 1280.0, + height: 720.0, + }; + let frame = panel.frame(screen, 1.0, ScannerColor::default()); + assert!(frame.tiles.iter().any(|tile| tile.selected)); + assert!(frame + .tiles + .iter() + .all(|tile| tile.rect.x >= screen.x && tile.rect.y >= screen.y)); + } +} diff --git a/src-tauri/src/scan_preferences.rs b/src-tauri/src/scan_preferences.rs index 45b1d369..20cea2b8 100644 --- a/src-tauri/src/scan_preferences.rs +++ b/src-tauri/src/scan_preferences.rs @@ -37,6 +37,7 @@ pub enum Area { Point, Menu, Keyboard, + Mouse, } #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase", default)] @@ -59,6 +60,7 @@ pub struct Preferences { pub point: Overrides, pub menu: Overrides, pub keyboard: Overrides, + pub mouse: Overrides, } impl Default for Preferences { fn default() -> Self { @@ -70,6 +72,7 @@ impl Default for Preferences { point: Default::default(), menu: Default::default(), keyboard: Default::default(), + mouse: Default::default(), } } } @@ -108,6 +111,7 @@ impl Preferences { Area::Point => self.point, Area::Menu => self.menu, Area::Keyboard => self.keyboard, + Area::Mouse => self.mouse, }; Resolved { automatic: local.automatic.unwrap_or(automatic), @@ -145,6 +149,7 @@ pub fn deserialize_preferences<'de, D: serde::Deserializer<'de>>( &mut preferences.point, &mut preferences.menu, &mut preferences.keyboard, + &mut preferences.mouse, ] { local.interval_ms = local.interval_ms.filter(|v| (100..=10000).contains(v)); local.pass_limit = local.pass_limit.filter(|v| valid_passes(*v)); diff --git a/src-tauri/src/scan_tile.rs b/src-tauri/src/scan_tile.rs index a4a7a281..4358efb7 100644 --- a/src-tauri/src/scan_tile.rs +++ b/src-tauri/src/scan_tile.rs @@ -181,7 +181,8 @@ pub fn bitmap(tile: &FrameTile) -> Result { lines(&[(62., 46.), (106., 90.)]); lines(&[(106., 46.), (62., 90.)]); } - Keyboard | More | Group(_) | Command(_) | Setting(_) | Display(_) | Pause | Reverse => { + Keyboard | MousePanel | More | Group(_) | Command(_) | Setting(_) | Display(_) | Pause + | Reverse => { artwork(&mut path, tile.icon); } DragHere => { @@ -366,6 +367,7 @@ fn artwork(path: &mut PathBuilder, item: Item) { } line(path, &[(66., 81.), (102., 81.)]); } + Item::MousePanel => mouse(path, 65.), Item::More => { for x in [60., 84., 108.] { path.push_circle(x, 68., 5.); diff --git a/src-tauri/src/scanning.rs b/src-tauri/src/scanning.rs index 778b7e08..3318ee07 100644 --- a/src-tauri/src/scanning.rs +++ b/src-tauri/src/scanning.rs @@ -15,11 +15,13 @@ pub enum Action { Stop, Cancel, OpenKeyboard, + OpenMouse, } impl Action { pub fn label(self) -> &'static str { match self { Self::OpenKeyboard => "Open keyboard", + Self::OpenMouse => "Open mouse", Self::Select => "Select", Self::Next => "Next", Self::Back => "Previous", @@ -317,6 +319,10 @@ pub trait Technique { fn auto_selecting(&self) -> bool { false } + /// Consumes the next physical press while a technique is moving the pointer. + fn switch_pressed(&mut self) -> bool { + false + } fn execution_failed(&mut self, _message: String) {} fn execution_succeeded(&mut self) {} fn start(&mut self); @@ -380,7 +386,7 @@ impl Session { self.reset(); return None; } - if action == Action::OpenKeyboard { + if matches!(action, Action::OpenKeyboard | Action::OpenMouse) { let selection = self.technique.handle(action); if selection.is_some() { self.active = true; diff --git a/src-tauri/src/scanning_runtime.rs b/src-tauri/src/scanning_runtime.rs index c2041730..eb0140fd 100644 --- a/src-tauri/src/scanning_runtime.rs +++ b/src-tauri/src/scanning_runtime.rs @@ -16,6 +16,14 @@ pub trait Adapter: Send + Sync + 'static { fn preserve_visuals(_request: &::Selection) -> bool { false } + fn cursor_feedback(_technique: &Self::Technique) -> Option { + None + } + fn cursor_action_feedback( + _request: &::Selection, + ) -> Option { + None + } fn poll(_app: &AppHandle, _technique: &mut Self::Technique, _captured_keys: &[String]) {} fn cleanup(app: &AppHandle) -> Result<(), String>; fn validate(config: &Self::Config) -> Result<(), String>; @@ -70,6 +78,7 @@ pub struct Controller { } struct Data { cursor_suppression: Option<(u64, Instant)>, + cursor_mouse: Option<(crate::input::PointerFeedback, crate::state::AppSettings)>, config: A::Config, engine: Option>, display: Option, @@ -107,6 +116,7 @@ impl Controller { generation: AtomicU64::new(0), data: Mutex::new(Data { cursor_suppression: None, + cursor_mouse: None, config, engine: None, display: None, @@ -221,6 +231,7 @@ fn reset_scanner_for_capture(app: &AppHandle, message: &str, recover app.state::().stop(); } hide_scan_visuals(); + let cursor = app.state::(); let token = c .data .lock() @@ -228,9 +239,13 @@ fn reset_scanner_for_capture(app: &AppHandle, message: &str, recover .cursor_suppression .take(); if let Some((token, _)) = token { - app.state::() - .release_scan(token); + cursor.end_scan_mouse(token); + cursor.release_scan(token); } + c.data + .lock() + .unwrap_or_else(|p| p.into_inner()) + .cursor_mouse = None; publish::(app); } /// Pauses scanning so settings can be saved. Must run on the main thread, as @@ -411,7 +426,7 @@ fn switch(app: &AppHandle, action: Action, input_generation: u64, re hide_prompt(); let result = (|| -> Result<(), String> { let mut d = c.data.lock().unwrap_or_else(|p| p.into_inner()); - if action == Action::OpenKeyboard + if matches!(action, Action::OpenKeyboard | Action::OpenMouse) || (d.engine.as_ref().is_none_or(|e| !e.active()) && action == Action::Select) { let (engine, display) = A::create(app, d.config.clone())?; @@ -461,6 +476,7 @@ fn dispatch( hide_scan_visuals(); } let deferred = A::deferred(&request); + let pointer_feedback = A::cursor_action_feedback(&request); match A::activate(app, request) { Ok(environment) => { let mut d = c.data.lock().unwrap_or_else(|p| p.into_inner()); @@ -472,6 +488,16 @@ fn dispatch( engine.technique.execution_succeeded(); } } + let mouse_token = d + .cursor_suppression + .map(|(token, _)| token) + .filter(|_| d.cursor_mouse.is_some()); + drop(d); + if let (Some(token), Some(feedback)) = (mouse_token, pointer_feedback) { + let settings = app.state::().snapshot().settings; + app.state::() + .show_scan_mouse(token, feedback, settings)?; + } } Err(error) => { A::cleanup(app)?; @@ -591,8 +617,10 @@ fn render( if !active { hide_scan_visuals(); if let Some((token, _)) = d.cursor_suppression.take() { + cursor.end_scan_mouse(token); cursor.release_scan(token); } + d.cursor_mouse = None; return Ok(()); } if active { @@ -601,6 +629,7 @@ fn render( None => { let lease = (cursor.suppress_for_scan()?, Instant::now()); d.cursor_suppression = Some(lease); + d.cursor_mouse = None; lease } }; @@ -612,6 +641,22 @@ fn render( return Ok(()); } } + if let Some((token, _)) = d.cursor_suppression { + let feedback = d + .engine + .as_ref() + .and_then(|engine| A::cursor_feedback(&engine.technique)); + if let Some(feedback) = feedback { + let settings = app.state::().snapshot().settings; + let next = (feedback, settings); + if d.cursor_mouse.as_ref() != Some(&next) { + cursor.show_scan_mouse(token, feedback, next.1.clone())?; + d.cursor_mouse = Some(next); + } + } else if d.cursor_mouse.take().is_some() { + cursor.end_scan_mouse(token); + } + } let frame = d .engine .as_ref() @@ -684,6 +729,16 @@ fn tick(app: &AppHandle) { match edge { crate::remote_scan::Edge::Reset => unreachable!(), crate::remote_scan::Edge::Down(id) => { + if d.engine + .as_mut() + .is_some_and(|engine| engine.technique.switch_pressed()) + { + d.pressed.cancel(); + d.last_tick = Instant::now(); + drop(d); + publish::(app); + continue; + } if !d.pressed.held() { d.remote_hold_started = Some(now_ms); } @@ -755,6 +810,16 @@ fn tick(app: &AppHandle) { continue; } if action == crate::switch_input::Action::Pressed { + if d.engine + .as_mut() + .is_some_and(|engine| engine.technique.switch_pressed()) + { + d.pressed.cancel(); + d.last_tick = Instant::now(); + drop(d); + publish::(app); + continue; + } let settings = d.switches.clone(); let countdown = d .engine diff --git a/src-tauri/src/switch_gestures.rs b/src-tauri/src/switch_gestures.rs index bd7c7879..a2cb6300 100644 --- a/src-tauri/src/switch_gestures.rs +++ b/src-tauri/src/switch_gestures.rs @@ -34,9 +34,14 @@ impl Gestures { let mut binding = binding.clone(); if countdown && !binding.hold_actions.contains(&Action::OpenKeyboard) + && !binding.hold_actions.contains(&Action::OpenMouse) && !matches!( binding.press_action, - Action::Stop | Action::Pause | Action::Cancel | Action::OpenKeyboard + Action::Stop + | Action::Pause + | Action::Cancel + | Action::OpenKeyboard + | Action::OpenMouse ) { binding.press_action = Action::Select; @@ -108,7 +113,7 @@ mod tests { } } #[test] - fn countdown_preserves_direct_and_held_keyboard_actions() { + fn countdown_preserves_direct_and_held_panel_actions() { let mut settings = Settings { bindings: vec![Binding { id: "keyboard".into(), @@ -132,6 +137,10 @@ mod tests { gestures.released("keyboard", 2100), Some(Action::OpenKeyboard) ); + settings.bindings[0].press_action = Action::OpenMouse; + settings.bindings[0].hold_actions.clear(); + gestures.pressed_for_scan("keyboard", 3000, &settings, true); + assert_eq!(gestures.released("keyboard", 3100), Some(Action::OpenMouse)); } #[test] fn countdown_press_overrides_normal_actions_and_holds_once() { diff --git a/src-tauri/src/switches.rs b/src-tauri/src/switches.rs index 126526bd..fe621d94 100644 --- a/src-tauri/src/switches.rs +++ b/src-tauri/src/switches.rs @@ -122,7 +122,7 @@ impl Settings { mod tests { use super::*; #[test] - fn version_one_assignments_round_trip_with_additive_keyboard_action() { + fn version_one_assignments_round_trip_with_additive_keyboard_and_mouse_actions() { let legacy = serde_json::json!({ "schemaVersion": 1, "holdIntervalMs": 1000, "bindings": [{ "id": "one", "name": "One", "key": "Space", @@ -132,9 +132,11 @@ mod tests { settings.validate_actions(false).unwrap(); assert_eq!(serde_json::to_value(&settings).unwrap(), legacy); settings.bindings[0].hold_actions.push(Action::OpenKeyboard); + settings.bindings[0].hold_actions.push(Action::OpenMouse); settings.validate_actions(false).unwrap(); let updated = serde_json::to_value(&settings).unwrap(); assert_eq!(updated["bindings"][0]["holdActions"][5], "openKeyboard"); + assert_eq!(updated["bindings"][0]["holdActions"][6], "openMouse"); assert_eq!( serde_json::from_value::(updated).unwrap(), settings diff --git a/src/scanning/SwitchPractice.test.tsx b/src/scanning/SwitchPractice.test.tsx index 5983ec95..a7ca5e59 100644 --- a/src/scanning/SwitchPractice.test.tsx +++ b/src/scanning/SwitchPractice.test.tsx @@ -26,7 +26,7 @@ describe("safe switch practice", () => { render(); fireEvent.click(screen.getByRole("button", { name: "Test switches" })); const dialog = await screen.findByRole("dialog", { name: "Safe switch practice" }); expect(invoke).toHaveBeenCalledWith("begin_switch_practice", { remote: false }); - expect(dialog).toHaveFocus(); + await waitFor(() => expect(dialog).toHaveFocus()); expect(screen.queryByRole("button", { name: "Start another test" })).not.toBeInTheDocument(); fireEvent.click(screen.getByRole("button", { name: "Exit practice" })); await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); diff --git a/src/scanning/preferences.ts b/src/scanning/preferences.ts index f8c85d6b..5212a596 100644 --- a/src/scanning/preferences.ts +++ b/src/scanning/preferences.ts @@ -1,5 +1,5 @@ import type { PointScanConfig, ScannerColor } from './useScanning'; -export type ScanArea = 'point' | 'menu' | 'keyboard'; +export type ScanArea = 'point' | 'menu' | 'keyboard' | 'mouse'; export type ScanOptions = { automatic: boolean; intervalMs: number; @@ -12,7 +12,7 @@ export type ScanOptions = { export type ScanPreferences = Pick & Record>; export const defaultScanPreferences: ScanPreferences = { direction: 'forward', passLimit: 3, pattern: 'grouped', thickness: 'standard', - point: {}, menu: {}, keyboard: {}, + point: {}, menu: {}, keyboard: {}, mouse: {}, }; export function sharedOptions(config: PointScanConfig): ScanOptions { const settings = config.scanPreferences ?? defaultScanPreferences; @@ -21,7 +21,8 @@ export function sharedOptions(config: PointScanConfig): ScanOptions { } export function areaOptions(config: PointScanConfig, area: ScanArea): ScanOptions { const shared = sharedOptions(config); - const overrides = (config.scanPreferences ?? defaultScanPreferences)[area]; - return { ...shared, ...Object.fromEntries(Object.entries(overrides).filter(([, value]) => value != null)), + const overrides = (config.scanPreferences ?? defaultScanPreferences)[area] ?? {}; + return { ...shared, ...(area === 'mouse' && overrides.automatic == null ? { automatic: areaOptions(config, 'keyboard').automatic } : {}), + ...Object.fromEntries(Object.entries(overrides).filter(([, value]) => value != null)), ...(area === 'point' ? { pattern: 'grouped' as const } : {}) }; } diff --git a/src/scanning/useScanning.ts b/src/scanning/useScanning.ts index 2644ef18..e725d633 100644 --- a/src/scanning/useScanning.ts +++ b/src/scanning/useScanning.ts @@ -23,7 +23,7 @@ export type PointScanState = { remote?: boolean; config: PointScanConfig; enabled: boolean; - phase: "autoSelecting" | "idle" | "row" | "rowEscape" | "cell" | "x" | "y" | "menu" | "menuSuspended" | "dragDestination" | "dragConfirmation" | "executing" | "keyboard" | "keyboardSuspended" | "keyboardOpening"; + phase: "autoSelecting" | "idle" | "row" | "rowEscape" | "cell" | "x" | "y" | "menu" | "menuSuspended" | "dragDestination" | "dragConfirmation" | "executing" | "keyboard" | "keyboardSuspended" | "keyboardOpening" | "mouse" | "mouseSuspended" | "mouseMoving"; paused: boolean; message: string; supported: boolean; diff --git a/src/scanning/useSwitches.ts b/src/scanning/useSwitches.ts index 8b99a0bc..b8229e07 100644 --- a/src/scanning/useSwitches.ts +++ b/src/scanning/useSwitches.ts @@ -4,6 +4,7 @@ import { listen } from "@tauri-apps/api/event"; export const actions = { select: "Select", openKeyboard: "Open keyboard", + openMouse: "Open mouse", next: "Next", back: "Previous", reverse: "Reverse direction", diff --git a/src/settings/ScannerPreferences.tsx b/src/settings/ScannerPreferences.tsx index 18260d5f..e727c8bf 100644 --- a/src/settings/ScannerPreferences.tsx +++ b/src/settings/ScannerPreferences.tsx @@ -33,14 +33,14 @@ export function ScannerPreferences({ controller }: { controller: ScanningControl changedView.current = true; setArea(next); }; - const names = { point: 'Point scanning', menu: 'Menus', keyboard: 'Keyboard' }; + const names = { point: 'Point scanning', menu: 'Menus', keyboard: 'Keyboard', mouse: 'Mouse' }; const disabled = !state?.supported; const settings = config.scanPreferences ?? defaultScanPreferences; const shared = sharedOptions(config); const effective = area === 'shared' ? shared : areaOptions(config, area); const change = (key: K, value: ScanOptions[K] | undefined) => { if (area !== 'shared') { - const overrides = { ...settings[area], [key]: value }; + const overrides = { ...(settings[area] ?? {}), [key]: value }; update('scanPreferences', { ...settings, [area]: overrides }); } else if (key === 'automatic') update('automatic', value as boolean); else if (key === 'intervalMs') update('blockIntervalMs', value as number); @@ -48,7 +48,7 @@ export function ScannerPreferences({ controller }: { controller: ScanningControl else update('scanPreferences', { ...settings, [key]: value }); }; const field = (key: keyof ScanOptions, label: string, children: (locked: boolean) => ReactNode) => { - const custom = area !== 'shared' && settings[area][key] != null; + const custom = area !== 'shared' && settings[area]?.[key] != null; return
{area !== 'shared' &&
{custom ? 'Custom' : 'Default'} {custom && } @@ -157,9 +157,9 @@ export function ScannerPreferences({ controller }: { controller: ScanningControl {area === 'shared' &&

Customise an area

Give an area different settings, or keep using your defaults.

-
{(['point', 'menu', 'keyboard'] as const).map(key => { +
{(['point', 'menu', 'keyboard', 'mouse'] as const).map(key => { const options = areaOptions(config, key); - const count = Object.entries(settings[key]).filter(([name, value]) => value != null && !(key === 'point' && name === 'pattern')).length; + const count = Object.entries(settings[key] ?? {}).filter(([name, value]) => value != null && !(key === 'point' && name === 'pattern')).length; return
} - {area !== 'shared' && } + {area !== 'shared' && }

Saved automatically. Press Select to scan again.

; } diff --git a/src/settings/ScanningSection.tsx b/src/settings/ScanningSection.tsx index 740bcb67..428bde95 100644 --- a/src/settings/ScanningSection.tsx +++ b/src/settings/ScanningSection.tsx @@ -7,6 +7,9 @@ const phases = { keyboard: "Choose a keyboard row, then a key", keyboardSuspended: "Select to resume the keyboard", keyboardOpening: "Opening the keyboard", + mouse: "Choose a mouse control", + mouseSuspended: "Select to resume the mouse", + mouseMoving: "Moving pointer. Press any switch to stop", idle: "Ready to begin", autoSelecting: "Waiting to click. Press a switch for the action menu", menu: "Choose an action at the selected point", @@ -34,6 +37,7 @@ export function ScanningSection({ controller }: { controller: ScanningController
How scanning works

Focus the application you want to use, then press Select to scan the display under the pointer. Choose a point, then use the action menu for clicks, scrolling and dragging.

+

Choose Mouse from the action menu, or assign Open mouse to a switch. The pointer ring stays visible while Mouse is open. Select a direction to move with the saved Remote repeat settings; press any switch to stop repeating and scan the mouse panel again. If repeat is off, each selection moves one step. The panel also offers clicks, dragging, scrolling, speed, monitor navigation and Keyboard.

Assign switch actions in the Switches page. Actions run on release; holding a switch freezes movement. Manual scanning needs Select, Next and Previous. After clicking or reaching the pass limit, use Select to start again.

Assigned keys stay reserved while Switchify runs. Escape resets the scan. Mobile connections pause local scanning until they end.

On Windows, assigned keys remain switches with Shift, Ctrl, Alt or Windows held, even in the background. Other keys and Switchify-generated shortcuts still work normally.