diff --git a/docs/qwerty-keyboard.md b/docs/qwerty-keyboard.md new file mode 100644 index 00000000..0238044b --- /dev/null +++ b/docs/qwerty-keyboard.md @@ -0,0 +1,32 @@ +# QWERTY scanning keyboard + +Switchify has its own UK English keyboard on Windows and macOS. It uses the saved local switches or the existing Android remote scanning actions. It does not use the operating system's on-screen keyboard. + +Choose a point and select **Type here** to click that point once and open the keyboard. To keep the existing input focus without clicking, select **More → Keyboard**. The keyboard does not take focus or accept mouse clicks. + +Scan a row, select it, then scan and select a key. The escape slot returns to rows. After typing, scanning starts again at the first row. Existing automatic/manual movement, speed, pause, reverse and inactivity suspension apply. The keyboard stays open after Space, Backspace, Enter and Tab. **Close** returns to point scanning. + +Pages: + +- **Letters:** UK QWERTY and punctuation, with Backspace beside the top letter row. +- **Functions:** Esc, F1–F12, arrows, Home, End, Page Up, Page Down and Delete. Windows also has Insert, Print Screen, Scroll Lock and Pause. +- **Numbers:** digits, decimal point, arithmetic operators, UK number-row symbols (including £), Enter and Backspace. Digits are independent of hardware Num Lock. + +The bottom control row starts with Close keyboard, followed by Letters, Navigation, Numbers and docking. Space has a dedicated wide key. Editing and modifier keys are wider than ordinary characters; duplicate Shift and Caps controls are removed from the letters page. A header names the current page and scan target. The return-to-rows slot highlights only the header. Active modifiers and the current page have an indicator separate from the scan highlight. Modifiers are available on every page. Shift, Ctrl, Alt/Option and Windows/Command cycle through **off → once → locked → off**. The label shows once or locked. Caps is a keyboard-local uppercase setting; Shift reverses its letter case. Page and position controls do not consume a one-shot modifier. + +Modifier keys are pressed only around each emitted shortcut and immediately released. Selecting a locked modifier does not hold that operating-system key while the scanner runs. Ordinary characters use text injection; command combinations and navigation use native key events. Native shortcuts retain the operating system's layout semantics. + +The keyboard closes when the foreground target or display environment changes, or its scan session ends. Failed input clears keyboard modifiers and requires Select to resume. Stop, disconnect and application exit use the shared deterministic input cleanup. No typed text is logged, and no prediction data, context reader, protocol extension or persistent keyboard settings are added. + +## Manual validation + +Use synthetic text in Notepad and a browser on Windows, and TextEdit and a browser on macOS. Launch macOS through `npm run macos:run` to retain its Accessibility identity. + +1. Use Type here and verify exactly one click focuses the intended field. Use More → Keyboard and verify no click or focus change occurs. +2. Type lowercase letters, uppercase letters with Shift/Caps, and UK punctuation including £, @ and double quotes. Test Space, Backspace, Enter and Tab. +3. Test Ctrl+A/C/V on Windows and Command+A/C/V on macOS. Cycle once/locked/off, including Shift with another modifier, and confirm no modifier remains physically held between selections. +4. Visit every page. Verify row/key highlighting, reverse movement, row escape, suspension/resume, returning to the first row after a key and Close. +5. Move the keyboard between the top and bottom. Check a scaled display and a secondary display, including negative coordinates. Verify the keyboard fits the work area and never activates its native windows. +6. Switch foreground apps, disconnect remote scanning, change display configuration and close Switchify. Verify overlays disappear and owned input is released. + +Automated tests use fake input adapters only. Native manual results must be recorded separately; compilation and unit tests do not establish live application compatibility. diff --git a/src-tauri/src/input.rs b/src-tauri/src/input.rs index 01315d27..3df76983 100644 --- a/src-tauri/src/input.rs +++ b/src-tauri/src/input.rs @@ -154,6 +154,14 @@ fn pointer_button(button: MouseButton) -> Button { #[cfg(target_os = "windows")] fn windows_alphanumeric_key(name: &str) -> Option { Some(match name { + #[cfg(target_os = "windows")] + "Insert" => Key::Insert, + #[cfg(target_os = "windows")] + "PrintScreen" => Key::PrintScr, + #[cfg(target_os = "windows")] + "ScrollLock" => Key::Scroll, + #[cfg(target_os = "windows")] + "Pause" => Key::Pause, "=" => Key::OEMPlus, "-" => Key::OEMMinus, "[" => Key::OEM4, diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 317acb87..57b7ebc1 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -24,6 +24,7 @@ mod protocol; mod remote_scan; mod scan_executor; mod scan_host; +mod scan_keyboard; mod scan_menu; mod scan_tile; mod scan_tree; diff --git a/src-tauri/src/modifier_overlay_windows.rs b/src-tauri/src/modifier_overlay_windows.rs index 0e96f01f..22ac5063 100644 --- a/src-tauri/src/modifier_overlay_windows.rs +++ b/src-tauri/src/modifier_overlay_windows.rs @@ -13,7 +13,7 @@ use windows::Win32::Graphics::Gdi::{ CreateCompatibleDC, CreateDIBSection, CreateFontW, DeleteDC, DeleteObject, DrawTextW, GetDC, GetMonitorInfoW, MonitorFromPoint, ReleaseDC, SelectObject, SetBkMode, SetTextColor, AC_SRC_ALPHA, AC_SRC_OVER, BITMAPINFO, BITMAPINFOHEADER, BI_RGB, BLENDFUNCTION, - CLIP_DEFAULT_PRECIS, DEFAULT_CHARSET, DEFAULT_QUALITY, DIB_RGB_COLORS, DT_CENTER, + CLIP_DEFAULT_PRECIS, DEFAULT_CHARSET, DEFAULT_QUALITY, DIB_RGB_COLORS, DT_CALCRECT, DT_CENTER, DT_SINGLELINE, DT_VCENTER, DT_WORDBREAK, FF_DONTCARE, FW_BOLD, HDC, HGDIOBJ, MONITORINFO, MONITOR_DEFAULTTONEAREST, OUT_DEFAULT_PRECIS, TRANSPARENT, }; @@ -411,6 +411,7 @@ fn present(window: HWND, labels: &[String], layout: &Layout) -> Result<(), Strin layout, pixmap.data(), labels.iter().any(|label| label.contains('\n')), + false, ) } @@ -466,6 +467,7 @@ fn present_pixmap_with_text( layout: &Layout, rgba: &[u8], wrap: bool, + center_wrapped: bool, ) -> Result<(), String> { unsafe { let screen = ScreenDc(GetDC(None)); @@ -534,8 +536,24 @@ fn present_pixmap_with_text( SetBkMode(memory.0, TRANSPARENT); SetTextColor(memory.0, COLORREF(0x00ff_ffff)); for (label, chip) in labels.iter().zip(&layout.chips) { + if label.is_empty() { + continue; + } let mut text = label.encode_utf16().collect::>(); let mut text_rect = *chip; + if center_wrapped { + let mut measured = text_rect; + let height = DrawTextW( + memory.0, + &mut text, + &mut measured, + DT_CENTER | DT_WORDBREAK | DT_CALCRECT, + ); + let vertical_padding = text_rect.top.min((layout.height - height).max(0) / 2); + text_rect.top = vertical_padding; + text_rect.bottom = layout.height - vertical_padding; + text_rect.top += ((text_rect.bottom - text_rect.top - height) / 2).max(0); + } let _ = DrawTextW( memory.0, &mut text, @@ -595,6 +613,35 @@ pub(crate) fn present_scan_tile( tile: &crate::scanning::FrameTile, ) -> Result<(), String> { let pixmap = crate::scan_tile::bitmap(tile)?; + if tile.icon == crate::scan_menu::Item::KeyboardKey { + let width = pixmap.width() as i32; + let height = pixmap.height() as i32; + let horizontal_padding = (8.0 * tile.scale).round().max(1.0) as i32; + let vertical_padding = (6.0 * tile.scale).round().max(1.0) as i32; + let horizontal_padding = horizontal_padding.min((width - 1).max(0) / 2); + let vertical_padding = vertical_padding.min((height - 1).max(0) / 2); + let layout = Layout { + x: tile.rect.x.round() as i32, + y: tile.rect.y.round() as i32, + width, + height, + scale: tile.scale * crate::scan_tile::keyboard_font_size(tile) / FONT_SIZE, + chips: vec![RECT { + left: horizontal_padding, + top: vertical_padding, + right: width - horizontal_padding, + bottom: height - vertical_padding, + }], + }; + return present_pixmap_with_text( + window, + std::slice::from_ref(&tile.text), + &layout, + pixmap.data(), + true, + true, + ); + } let size = pixmap.width() as i32; let layout = Layout { x: tile.rect.x.round() as i32, @@ -615,6 +662,7 @@ pub(crate) fn present_scan_tile( &layout, pixmap.data(), true, + false, ) } @@ -646,6 +694,34 @@ pub(crate) fn present_scan_prompt( #[cfg(test)] mod tests { + #[test] + fn empty_keyboard_backdrop_label_never_reaches_native_text_drawing() { + let layout = super::Layout { + x: 0, + y: 0, + width: 100, + height: 50, + scale: 1.0, + chips: vec![windows::Win32::Foundation::RECT { + left: 8, + top: 6, + right: 92, + bottom: 44, + }], + }; + let pixels = vec![255; 100 * 50 * 4]; + for wrapped in [false, true] { + let result = super::present_pixmap_with_text( + windows::Win32::Foundation::HWND::default(), + &[String::new()], + &layout, + &pixels, + wrapped, + wrapped, + ); + assert!(result.is_err()); + } + } use super::*; use windows::Win32::Foundation::{LPARAM, WPARAM}; use windows::Win32::UI::WindowsAndMessaging::{ @@ -800,6 +876,24 @@ mod tests { #[test] fn native_window_render_smoke_becomes_visible_and_hides() { let mut host = NativeHost::new().unwrap(); + let keyboard = crate::scan_keyboard::Keyboard::new(false); + let frame = keyboard.frame( + crate::scanning::Rect { + x: 0.0, + y: 0.0, + width: 1280.0, + height: 720.0, + }, + 1.0, + crate::scanning::ScannerColor::default(), + ); + for tile in &frame.tiles { + super::present_scan_tile(host.window, tile).unwrap(); + unsafe { + assert!(IsWindowVisible(host.window).as_bool()); + } + } + host.hide(); host.render(&["Ctrl".into(), "Shift".into()]).unwrap(); unsafe { assert!(IsWindowVisible(host.window).as_bool()); diff --git a/src-tauri/src/overlay_macos.rs b/src-tauri/src/overlay_macos.rs index fd9a5507..ee166324 100644 --- a/src-tauri/src/overlay_macos.rs +++ b/src-tauri/src/overlay_macos.rs @@ -228,6 +228,20 @@ pub(crate) fn image_from_rgba( height: usize, logical_size: f64, ) -> Result, String> { + image_from_rgba_rect(rgba, width, height, logical_size, logical_size) +} + +pub(crate) fn image_from_rgba_rect( + rgba: &[u8], + width: usize, + height: usize, + logical_width: f64, + logical_height: f64, +) -> Result, String> { + if !logical_height.is_finite() || logical_height <= 0.0 { + return Err("Invalid overlay height.".into()); + } + let logical_size = logical_width; let layout = MacBitmapLayout::new(rgba.len(), width, height, logical_size)?; let representation = unsafe { NSBitmapImageRep::initWithBitmapDataPlanes_pixelsWide_pixelsHigh_bitsPerSample_samplesPerPixel_hasAlpha_isPlanar_colorSpaceName_bitmapFormat_bytesPerRow_bitsPerPixel( @@ -251,13 +265,13 @@ pub(crate) fn image_from_rgba( } representation.setSize(NSSize { width: layout.logical_size, - height: layout.logical_size, + height: logical_height, }); let image = NSImage::initWithSize( NSImage::alloc(), NSSize { width: layout.logical_size, - height: layout.logical_size, + height: logical_height, }, ); image.addRepresentation(&representation as &NSImageRep); diff --git a/src-tauri/src/point_scan_runtime.rs b/src-tauri/src/point_scan_runtime.rs index 317c7d2a..a3e32724 100644 --- a/src-tauri/src/point_scan_runtime.rs +++ b/src-tauri/src/point_scan_runtime.rs @@ -11,6 +11,26 @@ use tauri::{AppHandle, Manager}; pub struct Environment { display: Display, foreground: usize, + keyboard_area: Rect, + handoff: Option, +} +#[derive(Clone)] +struct FocusHandoff { + target: usize, + ready_at: std::time::Instant, +} +impl FocusHandoff { + fn verify(&self, now: std::time::Instant, foreground: usize) -> Result { + if now < self.ready_at { + return Ok(false); + } + if foreground != self.target { + return Err( + "The selected target did not receive focus. Please select it again.".into(), + ); + } + Ok(true) + } } pub struct PointScan; pub type Controller = scanning_runtime::Controller; @@ -59,13 +79,43 @@ impl Adapter for PointScan { crate::scan_executor::cleanup()?; crate::point_scan_prepare(app) } - fn activate(app: &AppHandle, request: Request) -> Result<(), String> { + fn activate(app: &AppHandle, request: Request) -> Result, String> { crate::point_scan_ready(app)?; match request { + Request::OpenKeyboard { point: Some(point) } => { + let target = crate::scan_host::target_at(point)?; + crate::scan_executor::activate(request)?; + let (_, mut environment) = + new_engine(app, app.state::().view().config)?; + // Allow the one requested click to establish focus before accepting + // any key. No typing or scan advancement occurs during this handoff. + environment.handoff = Some(FocusHandoff { + target, + ready_at: std::time::Instant::now() + std::time::Duration::from_millis(150), + }); + return Ok(Some(environment)); + } Request::Setting(setting) => scanning_runtime::update_point_setting(app, setting), Request::Display(next) => scanning_runtime::restart_point_on_display(app, next), request => crate::scan_executor::activate(request), } + .map(|()| None) + } + fn settle_environment( + app: &AppHandle, + environment: Option<&mut Environment>, + ) -> Result { + if let Some(environment) = environment { + if let Some(handoff) = &environment.handoff { + crate::point_scan_ready(app)?; + if !handoff.verify(std::time::Instant::now(), crate::scan_host::foreground()?)? { + return Ok(false); + } + environment.foreground = handoff.target; + environment.handoff = None; + } + } + Ok(true) } fn cleanup(_app: &AppHandle) -> Result<(), String> { crate::scan_executor::cleanup() @@ -81,7 +131,7 @@ fn new_engine(app: &AppHandle, config: Config) -> Result<(Workflow, Environment) } else { 1.0 }; - let e = Workflow::new( + let mut e = Workflow::new( config.point(), Rect { x: display.x.into(), @@ -91,11 +141,20 @@ fn new_engine(app: &AppHandle, config: Config) -> Result<(Workflow, Environment) }, units, )?; + let keyboard_area = crate::scan_host::work_area(Rect { + x: display.x.into(), + y: display.y.into(), + width: display.width.into(), + height: display.height.into(), + })?; + e.set_keyboard_area(keyboard_area); Ok(( e, Environment { display, foreground: crate::scan_host::foreground()?, + keyboard_area, + handoff: None, }, )) } @@ -109,6 +168,36 @@ fn validate_display(app: &AppHandle, display: Option<&Environment>) -> Result<() if !displays.contains(&expected.display) { return Err("Display geometry changed. Scanning restarts.".into()); } + let d = &expected.display; + if crate::scan_host::work_area(Rect { + x: d.x.into(), + y: d.y.into(), + width: d.width.into(), + height: d.height.into(), + })? != expected.keyboard_area + { + return Err("Display work area changed. Scanning restarts.".into()); + } } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn focus_handoff_waits_and_only_accepts_the_clicked_target() { + let start = std::time::Instant::now(); + let handoff = FocusHandoff { + target: 42, + ready_at: start + std::time::Duration::from_millis(150), + }; + assert_eq!(handoff.verify(start, 7), Ok(false)); + assert_eq!(handoff.verify(start, 42), Ok(false)); + assert_eq!(handoff.verify(handoff.ready_at, 42), Ok(true)); + // A failed activation and an unrelated app stealing focus both fail closed. + assert!(handoff.verify(handoff.ready_at, 7).is_err()); + assert!(handoff.verify(handoff.ready_at, 99).is_err()); + } +} diff --git a/src-tauri/src/point_workflow.rs b/src-tauri/src/point_workflow.rs index 0021ce53..52bd2f17 100644 --- a/src-tauri/src/point_workflow.rs +++ b/src-tauri/src/point_workflow.rs @@ -8,6 +8,10 @@ use serde::Serialize; pub type Point = (i32, i32); #[derive(Debug, Clone, Copy, PartialEq)] pub enum Request { + OpenKeyboard { + point: Option, + }, + Keyboard(crate::scan_keyboard::Stroke), Click { point: Point, right: bool, @@ -52,6 +56,9 @@ impl Default for Phase { #[derive(Debug, Clone, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum WorkflowPhase { + Keyboard, + KeyboardSuspended, + KeyboardOpening, Menu, MenuSuspended, AutoSelecting, @@ -61,6 +68,8 @@ pub enum WorkflowPhase { } #[derive(PartialEq)] enum Stage { + Keyboard, + KeyboardOpening, Idle, Countdown, Point, @@ -69,6 +78,8 @@ enum Stage { Executing, } pub struct Workflow { + keyboard: crate::scan_keyboard::Keyboard, + keyboard_area: Rect, point: Engine, stage: Stage, menu: Menu, @@ -83,6 +94,8 @@ impl Workflow { pub fn new(config: PointSettings, screen: Rect, scale: f64) -> Result { let period = config.block_interval_ms; Ok(Self { + keyboard: crate::scan_keyboard::Keyboard::new(cfg!(target_os = "macos")), + keyboard_area: screen, point: Engine::new(config, screen, scale)?, stage: Stage::Idle, menu: Menu::new(Kind::Actions, period), @@ -106,11 +119,15 @@ impl Workflow { menu.set_period(self.point.config.block_interval_ms); } if restart { + self.keyboard = crate::scan_keyboard::Keyboard::new(cfg!(target_os = "macos")); self.parent_menu.clear(); self.stage = Stage::Point; self.point.start(); } } + pub fn set_keyboard_area(&mut self, area: Rect) { + self.keyboard_area = area; + } fn open(&mut self, kind: Kind) { self.menu = Menu::new(kind, self.point.config.block_interval_ms); self.stage = Stage::Menu; @@ -126,6 +143,14 @@ impl Workflow { } fn selected(&mut self, item: Item) -> Option { match item { + Item::TypeHere | Item::Keyboard => { + self.stage = Stage::KeyboardOpening; + self.keyboard = crate::scan_keyboard::Keyboard::new(cfg!(target_os = "macos")); + return Some(Request::OpenKeyboard { + point: (item == Item::TypeHere).then_some(self.source), + }); + } + Item::KeyboardKey => {} Item::More | Item::Group(_) => { let kind = if let Item::Group(kind) = item { kind @@ -211,6 +236,10 @@ impl Technique for Workflow { type Selection = Request; type Phase = Phase; fn execution_failed(&mut self, message: String) { + if self.stage == Stage::Keyboard { + self.keyboard.failed(); + return; + } self.pending = None; self.stage = Stage::Menu; self.menu = Menu::new(Kind::Actions, self.point.config.block_interval_ms); @@ -218,12 +247,20 @@ impl Technique for Workflow { self.menu.suspended = true; self.error = Some(message); } + fn execution_succeeded(&mut self) { + if self.stage == Stage::KeyboardOpening { + self.stage = Stage::Keyboard; + } else if self.stage == Stage::Keyboard { + self.keyboard.succeeded(); + } + } fn start(&mut self) { self.reset(); self.stage = Stage::Point; self.point.start(); } fn reset(&mut self) { + self.keyboard = crate::scan_keyboard::Keyboard::new(cfg!(target_os = "macos")); self.stage = Stage::Idle; self.parent_menu.clear(); self.pending = None; @@ -247,6 +284,14 @@ impl Technique for Workflow { } fn handle(&mut self, action: Action) -> Option { match self.stage { + Stage::Keyboard => match self.keyboard.handle(action) { + Some(crate::scan_keyboard::Output::Stroke(stroke)) => { + return Some(Request::Keyboard(stroke)) + } + Some(crate::scan_keyboard::Output::Close) => self.start(), + None => {} + }, + Stage::KeyboardOpening => {} Stage::Point | Stage::Destination => { if let Some(point) = self.point.handle(action) { if self.stage == Stage::Destination { @@ -288,6 +333,9 @@ impl Technique for Workflow { } fn advance(&mut self, ms: u64) { match self.stage { + Stage::Keyboard => self + .keyboard + .advance(ms, self.point.config.block_interval_ms), Stage::Point | Stage::Destination => { self.point.advance(ms); if self.point.exhausted() { @@ -339,6 +387,12 @@ impl Technique for Workflow { } fn phase(&self) -> Phase { match self.stage { + Stage::KeyboardOpening => Phase::Workflow(WorkflowPhase::KeyboardOpening), + Stage::Keyboard => Phase::Workflow(if self.keyboard.suspended { + WorkflowPhase::KeyboardSuspended + } else { + WorkflowPhase::Keyboard + }), Stage::Idle => Phase::default(), Stage::Countdown => Phase::Workflow(WorkflowPhase::AutoSelecting), Stage::Point => Phase::Point(self.point.phase()), @@ -355,6 +409,11 @@ impl Technique for Workflow { } fn frame(&self) -> Frame { let mut frame = match self.stage { + Stage::Keyboard => self.keyboard.frame( + self.keyboard_area, + self.point.units_per_logical_pixel, + self.point.config.scanner_color, + ), Stage::Countdown => { let scale = self.point.units_per_logical_pixel; let screen = self.point.screen; @@ -466,6 +525,57 @@ mod tests { automatic, ) } + #[test] + fn keyboard_opening_waits_for_click_success_and_failure_stays_in_menu() { + let mut s = session(false); + let w = &mut s.technique; + w.source = (120, 230); + assert_eq!( + w.selected(Item::TypeHere), + Some(Request::OpenKeyboard { + point: Some((120, 230)) + }) + ); + assert_eq!(w.phase(), Phase::Workflow(WorkflowPhase::KeyboardOpening)); + assert!(w.handle(Action::Select).is_none()); + w.execution_failed("Click failed".into()); + assert_eq!(w.phase(), Phase::Workflow(WorkflowPhase::MenuSuspended)); + w.error = None; + assert_eq!( + w.selected(Item::Keyboard), + Some(Request::OpenKeyboard { point: None }) + ); + w.execution_succeeded(); + assert_eq!(w.phase(), Phase::Workflow(WorkflowPhase::Keyboard)); + w.keyboard.modifiers[0] = crate::scan_keyboard::Modifier::Locked; + w.keyboard.caps = true; + w.execution_failed("Do not expose provider details".into()); + assert_eq!(w.phase(), Phase::Workflow(WorkflowPhase::KeyboardSuspended)); + assert_eq!( + w.keyboard.modifiers, + [crate::scan_keyboard::Modifier::Off; 4] + ); + w.reset(); + assert!(!w.keyboard.caps); + assert!(w.frame().tiles.is_empty()); + } + #[test] + fn keyboard_key_acknowledgement_is_exactly_once_and_reset_discards_state() { + let mut s = session(false); + let w = &mut s.technique; + w.selected(Item::Keyboard); + w.execution_succeeded(); + assert!(w.handle(Action::Select).is_none()); + assert!(matches!( + w.handle(Action::Select), + Some(Request::Keyboard(_)) + )); + assert!(w.handle(Action::Select).is_none()); + w.execution_succeeded(); + assert!(w.handle(Action::Select).is_none()); + w.reset(); + assert_eq!(w.phase(), Phase::default()); + } fn open(s: &mut Session) { for _ in 0..3 { assert_eq!(s.action(Action::Select), None); @@ -749,7 +859,7 @@ mod tests { for (column, right, count) in [(0, false, 1), (1, true, 1), (2, false, 2)] { let mut s = session(false); open(&mut s); - assert_eq!(s.frame().tiles.len(), 8); + assert_eq!(s.frame().tiles.len(), 9); assert_eq!( choose(&mut s, 0, column), Some(Request::Click { @@ -836,7 +946,7 @@ mod tests { fn automatic_menu_pauses_preserving_target_and_select_only_resumes() { let mut s = session(true); open(&mut s); - for _ in 0..9 { + for _ in 0..12 { s.tick(250, false); s.take_selection(); } @@ -882,14 +992,14 @@ mod tests { fn point_reselection_cancel_and_drag_confirmation_back_paths() { let mut s = session(false); open(&mut s); - assert_eq!(choose(&mut s, 2, 0), None); + assert_eq!(choose(&mut s, 3, 0), None); assert_eq!( s.technique.phase(), Phase::Point(crate::point_scan::Phase::X) ); s.action(Action::Select); s.action(Action::Select); - assert_eq!(choose(&mut s, 2, 1), None); + assert_eq!(choose(&mut s, 3, 1), None); assert!(!s.active()); open(&mut s); choose(&mut s, 1, 1); diff --git a/src-tauri/src/scan_executor.rs b/src-tauri/src/scan_executor.rs index 5600f522..988699ff 100644 --- a/src-tauri/src/scan_executor.rs +++ b/src-tauri/src/scan_executor.rs @@ -15,6 +15,47 @@ pub fn execute( ); } match request { + Request::OpenKeyboard { point } => { + if input.has_active_drag() { + return Err("End the active drag before typing.".into()); + } + match point { + Some(point) => input.click_pointer_at(point, MouseButton::Left, 1, &[]), + None => Ok(()), + } + } + Request::Keyboard(stroke) => { + if input.has_active_drag() { + return Err("End the active drag before typing.".into()); + } + input.release_all()?; + if let Some(character) = stroke.character().filter(|_| !stroke.shortcut()) { + return input.type_text(&character.to_string()); + } + let mut modifiers = stroke.modifiers; + let name = match stroke.key { + crate::scan_keyboard::Key::Character('+', '+') => { + modifiers[0] = true; + "=".into() + } + crate::scan_keyboard::Key::Character('*', '*') => { + modifiers[0] = true; + "8".into() + } + crate::scan_keyboard::Key::Character(base, _) => { + base.to_ascii_uppercase().to_string() + } + crate::scan_keyboard::Key::Named(name) => name.into(), + _ => return Err("Invalid keyboard action.".into()), + }; + let mut keys: Vec<&str> = ["Shift", "Ctrl", "Alt", "Meta"] + .into_iter() + .zip(modifiers) + .filter_map(|(key, on)| on.then_some(key)) + .collect(); + keys.push(&name); + input.scan_chord(&keys, None) + } Request::Command { command, point } => execute_command(input, command, point), Request::Setting(_) | Request::Display(_) => { Err("Scanning action requires the scan controller.".into()) @@ -221,8 +262,9 @@ mod tests { legacy_clicks: usize, } impl InputInjector for Fake { - fn inject_text(&mut self, _: &str) -> Result<(), String> { - panic!("No typing keyboard") + fn inject_text(&mut self, text: &str) -> Result<(), String> { + self.events.push(format!("text {text}")); + Ok(()) } fn move_pointer(&mut self, _: i32, _: i32) -> Result<(), String> { panic!("No relative movement") @@ -296,6 +338,134 @@ mod tests { } } #[test] + fn keyboard_text_chords_and_current_focus_use_fake_input() { + use crate::scan_keyboard::{Key, Stroke}; + let mut input = DesktopInput::new(Fake::default()); + execute(&mut input, Request::OpenKeyboard { point: None }, true).unwrap(); + assert!(input.injector.events.is_empty()); + execute( + &mut input, + Request::Keyboard(Stroke { + key: Key::Character('3', '£'), + modifiers: [true, false, false, false], + caps: false, + }), + true, + ) + .unwrap(); + assert_eq!(input.injector.events, ["text £"]); + input.injector.events.clear(); + execute( + &mut input, + Request::Keyboard(Stroke { + key: Key::Character('a', 'A'), + modifiers: [false, true, false, false], + caps: false, + }), + true, + ) + .unwrap(); + assert_eq!( + input.injector.events, + [ + "key Ctrl true", + "key A true", + "key A false", + "key Ctrl false" + ] + ); + input.injector.events.clear(); + execute( + &mut input, + Request::Keyboard(Stroke { + key: Key::Named("Escape"), + modifiers: [false; 4], + caps: false, + }), + true, + ) + .unwrap(); + assert_eq!( + input.injector.events, + ["key Escape true", "key Escape false"] + ); + } + #[test] + fn numeric_operator_shortcuts_include_required_shift_and_release_every_key() { + use crate::scan_keyboard::{Key, Stroke}; + for (operator, native_key) in [('+', "="), ('*', "8")] { + let mut input = DesktopInput::new(Fake::default()); + execute( + &mut input, + Request::Keyboard(Stroke { + key: Key::Character(operator, operator), + modifiers: [false, true, false, false], + caps: false, + }), + true, + ) + .unwrap(); + assert_eq!( + input.injector.events, + [ + "key Shift true".to_string(), + "key Ctrl true".to_string(), + format!("key {native_key} true"), + format!("key {native_key} false"), + "key Ctrl false".to_string(), + "key Shift false".to_string(), + ] + ); + } + } + #[test] + fn keyboard_releases_failed_keys_and_rejects_external_modifiers() { + use crate::scan_keyboard::{Key, Stroke}; + let request = Request::Keyboard(Stroke { + key: Key::Named("Tab"), + modifiers: [true, false, false, false], + caps: false, + }); + let mut input = DesktopInput::new(Fake { + fail_release: true, + ..Default::default() + }); + assert!(execute(&mut input, request, true).is_err()); + input.injector.fail_release = false; + input.release_all().unwrap(); + assert!(input + .injector + .events + .ends_with(&["key Tab false".into(), "key Shift false".into()])); + input.injector.events.clear(); + assert!(execute(&mut input, request, false).is_err()); + assert!(input.injector.events.is_empty()); + } + #[test] + fn type_here_clicks_exactly_once_and_propagates_failure() { + let mut input = DesktopInput::new(Fake::default()); + execute( + &mut input, + Request::OpenKeyboard { + point: Some((42, 70)), + }, + true, + ) + .unwrap(); + assert_eq!(input.injector.target_clicks.len(), 1); + assert_eq!(input.injector.target_clicks[0].0, (42, 70)); + input.injector.fail_click = true; + assert!(execute( + &mut input, + Request::OpenKeyboard { + point: Some((42, 70)) + }, + true + ) + .is_err()); + assert_eq!(input.injector.target_clicks.len(), 1); + } + #[test] fn every_scan_click_uses_target_even_while_cursor_readback_is_stale() { for point in [(-1600, -200), (3600, 1200)] { let mut cases = vec![ diff --git a/src-tauri/src/scan_host.rs b/src-tauri/src/scan_host.rs index c7e8fd87..b8f6541d 100644 --- a/src-tauri/src/scan_host.rs +++ b/src-tauri/src/scan_host.rs @@ -1,6 +1,68 @@ //! Native, nonactivating strips, using the same display units as input injection. use crate::scanning::Rect; +/// Work area in the same native coordinates used by the scan engine. +pub fn work_area(screen: Rect) -> Result { + #[cfg(target_os = "windows")] + unsafe { + use windows::Win32::{ + Foundation::POINT, + Graphics::Gdi::{ + GetMonitorInfoW, MonitorFromPoint, MONITORINFO, MONITOR_DEFAULTTONEAREST, + }, + }; + let monitor = MonitorFromPoint( + POINT { + x: (screen.x + screen.width / 2.0) as i32, + y: (screen.y + screen.height / 2.0) as i32, + }, + MONITOR_DEFAULTTONEAREST, + ); + let mut info = MONITORINFO { + cbSize: std::mem::size_of::() as u32, + ..Default::default() + }; + if !GetMonitorInfoW(monitor, &mut info).as_bool() { + return Err("Display work area is unavailable.".into()); + } + let r = info.rcWork; + Ok(Rect { + x: r.left.into(), + y: r.top.into(), + width: (r.right - r.left).into(), + height: (r.bottom - r.top).into(), + }) + } + #[cfg(target_os = "macos")] + { + use objc2::MainThreadMarker; + use objc2_app_kit::NSScreen; + let mtm = MainThreadMarker::new().ok_or("Keyboard requires the main thread.")?; + let screens = NSScreen::screens(mtm); + let primary = screens + .firstObject() + .ok_or("Display work area is unavailable.")?; + let top = primary.frame().origin.y + primary.frame().size.height; + for s in screens.iter() { + let r = s.frame(); + if (r.origin.x - screen.x).abs() < 1.0 + && (top - r.origin.y - r.size.height - screen.y).abs() < 1.0 + { + let r = s.visibleFrame(); + return Ok(Rect { + x: r.origin.x, + y: top - r.origin.y - r.size.height, + width: r.size.width, + height: r.size.height, + }); + } + } + Err("Display work area is unavailable.".into()) + } + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + Ok(screen) +} + #[cfg(target_os = "windows")] mod platform { use super::*; @@ -333,11 +395,12 @@ mod platform { use objc2_foundation::NSString; let mtm = MainThreadMarker::new().ok_or("Action tile requires the main thread")?; let pixels = crate::scan_tile::bitmap(tile)?; - let image = crate::overlay::platform::image_from_rgba( + let image = crate::overlay::platform::image_from_rgba_rect( pixels.data(), pixels.width() as usize, pixels.height() as usize, tile.rect.width, + tile.rect.height, )?; self.render(&[crate::scanning::PaintedRect { rect: tile.rect, @@ -345,6 +408,12 @@ mod platform { opacity: 255, role: crate::scanning::VisualRole::Accent, }])?; + if tile + .keyboard + .is_some_and(|s| s.role == crate::scanning::KeyboardRole::Background) + { + self.panels[0].setBackgroundColor(Some(&NSColor::clearColor())); + } let bounds = NSRect::new( NSPoint::new(0.0, 0.0), NSSize::new(tile.rect.width, tile.rect.height), @@ -354,13 +423,37 @@ mod platform { artwork.setImage(Some(&image)); view.addSubview(&artwork); let label = NSTextField::wrappingLabelWithString(&NSString::from_str(&tile.text), mtm); - label.setFont(Some(&NSFont::boldSystemFontOfSize(15.0 * tile.scale))); + let key = tile.icon == crate::scan_menu::Item::KeyboardKey; + label.setFont(Some(&NSFont::boldSystemFontOfSize( + if key { + crate::scan_tile::keyboard_font_size(tile) + } else { + 15.0 + } * tile.scale, + ))); label.setTextColor(Some(&NSColor::whiteColor())); label.setAlignment(NSTextAlignment::Center); - label.setFrame(NSRect::new( - NSPoint::new(6.0 * tile.scale, 12.0 * tile.scale), - NSSize::new(tile.rect.width - 12.0 * tile.scale, 40.0 * tile.scale), - )); + label.setFrame(if key { + let padding = 8.0 * tile.scale; + let width = (tile.rect.width - padding * 2.0).max(1.0); + let measured = label + .cell() + .ok_or("Keyboard label is unavailable.")? + .cellSizeForBounds(NSRect::new( + NSPoint::new(0.0, 0.0), + NSSize::new(width, f64::MAX), + )); + let height = measured.height.min(tile.rect.height); + NSRect::new( + NSPoint::new(padding, (tile.rect.height - height) / 2.0), + NSSize::new(width, height), + ) + } else { + NSRect::new( + NSPoint::new(6.0 * tile.scale, 12.0 * tile.scale), + NSSize::new(tile.rect.width - 12.0 * tile.scale, 40.0 * tile.scale), + ) + }); view.addSubview(&label); self.panels[0].setContentView(Some(&view)); Ok(()) @@ -418,6 +511,70 @@ pub fn modifiers_released() -> bool { } } +/// Resolve the intended target while scan overlays are hidden, before clicking. +/// Uses the same identity as `foreground`: a Windows root window or macOS PID. +pub fn target_at(point: (i32, i32)) -> Result { + #[cfg(target_os = "windows")] + unsafe { + use windows::Win32::{ + Foundation::POINT, + UI::WindowsAndMessaging::{GetAncestor, WindowFromPoint, GA_ROOT}, + }; + let child = WindowFromPoint(POINT { + x: point.0, + y: point.1, + }); + let root = GetAncestor(child, GA_ROOT); + if root.0.is_null() { + return Err("The selected target is unavailable.".into()); + } + Ok(root.0 as usize) + } + #[cfg(target_os = "macos")] + { + use core_foundation::base::{CFType, CFTypeRef, TCFType}; + #[link(name = "ApplicationServices", kind = "framework")] + unsafe extern "C" { + fn AXUIElementCreateSystemWide() -> CFTypeRef; + fn AXUIElementCopyElementAtPosition( + element: CFTypeRef, + x: f32, + y: f32, + result: *mut CFTypeRef, + ) -> i32; + fn AXUIElementGetPid(element: CFTypeRef, pid: *mut i32) -> i32; + } + let system = unsafe { AXUIElementCreateSystemWide() }; + if system.is_null() { + return Err("The selected target is unavailable.".into()); + } + let system = unsafe { CFType::wrap_under_create_rule(system) }; + let mut element = std::ptr::null(); + let status = unsafe { + AXUIElementCopyElementAtPosition( + system.as_CFTypeRef(), + point.0 as f32, + point.1 as f32, + &mut element, + ) + }; + if status != 0 || element.is_null() { + return Err("The selected target is unavailable.".into()); + } + let element = unsafe { CFType::wrap_under_create_rule(element) }; + let mut pid = 0; + if unsafe { AXUIElementGetPid(element.as_CFTypeRef(), &mut pid) } != 0 || pid <= 0 { + return Err("The selected target is unavailable.".into()); + } + Ok(pid as usize) + } + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + { + let _ = point; + Err("Local scanning is unavailable.".into()) + } +} + /// Opaque foreground identity; never emitted or persisted. pub fn foreground() -> Result { #[cfg(target_os = "windows")] diff --git a/src-tauri/src/scan_keyboard.rs b/src-tauri/src/scan_keyboard.rs new file mode 100644 index 00000000..c29fbf45 --- /dev/null +++ b/src-tauri/src/scan_keyboard.rs @@ -0,0 +1,711 @@ +//! Switch-owned keyboard state and geometry. No native input or captured text. +use crate::{ + scan_menu::Item, + scan_tree::{Navigator, Node, Selection}, + scanning::{ + Action, Frame, FrameTile, Interval, KeyboardRole, KeyboardTileStyle, Rect, ScannerColor, + MAX_SCAN_CYCLES, + }, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Page { + Letters, + Functions, + Numbers, +} +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Modifier { + #[default] + Off, + Once, + Locked, +} +impl Modifier { + fn next(self) -> Self { + match self { + Self::Off => Self::Once, + Self::Once => Self::Locked, + Self::Locked => Self::Off, + } + } + fn suffix(self) -> &'static str { + match self { + Self::Off => "", + Self::Once => "\nNext key", + Self::Locked => "\nLocked", + } + } +} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Key { + Character(char, char), + Named(&'static str), + Modifier(usize), + Caps, + Page(Page), + Dock, + Close, +} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Stroke { + pub key: Key, + pub modifiers: [bool; 4], + pub caps: bool, +} +impl Stroke { + pub fn character(self) -> Option { + let Key::Character(base, shifted) = self.key else { + return None; + }; + Some( + if self.modifiers[0] ^ (self.caps && base.is_ascii_alphabetic()) { + shifted + } else { + base + }, + ) + } + pub fn shortcut(self) -> bool { + self.modifiers[1..].iter().any(|m| *m) + } +} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Output { + Stroke(Stroke), + Close, +} + +pub fn rows(page: Page, mac: bool) -> Vec> { + use Key::{Character as C, Named as N}; + let chars = |normal: &str, shifted: &str| { + normal + .chars() + .zip(shifted.chars()) + .map(|(a, b)| C(a, b)) + .collect::>() + }; + let mut result = match page { + Page::Letters => { + let mut top = vec![N("Tab")]; + top.extend(chars("qwertyuiop[]", "QWERTYUIOP{}")); + top.push(N("Backspace")); + let mut middle = vec![Key::Caps]; + middle.extend(chars("asdfghjkl;'#", "ASDFGHJKL:@~")); + middle.push(N("Enter")); + let mut bottom = vec![Key::Modifier(0)]; + bottom.extend(chars("\\zxcvbnm,./", "|ZXCVBNM<>?")); + vec![top, middle, bottom] + } + Page::Functions => { + let mut result = vec![ + vec![ + N("Escape"), + N("F1"), + N("F2"), + N("F3"), + N("F4"), + N("F5"), + N("F6"), + ], + vec![N("F7"), N("F8"), N("F9"), N("F10"), N("F11"), N("F12")], + vec![N("Home"), N("ArrowUp"), N("End"), N("PageUp"), N("Delete")], + vec![ + N("ArrowLeft"), + N("ArrowDown"), + N("ArrowRight"), + N("PageDown"), + N("Tab"), + N("Enter"), + ], + ]; + if !mac { + result.push(vec![ + N("Insert"), + N("PrintScreen"), + N("ScrollLock"), + N("Pause"), + ]); + } + result + } + Page::Numbers => vec![ + vec![C('7', '7'), C('8', '8'), C('9', '9'), C('/', '/')], + vec![C('4', '4'), C('5', '5'), C('6', '6'), C('*', '*')], + vec![C('1', '1'), C('2', '2'), C('3', '3'), C('-', '-')], + vec![ + C('0', '0'), + C('.', '.'), + C('+', '+'), + N("Enter"), + N("Backspace"), + ], + chars("`¬!\"£$%^&()_=", "`¬!\"£$%^&()_="), + ], + }; + result.push(if page == Page::Letters { + vec![ + Key::Modifier(1), + Key::Modifier(2), + C(' ', ' '), + Key::Modifier(3), + ] + } else { + vec![ + Key::Modifier(0), + Key::Modifier(1), + Key::Modifier(2), + C(' ', ' '), + Key::Modifier(3), + Key::Caps, + ] + }); + result.push(vec![ + Key::Close, + Key::Page(Page::Letters), + Key::Page(Page::Functions), + Key::Page(Page::Numbers), + Key::Dock, + ]); + result +} + +pub struct Keyboard { + pub page: Page, + pub modifiers: [Modifier; 4], + pub caps: bool, + pub top: bool, + pub suspended: bool, + pub error: bool, + mac: bool, + rows: Vec>, + nav: Navigator, + interval: Interval, + cycles: usize, + forward: bool, + pending: bool, +} +impl Keyboard { + pub fn new(mac: bool) -> Self { + let rows = rows(Page::Letters, mac); + let nav = Self::navigator(&rows); + Self { + page: Page::Letters, + modifiers: [Modifier::Off; 4], + caps: false, + top: false, + suspended: false, + error: false, + mac, + rows, + nav, + interval: Interval::default(), + cycles: 0, + forward: true, + pending: false, + } + } + fn navigator(rows: &[Vec]) -> Navigator { + Navigator::new( + rows.iter() + .map(|row| Node::Branch(row.iter().copied().map(Node::Leaf).collect())) + .collect(), + ) + } + fn restart(&mut self) { + self.nav.reset(); + self.interval.reset(); + self.cycles = 0; + self.forward = true; + self.suspended = false; + } + pub fn advance(&mut self, ms: u64, period: u64) { + if !self.pending + && !self.suspended + && self.interval.elapsed(ms, period) + && self.nav.step(self.forward) + { + self.cycles += 1; + self.suspended = self.cycles >= MAX_SCAN_CYCLES; + } + } + pub fn handle(&mut self, action: Action) -> Option { + if self.pending { + return None; + } + if self.suspended { + if action == Action::Select { + self.error = false; + self.restart(); + } + return None; + } + match action { + Action::Select => { + self.interval.reset(); + self.cycles = 0; + match self.nav.select() { + Selection::Leaf(key) => return self.choose(key), + Selection::Entered | Selection::Escaped => self.forward = true, + Selection::None => {} + } + } + Action::Next | Action::Back => { + self.interval.reset(); + self.cycles = 0; + self.forward = action == Action::Next; + self.nav.step(self.forward); + } + Action::Reverse => { + self.forward = !self.forward; + self.interval.reset(); + self.cycles = 0; + } + _ => {} + } + None + } + fn choose(&mut self, key: Key) -> Option { + match key { + Key::Modifier(i) => self.modifiers[i] = self.modifiers[i].next(), + Key::Caps => self.caps = !self.caps, + Key::Dock => self.top = !self.top, + Key::Page(page) => { + self.page = page; + self.rows = rows(page, self.mac); + self.nav = Self::navigator(&self.rows); + } + Key::Close => return Some(Output::Close), + Key::Character(..) | Key::Named(_) => { + self.pending = true; + return Some(Output::Stroke(Stroke { + key, + modifiers: self.modifiers.map(|m| m != Modifier::Off), + caps: self.caps, + })); + } + } + self.restart(); + None + } + pub fn succeeded(&mut self) { + if self.pending { + for modifier in &mut self.modifiers { + if *modifier == Modifier::Once { + *modifier = Modifier::Off; + } + } + self.pending = false; + self.restart(); + } + } + pub fn failed(&mut self) { + self.modifiers = [Modifier::Off; 4]; + self.caps = false; + self.pending = false; + self.restart(); + self.error = true; + self.suspended = true; + } + fn label(&self, key: Key) -> String { + match key { + Key::Character(' ', _) => "Space".into(), + Key::Character(..) => Stroke { + key, + modifiers: self.modifiers.map(|m| m != Modifier::Off), + caps: self.caps, + } + .character() + .unwrap() + .to_string(), + Key::Named(name) => match name { + "ArrowUp" => "↑", + "ArrowDown" => "↓", + "ArrowLeft" => "←", + "ArrowRight" => "→", + "PageUp" => "Page Up", + "PageDown" => "Page Down", + "PrintScreen" => "Print Screen", + "ScrollLock" => "Scroll Lock", + "Escape" => "Esc", + _ => name, + } + .into(), + Key::Modifier(i) => format!( + "{}{}", + match i { + 0 => "Shift", + 1 => "Ctrl", + 2 => + if self.mac { + "Option" + } else { + "Alt" + }, + _ => + if self.mac { + "Command" + } else { + "Windows" + }, + }, + self.modifiers[i].suffix() + ), + Key::Caps => if self.caps { + "Caps lock\nOn" + } else { + "Caps lock" + } + .into(), + Key::Page(page) => format!( + "{}{}", + match page { + Page::Letters => "Letters", + Page::Functions => "Navigation", + Page::Numbers => "Numbers", + }, + if page == self.page { " •" } else { "" } + ), + Key::Dock => if self.top { "Dock bottom" } else { "Dock top" }.into(), + Key::Close => "Close keyboard".into(), + } + } + fn weight(key: Key) -> f64 { + match key { + Key::Character(' ', _) => 4.0, + Key::Named("Backspace" | "Enter") => 1.9, + Key::Named("Tab") | Key::Caps | Key::Modifier(_) => 1.6, + Key::Close | Key::Dock => 1.5, + _ => 1.0, + } + } + fn style(&self, key: Key, row_scan: bool) -> KeyboardTileStyle { + KeyboardTileStyle { + role: match key { + Key::Character(..) => KeyboardRole::Character, + Key::Page(_) | Key::Dock | Key::Close => KeyboardRole::Toolbar, + _ => KeyboardRole::Utility, + }, + active: match key { + Key::Modifier(i) => self.modifiers[i] != Modifier::Off, + Key::Caps => self.caps, + Key::Page(page) => self.page == page, + _ => false, + }, + row_scan, + } + } + pub fn frame(&self, screen: Rect, units: f64, color: ScannerColor) -> Frame { + let width = (1180.0 * units).min((screen.width - 24.0 * units).max(1.0)); + let height = (460.0 * units).min(screen.height * 0.55).max(1.0); + let x = screen.x + (screen.width - width) / 2.0; + let y = if self.top { + screen.y + } else { + screen.y + screen.height - height + }; + let outer_scale = units.min(height / 460.0).min(width / 1180.0); + let padding = 16.0 * outer_scale; + let content_width = (width - padding * 2.0).max(1.0); + let content_height = (height - padding * 2.0).max(1.0); + let row_height = content_height / (self.rows.len() as f64 + 0.9); + let scale = units.min(row_height / 60.0).min(content_width / 1040.0); + let gap = (6.0 * scale).min(row_height / 8.0); + let header_height = row_height * 0.9; + let mut frame = Frame::default(); + frame.tiles.push(FrameTile { + color, + text: String::new(), + icon: Item::KeyboardKey, + keyboard: Some(KeyboardTileStyle { + role: KeyboardRole::Background, + active: false, + row_scan: false, + }), + rect: Rect { + x, + y, + width, + height, + }, + scale: outer_scale, + selected: false, + }); + let x = x + padding; + let y = y + padding; + let width = content_width; + let row_scan = self.nav.path().is_empty(); + let active_row = self.nav.path().first().copied().unwrap_or(self.nav.index()); + for (r, row) in self.rows.iter().enumerate() { + let total_weight: f64 = row.iter().copied().map(Self::weight).sum(); + let cell_unit = (width - gap * (row.len() - 1) as f64).max(1.0) / total_weight; + let mut left = x; + for (c, key) in row.iter().enumerate() { + let key_width = cell_unit * Self::weight(*key); + frame.tiles.push(FrameTile { + color, + text: self.label(*key), + icon: Item::KeyboardKey, + keyboard: Some(self.style(*key, row_scan)), + rect: Rect { + x: left, + y: y + header_height + r as f64 * row_height, + width: key_width, + height: (row_height - gap).max(1.0), + }, + scale, + selected: !self.suspended + && !self.nav.escaping() + && r == active_row + && (row_scan || c == self.nav.index()), + }); + left += key_width + gap; + } + } + let page = match self.page { + Page::Letters => "Letters", + Page::Functions => "Navigation", + Page::Numbers => "Numbers", + }; + let text = if self.error { + "Input failed · Select to try again".to_owned() + } else if self.suspended { + "Keyboard paused · Select to resume".to_owned() + } else if self.nav.escaping() { + "Back to rows · Select to return".to_owned() + } else if row_scan { + format!("{} · Select a row", page) + } else { + format!( + "{} · Select {}", + page, + self.label(self.rows[active_row][self.nav.index()]) + .replace('\n', " ") + ) + }; + frame.tiles.push(FrameTile { + color, + text, + icon: Item::KeyboardKey, + keyboard: Some(KeyboardTileStyle { + role: KeyboardRole::Status, + active: false, + row_scan: false, + }), + rect: Rect { + x, + y, + width, + height: (header_height - gap).max(1.0), + }, + scale, + selected: !self.suspended && self.nav.escaping(), + }); + frame + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn rounded_panel_has_clear_corners_and_insets_all_controls() { + let frame = Keyboard::new(false).frame( + Rect { + x: -1280.0, + y: 0.0, + width: 1280.0, + height: 720.0, + }, + 1.0, + ScannerColor::default(), + ); + let panel = &frame.tiles[0]; + assert_eq!(panel.keyboard.unwrap().role, KeyboardRole::Background); + for key in &frame.tiles[1..] { + assert!(key.rect.x > panel.rect.x); + assert!(key.rect.y > panel.rect.y); + assert!(key.rect.x + key.rect.width < panel.rect.x + panel.rect.width); + assert!(key.rect.y + key.rect.height < panel.rect.y + panel.rect.height); + } + let pixels = crate::scan_tile::bitmap(panel).unwrap(); + assert_eq!(pixels.data()[3], 0); + let center = ((pixels.height() / 2 * pixels.width() + pixels.width() / 2) * 4 + 3) as usize; + assert_eq!(pixels.data()[center], 255); + } + #[test] + fn everyday_keys_are_unique_and_space_is_wide() { + let k = Keyboard::new(false); + let keys = k.rows.concat(); + for key in [ + Key::Caps, + Key::Modifier(0), + Key::Character(' ', ' '), + Key::Close, + ] { + assert_eq!( + keys.iter().filter(|candidate| **candidate == key).count(), + 1 + ); + } + let frame = k.frame( + Rect { + x: 0.0, + y: 0.0, + width: 1280.0, + height: 680.0, + }, + 1.0, + ScannerColor::default(), + ); + let space = frame.tiles.iter().find(|t| t.text == "Space").unwrap(); + let letter = frame.tiles.iter().find(|t| t.text == "a").unwrap(); + assert!(space.rect.width > letter.rect.width * 3.0); + let backspace = frame.tiles.iter().find(|t| t.text == "Backspace").unwrap(); + assert!(backspace.rect.width > letter.rect.width); + assert_eq!(k.rows.last().unwrap()[0], Key::Close); + } + #[test] + fn row_escape_only_highlights_return_and_modifier_state_is_separate() { + let screen = Rect { + x: 0.0, + y: 0.0, + width: 1280.0, + height: 680.0, + }; + let mut k = Keyboard::new(false); + k.choose(Key::Modifier(0)); + let active = k.frame(screen, 1.0, ScannerColor::default()); + let shift = active + .tiles + .iter() + .find(|t| t.text.starts_with("Shift")) + .unwrap(); + assert!(shift.keyboard.unwrap().active); + assert!(!shift.selected); + k.handle(Action::Select); + let keys = k.frame(screen, 1.0, ScannerColor::default()); + assert_eq!(keys.tiles.iter().filter(|t| t.selected).count(), 1); + k.handle(Action::Back); + let escaping = k.frame(screen, 1.0, ScannerColor::default()); + let selected: Vec<_> = escaping.tiles.iter().filter(|t| t.selected).collect(); + assert_eq!(selected.len(), 1); + assert!(selected[0].text.starts_with("Back to rows")); + k.handle(Action::Select); + assert!(k.nav.path().is_empty()); + } + #[test] + fn uk_layout_and_platform_keys() { + let letters = rows(Page::Letters, false).concat(); + for c in 'a'..='z' { + assert!(letters.contains(&Key::Character(c, c.to_ascii_uppercase()))); + } + assert!(!letters + .iter() + .any(|key| matches!(key, Key::Character(c, _) if c.is_ascii_digit()))); + assert!(rows(Page::Letters, false)[0].contains(&Key::Named("Backspace"))); + let numbers = rows(Page::Numbers, false).concat(); + for symbol in "`1234567890-=¬!\"£$%^&*()_+".chars() { + assert!( + numbers + .iter() + .any(|key| matches!(key, Key::Character(a, b) if *a == symbol || *b == symbol)), + "Missing {symbol}" + ); + } + assert!(rows(Page::Functions, false) + .concat() + .contains(&Key::Named("Insert"))); + assert!(!rows(Page::Functions, true) + .concat() + .contains(&Key::Named("Insert"))); + } + #[test] + fn modifiers_are_consumed_only_after_success_and_lock_persists() { + let mut k = Keyboard::new(false); + k.choose(Key::Modifier(0)); + k.choose(Key::Modifier(1)); + k.choose(Key::Modifier(1)); + k.choose(Key::Page(Page::Numbers)); + k.choose(Key::Dock); + assert_eq!( + k.modifiers, + [ + Modifier::Once, + Modifier::Locked, + Modifier::Off, + Modifier::Off + ] + ); + k.choose(Key::Named("Enter")); + assert!(k.handle(Action::Select).is_none()); + k.succeeded(); + assert_eq!(k.modifiers[0], Modifier::Off); + assert_eq!(k.modifiers[1], Modifier::Locked); + k.choose(Key::Modifier(1)); + assert_eq!(k.modifiers[1], Modifier::Off); + k.choose(Key::Modifier(2)); + k.failed(); + assert_eq!(k.modifiers, [Modifier::Off; 4]); + assert!(k.suspended); + k.handle(Action::Select); + assert!(!k.suspended); + } + #[test] + fn caps_xor_shift_only_affects_letters() { + let mut s = Stroke { + key: Key::Character('a', 'A'), + caps: true, + modifiers: [false; 4], + }; + assert_eq!(s.character(), Some('A')); + s.modifiers[0] = true; + assert_eq!(s.character(), Some('a')); + s.key = Key::Character('3', '£'); + assert_eq!(s.character(), Some('£')); + } + #[test] + fn scan_returns_to_top_and_has_row_escape_and_suspension() { + let mut k = Keyboard::new(false); + k.handle(Action::Select); + assert!(!k.nav.path().is_empty()); + k.handle(Action::Back); + assert!(k.nav.escaping()); + k.handle(Action::Select); + assert!(k.nav.path().is_empty()); + k.handle(Action::Select); + assert!(matches!(k.handle(Action::Select), Some(Output::Stroke(_)))); + k.succeeded(); + assert!(k.nav.path().is_empty()); + assert_eq!(k.nav.index(), 0); + for _ in 0..k.rows.len() * MAX_SCAN_CYCLES { + k.advance(100, 100); + } + assert!(k.suspended); + assert!(k.handle(Action::Select).is_none()); + assert!(!k.suspended); + } + #[test] + fn geometry_fits_work_area_at_both_docks_and_scales() { + for units in [0.75, 1.0, 1.5, 2.0, 3.0] { + for top in [false, true] { + for page in [Page::Letters, Page::Functions, Page::Numbers] { + let mut k = Keyboard::new(false); + k.choose(Key::Page(page)); + k.top = top; + let screen = Rect { + x: -1600.0, + y: -200.0, + width: 800.0, + height: 600.0, + }; + for tile in k.frame(screen, units, ScannerColor::default()).tiles { + assert!(tile.rect.x >= screen.x && tile.rect.y >= screen.y); + assert!(tile.rect.x + tile.rect.width <= screen.x + screen.width + 0.01); + assert!(tile.rect.y + tile.rect.height <= screen.y + screen.height + 0.01); + } + } + } + } + } +} diff --git a/src-tauri/src/scan_menu.rs b/src-tauri/src/scan_menu.rs index 001b4a5d..1c7f1507 100644 --- a/src-tauri/src/scan_menu.rs +++ b/src-tauri/src/scan_menu.rs @@ -5,6 +5,9 @@ use crate::{ }; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Item { + TypeHere, + Keyboard, + KeyboardKey, More, Group(Kind), Command(Command), @@ -31,6 +34,9 @@ pub enum Item { impl Item { pub fn label(self) -> &'static str { match self { + Self::TypeHere => "Type here", + Self::Keyboard => "Keyboard", + Self::KeyboardKey => "Key", Self::More => "More", Self::Group(kind) => kind.label(), Self::Command(command) => command.label(), @@ -187,6 +193,7 @@ impl Menu { let selected = r == active_row && (self.nav.path().is_empty() || self.nav.escaping() || c == self.nav.index()); frame.tiles.push(FrameTile { + keyboard: None, color: Default::default(), text: item.label().into(), rect, @@ -448,6 +455,7 @@ impl Kind { return vec![ vec![LeftClick, RightClick, DoubleClick], vec![Scroll, Drag, More], + vec![TypeHere], vec![NewPoint, Cancel], ] } @@ -456,6 +464,7 @@ impl Kind { return vec![vec![DragHere, DestinationAgain], vec![CancelDrag, Cancel]] } Self::More => vec![ + Keyboard, Group(Self::Mouse), Group(Self::Editing), Group(Self::Windows), @@ -738,6 +747,6 @@ mod tests { assert_eq!(m.nav.index(), 0); m.handle(Action::Reverse); m.advance(250); - assert_eq!(m.nav.index(), 2); + assert_eq!(m.nav.index(), 3); } } diff --git a/src-tauri/src/scan_tile.rs b/src-tauri/src/scan_tile.rs index 2e1e24b1..e9a4fb08 100644 --- a/src-tauri/src/scan_tile.rs +++ b/src-tauri/src/scan_tile.rs @@ -5,6 +5,91 @@ use tiny_skia::{ }; pub fn bitmap(tile: &FrameTile) -> Result { + if tile.icon == Item::KeyboardKey { + let mut bitmap = Pixmap::new( + tile.rect.width.round().max(1.0) as u32, + tile.rect.height.round().max(1.0) as u32, + ) + .ok_or("Cannot allocate keyboard key")?; + use crate::scanning::KeyboardRole; + let style = tile.keyboard; + if style.is_some_and(|s| s.role == KeyboardRole::Background) { + let mut paint = Paint::default(); + paint.set_color_rgba8(20, 24, 32, 255); + let path = key_outline( + bitmap.width() as f32, + bitmap.height() as f32, + 0.0, + (18.0 * tile.scale) as f32, + ); + bitmap.fill_path( + &path, + &paint, + FillRule::Winding, + Transform::identity(), + None, + ); + return Ok(bitmap); + } + let base = match style.map(|s| s.role) { + Some(KeyboardRole::Character) => [43, 51, 66], + Some(KeyboardRole::Utility) => [34, 41, 54], + _ => [25, 30, 40], + }; + bitmap.fill(Color::from_rgba8(20, 24, 32, 255)); + let rgb = if tile.selected { + tile.color.menu_fill() + } else { + base + }; + let inset = (2.0 * tile.scale) as f32; + let radius = (8.0 * tile.scale) as f32; + let width = bitmap.width() as f32; + let height = bitmap.height() as f32; + let path = key_outline(width, height, inset.min(width.min(height) / 4.0), radius); + let mut paint = Paint::default(); + paint.set_color_rgba8(rgb[0], rgb[1], rgb[2], 255); + bitmap.fill_path( + &path, + &paint, + FillRule::Winding, + Transform::identity(), + None, + ); + let [r, g, b] = if tile.selected { + tile.color.rgb() + } else { + [62, 72, 89] + }; + paint.set_color_rgba8(r, g, b, 255); + bitmap.stroke_path( + &path, + &paint, + &Stroke { + width: if tile.selected && !style.is_some_and(|s| s.row_scan) { + 3.0 + } else { + 1.0 + } * tile.scale as f32, + ..Default::default() + }, + Transform::identity(), + None, + ); + if style.is_some_and(|s| s.active) { + let [r, g, b] = tile.color.rgb(); + paint.set_color_rgba8(r, g, b, 255); + if let Some(rect) = tiny_skia::Rect::from_xywh( + width * 0.35, + height - 5.0 * tile.scale as f32, + width * 0.3, + (2.0 * tile.scale) as f32, + ) { + bitmap.fill_rect(rect, &paint, Transform::identity(), None); + } + } + return Ok(bitmap); + } let size = tile.rect.width.round().max(1.0) as u32; let mut pixmap = Pixmap::new(size, size).ok_or("Cannot allocate action tile")?; pixmap.fill(if tile.selected { @@ -81,12 +166,14 @@ pub fn bitmap(tile: &FrameTile) -> Result { lines(&[(62., 46.), (106., 90.)]); lines(&[(106., 46.), (62., 90.)]); } - More | Group(_) | Command(_) | Setting(_) | Display(_) | Pause | Reverse => { + TypeHere | Keyboard | More | Group(_) | Command(_) | Setting(_) | Display(_) | Pause + | Reverse => { artwork(&mut path, tile.icon); } DragHere => { lines(&[(55., 69.), (75., 89.), (113., 47.)]); } + KeyboardKey => unreachable!("Keyboard keys are rendered without artwork"), } if matches!(tile.icon, LeftClick | RightClick | DoubleClick) { path.move_to(62., 64.); @@ -146,6 +233,30 @@ pub fn bitmap(tile: &FrameTile) -> Result { Ok(pixmap) } +pub fn keyboard_font_size(tile: &FrameTile) -> f64 { + match tile.keyboard.map(|s| s.role) { + Some(crate::scanning::KeyboardRole::Character) if tile.text.chars().count() == 1 => 24.0, + Some(crate::scanning::KeyboardRole::Status) => 17.0, + _ => 16.0, + } +} +fn key_outline(width: f32, height: f32, inset: f32, radius: f32) -> tiny_skia::Path { + let (left, top, right, bottom) = (inset, inset, width - inset, height - inset); + let r = radius.min((right - left) / 2.0).min((bottom - top) / 2.0); + let mut p = PathBuilder::new(); + p.move_to(left + r, top); + p.line_to(right - r, top); + p.quad_to(right, top, right, top + r); + p.line_to(right, bottom - r); + p.quad_to(right, bottom, right - r, bottom); + p.line_to(left + r, bottom); + p.quad_to(left, bottom, left, bottom - r); + p.line_to(left, top + r); + p.quad_to(left, top, left + r, top); + p.close(); + p.finish().unwrap() +} + fn line(path: &mut PathBuilder, points: &[(f32, f32)]) { path.move_to(points[0].0, points[0].1); for &(x, y) in &points[1..] { @@ -232,6 +343,19 @@ fn return_arrow(path: &mut PathBuilder, forward: bool) { fn artwork(path: &mut PathBuilder, item: Item) { use crate::scan_menu::{Command as C, Kind as K, Setting as S}; match item { + Item::TypeHere | Item::Keyboard => { + rect(path, 46., 44., 76., 48.); + for y in [55., 67.] { + for x in [58., 74., 90., 106.] { + rect(path, x, y, 4., 4.); + } + } + line(path, &[(66., 81.), (102., 81.)]); + if item == Item::TypeHere { + line(path, &[(84., 100.), (84., 114.)]); + line(path, &[(77., 107.), (91., 107.)]); + } + } Item::More => { for x in [60., 84., 108.] { path.push_circle(x, 68., 5.); @@ -579,6 +703,7 @@ mod tests { use crate::scanning::{Rect, ScannerColor::*}; for color in [Red, Green, Blue, Yellow, White] { let mut tile = FrameTile { + keyboard: None, color, text: "Click".into(), rect: Rect { @@ -679,6 +804,7 @@ mod tests { Setting(S::GridMode), ] { let mut tile = FrameTile { + keyboard: None, color: Default::default(), text: icon.label().into(), rect: crate::scanning::Rect { diff --git a/src-tauri/src/scanning.rs b/src-tauri/src/scanning.rs index 948d262b..3bc1251d 100644 --- a/src-tauri/src/scanning.rs +++ b/src-tauri/src/scanning.rs @@ -244,6 +244,20 @@ pub struct UpdateContext { pub switch_held: bool, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeyboardRole { + Background, + Character, + Utility, + Toolbar, + Status, +} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeyboardTileStyle { + pub role: KeyboardRole, + pub active: bool, + pub row_scan: bool, +} #[derive(Debug, Clone, PartialEq)] pub struct FrameTile { pub color: ScannerColor, @@ -251,6 +265,7 @@ pub struct FrameTile { pub rect: Rect, pub scale: f64, pub icon: crate::scan_menu::Item, + pub keyboard: Option, pub selected: bool, } #[derive(Debug, Clone, PartialEq)] @@ -278,6 +293,7 @@ pub trait Technique { false } fn execution_failed(&mut self, _message: String) {} + fn execution_succeeded(&mut self) {} fn start(&mut self); fn advance(&mut self, elapsed_ms: u64); fn handle(&mut self, action: Action) -> Option; diff --git a/src-tauri/src/scanning_runtime.rs b/src-tauri/src/scanning_runtime.rs index 62c6f555..f8cf7471 100644 --- a/src-tauri/src/scanning_runtime.rs +++ b/src-tauri/src/scanning_runtime.rs @@ -28,7 +28,11 @@ pub trait Adapter: Send + Sync + 'static { fn activate( app: &AppHandle, selection: ::Selection, - ) -> Result<(), String>; + ) -> Result, String>; + fn settle_environment( + app: &AppHandle, + environment: Option<&mut Self::Environment>, + ) -> Result; } use crate::switch_input::Event; @@ -398,6 +402,9 @@ fn switch(app: &AppHandle, action: Action, input_generation: u64, re d.display = Some(display); A::prepare(app)?; } + if !A::settle_environment(app, d.display.as_mut())? { + return Ok(()); + } A::validate_environment(app, d.display.as_ref())?; let point = d.engine.as_mut().and_then(|e| e.action(action)); d.last_tick = Instant::now(); @@ -427,12 +434,23 @@ fn dispatch( return Err("Scan action was cancelled.".into()); } hide_scan_visuals(); - if let Err(error) = A::activate(app, request) { - A::cleanup(app)?; - let mut d = c.data.lock().unwrap_or_else(|p| p.into_inner()); - d.message = error.clone(); - if let Some(engine) = d.engine.as_mut() { - engine.execution_failed(error); + match A::activate(app, request) { + Ok(environment) => { + let mut d = c.data.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(environment) = environment { + d.display = Some(environment); + } + if let Some(engine) = d.engine.as_mut() { + engine.technique.execution_succeeded(); + } + } + Err(error) => { + A::cleanup(app)?; + let mut d = c.data.lock().unwrap_or_else(|p| p.into_inner()); + d.message = error.clone(); + if let Some(engine) = d.engine.as_mut() { + engine.execution_failed(error); + } } } { @@ -501,8 +519,16 @@ fn render_tiles(tiles: &[crate::scanning::FrameTile]) -> Result<(), String> { while slot.0.len() < tiles.len() { slot.0.push(Host::new()?); } - for (index, host) in slot.0.iter_mut().enumerate() { + let (hosts, previous) = &mut *slot; + for (index, host) in hosts.iter_mut().enumerate() { if let Some(tile) = tiles.get(index) { + if tile + .keyboard + .is_some_and(|s| s.role == crate::scanning::KeyboardRole::Background) + && previous.get(index) == Some(tile) + { + continue; + } host.tile(tile)?; } else { host.hide(); @@ -718,6 +744,10 @@ fn tick(app: &AppHandle) { } let result = (|| -> Result<(), String> { let mut d = c.data.lock().unwrap_or_else(|p| p.into_inner()); + if !A::settle_environment(app, d.display.as_mut())? { + d.last_tick = Instant::now(); + return Ok(()); + } A::validate_environment(app, d.display.as_ref())?; let now = Instant::now(); let elapsed = if d.cursor_suppression.is_some_and(|(token, _)| { diff --git a/src/PointScan.test.tsx b/src/PointScan.test.tsx index 31db9677..c82aa237 100644 --- a/src/PointScan.test.tsx +++ b/src/PointScan.test.tsx @@ -195,6 +195,9 @@ it.each([ ["dragDestination", "Choose drag destination."], ["dragConfirmation", "Confirm drag."], ["executing", "Performing drag."], + ["keyboard", "Choose a keyboard row, then a key."], + ["keyboardSuspended", "Select to resume the keyboard."], + ["keyboardOpening", "Opening the keyboard."], ])("explains the %s workflow phase", async (phase, message) => { mocks.invoke.mockResolvedValue({ ...initial, enabled: true, phase }); render(); diff --git a/src/scanning/useScanning.ts b/src/scanning/useScanning.ts index 77dc1046..0fe324ab 100644 --- a/src/scanning/useScanning.ts +++ b/src/scanning/useScanning.ts @@ -20,7 +20,7 @@ export type PointScanState = { remote?: boolean; config: PointScanConfig; enabled: boolean; - phase: "autoSelecting" | "idle" | "row" | "rowEscape" | "cell" | "x" | "y" | "menu" | "menuSuspended" | "dragDestination" | "dragConfirmation" | "executing"; + phase: "autoSelecting" | "idle" | "row" | "rowEscape" | "cell" | "x" | "y" | "menu" | "menuSuspended" | "dragDestination" | "dragConfirmation" | "executing" | "keyboard" | "keyboardSuspended" | "keyboardOpening"; paused: boolean; message: string; supported: boolean; diff --git a/src/settings/ScanningSection.tsx b/src/settings/ScanningSection.tsx index 86dfbdf4..4d89122e 100644 --- a/src/settings/ScanningSection.tsx +++ b/src/settings/ScanningSection.tsx @@ -7,6 +7,9 @@ import { import { SettingGroup, Toggle, OptionGroup, secondsOptions } from "./controls"; const phases = { + keyboard: "Choose a keyboard row, then a key", + keyboardSuspended: "Select to resume the keyboard", + keyboardOpening: "Opening the keyboard", idle: "Ready to begin", autoSelecting: "Waiting to click. Press a switch for the action menu", menu: "Choose an action at the selected point",