From ce47adfd3f6035cee9332c7080bba0cbaceab48f Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Fri, 18 Sep 2026 15:29:51 +0100 Subject: [PATCH 1/9] Add native QWERTY switch-scanning keyboard (#789) --- docs/qwerty-keyboard.md | 32 ++ src-tauri/src/input.rs | 8 + src-tauri/src/lib.rs | 1 + src-tauri/src/modifier_overlay_windows.rs | 24 + src-tauri/src/point_scan_runtime.rs | 52 ++- src-tauri/src/point_workflow.rs | 118 ++++- src-tauri/src/scan_executor.rs | 137 +++++- src-tauri/src/scan_host.rs | 85 +++- src-tauri/src/scan_keyboard.rs | 528 ++++++++++++++++++++++ src-tauri/src/scan_menu.rs | 10 +- src-tauri/src/scan_tile.rs | 55 ++- src-tauri/src/scanning.rs | 1 + src-tauri/src/scanning_runtime.rs | 36 +- src/PointScan.test.tsx | 3 + src/scanning/useScanning.ts | 2 +- src/settings/ScanningSection.tsx | 3 + 16 files changed, 1072 insertions(+), 23 deletions(-) create mode 100644 docs/qwerty-keyboard.md create mode 100644 src-tauri/src/scan_keyboard.rs diff --git a/docs/qwerty-keyboard.md b/docs/qwerty-keyboard.md new file mode 100644 index 00000000..4b9342a3 --- /dev/null +++ b/docs/qwerty-keyboard.md @@ -0,0 +1,32 @@ +# QWERTY scanning keyboard + +Switchify has its own UK English keyboard on Windows and macOS. It uses the saved local switches or the existing Android remote scanning actions. It does not use the operating system's on-screen keyboard. + +Choose a point and select **Type here** to click that point once and open the keyboard. To keep the existing input focus without clicking, select **More → Keyboard**. The keyboard does not take focus or accept mouse clicks. + +Scan a row, select it, then scan and select a key. The escape slot returns to rows. After typing, scanning starts again at the first row. Existing automatic/manual movement, speed, pause, reverse and inactivity suspension apply. The keyboard stays open after Space, Backspace, Enter and Tab. **Close** returns to point scanning. + +Pages: + +- **Letters:** UK QWERTY, numbers and punctuation, including £ and the UK shifted symbols. +- **Functions:** Esc, F1–F12, arrows, Home, End, Page Up, Page Down and Delete. Windows also has Insert, Print Screen, Scroll Lock and Pause. +- **Numbers:** digits, decimal point, arithmetic operators, Enter and Backspace. Digits are independent of hardware Num Lock. + +The bottom row switches pages, moves the keyboard between the top and bottom of the display work area, and closes it. Modifiers are available on every page. Shift, Ctrl, Alt/Option and Windows/Command cycle through **off → once → locked → off**. The label shows once or locked. Caps is a keyboard-local uppercase setting; Shift reverses its letter case. Page and position controls do not consume a one-shot modifier. + +Modifier keys are pressed only around each emitted shortcut and immediately released. Selecting a locked modifier does not hold that operating-system key while the scanner runs. Ordinary characters use text injection; command combinations and navigation use native key events. Native shortcuts retain the operating system's layout semantics. + +The keyboard closes when the foreground target or display environment changes, or its scan session ends. Failed input clears keyboard modifiers and requires Select to resume. Stop, disconnect and application exit use the shared deterministic input cleanup. No typed text is logged, and no prediction data, context reader, protocol extension or persistent keyboard settings are added. + +## Manual validation + +Use synthetic text in Notepad and a browser on Windows, and TextEdit and a browser on macOS. Launch macOS through `npm run macos:run` to retain its Accessibility identity. + +1. Use Type here and verify exactly one click focuses the intended field. Use More → Keyboard and verify no click or focus change occurs. +2. Type lowercase letters, uppercase letters with Shift/Caps, and UK punctuation including £, @ and double quotes. Test Space, Backspace, Enter and Tab. +3. Test Ctrl+A/C/V on Windows and Command+A/C/V on macOS. Cycle once/locked/off, including Shift with another modifier, and confirm no modifier remains physically held between selections. +4. Visit every page. Verify row/key highlighting, reverse movement, row escape, suspension/resume, returning to the first row after a key and Close. +5. Move the keyboard between the top and bottom. Check a scaled display and a secondary display, including negative coordinates. Verify the keyboard fits the work area and never activates its native windows. +6. Switch foreground apps, disconnect remote scanning, change display configuration and close Switchify. Verify overlays disappear and owned input is released. + +Automated tests use fake input adapters only. Native manual results must be recorded separately; compilation and unit tests do not establish live application compatibility. diff --git a/src-tauri/src/input.rs b/src-tauri/src/input.rs index 01315d27..3df76983 100644 --- a/src-tauri/src/input.rs +++ b/src-tauri/src/input.rs @@ -154,6 +154,14 @@ fn pointer_button(button: MouseButton) -> Button { #[cfg(target_os = "windows")] fn windows_alphanumeric_key(name: &str) -> Option { Some(match name { + #[cfg(target_os = "windows")] + "Insert" => Key::Insert, + #[cfg(target_os = "windows")] + "PrintScreen" => Key::PrintScr, + #[cfg(target_os = "windows")] + "ScrollLock" => Key::Scroll, + #[cfg(target_os = "windows")] + "Pause" => Key::Pause, "=" => Key::OEMPlus, "-" => Key::OEMMinus, "[" => Key::OEM4, diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 3c14020f..25290e87 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -24,6 +24,7 @@ mod protocol; mod remote_scan; mod scan_executor; mod scan_host; +mod scan_keyboard; mod scan_menu; mod scan_tile; mod scan_tree; diff --git a/src-tauri/src/modifier_overlay_windows.rs b/src-tauri/src/modifier_overlay_windows.rs index 0e96f01f..92982dfc 100644 --- a/src-tauri/src/modifier_overlay_windows.rs +++ b/src-tauri/src/modifier_overlay_windows.rs @@ -595,6 +595,30 @@ pub(crate) fn present_scan_tile( tile: &crate::scanning::FrameTile, ) -> Result<(), String> { let pixmap = crate::scan_tile::bitmap(tile)?; + if tile.icon == crate::scan_menu::Item::KeyboardKey { + let width = pixmap.width() as i32; + let height = pixmap.height() as i32; + let layout = Layout { + x: tile.rect.x.round() as i32, + y: tile.rect.y.round() as i32, + width, + height, + scale: tile.scale * 18.0 / FONT_SIZE, + chips: vec![RECT { + left: 2, + top: 0, + right: width - 2, + bottom: height, + }], + }; + return present_pixmap_with_text( + window, + std::slice::from_ref(&tile.text), + &layout, + pixmap.data(), + true, + ); + } let size = pixmap.width() as i32; let layout = Layout { x: tile.rect.x.round() as i32, diff --git a/src-tauri/src/point_scan_runtime.rs b/src-tauri/src/point_scan_runtime.rs index 317c7d2a..9518281a 100644 --- a/src-tauri/src/point_scan_runtime.rs +++ b/src-tauri/src/point_scan_runtime.rs @@ -11,6 +11,8 @@ use tauri::{AppHandle, Manager}; pub struct Environment { display: Display, foreground: usize, + keyboard_area: Rect, + handoff_until: Option, } pub struct PointScan; pub type Controller = scanning_runtime::Controller; @@ -59,13 +61,40 @@ impl Adapter for PointScan { crate::scan_executor::cleanup()?; crate::point_scan_prepare(app) } - fn activate(app: &AppHandle, request: Request) -> Result<(), String> { + fn activate(app: &AppHandle, request: Request) -> Result, String> { crate::point_scan_ready(app)?; match request { + Request::OpenKeyboard { point: Some(_) } => { + crate::scan_executor::activate(request)?; + let (_, mut environment) = + new_engine(app, app.state::().view().config)?; + // Allow the one requested click to establish focus before accepting + // any key. No typing or scan advancement occurs during this handoff. + environment.handoff_until = + Some(std::time::Instant::now() + std::time::Duration::from_millis(150)); + return Ok(Some(environment)); + } Request::Setting(setting) => scanning_runtime::update_point_setting(app, setting), Request::Display(next) => scanning_runtime::restart_point_on_display(app, next), request => crate::scan_executor::activate(request), } + .map(|()| None) + } + fn settle_environment( + app: &AppHandle, + environment: Option<&mut Environment>, + ) -> Result { + if let Some(environment) = environment { + if let Some(until) = environment.handoff_until { + crate::point_scan_ready(app)?; + if std::time::Instant::now() < until { + return Ok(false); + } + environment.foreground = crate::scan_host::foreground()?; + environment.handoff_until = None; + } + } + Ok(true) } fn cleanup(_app: &AppHandle) -> Result<(), String> { crate::scan_executor::cleanup() @@ -81,7 +110,7 @@ fn new_engine(app: &AppHandle, config: Config) -> Result<(Workflow, Environment) } else { 1.0 }; - let e = Workflow::new( + let mut e = Workflow::new( config.point(), Rect { x: display.x.into(), @@ -91,11 +120,20 @@ fn new_engine(app: &AppHandle, config: Config) -> Result<(Workflow, Environment) }, units, )?; + let keyboard_area = crate::scan_host::work_area(Rect { + x: display.x.into(), + y: display.y.into(), + width: display.width.into(), + height: display.height.into(), + })?; + e.set_keyboard_area(keyboard_area); Ok(( e, Environment { display, foreground: crate::scan_host::foreground()?, + keyboard_area, + handoff_until: None, }, )) } @@ -109,6 +147,16 @@ fn validate_display(app: &AppHandle, display: Option<&Environment>) -> Result<() if !displays.contains(&expected.display) { return Err("Display geometry changed. Scanning restarts.".into()); } + let d = &expected.display; + if crate::scan_host::work_area(Rect { + x: d.x.into(), + y: d.y.into(), + width: d.width.into(), + height: d.height.into(), + })? != expected.keyboard_area + { + return Err("Display work area changed. Scanning restarts.".into()); + } } Ok(()) } diff --git a/src-tauri/src/point_workflow.rs b/src-tauri/src/point_workflow.rs index 0021ce53..52bd2f17 100644 --- a/src-tauri/src/point_workflow.rs +++ b/src-tauri/src/point_workflow.rs @@ -8,6 +8,10 @@ use serde::Serialize; pub type Point = (i32, i32); #[derive(Debug, Clone, Copy, PartialEq)] pub enum Request { + OpenKeyboard { + point: Option, + }, + Keyboard(crate::scan_keyboard::Stroke), Click { point: Point, right: bool, @@ -52,6 +56,9 @@ impl Default for Phase { #[derive(Debug, Clone, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum WorkflowPhase { + Keyboard, + KeyboardSuspended, + KeyboardOpening, Menu, MenuSuspended, AutoSelecting, @@ -61,6 +68,8 @@ pub enum WorkflowPhase { } #[derive(PartialEq)] enum Stage { + Keyboard, + KeyboardOpening, Idle, Countdown, Point, @@ -69,6 +78,8 @@ enum Stage { Executing, } pub struct Workflow { + keyboard: crate::scan_keyboard::Keyboard, + keyboard_area: Rect, point: Engine, stage: Stage, menu: Menu, @@ -83,6 +94,8 @@ impl Workflow { pub fn new(config: PointSettings, screen: Rect, scale: f64) -> Result { let period = config.block_interval_ms; Ok(Self { + keyboard: crate::scan_keyboard::Keyboard::new(cfg!(target_os = "macos")), + keyboard_area: screen, point: Engine::new(config, screen, scale)?, stage: Stage::Idle, menu: Menu::new(Kind::Actions, period), @@ -106,11 +119,15 @@ impl Workflow { menu.set_period(self.point.config.block_interval_ms); } if restart { + self.keyboard = crate::scan_keyboard::Keyboard::new(cfg!(target_os = "macos")); self.parent_menu.clear(); self.stage = Stage::Point; self.point.start(); } } + pub fn set_keyboard_area(&mut self, area: Rect) { + self.keyboard_area = area; + } fn open(&mut self, kind: Kind) { self.menu = Menu::new(kind, self.point.config.block_interval_ms); self.stage = Stage::Menu; @@ -126,6 +143,14 @@ impl Workflow { } fn selected(&mut self, item: Item) -> Option { match item { + Item::TypeHere | Item::Keyboard => { + self.stage = Stage::KeyboardOpening; + self.keyboard = crate::scan_keyboard::Keyboard::new(cfg!(target_os = "macos")); + return Some(Request::OpenKeyboard { + point: (item == Item::TypeHere).then_some(self.source), + }); + } + Item::KeyboardKey => {} Item::More | Item::Group(_) => { let kind = if let Item::Group(kind) = item { kind @@ -211,6 +236,10 @@ impl Technique for Workflow { type Selection = Request; type Phase = Phase; fn execution_failed(&mut self, message: String) { + if self.stage == Stage::Keyboard { + self.keyboard.failed(); + return; + } self.pending = None; self.stage = Stage::Menu; self.menu = Menu::new(Kind::Actions, self.point.config.block_interval_ms); @@ -218,12 +247,20 @@ impl Technique for Workflow { self.menu.suspended = true; self.error = Some(message); } + fn execution_succeeded(&mut self) { + if self.stage == Stage::KeyboardOpening { + self.stage = Stage::Keyboard; + } else if self.stage == Stage::Keyboard { + self.keyboard.succeeded(); + } + } fn start(&mut self) { self.reset(); self.stage = Stage::Point; self.point.start(); } fn reset(&mut self) { + self.keyboard = crate::scan_keyboard::Keyboard::new(cfg!(target_os = "macos")); self.stage = Stage::Idle; self.parent_menu.clear(); self.pending = None; @@ -247,6 +284,14 @@ impl Technique for Workflow { } fn handle(&mut self, action: Action) -> Option { match self.stage { + Stage::Keyboard => match self.keyboard.handle(action) { + Some(crate::scan_keyboard::Output::Stroke(stroke)) => { + return Some(Request::Keyboard(stroke)) + } + Some(crate::scan_keyboard::Output::Close) => self.start(), + None => {} + }, + Stage::KeyboardOpening => {} Stage::Point | Stage::Destination => { if let Some(point) = self.point.handle(action) { if self.stage == Stage::Destination { @@ -288,6 +333,9 @@ impl Technique for Workflow { } fn advance(&mut self, ms: u64) { match self.stage { + Stage::Keyboard => self + .keyboard + .advance(ms, self.point.config.block_interval_ms), Stage::Point | Stage::Destination => { self.point.advance(ms); if self.point.exhausted() { @@ -339,6 +387,12 @@ impl Technique for Workflow { } fn phase(&self) -> Phase { match self.stage { + Stage::KeyboardOpening => Phase::Workflow(WorkflowPhase::KeyboardOpening), + Stage::Keyboard => Phase::Workflow(if self.keyboard.suspended { + WorkflowPhase::KeyboardSuspended + } else { + WorkflowPhase::Keyboard + }), Stage::Idle => Phase::default(), Stage::Countdown => Phase::Workflow(WorkflowPhase::AutoSelecting), Stage::Point => Phase::Point(self.point.phase()), @@ -355,6 +409,11 @@ impl Technique for Workflow { } fn frame(&self) -> Frame { let mut frame = match self.stage { + Stage::Keyboard => self.keyboard.frame( + self.keyboard_area, + self.point.units_per_logical_pixel, + self.point.config.scanner_color, + ), Stage::Countdown => { let scale = self.point.units_per_logical_pixel; let screen = self.point.screen; @@ -466,6 +525,57 @@ mod tests { automatic, ) } + #[test] + fn keyboard_opening_waits_for_click_success_and_failure_stays_in_menu() { + let mut s = session(false); + let w = &mut s.technique; + w.source = (120, 230); + assert_eq!( + w.selected(Item::TypeHere), + Some(Request::OpenKeyboard { + point: Some((120, 230)) + }) + ); + assert_eq!(w.phase(), Phase::Workflow(WorkflowPhase::KeyboardOpening)); + assert!(w.handle(Action::Select).is_none()); + w.execution_failed("Click failed".into()); + assert_eq!(w.phase(), Phase::Workflow(WorkflowPhase::MenuSuspended)); + w.error = None; + assert_eq!( + w.selected(Item::Keyboard), + Some(Request::OpenKeyboard { point: None }) + ); + w.execution_succeeded(); + assert_eq!(w.phase(), Phase::Workflow(WorkflowPhase::Keyboard)); + w.keyboard.modifiers[0] = crate::scan_keyboard::Modifier::Locked; + w.keyboard.caps = true; + w.execution_failed("Do not expose provider details".into()); + assert_eq!(w.phase(), Phase::Workflow(WorkflowPhase::KeyboardSuspended)); + assert_eq!( + w.keyboard.modifiers, + [crate::scan_keyboard::Modifier::Off; 4] + ); + w.reset(); + assert!(!w.keyboard.caps); + assert!(w.frame().tiles.is_empty()); + } + #[test] + fn keyboard_key_acknowledgement_is_exactly_once_and_reset_discards_state() { + let mut s = session(false); + let w = &mut s.technique; + w.selected(Item::Keyboard); + w.execution_succeeded(); + assert!(w.handle(Action::Select).is_none()); + assert!(matches!( + w.handle(Action::Select), + Some(Request::Keyboard(_)) + )); + assert!(w.handle(Action::Select).is_none()); + w.execution_succeeded(); + assert!(w.handle(Action::Select).is_none()); + w.reset(); + assert_eq!(w.phase(), Phase::default()); + } fn open(s: &mut Session) { for _ in 0..3 { assert_eq!(s.action(Action::Select), None); @@ -749,7 +859,7 @@ mod tests { for (column, right, count) in [(0, false, 1), (1, true, 1), (2, false, 2)] { let mut s = session(false); open(&mut s); - assert_eq!(s.frame().tiles.len(), 8); + assert_eq!(s.frame().tiles.len(), 9); assert_eq!( choose(&mut s, 0, column), Some(Request::Click { @@ -836,7 +946,7 @@ mod tests { fn automatic_menu_pauses_preserving_target_and_select_only_resumes() { let mut s = session(true); open(&mut s); - for _ in 0..9 { + for _ in 0..12 { s.tick(250, false); s.take_selection(); } @@ -882,14 +992,14 @@ mod tests { fn point_reselection_cancel_and_drag_confirmation_back_paths() { let mut s = session(false); open(&mut s); - assert_eq!(choose(&mut s, 2, 0), None); + assert_eq!(choose(&mut s, 3, 0), None); assert_eq!( s.technique.phase(), Phase::Point(crate::point_scan::Phase::X) ); s.action(Action::Select); s.action(Action::Select); - assert_eq!(choose(&mut s, 2, 1), None); + assert_eq!(choose(&mut s, 3, 1), None); assert!(!s.active()); open(&mut s); choose(&mut s, 1, 1); diff --git a/src-tauri/src/scan_executor.rs b/src-tauri/src/scan_executor.rs index 5600f522..b641c9bd 100644 --- a/src-tauri/src/scan_executor.rs +++ b/src-tauri/src/scan_executor.rs @@ -15,6 +15,38 @@ pub fn execute( ); } match request { + Request::OpenKeyboard { point } => { + if input.has_active_drag() { + return Err("End the active drag before typing.".into()); + } + match point { + Some(point) => input.click_pointer_at(point, MouseButton::Left, 1, &[]), + None => Ok(()), + } + } + Request::Keyboard(stroke) => { + if input.has_active_drag() { + return Err("End the active drag before typing.".into()); + } + input.release_all()?; + if let Some(character) = stroke.character().filter(|_| !stroke.shortcut()) { + return input.type_text(&character.to_string()); + } + let name = match stroke.key { + crate::scan_keyboard::Key::Character(base, _) => { + base.to_ascii_uppercase().to_string() + } + crate::scan_keyboard::Key::Named(name) => name.into(), + _ => return Err("Invalid keyboard action.".into()), + }; + let mut keys: Vec<&str> = ["Shift", "Ctrl", "Alt", "Meta"] + .into_iter() + .zip(stroke.modifiers) + .filter_map(|(key, on)| on.then_some(key)) + .collect(); + keys.push(&name); + input.scan_chord(&keys, None) + } Request::Command { command, point } => execute_command(input, command, point), Request::Setting(_) | Request::Display(_) => { Err("Scanning action requires the scan controller.".into()) @@ -221,8 +253,9 @@ mod tests { legacy_clicks: usize, } impl InputInjector for Fake { - fn inject_text(&mut self, _: &str) -> Result<(), String> { - panic!("No typing keyboard") + fn inject_text(&mut self, text: &str) -> Result<(), String> { + self.events.push(format!("text {text}")); + Ok(()) } fn move_pointer(&mut self, _: i32, _: i32) -> Result<(), String> { panic!("No relative movement") @@ -296,6 +329,106 @@ mod tests { } } #[test] + fn keyboard_text_chords_and_current_focus_use_fake_input() { + use crate::scan_keyboard::{Key, Stroke}; + let mut input = DesktopInput::new(Fake::default()); + execute(&mut input, Request::OpenKeyboard { point: None }, true).unwrap(); + assert!(input.injector.events.is_empty()); + execute( + &mut input, + Request::Keyboard(Stroke { + key: Key::Character('3', '£'), + modifiers: [true, false, false, false], + caps: false, + }), + true, + ) + .unwrap(); + assert_eq!(input.injector.events, ["text £"]); + input.injector.events.clear(); + execute( + &mut input, + Request::Keyboard(Stroke { + key: Key::Character('a', 'A'), + modifiers: [false, true, false, false], + caps: false, + }), + true, + ) + .unwrap(); + assert_eq!( + input.injector.events, + [ + "key Ctrl true", + "key A true", + "key A false", + "key Ctrl false" + ] + ); + input.injector.events.clear(); + execute( + &mut input, + Request::Keyboard(Stroke { + key: Key::Named("Escape"), + modifiers: [false; 4], + caps: false, + }), + true, + ) + .unwrap(); + assert_eq!( + input.injector.events, + ["key Escape true", "key Escape false"] + ); + } + #[test] + fn keyboard_releases_failed_keys_and_rejects_external_modifiers() { + use crate::scan_keyboard::{Key, Stroke}; + let request = Request::Keyboard(Stroke { + key: Key::Named("Tab"), + modifiers: [true, false, false, false], + caps: false, + }); + let mut input = DesktopInput::new(Fake { + fail_release: true, + ..Default::default() + }); + assert!(execute(&mut input, request, true).is_err()); + input.injector.fail_release = false; + input.release_all().unwrap(); + assert!(input + .injector + .events + .ends_with(&["key Tab false".into(), "key Shift false".into()])); + input.injector.events.clear(); + assert!(execute(&mut input, request, false).is_err()); + assert!(input.injector.events.is_empty()); + } + #[test] + fn type_here_clicks_exactly_once_and_propagates_failure() { + let mut input = DesktopInput::new(Fake::default()); + execute( + &mut input, + Request::OpenKeyboard { + point: Some((42, 70)), + }, + true, + ) + .unwrap(); + assert_eq!(input.injector.target_clicks.len(), 1); + assert_eq!(input.injector.target_clicks[0].0, (42, 70)); + input.injector.fail_click = true; + assert!(execute( + &mut input, + Request::OpenKeyboard { + point: Some((42, 70)) + }, + true + ) + .is_err()); + assert_eq!(input.injector.target_clicks.len(), 1); + } + #[test] fn every_scan_click_uses_target_even_while_cursor_readback_is_stale() { for point in [(-1600, -200), (3600, 1200)] { let mut cases = vec![ diff --git a/src-tauri/src/scan_host.rs b/src-tauri/src/scan_host.rs index c7e8fd87..6d4aca02 100644 --- a/src-tauri/src/scan_host.rs +++ b/src-tauri/src/scan_host.rs @@ -1,6 +1,68 @@ //! Native, nonactivating strips, using the same display units as input injection. use crate::scanning::Rect; +/// Work area in the same native coordinates used by the scan engine. +pub fn work_area(screen: Rect) -> Result { + #[cfg(target_os = "windows")] + unsafe { + use windows::Win32::{ + Foundation::POINT, + Graphics::Gdi::{ + GetMonitorInfoW, MonitorFromPoint, MONITORINFO, MONITOR_DEFAULTTONEAREST, + }, + }; + let monitor = MonitorFromPoint( + POINT { + x: (screen.x + screen.width / 2.0) as i32, + y: (screen.y + screen.height / 2.0) as i32, + }, + MONITOR_DEFAULTTONEAREST, + ); + let mut info = MONITORINFO { + cbSize: std::mem::size_of::() as u32, + ..Default::default() + }; + if !GetMonitorInfoW(monitor, &mut info).as_bool() { + return Err("Display work area is unavailable.".into()); + } + let r = info.rcWork; + Ok(Rect { + x: r.left.into(), + y: r.top.into(), + width: (r.right - r.left).into(), + height: (r.bottom - r.top).into(), + }) + } + #[cfg(target_os = "macos")] + { + use objc2::MainThreadMarker; + use objc2_app_kit::NSScreen; + let mtm = MainThreadMarker::new().ok_or("Keyboard requires the main thread.")?; + let screens = NSScreen::screens(mtm); + let primary = screens + .firstObject() + .ok_or("Display work area is unavailable.")?; + let top = primary.frame().origin.y + primary.frame().size.height; + for s in screens.iter() { + let r = s.frame(); + if (r.origin.x - screen.x).abs() < 1.0 + && (top - r.origin.y - r.size.height - screen.y).abs() < 1.0 + { + let r = s.visibleFrame(); + return Ok(Rect { + x: r.origin.x, + y: top - r.origin.y - r.size.height, + width: r.size.width, + height: r.size.height, + }); + } + } + Err("Display work area is unavailable.".into()) + } + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + Ok(screen) +} + #[cfg(target_os = "windows")] mod platform { use super::*; @@ -354,13 +416,26 @@ mod platform { artwork.setImage(Some(&image)); view.addSubview(&artwork); let label = NSTextField::wrappingLabelWithString(&NSString::from_str(&tile.text), mtm); - label.setFont(Some(&NSFont::boldSystemFontOfSize(15.0 * tile.scale))); + let key = tile.icon == crate::scan_menu::Item::KeyboardKey; + label.setFont(Some(&NSFont::boldSystemFontOfSize( + if key { 18.0 } else { 15.0 } * tile.scale, + ))); label.setTextColor(Some(&NSColor::whiteColor())); label.setAlignment(NSTextAlignment::Center); - label.setFrame(NSRect::new( - NSPoint::new(6.0 * tile.scale, 12.0 * tile.scale), - NSSize::new(tile.rect.width - 12.0 * tile.scale, 40.0 * tile.scale), - )); + label.setFrame(if key { + NSRect::new( + NSPoint::new(2.0, (tile.rect.height - 42.0 * tile.scale).max(0.0) / 2.0), + NSSize::new( + tile.rect.width - 4.0, + (42.0 * tile.scale).min(tile.rect.height), + ), + ) + } else { + NSRect::new( + NSPoint::new(6.0 * tile.scale, 12.0 * tile.scale), + NSSize::new(tile.rect.width - 12.0 * tile.scale, 40.0 * tile.scale), + ) + }); view.addSubview(&label); self.panels[0].setContentView(Some(&view)); Ok(()) diff --git a/src-tauri/src/scan_keyboard.rs b/src-tauri/src/scan_keyboard.rs new file mode 100644 index 00000000..cd0cc1c7 --- /dev/null +++ b/src-tauri/src/scan_keyboard.rs @@ -0,0 +1,528 @@ +//! Switch-owned keyboard state and geometry. No native input or captured text. +use crate::{ + scan_menu::Item, + scan_tree::{Navigator, Node, Selection}, + scanning::{Action, Frame, FrameTile, Interval, Rect, ScannerColor, MAX_SCAN_CYCLES}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Page { + Letters, + Functions, + Numbers, +} +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Modifier { + #[default] + Off, + Once, + Locked, +} +impl Modifier { + fn next(self) -> Self { + match self { + Self::Off => Self::Once, + Self::Once => Self::Locked, + Self::Locked => Self::Off, + } + } + fn suffix(self) -> &'static str { + match self { + Self::Off => "", + Self::Once => "\nonce", + Self::Locked => "\nlocked", + } + } +} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Key { + Character(char, char), + Named(&'static str), + Modifier(usize), + Caps, + Page(Page), + Dock, + Close, +} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Stroke { + pub key: Key, + pub modifiers: [bool; 4], + pub caps: bool, +} +impl Stroke { + pub fn character(self) -> Option { + let Key::Character(base, shifted) = self.key else { + return None; + }; + Some( + if self.modifiers[0] ^ (self.caps && base.is_ascii_alphabetic()) { + shifted + } else { + base + }, + ) + } + pub fn shortcut(self) -> bool { + self.modifiers[1..].iter().any(|m| *m) + } +} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Output { + Stroke(Stroke), + Close, +} + +pub fn rows(page: Page, mac: bool) -> Vec> { + use Key::{Character as C, Named as N}; + let chars = |normal: &str, shifted: &str| { + normal + .chars() + .zip(shifted.chars()) + .map(|(a, b)| C(a, b)) + .collect::>() + }; + let mut result = match page { + Page::Letters => { + let mut number = chars("`1234567890-=", "¬!\"£$%^&*()_+"); + number.push(N("Backspace")); + let mut top = vec![N("Tab")]; + top.extend(chars("qwertyuiop[]", "QWERTYUIOP{}")); + let mut middle = vec![Key::Caps]; + middle.extend(chars("asdfghjkl;'#", "ASDFGHJKL:@~")); + middle.push(N("Enter")); + let mut bottom = vec![Key::Modifier(0)]; + bottom.extend(chars("\\zxcvbnm,./", "|ZXCVBNM<>?")); + bottom.push(C(' ', ' ')); + vec![number, top, middle, bottom] + } + Page::Functions => { + let mut result = vec![ + vec![ + N("Escape"), + N("F1"), + N("F2"), + N("F3"), + N("F4"), + N("F5"), + N("F6"), + ], + vec![N("F7"), N("F8"), N("F9"), N("F10"), N("F11"), N("F12")], + vec![N("Home"), N("ArrowUp"), N("End"), N("PageUp"), N("Delete")], + vec![ + N("ArrowLeft"), + N("ArrowDown"), + N("ArrowRight"), + N("PageDown"), + N("Tab"), + N("Enter"), + ], + ]; + if !mac { + result.push(vec![ + N("Insert"), + N("PrintScreen"), + N("ScrollLock"), + N("Pause"), + ]); + } + result + } + Page::Numbers => vec![ + vec![C('7', '7'), C('8', '8'), C('9', '9'), C('/', '/')], + vec![C('4', '4'), C('5', '5'), C('6', '6'), C('*', '*')], + vec![C('1', '1'), C('2', '2'), C('3', '3'), C('-', '-')], + vec![ + C('0', '0'), + C('.', '.'), + C('+', '+'), + N("Enter"), + N("Backspace"), + ], + ], + }; + result.push(vec![ + Key::Modifier(0), + Key::Modifier(1), + Key::Modifier(2), + Key::Modifier(3), + Key::Caps, + ]); + result.push(vec![ + Key::Page(Page::Letters), + Key::Page(Page::Functions), + Key::Page(Page::Numbers), + Key::Dock, + Key::Close, + ]); + result +} + +pub struct Keyboard { + pub page: Page, + pub modifiers: [Modifier; 4], + pub caps: bool, + pub top: bool, + pub suspended: bool, + pub error: bool, + mac: bool, + rows: Vec>, + nav: Navigator, + interval: Interval, + cycles: usize, + forward: bool, + pending: bool, +} +impl Keyboard { + pub fn new(mac: bool) -> Self { + let rows = rows(Page::Letters, mac); + let nav = Self::navigator(&rows); + Self { + page: Page::Letters, + modifiers: [Modifier::Off; 4], + caps: false, + top: false, + suspended: false, + error: false, + mac, + rows, + nav, + interval: Interval::default(), + cycles: 0, + forward: true, + pending: false, + } + } + fn navigator(rows: &[Vec]) -> Navigator { + Navigator::new( + rows.iter() + .map(|row| Node::Branch(row.iter().copied().map(Node::Leaf).collect())) + .collect(), + ) + } + fn restart(&mut self) { + self.nav.reset(); + self.interval.reset(); + self.cycles = 0; + self.forward = true; + self.suspended = false; + } + pub fn advance(&mut self, ms: u64, period: u64) { + if !self.pending + && !self.suspended + && self.interval.elapsed(ms, period) + && self.nav.step(self.forward) + { + self.cycles += 1; + self.suspended = self.cycles >= MAX_SCAN_CYCLES; + } + } + pub fn handle(&mut self, action: Action) -> Option { + if self.pending { + return None; + } + if self.suspended { + if action == Action::Select { + self.error = false; + self.restart(); + } + return None; + } + match action { + Action::Select => { + self.interval.reset(); + self.cycles = 0; + match self.nav.select() { + Selection::Leaf(key) => return self.choose(key), + Selection::Entered | Selection::Escaped => self.forward = true, + Selection::None => {} + } + } + Action::Next | Action::Back => { + self.interval.reset(); + self.cycles = 0; + self.forward = action == Action::Next; + self.nav.step(self.forward); + } + Action::Reverse => { + self.forward = !self.forward; + self.interval.reset(); + self.cycles = 0; + } + _ => {} + } + None + } + fn choose(&mut self, key: Key) -> Option { + match key { + Key::Modifier(i) => self.modifiers[i] = self.modifiers[i].next(), + Key::Caps => self.caps = !self.caps, + Key::Dock => self.top = !self.top, + Key::Page(page) => { + self.page = page; + self.rows = rows(page, self.mac); + self.nav = Self::navigator(&self.rows); + } + Key::Close => return Some(Output::Close), + Key::Character(..) | Key::Named(_) => { + self.pending = true; + return Some(Output::Stroke(Stroke { + key, + modifiers: self.modifiers.map(|m| m != Modifier::Off), + caps: self.caps, + })); + } + } + self.restart(); + None + } + pub fn succeeded(&mut self) { + if self.pending { + for modifier in &mut self.modifiers { + if *modifier == Modifier::Once { + *modifier = Modifier::Off; + } + } + self.pending = false; + self.restart(); + } + } + pub fn failed(&mut self) { + self.modifiers = [Modifier::Off; 4]; + self.caps = false; + self.pending = false; + self.restart(); + self.error = true; + self.suspended = true; + } + fn label(&self, key: Key) -> String { + match key { + Key::Character(' ', _) => "Space".into(), + Key::Character(..) => Stroke { + key, + modifiers: self.modifiers.map(|m| m != Modifier::Off), + caps: self.caps, + } + .character() + .unwrap() + .to_string(), + Key::Named(name) => match name { + "ArrowUp" => "↑", + "ArrowDown" => "↓", + "ArrowLeft" => "←", + "ArrowRight" => "→", + "PageUp" => "Page Up", + "PageDown" => "Page Down", + "PrintScreen" => "Print Screen", + "ScrollLock" => "Scroll Lock", + "Escape" => "Esc", + _ => name, + } + .into(), + Key::Modifier(i) => format!( + "{}{}", + match i { + 0 => "Shift", + 1 => "Ctrl", + 2 => + if self.mac { + "Option" + } else { + "Alt" + }, + _ => + if self.mac { + "Command" + } else { + "Windows" + }, + }, + self.modifiers[i].suffix() + ), + Key::Caps => if self.caps { "Caps\non" } else { "Caps\noff" }.into(), + Key::Page(page) => format!( + "{}{}", + match page { + Page::Letters => "Letters", + Page::Functions => "Functions", + Page::Numbers => "Numbers", + }, + if page == self.page { " •" } else { "" } + ), + Key::Dock => if self.top { + "Move to bottom" + } else { + "Move to top" + } + .into(), + Key::Close => "Close".into(), + } + } + pub fn frame(&self, screen: Rect, units: f64, color: ScannerColor) -> Frame { + let width = (1200.0 * units).min(screen.width - 16.0 * units).max(1.0); + let height = (420.0 * units).min(screen.height / 2.0).max(1.0); + let x = screen.x + (screen.width - width) / 2.0; + let y = if self.top { + screen.y + } else { + screen.y + screen.height - height + }; + let row_height = height / (self.rows.len() as f64 + 0.5); + let mut frame = Frame::default(); + let active_row = self.nav.path().first().copied().unwrap_or(self.nav.index()); + for (r, row) in self.rows.iter().enumerate() { + let key_width = width / row.len() as f64; + for (c, key) in row.iter().enumerate() { + frame.tiles.push(FrameTile { + color, + text: self.label(*key), + icon: Item::KeyboardKey, + rect: Rect { + x: x + c as f64 * key_width + 2.0 * units, + y: y + r as f64 * row_height + 2.0 * units, + width: (key_width - 4.0 * units).max(1.0), + height: (row_height - 4.0 * units).max(1.0), + }, + scale: units.min(row_height / 44.0).min(key_width / 65.0), + selected: !self.suspended + && r == active_row + && (self.nav.path().is_empty() + || self.nav.escaping() + || c == self.nav.index()), + }); + } + } + let text = if self.error { + "Input failed. Select to resume." + } else if self.suspended { + "Select to resume keyboard." + } else if self.nav.escaping() { + "Back to rows" + } else if self.nav.path().is_empty() { + "Choose a row" + } else { + "Choose a key" + }; + frame.tiles.push(FrameTile { + color, + text: text.into(), + icon: Item::KeyboardKey, + rect: Rect { + x, + y: y + height - row_height * 0.5, + width, + height: row_height * 0.5, + }, + scale: units.min(row_height / 44.0), + selected: self.nav.escaping(), + }); + frame + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn uk_layout_and_platform_keys() { + let letters = rows(Page::Letters, false).concat(); + for c in 'a'..='z' { + assert!(letters.contains(&Key::Character(c, c.to_ascii_uppercase()))); + } + assert!(letters.contains(&Key::Character('3', '£'))); + assert!(rows(Page::Functions, false) + .concat() + .contains(&Key::Named("Insert"))); + assert!(!rows(Page::Functions, true) + .concat() + .contains(&Key::Named("Insert"))); + } + #[test] + fn modifiers_are_consumed_only_after_success_and_lock_persists() { + let mut k = Keyboard::new(false); + k.choose(Key::Modifier(0)); + k.choose(Key::Modifier(1)); + k.choose(Key::Modifier(1)); + k.choose(Key::Page(Page::Numbers)); + k.choose(Key::Dock); + assert_eq!( + k.modifiers, + [ + Modifier::Once, + Modifier::Locked, + Modifier::Off, + Modifier::Off + ] + ); + k.choose(Key::Named("Enter")); + assert!(k.handle(Action::Select).is_none()); + k.succeeded(); + assert_eq!(k.modifiers[0], Modifier::Off); + assert_eq!(k.modifiers[1], Modifier::Locked); + k.choose(Key::Modifier(1)); + assert_eq!(k.modifiers[1], Modifier::Off); + k.choose(Key::Modifier(2)); + k.failed(); + assert_eq!(k.modifiers, [Modifier::Off; 4]); + assert!(k.suspended); + k.handle(Action::Select); + assert!(!k.suspended); + } + #[test] + fn caps_xor_shift_only_affects_letters() { + let mut s = Stroke { + key: Key::Character('a', 'A'), + caps: true, + modifiers: [false; 4], + }; + assert_eq!(s.character(), Some('A')); + s.modifiers[0] = true; + assert_eq!(s.character(), Some('a')); + s.key = Key::Character('3', '£'); + assert_eq!(s.character(), Some('£')); + } + #[test] + fn scan_returns_to_top_and_has_row_escape_and_suspension() { + let mut k = Keyboard::new(false); + k.handle(Action::Select); + assert!(!k.nav.path().is_empty()); + k.handle(Action::Back); + assert!(k.nav.escaping()); + k.handle(Action::Select); + assert!(k.nav.path().is_empty()); + k.handle(Action::Select); + assert!(matches!(k.handle(Action::Select), Some(Output::Stroke(_)))); + k.succeeded(); + assert!(k.nav.path().is_empty()); + assert_eq!(k.nav.index(), 0); + for _ in 0..k.rows.len() * MAX_SCAN_CYCLES { + k.advance(100, 100); + } + assert!(k.suspended); + assert!(k.handle(Action::Select).is_none()); + assert!(!k.suspended); + } + #[test] + fn geometry_fits_work_area_at_both_docks_and_scales() { + for units in [1.0, 1.5, 2.0] { + for top in [false, true] { + for page in [Page::Letters, Page::Functions, Page::Numbers] { + let mut k = Keyboard::new(false); + k.choose(Key::Page(page)); + k.top = top; + let screen = Rect { + x: -1600.0, + y: -200.0, + width: 1600.0, + height: 900.0, + }; + for tile in k.frame(screen, units, ScannerColor::default()).tiles { + assert!(tile.rect.x >= screen.x && tile.rect.y >= screen.y); + assert!(tile.rect.x + tile.rect.width <= screen.x + screen.width + 0.01); + assert!(tile.rect.y + tile.rect.height <= screen.y + screen.height + 0.01); + } + } + } + } + } +} diff --git a/src-tauri/src/scan_menu.rs b/src-tauri/src/scan_menu.rs index 001b4a5d..0e01a4ac 100644 --- a/src-tauri/src/scan_menu.rs +++ b/src-tauri/src/scan_menu.rs @@ -5,6 +5,9 @@ use crate::{ }; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Item { + TypeHere, + Keyboard, + KeyboardKey, More, Group(Kind), Command(Command), @@ -31,6 +34,9 @@ pub enum Item { impl Item { pub fn label(self) -> &'static str { match self { + Self::TypeHere => "Type here", + Self::Keyboard => "Keyboard", + Self::KeyboardKey => "Key", Self::More => "More", Self::Group(kind) => kind.label(), Self::Command(command) => command.label(), @@ -448,6 +454,7 @@ impl Kind { return vec![ vec![LeftClick, RightClick, DoubleClick], vec![Scroll, Drag, More], + vec![TypeHere], vec![NewPoint, Cancel], ] } @@ -456,6 +463,7 @@ impl Kind { return vec![vec![DragHere, DestinationAgain], vec![CancelDrag, Cancel]] } Self::More => vec![ + Keyboard, Group(Self::Mouse), Group(Self::Editing), Group(Self::Windows), @@ -738,6 +746,6 @@ mod tests { assert_eq!(m.nav.index(), 0); m.handle(Action::Reverse); m.advance(250); - assert_eq!(m.nav.index(), 2); + assert_eq!(m.nav.index(), 3); } } diff --git a/src-tauri/src/scan_tile.rs b/src-tauri/src/scan_tile.rs index 2e1e24b1..db35cc58 100644 --- a/src-tauri/src/scan_tile.rs +++ b/src-tauri/src/scan_tile.rs @@ -5,6 +5,44 @@ use tiny_skia::{ }; pub fn bitmap(tile: &FrameTile) -> Result { + if tile.icon == Item::KeyboardKey { + let mut bitmap = Pixmap::new( + tile.rect.width.round().max(1.0) as u32, + tile.rect.height.round().max(1.0) as u32, + ) + .ok_or("Cannot allocate keyboard key")?; + let rgb = if tile.selected { + tile.color.menu_fill() + } else { + [30, 35, 46] + }; + bitmap.fill(Color::from_rgba8(rgb[0], rgb[1], rgb[2], 255)); + if tile.selected { + let mut paint = Paint::default(); + let [r, g, b] = tile.color.rgb(); + paint.set_color_rgba8(r, g, b, 255); + let mut path = PathBuilder::new(); + if let Some(rect) = tiny_skia::Rect::from_xywh( + 1.0, + 1.0, + (bitmap.width() as f32 - 2.0).max(0.1), + (bitmap.height() as f32 - 2.0).max(0.1), + ) { + path.push_rect(rect); + bitmap.stroke_path( + &path.finish().unwrap(), + &paint, + &Stroke { + width: 2.0, + ..Default::default() + }, + Transform::identity(), + None, + ); + } + } + return Ok(bitmap); + } let size = tile.rect.width.round().max(1.0) as u32; let mut pixmap = Pixmap::new(size, size).ok_or("Cannot allocate action tile")?; pixmap.fill(if tile.selected { @@ -81,12 +119,14 @@ pub fn bitmap(tile: &FrameTile) -> Result { lines(&[(62., 46.), (106., 90.)]); lines(&[(106., 46.), (62., 90.)]); } - More | Group(_) | Command(_) | Setting(_) | Display(_) | Pause | Reverse => { + TypeHere | Keyboard | More | Group(_) | Command(_) | Setting(_) | Display(_) | Pause + | Reverse => { artwork(&mut path, tile.icon); } DragHere => { lines(&[(55., 69.), (75., 89.), (113., 47.)]); } + KeyboardKey => unreachable!("Keyboard keys are rendered without artwork"), } if matches!(tile.icon, LeftClick | RightClick | DoubleClick) { path.move_to(62., 64.); @@ -232,6 +272,19 @@ fn return_arrow(path: &mut PathBuilder, forward: bool) { fn artwork(path: &mut PathBuilder, item: Item) { use crate::scan_menu::{Command as C, Kind as K, Setting as S}; match item { + Item::TypeHere | Item::Keyboard => { + rect(path, 46., 44., 76., 48.); + for y in [55., 67.] { + for x in [58., 74., 90., 106.] { + rect(path, x, y, 4., 4.); + } + } + line(path, &[(66., 81.), (102., 81.)]); + if item == Item::TypeHere { + line(path, &[(84., 100.), (84., 114.)]); + line(path, &[(77., 107.), (91., 107.)]); + } + } Item::More => { for x in [60., 84., 108.] { path.push_circle(x, 68., 5.); diff --git a/src-tauri/src/scanning.rs b/src-tauri/src/scanning.rs index 948d262b..975cc416 100644 --- a/src-tauri/src/scanning.rs +++ b/src-tauri/src/scanning.rs @@ -278,6 +278,7 @@ pub trait Technique { false } fn execution_failed(&mut self, _message: String) {} + fn execution_succeeded(&mut self) {} fn start(&mut self); fn advance(&mut self, elapsed_ms: u64); fn handle(&mut self, action: Action) -> Option; diff --git a/src-tauri/src/scanning_runtime.rs b/src-tauri/src/scanning_runtime.rs index 65535ba1..c17191d1 100644 --- a/src-tauri/src/scanning_runtime.rs +++ b/src-tauri/src/scanning_runtime.rs @@ -28,7 +28,11 @@ pub trait Adapter: Send + Sync + 'static { fn activate( app: &AppHandle, selection: ::Selection, - ) -> Result<(), String>; + ) -> Result, String>; + fn settle_environment( + app: &AppHandle, + environment: Option<&mut Self::Environment>, + ) -> Result; } use crate::switch_input::Event; @@ -389,6 +393,9 @@ fn switch(app: &AppHandle, action: Action, input_generation: u64, re d.display = Some(display); A::prepare(app)?; } + if !A::settle_environment(app, d.display.as_mut())? { + return Ok(()); + } A::validate_environment(app, d.display.as_ref())?; let point = d.engine.as_mut().and_then(|e| e.action(action)); d.last_tick = Instant::now(); @@ -418,12 +425,23 @@ fn dispatch( return Err("Scan action was cancelled.".into()); } hide_scan_visuals(); - if let Err(error) = A::activate(app, request) { - A::cleanup(app)?; - let mut d = c.data.lock().unwrap_or_else(|p| p.into_inner()); - d.message = error.clone(); - if let Some(engine) = d.engine.as_mut() { - engine.execution_failed(error); + match A::activate(app, request) { + Ok(environment) => { + let mut d = c.data.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(environment) = environment { + d.display = Some(environment); + } + if let Some(engine) = d.engine.as_mut() { + engine.technique.execution_succeeded(); + } + } + Err(error) => { + A::cleanup(app)?; + let mut d = c.data.lock().unwrap_or_else(|p| p.into_inner()); + d.message = error.clone(); + if let Some(engine) = d.engine.as_mut() { + engine.execution_failed(error); + } } } { @@ -703,6 +721,10 @@ fn tick(app: &AppHandle) { } let result = (|| -> Result<(), String> { let mut d = c.data.lock().unwrap_or_else(|p| p.into_inner()); + if !A::settle_environment(app, d.display.as_mut())? { + d.last_tick = Instant::now(); + return Ok(()); + } A::validate_environment(app, d.display.as_ref())?; let now = Instant::now(); let elapsed = if d.cursor_suppression.is_some_and(|(token, _)| { diff --git a/src/PointScan.test.tsx b/src/PointScan.test.tsx index 31db9677..c82aa237 100644 --- a/src/PointScan.test.tsx +++ b/src/PointScan.test.tsx @@ -195,6 +195,9 @@ it.each([ ["dragDestination", "Choose drag destination."], ["dragConfirmation", "Confirm drag."], ["executing", "Performing drag."], + ["keyboard", "Choose a keyboard row, then a key."], + ["keyboardSuspended", "Select to resume the keyboard."], + ["keyboardOpening", "Opening the keyboard."], ])("explains the %s workflow phase", async (phase, message) => { mocks.invoke.mockResolvedValue({ ...initial, enabled: true, phase }); render(); diff --git a/src/scanning/useScanning.ts b/src/scanning/useScanning.ts index 77dc1046..0fe324ab 100644 --- a/src/scanning/useScanning.ts +++ b/src/scanning/useScanning.ts @@ -20,7 +20,7 @@ export type PointScanState = { remote?: boolean; config: PointScanConfig; enabled: boolean; - phase: "autoSelecting" | "idle" | "row" | "rowEscape" | "cell" | "x" | "y" | "menu" | "menuSuspended" | "dragDestination" | "dragConfirmation" | "executing"; + phase: "autoSelecting" | "idle" | "row" | "rowEscape" | "cell" | "x" | "y" | "menu" | "menuSuspended" | "dragDestination" | "dragConfirmation" | "executing" | "keyboard" | "keyboardSuspended" | "keyboardOpening"; paused: boolean; message: string; supported: boolean; diff --git a/src/settings/ScanningSection.tsx b/src/settings/ScanningSection.tsx index 86dfbdf4..4d89122e 100644 --- a/src/settings/ScanningSection.tsx +++ b/src/settings/ScanningSection.tsx @@ -7,6 +7,9 @@ import { import { SettingGroup, Toggle, OptionGroup, secondsOptions } from "./controls"; const phases = { + keyboard: "Choose a keyboard row, then a key", + keyboardSuspended: "Select to resume the keyboard", + keyboardOpening: "Opening the keyboard", idle: "Ready to begin", autoSelecting: "Waiting to click. Press a switch for the action menu", menu: "Choose an action at the selected point", From 4cc5ba7b7a9ee3d2598d6b1477889e435a1ed5a2 Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Fri, 18 Sep 2026 15:34:21 +0100 Subject: [PATCH 2/9] Verify keyboard focus handoff and fix native key rendering --- src-tauri/src/overlay_macos.rs | 18 +++++++- src-tauri/src/point_scan_runtime.rs | 59 +++++++++++++++++++++---- src-tauri/src/scan_executor.rs | 39 ++++++++++++++++- src-tauri/src/scan_host.rs | 67 ++++++++++++++++++++++++++++- 4 files changed, 170 insertions(+), 13 deletions(-) diff --git a/src-tauri/src/overlay_macos.rs b/src-tauri/src/overlay_macos.rs index fd9a5507..ee166324 100644 --- a/src-tauri/src/overlay_macos.rs +++ b/src-tauri/src/overlay_macos.rs @@ -228,6 +228,20 @@ pub(crate) fn image_from_rgba( height: usize, logical_size: f64, ) -> Result, String> { + image_from_rgba_rect(rgba, width, height, logical_size, logical_size) +} + +pub(crate) fn image_from_rgba_rect( + rgba: &[u8], + width: usize, + height: usize, + logical_width: f64, + logical_height: f64, +) -> Result, String> { + if !logical_height.is_finite() || logical_height <= 0.0 { + return Err("Invalid overlay height.".into()); + } + let logical_size = logical_width; let layout = MacBitmapLayout::new(rgba.len(), width, height, logical_size)?; let representation = unsafe { NSBitmapImageRep::initWithBitmapDataPlanes_pixelsWide_pixelsHigh_bitsPerSample_samplesPerPixel_hasAlpha_isPlanar_colorSpaceName_bitmapFormat_bytesPerRow_bitsPerPixel( @@ -251,13 +265,13 @@ pub(crate) fn image_from_rgba( } representation.setSize(NSSize { width: layout.logical_size, - height: layout.logical_size, + height: logical_height, }); let image = NSImage::initWithSize( NSImage::alloc(), NSSize { width: layout.logical_size, - height: layout.logical_size, + height: logical_height, }, ); image.addRepresentation(&representation as &NSImageRep); diff --git a/src-tauri/src/point_scan_runtime.rs b/src-tauri/src/point_scan_runtime.rs index 9518281a..a3e32724 100644 --- a/src-tauri/src/point_scan_runtime.rs +++ b/src-tauri/src/point_scan_runtime.rs @@ -12,7 +12,25 @@ pub struct Environment { display: Display, foreground: usize, keyboard_area: Rect, - handoff_until: Option, + handoff: Option, +} +#[derive(Clone)] +struct FocusHandoff { + target: usize, + ready_at: std::time::Instant, +} +impl FocusHandoff { + fn verify(&self, now: std::time::Instant, foreground: usize) -> Result { + if now < self.ready_at { + return Ok(false); + } + if foreground != self.target { + return Err( + "The selected target did not receive focus. Please select it again.".into(), + ); + } + Ok(true) + } } pub struct PointScan; pub type Controller = scanning_runtime::Controller; @@ -64,14 +82,17 @@ impl Adapter for PointScan { fn activate(app: &AppHandle, request: Request) -> Result, String> { crate::point_scan_ready(app)?; match request { - Request::OpenKeyboard { point: Some(_) } => { + Request::OpenKeyboard { point: Some(point) } => { + let target = crate::scan_host::target_at(point)?; crate::scan_executor::activate(request)?; let (_, mut environment) = new_engine(app, app.state::().view().config)?; // Allow the one requested click to establish focus before accepting // any key. No typing or scan advancement occurs during this handoff. - environment.handoff_until = - Some(std::time::Instant::now() + std::time::Duration::from_millis(150)); + environment.handoff = Some(FocusHandoff { + target, + ready_at: std::time::Instant::now() + std::time::Duration::from_millis(150), + }); return Ok(Some(environment)); } Request::Setting(setting) => scanning_runtime::update_point_setting(app, setting), @@ -85,13 +106,13 @@ impl Adapter for PointScan { environment: Option<&mut Environment>, ) -> Result { if let Some(environment) = environment { - if let Some(until) = environment.handoff_until { + if let Some(handoff) = &environment.handoff { crate::point_scan_ready(app)?; - if std::time::Instant::now() < until { + if !handoff.verify(std::time::Instant::now(), crate::scan_host::foreground()?)? { return Ok(false); } - environment.foreground = crate::scan_host::foreground()?; - environment.handoff_until = None; + environment.foreground = handoff.target; + environment.handoff = None; } } Ok(true) @@ -133,7 +154,7 @@ fn new_engine(app: &AppHandle, config: Config) -> Result<(Workflow, Environment) display, foreground: crate::scan_host::foreground()?, keyboard_area, - handoff_until: None, + handoff: None, }, )) } @@ -160,3 +181,23 @@ fn validate_display(app: &AppHandle, display: Option<&Environment>) -> Result<() } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn focus_handoff_waits_and_only_accepts_the_clicked_target() { + let start = std::time::Instant::now(); + let handoff = FocusHandoff { + target: 42, + ready_at: start + std::time::Duration::from_millis(150), + }; + assert_eq!(handoff.verify(start, 7), Ok(false)); + assert_eq!(handoff.verify(start, 42), Ok(false)); + assert_eq!(handoff.verify(handoff.ready_at, 42), Ok(true)); + // A failed activation and an unrelated app stealing focus both fail closed. + assert!(handoff.verify(handoff.ready_at, 7).is_err()); + assert!(handoff.verify(handoff.ready_at, 99).is_err()); + } +} diff --git a/src-tauri/src/scan_executor.rs b/src-tauri/src/scan_executor.rs index b641c9bd..988699ff 100644 --- a/src-tauri/src/scan_executor.rs +++ b/src-tauri/src/scan_executor.rs @@ -32,7 +32,16 @@ pub fn execute( if let Some(character) = stroke.character().filter(|_| !stroke.shortcut()) { return input.type_text(&character.to_string()); } + let mut modifiers = stroke.modifiers; let name = match stroke.key { + crate::scan_keyboard::Key::Character('+', '+') => { + modifiers[0] = true; + "=".into() + } + crate::scan_keyboard::Key::Character('*', '*') => { + modifiers[0] = true; + "8".into() + } crate::scan_keyboard::Key::Character(base, _) => { base.to_ascii_uppercase().to_string() } @@ -41,7 +50,7 @@ pub fn execute( }; let mut keys: Vec<&str> = ["Shift", "Ctrl", "Alt", "Meta"] .into_iter() - .zip(stroke.modifiers) + .zip(modifiers) .filter_map(|(key, on)| on.then_some(key)) .collect(); keys.push(&name); @@ -382,6 +391,34 @@ mod tests { ); } #[test] + fn numeric_operator_shortcuts_include_required_shift_and_release_every_key() { + use crate::scan_keyboard::{Key, Stroke}; + for (operator, native_key) in [('+', "="), ('*', "8")] { + let mut input = DesktopInput::new(Fake::default()); + execute( + &mut input, + Request::Keyboard(Stroke { + key: Key::Character(operator, operator), + modifiers: [false, true, false, false], + caps: false, + }), + true, + ) + .unwrap(); + assert_eq!( + input.injector.events, + [ + "key Shift true".to_string(), + "key Ctrl true".to_string(), + format!("key {native_key} true"), + format!("key {native_key} false"), + "key Ctrl false".to_string(), + "key Shift false".to_string(), + ] + ); + } + } + #[test] fn keyboard_releases_failed_keys_and_rejects_external_modifiers() { use crate::scan_keyboard::{Key, Stroke}; let request = Request::Keyboard(Stroke { diff --git a/src-tauri/src/scan_host.rs b/src-tauri/src/scan_host.rs index 6d4aca02..e3e0d655 100644 --- a/src-tauri/src/scan_host.rs +++ b/src-tauri/src/scan_host.rs @@ -373,7 +373,7 @@ mod platform { }])?; let ratio = self.panels[0].backingScaleFactor(); let pixels = countdown.bitmap(ratio)?; - let image = crate::overlay::platform::image_from_rgba( + let image = crate::overlay::platform::image_from_rgba_rect( pixels.data(), pixels.width() as usize, pixels.height() as usize, @@ -400,6 +400,7 @@ mod platform { pixels.width() as usize, pixels.height() as usize, tile.rect.width, + tile.rect.height, )?; self.render(&[crate::scanning::PaintedRect { rect: tile.rect, @@ -493,6 +494,70 @@ pub fn modifiers_released() -> bool { } } +/// Resolve the intended target while scan overlays are hidden, before clicking. +/// Uses the same identity as `foreground`: a Windows root window or macOS PID. +pub fn target_at(point: (i32, i32)) -> Result { + #[cfg(target_os = "windows")] + unsafe { + use windows::Win32::{ + Foundation::POINT, + UI::WindowsAndMessaging::{GetAncestor, WindowFromPoint, GA_ROOT}, + }; + let child = WindowFromPoint(POINT { + x: point.0, + y: point.1, + }); + let root = GetAncestor(child, GA_ROOT); + if root.0.is_null() { + return Err("The selected target is unavailable.".into()); + } + Ok(root.0 as usize) + } + #[cfg(target_os = "macos")] + { + use core_foundation::base::{CFType, CFTypeRef, TCFType}; + #[link(name = "ApplicationServices", kind = "framework")] + unsafe extern "C" { + fn AXUIElementCreateSystemWide() -> CFTypeRef; + fn AXUIElementCopyElementAtPosition( + element: CFTypeRef, + x: f32, + y: f32, + result: *mut CFTypeRef, + ) -> i32; + fn AXUIElementGetPid(element: CFTypeRef, pid: *mut i32) -> i32; + } + let system = unsafe { AXUIElementCreateSystemWide() }; + if system.is_null() { + return Err("The selected target is unavailable.".into()); + } + let system = unsafe { CFType::wrap_under_create_rule(system) }; + let mut element = std::ptr::null(); + let status = unsafe { + AXUIElementCopyElementAtPosition( + system.as_CFTypeRef(), + point.0 as f32, + point.1 as f32, + &mut element, + ) + }; + if status != 0 || element.is_null() { + return Err("The selected target is unavailable.".into()); + } + let element = unsafe { CFType::wrap_under_create_rule(element) }; + let mut pid = 0; + if unsafe { AXUIElementGetPid(element.as_CFTypeRef(), &mut pid) } != 0 || pid <= 0 { + return Err("The selected target is unavailable.".into()); + } + Ok(pid as usize) + } + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + { + let _ = point; + Err("Local scanning is unavailable.".into()) + } +} + /// Opaque foreground identity; never emitted or persisted. pub fn foreground() -> Result { #[cfg(target_os = "windows")] From 3713b87fbc5e30ccf41631e5eb318f906864e73b Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Fri, 18 Sep 2026 15:35:07 +0100 Subject: [PATCH 3/9] Use rectangular image helper for macOS keyboard tiles --- src-tauri/src/scan_host.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src-tauri/src/scan_host.rs b/src-tauri/src/scan_host.rs index e3e0d655..4ef71d64 100644 --- a/src-tauri/src/scan_host.rs +++ b/src-tauri/src/scan_host.rs @@ -373,7 +373,7 @@ mod platform { }])?; let ratio = self.panels[0].backingScaleFactor(); let pixels = countdown.bitmap(ratio)?; - let image = crate::overlay::platform::image_from_rgba_rect( + let image = crate::overlay::platform::image_from_rgba( pixels.data(), pixels.width() as usize, pixels.height() as usize, @@ -395,7 +395,7 @@ mod platform { use objc2_foundation::NSString; let mtm = MainThreadMarker::new().ok_or("Action tile requires the main thread")?; let pixels = crate::scan_tile::bitmap(tile)?; - let image = crate::overlay::platform::image_from_rgba( + let image = crate::overlay::platform::image_from_rgba_rect( pixels.data(), pixels.width() as usize, pixels.height() as usize, From fdc3963298ba27435757da290fa594b87159410e Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Fri, 18 Sep 2026 20:06:20 +0100 Subject: [PATCH 4/9] Pad and vertically centre Windows keyboard labels Measure wrapped key text before drawing and centre it inside scaled horizontal and vertical margins. Keep action-menu and modifier-overlay placement unchanged. --- src-tauri/src/modifier_overlay_windows.rs | 28 +++++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src-tauri/src/modifier_overlay_windows.rs b/src-tauri/src/modifier_overlay_windows.rs index 92982dfc..fd8a9281 100644 --- a/src-tauri/src/modifier_overlay_windows.rs +++ b/src-tauri/src/modifier_overlay_windows.rs @@ -13,7 +13,7 @@ use windows::Win32::Graphics::Gdi::{ CreateCompatibleDC, CreateDIBSection, CreateFontW, DeleteDC, DeleteObject, DrawTextW, GetDC, GetMonitorInfoW, MonitorFromPoint, ReleaseDC, SelectObject, SetBkMode, SetTextColor, AC_SRC_ALPHA, AC_SRC_OVER, BITMAPINFO, BITMAPINFOHEADER, BI_RGB, BLENDFUNCTION, - CLIP_DEFAULT_PRECIS, DEFAULT_CHARSET, DEFAULT_QUALITY, DIB_RGB_COLORS, DT_CENTER, + CLIP_DEFAULT_PRECIS, DEFAULT_CHARSET, DEFAULT_QUALITY, DIB_RGB_COLORS, DT_CALCRECT, DT_CENTER, DT_SINGLELINE, DT_VCENTER, DT_WORDBREAK, FF_DONTCARE, FW_BOLD, HDC, HGDIOBJ, MONITORINFO, MONITOR_DEFAULTTONEAREST, OUT_DEFAULT_PRECIS, TRANSPARENT, }; @@ -411,6 +411,7 @@ fn present(window: HWND, labels: &[String], layout: &Layout) -> Result<(), Strin layout, pixmap.data(), labels.iter().any(|label| label.contains('\n')), + false, ) } @@ -466,6 +467,7 @@ fn present_pixmap_with_text( layout: &Layout, rgba: &[u8], wrap: bool, + center_wrapped: bool, ) -> Result<(), String> { unsafe { let screen = ScreenDc(GetDC(None)); @@ -536,6 +538,16 @@ fn present_pixmap_with_text( for (label, chip) in labels.iter().zip(&layout.chips) { let mut text = label.encode_utf16().collect::>(); let mut text_rect = *chip; + if center_wrapped { + let mut measured = text_rect; + let height = DrawTextW( + memory.0, + &mut text, + &mut measured, + DT_CENTER | DT_WORDBREAK | DT_CALCRECT, + ); + text_rect.top += ((text_rect.bottom - text_rect.top - height) / 2).max(0); + } let _ = DrawTextW( memory.0, &mut text, @@ -598,6 +610,10 @@ pub(crate) fn present_scan_tile( if tile.icon == crate::scan_menu::Item::KeyboardKey { let width = pixmap.width() as i32; let height = pixmap.height() as i32; + let horizontal_padding = (8.0 * tile.scale).round().max(1.0) as i32; + let vertical_padding = (6.0 * tile.scale).round().max(1.0) as i32; + let horizontal_padding = horizontal_padding.min((width - 1).max(0) / 2); + let vertical_padding = vertical_padding.min((height - 1).max(0) / 2); let layout = Layout { x: tile.rect.x.round() as i32, y: tile.rect.y.round() as i32, @@ -605,10 +621,10 @@ pub(crate) fn present_scan_tile( height, scale: tile.scale * 18.0 / FONT_SIZE, chips: vec![RECT { - left: 2, - top: 0, - right: width - 2, - bottom: height, + left: horizontal_padding, + top: vertical_padding, + right: width - horizontal_padding, + bottom: height - vertical_padding, }], }; return present_pixmap_with_text( @@ -617,6 +633,7 @@ pub(crate) fn present_scan_tile( &layout, pixmap.data(), true, + true, ); } let size = pixmap.width() as i32; @@ -639,6 +656,7 @@ pub(crate) fn present_scan_tile( &layout, pixmap.data(), true, + false, ) } From 17d2a6e26e708a514ddf99bd2e4971e94eb9109e Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Fri, 18 Sep 2026 20:07:40 +0100 Subject: [PATCH 5/9] Preserve room for wrapped keyboard labels on smaller displays --- src-tauri/src/modifier_overlay_windows.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src-tauri/src/modifier_overlay_windows.rs b/src-tauri/src/modifier_overlay_windows.rs index fd8a9281..1a8548e7 100644 --- a/src-tauri/src/modifier_overlay_windows.rs +++ b/src-tauri/src/modifier_overlay_windows.rs @@ -546,6 +546,9 @@ fn present_pixmap_with_text( &mut measured, DT_CENTER | DT_WORDBREAK | DT_CALCRECT, ); + let vertical_padding = text_rect.top.min((layout.height - height).max(0) / 2); + text_rect.top = vertical_padding; + text_rect.bottom = layout.height - vertical_padding; text_rect.top += ((text_rect.bottom - text_rect.top - height) / 2).max(0); } let _ = DrawTextW( From 7997104cfff4015c4e5e72192f38ef2321160527 Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Fri, 18 Sep 2026 20:50:37 +0100 Subject: [PATCH 6/9] Refine switch keyboard layout and scan feedback Use weighted editing keys, a dedicated wide Space bar and one copy of each modifier. Add keyboard-specific visual roles, active-state indicators, rounded key artwork and a solid panel. Clarify row return and scan target feedback while preserving input and switch timing. --- docs/qwerty-keyboard.md | 2 +- src-tauri/src/modifier_overlay_windows.rs | 2 +- src-tauri/src/point_scan.rs | 1 + src-tauri/src/scan_host.rs | 23 ++- src-tauri/src/scan_keyboard.rs | 225 +++++++++++++++++----- src-tauri/src/scan_menu.rs | 1 + src-tauri/src/scan_tile.rs | 95 +++++++-- src-tauri/src/scanning.rs | 23 +++ 8 files changed, 296 insertions(+), 76 deletions(-) diff --git a/docs/qwerty-keyboard.md b/docs/qwerty-keyboard.md index 4b9342a3..c58b8f21 100644 --- a/docs/qwerty-keyboard.md +++ b/docs/qwerty-keyboard.md @@ -12,7 +12,7 @@ Pages: - **Functions:** Esc, F1–F12, arrows, Home, End, Page Up, Page Down and Delete. Windows also has Insert, Print Screen, Scroll Lock and Pause. - **Numbers:** digits, decimal point, arithmetic operators, Enter and Backspace. Digits are independent of hardware Num Lock. -The bottom row switches pages, moves the keyboard between the top and bottom of the display work area, and closes it. Modifiers are available on every page. Shift, Ctrl, Alt/Option and Windows/Command cycle through **off → once → locked → off**. The label shows once or locked. Caps is a keyboard-local uppercase setting; Shift reverses its letter case. Page and position controls do not consume a one-shot modifier. +The bottom control row starts with Close keyboard, followed by Letters, Navigation, Numbers and docking. Space has a dedicated wide key. Editing and modifier keys are wider than ordinary characters; duplicate Shift and Caps controls are removed from the letters page. A header names the current page and scan target. The return-to-rows slot highlights only the header. Active modifiers and the current page have an indicator separate from the scan highlight. Modifiers are available on every page. Shift, Ctrl, Alt/Option and Windows/Command cycle through **off → once → locked → off**. The label shows once or locked. Caps is a keyboard-local uppercase setting; Shift reverses its letter case. Page and position controls do not consume a one-shot modifier. Modifier keys are pressed only around each emitted shortcut and immediately released. Selecting a locked modifier does not hold that operating-system key while the scanner runs. Ordinary characters use text injection; command combinations and navigation use native key events. Native shortcuts retain the operating system's layout semantics. diff --git a/src-tauri/src/modifier_overlay_windows.rs b/src-tauri/src/modifier_overlay_windows.rs index 1a8548e7..6f8bc1bb 100644 --- a/src-tauri/src/modifier_overlay_windows.rs +++ b/src-tauri/src/modifier_overlay_windows.rs @@ -622,7 +622,7 @@ pub(crate) fn present_scan_tile( y: tile.rect.y.round() as i32, width, height, - scale: tile.scale * 18.0 / FONT_SIZE, + scale: tile.scale * crate::scan_tile::keyboard_font_size(tile) / FONT_SIZE, chips: vec![RECT { left: horizontal_padding, top: vertical_padding, diff --git a/src-tauri/src/point_scan.rs b/src-tauri/src/point_scan.rs index 88b4b1cd..77ba8320 100644 --- a/src-tauri/src/point_scan.rs +++ b/src-tauri/src/point_scan.rs @@ -417,6 +417,7 @@ impl Technique for Engine { }); } Frame { + backdrop: None, color: self.config.scanner_color, fills, grid, diff --git a/src-tauri/src/scan_host.rs b/src-tauri/src/scan_host.rs index 4ef71d64..3090ebfa 100644 --- a/src-tauri/src/scan_host.rs +++ b/src-tauri/src/scan_host.rs @@ -419,17 +419,28 @@ mod platform { let label = NSTextField::wrappingLabelWithString(&NSString::from_str(&tile.text), mtm); let key = tile.icon == crate::scan_menu::Item::KeyboardKey; label.setFont(Some(&NSFont::boldSystemFontOfSize( - if key { 18.0 } else { 15.0 } * tile.scale, + if key { + crate::scan_tile::keyboard_font_size(tile) + } else { + 15.0 + } * tile.scale, ))); label.setTextColor(Some(&NSColor::whiteColor())); label.setAlignment(NSTextAlignment::Center); label.setFrame(if key { + let padding = 8.0 * tile.scale; + let width = (tile.rect.width - padding * 2.0).max(1.0); + let measured = label + .cell() + .ok_or("Keyboard label is unavailable.")? + .cellSizeForBounds(NSRect::new( + NSPoint::new(0.0, 0.0), + NSSize::new(width, f64::MAX), + )); + let height = measured.height.min(tile.rect.height); NSRect::new( - NSPoint::new(2.0, (tile.rect.height - 42.0 * tile.scale).max(0.0) / 2.0), - NSSize::new( - tile.rect.width - 4.0, - (42.0 * tile.scale).min(tile.rect.height), - ), + NSPoint::new(padding, (tile.rect.height - height) / 2.0), + NSSize::new(width, height), ) } else { NSRect::new( diff --git a/src-tauri/src/scan_keyboard.rs b/src-tauri/src/scan_keyboard.rs index cd0cc1c7..c87cf4d3 100644 --- a/src-tauri/src/scan_keyboard.rs +++ b/src-tauri/src/scan_keyboard.rs @@ -2,7 +2,10 @@ use crate::{ scan_menu::Item, scan_tree::{Navigator, Node, Selection}, - scanning::{Action, Frame, FrameTile, Interval, Rect, ScannerColor, MAX_SCAN_CYCLES}, + scanning::{ + Action, Frame, FrameTile, Interval, KeyboardRole, KeyboardTileStyle, Rect, ScannerColor, + MAX_SCAN_CYCLES, + }, }; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -29,8 +32,8 @@ impl Modifier { fn suffix(self) -> &'static str { match self { Self::Off => "", - Self::Once => "\nonce", - Self::Locked => "\nlocked", + Self::Once => "\nNext key", + Self::Locked => "\nLocked", } } } @@ -93,7 +96,6 @@ pub fn rows(page: Page, mac: bool) -> Vec> { middle.push(N("Enter")); let mut bottom = vec![Key::Modifier(0)]; bottom.extend(chars("\\zxcvbnm,./", "|ZXCVBNM<>?")); - bottom.push(C(' ', ' ')); vec![number, top, middle, bottom] } Page::Functions => { @@ -141,19 +143,29 @@ pub fn rows(page: Page, mac: bool) -> Vec> { ], ], }; + result.push(if page == Page::Letters { + vec![ + Key::Modifier(1), + Key::Modifier(2), + C(' ', ' '), + Key::Modifier(3), + ] + } else { + vec![ + Key::Modifier(0), + Key::Modifier(1), + Key::Modifier(2), + C(' ', ' '), + Key::Modifier(3), + Key::Caps, + ] + }); result.push(vec![ - Key::Modifier(0), - Key::Modifier(1), - Key::Modifier(2), - Key::Modifier(3), - Key::Caps, - ]); - result.push(vec![ + Key::Close, Key::Page(Page::Letters), Key::Page(Page::Functions), Key::Page(Page::Numbers), Key::Dock, - Key::Close, ]); result } @@ -339,82 +351,138 @@ impl Keyboard { }, self.modifiers[i].suffix() ), - Key::Caps => if self.caps { "Caps\non" } else { "Caps\noff" }.into(), + Key::Caps => if self.caps { + "Caps lock\nOn" + } else { + "Caps lock" + } + .into(), Key::Page(page) => format!( "{}{}", match page { Page::Letters => "Letters", - Page::Functions => "Functions", + Page::Functions => "Navigation", Page::Numbers => "Numbers", }, if page == self.page { " •" } else { "" } ), - Key::Dock => if self.top { - "Move to bottom" - } else { - "Move to top" - } - .into(), - Key::Close => "Close".into(), + Key::Dock => if self.top { "Dock bottom" } else { "Dock top" }.into(), + Key::Close => "Close keyboard".into(), + } + } + fn weight(key: Key) -> f64 { + match key { + Key::Character(' ', _) => 4.0, + Key::Named("Backspace" | "Enter") => 1.9, + Key::Named("Tab") | Key::Caps | Key::Modifier(_) => 1.6, + Key::Close | Key::Dock => 1.5, + _ => 1.0, + } + } + fn style(&self, key: Key, row_scan: bool) -> KeyboardTileStyle { + KeyboardTileStyle { + role: match key { + Key::Character(..) => KeyboardRole::Character, + Key::Page(_) | Key::Dock | Key::Close => KeyboardRole::Toolbar, + _ => KeyboardRole::Utility, + }, + active: match key { + Key::Modifier(i) => self.modifiers[i] != Modifier::Off, + Key::Caps => self.caps, + Key::Page(page) => self.page == page, + _ => false, + }, + row_scan, } } pub fn frame(&self, screen: Rect, units: f64, color: ScannerColor) -> Frame { - let width = (1200.0 * units).min(screen.width - 16.0 * units).max(1.0); - let height = (420.0 * units).min(screen.height / 2.0).max(1.0); + let width = (1180.0 * units).min((screen.width - 24.0 * units).max(1.0)); + let height = (460.0 * units).min(screen.height * 0.55).max(1.0); let x = screen.x + (screen.width - width) / 2.0; let y = if self.top { screen.y } else { screen.y + screen.height - height }; - let row_height = height / (self.rows.len() as f64 + 0.5); - let mut frame = Frame::default(); + let row_height = height / (self.rows.len() as f64 + 0.9); + let scale = units.min(row_height / 60.0).min(width / 1040.0); + let gap = (6.0 * scale).min(row_height / 8.0); + let header_height = row_height * 0.9; + let mut frame = Frame { + backdrop: Some(Rect { + x, + y, + width, + height, + }), + ..Frame::default() + }; + let row_scan = self.nav.path().is_empty(); let active_row = self.nav.path().first().copied().unwrap_or(self.nav.index()); for (r, row) in self.rows.iter().enumerate() { - let key_width = width / row.len() as f64; + let total_weight: f64 = row.iter().copied().map(Self::weight).sum(); + let cell_unit = (width - gap * (row.len() - 1) as f64).max(1.0) / total_weight; + let mut left = x; for (c, key) in row.iter().enumerate() { + let key_width = cell_unit * Self::weight(*key); frame.tiles.push(FrameTile { color, text: self.label(*key), icon: Item::KeyboardKey, + keyboard: Some(self.style(*key, row_scan)), rect: Rect { - x: x + c as f64 * key_width + 2.0 * units, - y: y + r as f64 * row_height + 2.0 * units, - width: (key_width - 4.0 * units).max(1.0), - height: (row_height - 4.0 * units).max(1.0), + x: left, + y: y + header_height + r as f64 * row_height, + width: key_width, + height: (row_height - gap).max(1.0), }, - scale: units.min(row_height / 44.0).min(key_width / 65.0), + scale, selected: !self.suspended + && !self.nav.escaping() && r == active_row - && (self.nav.path().is_empty() - || self.nav.escaping() - || c == self.nav.index()), + && (row_scan || c == self.nav.index()), }); + left += key_width + gap; } } + let page = match self.page { + Page::Letters => "Letters", + Page::Functions => "Navigation", + Page::Numbers => "Numbers", + }; let text = if self.error { - "Input failed. Select to resume." + "Input failed · Select to try again".to_owned() } else if self.suspended { - "Select to resume keyboard." + "Keyboard paused · Select to resume".to_owned() } else if self.nav.escaping() { - "Back to rows" - } else if self.nav.path().is_empty() { - "Choose a row" + "Back to rows · Select to return".to_owned() + } else if row_scan { + format!("{} · Select a row", page) } else { - "Choose a key" + format!( + "{} · Select {}", + page, + self.label(self.rows[active_row][self.nav.index()]) + .replace('\n', " ") + ) }; frame.tiles.push(FrameTile { color, - text: text.into(), + text, icon: Item::KeyboardKey, + keyboard: Some(KeyboardTileStyle { + role: KeyboardRole::Status, + active: false, + row_scan: false, + }), rect: Rect { x, - y: y + height - row_height * 0.5, + y, width, - height: row_height * 0.5, + height: (header_height - gap).max(1.0), }, - scale: units.min(row_height / 44.0), - selected: self.nav.escaping(), + scale, + selected: !self.suspended && self.nav.escaping(), }); frame } @@ -424,6 +492,67 @@ impl Keyboard { mod tests { use super::*; #[test] + fn everyday_keys_are_unique_and_space_is_wide() { + let k = Keyboard::new(false); + let keys = k.rows.concat(); + for key in [ + Key::Caps, + Key::Modifier(0), + Key::Character(' ', ' '), + Key::Close, + ] { + assert_eq!( + keys.iter().filter(|candidate| **candidate == key).count(), + 1 + ); + } + let frame = k.frame( + Rect { + x: 0.0, + y: 0.0, + width: 1280.0, + height: 680.0, + }, + 1.0, + ScannerColor::default(), + ); + let space = frame.tiles.iter().find(|t| t.text == "Space").unwrap(); + let letter = frame.tiles.iter().find(|t| t.text == "a").unwrap(); + assert!(space.rect.width > letter.rect.width * 3.0); + let backspace = frame.tiles.iter().find(|t| t.text == "Backspace").unwrap(); + assert!(backspace.rect.width > letter.rect.width); + assert_eq!(k.rows.last().unwrap()[0], Key::Close); + } + #[test] + fn row_escape_only_highlights_return_and_modifier_state_is_separate() { + let screen = Rect { + x: 0.0, + y: 0.0, + width: 1280.0, + height: 680.0, + }; + let mut k = Keyboard::new(false); + k.choose(Key::Modifier(0)); + let active = k.frame(screen, 1.0, ScannerColor::default()); + let shift = active + .tiles + .iter() + .find(|t| t.text.starts_with("Shift")) + .unwrap(); + assert!(shift.keyboard.unwrap().active); + assert!(!shift.selected); + k.handle(Action::Select); + let keys = k.frame(screen, 1.0, ScannerColor::default()); + assert_eq!(keys.tiles.iter().filter(|t| t.selected).count(), 1); + k.handle(Action::Back); + let escaping = k.frame(screen, 1.0, ScannerColor::default()); + let selected: Vec<_> = escaping.tiles.iter().filter(|t| t.selected).collect(); + assert_eq!(selected.len(), 1); + assert!(selected[0].text.starts_with("Back to rows")); + k.handle(Action::Select); + assert!(k.nav.path().is_empty()); + } + #[test] fn uk_layout_and_platform_keys() { let letters = rows(Page::Letters, false).concat(); for c in 'a'..='z' { @@ -504,7 +633,7 @@ mod tests { } #[test] fn geometry_fits_work_area_at_both_docks_and_scales() { - for units in [1.0, 1.5, 2.0] { + for units in [0.75, 1.0, 1.5, 2.0, 3.0] { for top in [false, true] { for page in [Page::Letters, Page::Functions, Page::Numbers] { let mut k = Keyboard::new(false); @@ -513,8 +642,8 @@ mod tests { let screen = Rect { x: -1600.0, y: -200.0, - width: 1600.0, - height: 900.0, + width: 800.0, + height: 600.0, }; for tile in k.frame(screen, units, ScannerColor::default()).tiles { assert!(tile.rect.x >= screen.x && tile.rect.y >= screen.y); diff --git a/src-tauri/src/scan_menu.rs b/src-tauri/src/scan_menu.rs index 0e01a4ac..1c7f1507 100644 --- a/src-tauri/src/scan_menu.rs +++ b/src-tauri/src/scan_menu.rs @@ -193,6 +193,7 @@ impl Menu { let selected = r == active_row && (self.nav.path().is_empty() || self.nav.escaping() || c == self.nav.index()); frame.tiles.push(FrameTile { + keyboard: None, color: Default::default(), text: item.label().into(), rect, diff --git a/src-tauri/src/scan_tile.rs b/src-tauri/src/scan_tile.rs index db35cc58..307afa47 100644 --- a/src-tauri/src/scan_tile.rs +++ b/src-tauri/src/scan_tile.rs @@ -11,34 +11,63 @@ pub fn bitmap(tile: &FrameTile) -> Result { tile.rect.height.round().max(1.0) as u32, ) .ok_or("Cannot allocate keyboard key")?; + use crate::scanning::KeyboardRole; + let style = tile.keyboard; + let base = match style.map(|s| s.role) { + Some(KeyboardRole::Character) => [43, 51, 66], + Some(KeyboardRole::Utility) => [34, 41, 54], + _ => [25, 30, 40], + }; + bitmap.fill(Color::from_rgba8(20, 24, 32, 255)); let rgb = if tile.selected { tile.color.menu_fill() } else { - [30, 35, 46] + base + }; + let inset = (2.0 * tile.scale) as f32; + let radius = (8.0 * tile.scale) as f32; + let width = bitmap.width() as f32; + let height = bitmap.height() as f32; + let path = key_outline(width, height, inset.min(width.min(height) / 4.0), radius); + let mut paint = Paint::default(); + paint.set_color_rgba8(rgb[0], rgb[1], rgb[2], 255); + bitmap.fill_path( + &path, + &paint, + FillRule::Winding, + Transform::identity(), + None, + ); + let [r, g, b] = if tile.selected { + tile.color.rgb() + } else { + [62, 72, 89] }; - bitmap.fill(Color::from_rgba8(rgb[0], rgb[1], rgb[2], 255)); - if tile.selected { - let mut paint = Paint::default(); + paint.set_color_rgba8(r, g, b, 255); + bitmap.stroke_path( + &path, + &paint, + &Stroke { + width: if tile.selected && !style.is_some_and(|s| s.row_scan) { + 3.0 + } else { + 1.0 + } * tile.scale as f32, + ..Default::default() + }, + Transform::identity(), + None, + ); + if style.is_some_and(|s| s.active) { let [r, g, b] = tile.color.rgb(); paint.set_color_rgba8(r, g, b, 255); - let mut path = PathBuilder::new(); if let Some(rect) = tiny_skia::Rect::from_xywh( - 1.0, - 1.0, - (bitmap.width() as f32 - 2.0).max(0.1), - (bitmap.height() as f32 - 2.0).max(0.1), + width * 0.35, + height - 5.0 * tile.scale as f32, + width * 0.3, + (2.0 * tile.scale) as f32, ) { - path.push_rect(rect); - bitmap.stroke_path( - &path.finish().unwrap(), - &paint, - &Stroke { - width: 2.0, - ..Default::default() - }, - Transform::identity(), - None, - ); + bitmap.fill_rect(rect, &paint, Transform::identity(), None); } } return Ok(bitmap); @@ -186,6 +215,30 @@ pub fn bitmap(tile: &FrameTile) -> Result { Ok(pixmap) } +pub fn keyboard_font_size(tile: &FrameTile) -> f64 { + match tile.keyboard.map(|s| s.role) { + Some(crate::scanning::KeyboardRole::Character) if tile.text.chars().count() == 1 => 24.0, + Some(crate::scanning::KeyboardRole::Status) => 17.0, + _ => 16.0, + } +} +fn key_outline(width: f32, height: f32, inset: f32, radius: f32) -> tiny_skia::Path { + let (left, top, right, bottom) = (inset, inset, width - inset, height - inset); + let r = radius.min((right - left) / 2.0).min((bottom - top) / 2.0); + let mut p = PathBuilder::new(); + p.move_to(left + r, top); + p.line_to(right - r, top); + p.quad_to(right, top, right, top + r); + p.line_to(right, bottom - r); + p.quad_to(right, bottom, right - r, bottom); + p.line_to(left + r, bottom); + p.quad_to(left, bottom, left, bottom - r); + p.line_to(left, top + r); + p.quad_to(left, top, left + r, top); + p.close(); + p.finish().unwrap() +} + fn line(path: &mut PathBuilder, points: &[(f32, f32)]) { path.move_to(points[0].0, points[0].1); for &(x, y) in &points[1..] { @@ -632,6 +685,7 @@ mod tests { use crate::scanning::{Rect, ScannerColor::*}; for color in [Red, Green, Blue, Yellow, White] { let mut tile = FrameTile { + keyboard: None, color, text: "Click".into(), rect: Rect { @@ -732,6 +786,7 @@ mod tests { Setting(S::GridMode), ] { let mut tile = FrameTile { + keyboard: None, color: Default::default(), text: icon.label().into(), rect: crate::scanning::Rect { diff --git a/src-tauri/src/scanning.rs b/src-tauri/src/scanning.rs index 975cc416..b9d0660f 100644 --- a/src-tauri/src/scanning.rs +++ b/src-tauri/src/scanning.rs @@ -149,6 +149,7 @@ pub struct PaintedRect { } #[derive(Debug, Default, Clone, PartialEq)] pub struct Frame { + pub backdrop: Option, pub color: ScannerColor, pub fills: Vec, pub grid: Vec, @@ -160,6 +161,14 @@ pub struct Frame { impl Frame { pub fn rectangles(&self) -> Vec { let mut result = Vec::new(); + if let Some(rect) = self.backdrop { + result.push(PaintedRect { + rect, + color: [20, 24, 32], + opacity: 255, + role: VisualRole::Accent, + }); + } for rect in &self.fills { result.push(PaintedRect { rect: *rect, @@ -244,6 +253,19 @@ pub struct UpdateContext { pub switch_held: bool, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeyboardRole { + Character, + Utility, + Toolbar, + Status, +} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeyboardTileStyle { + pub role: KeyboardRole, + pub active: bool, + pub row_scan: bool, +} #[derive(Debug, Clone, PartialEq)] pub struct FrameTile { pub color: ScannerColor, @@ -251,6 +273,7 @@ pub struct FrameTile { pub rect: Rect, pub scale: f64, pub icon: crate::scan_menu::Item, + pub keyboard: Option, pub selected: bool, } #[derive(Debug, Clone, PartialEq)] From 45d325a6d9ff86c6b797ed8b325b1e8fe366b61a Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Fri, 18 Sep 2026 21:02:00 +0100 Subject: [PATCH 7/9] Round and pad the scanning keyboard panel Render a transparent rounded backdrop with scaled outer padding. Keep the backdrop cached during key scanning and verify transparent corners and inset control bounds. --- src-tauri/src/point_scan.rs | 1 - src-tauri/src/scan_host.rs | 6 ++++ src-tauri/src/scan_keyboard.rs | 56 +++++++++++++++++++++++++++---- src-tauri/src/scan_tile.rs | 18 ++++++++++ src-tauri/src/scanning.rs | 10 +----- src-tauri/src/scanning_runtime.rs | 10 +++++- 6 files changed, 83 insertions(+), 18 deletions(-) diff --git a/src-tauri/src/point_scan.rs b/src-tauri/src/point_scan.rs index 77ba8320..88b4b1cd 100644 --- a/src-tauri/src/point_scan.rs +++ b/src-tauri/src/point_scan.rs @@ -417,7 +417,6 @@ impl Technique for Engine { }); } Frame { - backdrop: None, color: self.config.scanner_color, fills, grid, diff --git a/src-tauri/src/scan_host.rs b/src-tauri/src/scan_host.rs index 3090ebfa..b8f6541d 100644 --- a/src-tauri/src/scan_host.rs +++ b/src-tauri/src/scan_host.rs @@ -408,6 +408,12 @@ mod platform { opacity: 255, role: crate::scanning::VisualRole::Accent, }])?; + if tile + .keyboard + .is_some_and(|s| s.role == crate::scanning::KeyboardRole::Background) + { + self.panels[0].setBackgroundColor(Some(&NSColor::clearColor())); + } let bounds = NSRect::new( NSPoint::new(0.0, 0.0), NSSize::new(tile.rect.width, tile.rect.height), diff --git a/src-tauri/src/scan_keyboard.rs b/src-tauri/src/scan_keyboard.rs index c87cf4d3..4b1d3189 100644 --- a/src-tauri/src/scan_keyboard.rs +++ b/src-tauri/src/scan_keyboard.rs @@ -404,19 +404,36 @@ impl Keyboard { } else { screen.y + screen.height - height }; - let row_height = height / (self.rows.len() as f64 + 0.9); - let scale = units.min(row_height / 60.0).min(width / 1040.0); + let outer_scale = units.min(height / 460.0).min(width / 1180.0); + let padding = 16.0 * outer_scale; + let content_width = (width - padding * 2.0).max(1.0); + let content_height = (height - padding * 2.0).max(1.0); + let row_height = content_height / (self.rows.len() as f64 + 0.9); + let scale = units.min(row_height / 60.0).min(content_width / 1040.0); let gap = (6.0 * scale).min(row_height / 8.0); let header_height = row_height * 0.9; - let mut frame = Frame { - backdrop: Some(Rect { + let mut frame = Frame::default(); + frame.tiles.push(FrameTile { + color, + text: String::new(), + icon: Item::KeyboardKey, + keyboard: Some(KeyboardTileStyle { + role: KeyboardRole::Background, + active: false, + row_scan: false, + }), + rect: Rect { x, y, width, height, - }), - ..Frame::default() - }; + }, + scale: outer_scale, + selected: false, + }); + let x = x + padding; + let y = y + padding; + let width = content_width; let row_scan = self.nav.path().is_empty(); let active_row = self.nav.path().first().copied().unwrap_or(self.nav.index()); for (r, row) in self.rows.iter().enumerate() { @@ -492,6 +509,31 @@ impl Keyboard { mod tests { use super::*; #[test] + fn rounded_panel_has_clear_corners_and_insets_all_controls() { + let frame = Keyboard::new(false).frame( + Rect { + x: -1280.0, + y: 0.0, + width: 1280.0, + height: 720.0, + }, + 1.0, + ScannerColor::default(), + ); + let panel = &frame.tiles[0]; + assert_eq!(panel.keyboard.unwrap().role, KeyboardRole::Background); + for key in &frame.tiles[1..] { + assert!(key.rect.x > panel.rect.x); + assert!(key.rect.y > panel.rect.y); + assert!(key.rect.x + key.rect.width < panel.rect.x + panel.rect.width); + assert!(key.rect.y + key.rect.height < panel.rect.y + panel.rect.height); + } + let pixels = crate::scan_tile::bitmap(panel).unwrap(); + assert_eq!(pixels.data()[3], 0); + let center = ((pixels.height() / 2 * pixels.width() + pixels.width() / 2) * 4 + 3) as usize; + assert_eq!(pixels.data()[center], 255); + } + #[test] fn everyday_keys_are_unique_and_space_is_wide() { let k = Keyboard::new(false); let keys = k.rows.concat(); diff --git a/src-tauri/src/scan_tile.rs b/src-tauri/src/scan_tile.rs index 307afa47..e9a4fb08 100644 --- a/src-tauri/src/scan_tile.rs +++ b/src-tauri/src/scan_tile.rs @@ -13,6 +13,24 @@ pub fn bitmap(tile: &FrameTile) -> Result { .ok_or("Cannot allocate keyboard key")?; use crate::scanning::KeyboardRole; let style = tile.keyboard; + if style.is_some_and(|s| s.role == KeyboardRole::Background) { + let mut paint = Paint::default(); + paint.set_color_rgba8(20, 24, 32, 255); + let path = key_outline( + bitmap.width() as f32, + bitmap.height() as f32, + 0.0, + (18.0 * tile.scale) as f32, + ); + bitmap.fill_path( + &path, + &paint, + FillRule::Winding, + Transform::identity(), + None, + ); + return Ok(bitmap); + } let base = match style.map(|s| s.role) { Some(KeyboardRole::Character) => [43, 51, 66], Some(KeyboardRole::Utility) => [34, 41, 54], diff --git a/src-tauri/src/scanning.rs b/src-tauri/src/scanning.rs index b9d0660f..3bc1251d 100644 --- a/src-tauri/src/scanning.rs +++ b/src-tauri/src/scanning.rs @@ -149,7 +149,6 @@ pub struct PaintedRect { } #[derive(Debug, Default, Clone, PartialEq)] pub struct Frame { - pub backdrop: Option, pub color: ScannerColor, pub fills: Vec, pub grid: Vec, @@ -161,14 +160,6 @@ pub struct Frame { impl Frame { pub fn rectangles(&self) -> Vec { let mut result = Vec::new(); - if let Some(rect) = self.backdrop { - result.push(PaintedRect { - rect, - color: [20, 24, 32], - opacity: 255, - role: VisualRole::Accent, - }); - } for rect in &self.fills { result.push(PaintedRect { rect: *rect, @@ -255,6 +246,7 @@ pub struct UpdateContext { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum KeyboardRole { + Background, Character, Utility, Toolbar, diff --git a/src-tauri/src/scanning_runtime.rs b/src-tauri/src/scanning_runtime.rs index a100a73b..f8cf7471 100644 --- a/src-tauri/src/scanning_runtime.rs +++ b/src-tauri/src/scanning_runtime.rs @@ -519,8 +519,16 @@ fn render_tiles(tiles: &[crate::scanning::FrameTile]) -> Result<(), String> { while slot.0.len() < tiles.len() { slot.0.push(Host::new()?); } - for (index, host) in slot.0.iter_mut().enumerate() { + let (hosts, previous) = &mut *slot; + for (index, host) in hosts.iter_mut().enumerate() { if let Some(tile) = tiles.get(index) { + if tile + .keyboard + .is_some_and(|s| s.role == crate::scanning::KeyboardRole::Background) + && previous.get(index) == Some(tile) + { + continue; + } host.tile(tile)?; } else { host.hide(); From 8da18e265936971dbf458f1d451856ea1e62f328 Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Fri, 18 Sep 2026 21:10:48 +0100 Subject: [PATCH 8/9] Skip empty native labels to prevent keyboard opening crash Avoid passing an empty UTF-16 slice to DrawTextW for the rounded keyboard backdrop. Cover empty labels through the native drawing path and render the full keyboard in the interactive overlay smoke test. --- src-tauri/src/modifier_overlay_windows.rs | 49 +++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src-tauri/src/modifier_overlay_windows.rs b/src-tauri/src/modifier_overlay_windows.rs index 6f8bc1bb..22ac5063 100644 --- a/src-tauri/src/modifier_overlay_windows.rs +++ b/src-tauri/src/modifier_overlay_windows.rs @@ -536,6 +536,9 @@ fn present_pixmap_with_text( SetBkMode(memory.0, TRANSPARENT); SetTextColor(memory.0, COLORREF(0x00ff_ffff)); for (label, chip) in labels.iter().zip(&layout.chips) { + if label.is_empty() { + continue; + } let mut text = label.encode_utf16().collect::>(); let mut text_rect = *chip; if center_wrapped { @@ -691,6 +694,34 @@ pub(crate) fn present_scan_prompt( #[cfg(test)] mod tests { + #[test] + fn empty_keyboard_backdrop_label_never_reaches_native_text_drawing() { + let layout = super::Layout { + x: 0, + y: 0, + width: 100, + height: 50, + scale: 1.0, + chips: vec![windows::Win32::Foundation::RECT { + left: 8, + top: 6, + right: 92, + bottom: 44, + }], + }; + let pixels = vec![255; 100 * 50 * 4]; + for wrapped in [false, true] { + let result = super::present_pixmap_with_text( + windows::Win32::Foundation::HWND::default(), + &[String::new()], + &layout, + &pixels, + wrapped, + wrapped, + ); + assert!(result.is_err()); + } + } use super::*; use windows::Win32::Foundation::{LPARAM, WPARAM}; use windows::Win32::UI::WindowsAndMessaging::{ @@ -845,6 +876,24 @@ mod tests { #[test] fn native_window_render_smoke_becomes_visible_and_hides() { let mut host = NativeHost::new().unwrap(); + let keyboard = crate::scan_keyboard::Keyboard::new(false); + let frame = keyboard.frame( + crate::scanning::Rect { + x: 0.0, + y: 0.0, + width: 1280.0, + height: 720.0, + }, + 1.0, + crate::scanning::ScannerColor::default(), + ); + for tile in &frame.tiles { + super::present_scan_tile(host.window, tile).unwrap(); + unsafe { + assert!(IsWindowVisible(host.window).as_bool()); + } + } + host.hide(); host.render(&["Ctrl".into(), "Shift".into()]).unwrap(); unsafe { assert!(IsWindowVisible(host.window).as_bool()); From 02bd8e8f8ea550f206a94577b36e0b523428f3fa Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Sat, 19 Sep 2026 08:54:57 +0100 Subject: [PATCH 9/9] Remove number row from the letters keyboard Move Backspace beside QWERTY and retain the removed digits and UK symbols on the Numbers page. Cover character availability and the letters layout in regression tests. --- docs/qwerty-keyboard.md | 4 ++-- src-tauri/src/scan_keyboard.rs | 20 ++++++++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/qwerty-keyboard.md b/docs/qwerty-keyboard.md index c58b8f21..0238044b 100644 --- a/docs/qwerty-keyboard.md +++ b/docs/qwerty-keyboard.md @@ -8,9 +8,9 @@ Scan a row, select it, then scan and select a key. The escape slot returns to ro Pages: -- **Letters:** UK QWERTY, numbers and punctuation, including £ and the UK shifted symbols. +- **Letters:** UK QWERTY and punctuation, with Backspace beside the top letter row. - **Functions:** Esc, F1–F12, arrows, Home, End, Page Up, Page Down and Delete. Windows also has Insert, Print Screen, Scroll Lock and Pause. -- **Numbers:** digits, decimal point, arithmetic operators, Enter and Backspace. Digits are independent of hardware Num Lock. +- **Numbers:** digits, decimal point, arithmetic operators, UK number-row symbols (including £), Enter and Backspace. Digits are independent of hardware Num Lock. The bottom control row starts with Close keyboard, followed by Letters, Navigation, Numbers and docking. Space has a dedicated wide key. Editing and modifier keys are wider than ordinary characters; duplicate Shift and Caps controls are removed from the letters page. A header names the current page and scan target. The return-to-rows slot highlights only the header. Active modifiers and the current page have an indicator separate from the scan highlight. Modifiers are available on every page. Shift, Ctrl, Alt/Option and Windows/Command cycle through **off → once → locked → off**. The label shows once or locked. Caps is a keyboard-local uppercase setting; Shift reverses its letter case. Page and position controls do not consume a one-shot modifier. diff --git a/src-tauri/src/scan_keyboard.rs b/src-tauri/src/scan_keyboard.rs index 4b1d3189..c29fbf45 100644 --- a/src-tauri/src/scan_keyboard.rs +++ b/src-tauri/src/scan_keyboard.rs @@ -87,16 +87,15 @@ pub fn rows(page: Page, mac: bool) -> Vec> { }; let mut result = match page { Page::Letters => { - let mut number = chars("`1234567890-=", "¬!\"£$%^&*()_+"); - number.push(N("Backspace")); let mut top = vec![N("Tab")]; top.extend(chars("qwertyuiop[]", "QWERTYUIOP{}")); + top.push(N("Backspace")); let mut middle = vec![Key::Caps]; middle.extend(chars("asdfghjkl;'#", "ASDFGHJKL:@~")); middle.push(N("Enter")); let mut bottom = vec![Key::Modifier(0)]; bottom.extend(chars("\\zxcvbnm,./", "|ZXCVBNM<>?")); - vec![number, top, middle, bottom] + vec![top, middle, bottom] } Page::Functions => { let mut result = vec![ @@ -141,6 +140,7 @@ pub fn rows(page: Page, mac: bool) -> Vec> { N("Enter"), N("Backspace"), ], + chars("`¬!\"£$%^&()_=", "`¬!\"£$%^&()_="), ], }; result.push(if page == Page::Letters { @@ -600,7 +600,19 @@ mod tests { for c in 'a'..='z' { assert!(letters.contains(&Key::Character(c, c.to_ascii_uppercase()))); } - assert!(letters.contains(&Key::Character('3', '£'))); + assert!(!letters + .iter() + .any(|key| matches!(key, Key::Character(c, _) if c.is_ascii_digit()))); + assert!(rows(Page::Letters, false)[0].contains(&Key::Named("Backspace"))); + let numbers = rows(Page::Numbers, false).concat(); + for symbol in "`1234567890-=¬!\"£$%^&*()_+".chars() { + assert!( + numbers + .iter() + .any(|key| matches!(key, Key::Character(a, b) if *a == symbol || *b == symbol)), + "Missing {symbol}" + ); + } assert!(rows(Page::Functions, false) .concat() .contains(&Key::Named("Insert")));