diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31b257d6..13040722 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: - name: Reject retired product identity run: | retired_identity="pre""view" - if git grep -n -i "$retired_identity" -- ':!src-tauri/Cargo.lock'; then + if git grep -I -n -i "$retired_identity" -- ':!src-tauri/Cargo.lock'; then exit 1 fi - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 diff --git a/docs/qwerty-keyboard.md b/docs/qwerty-keyboard.md index 0238044b..ed56a1b0 100644 --- a/docs/qwerty-keyboard.md +++ b/docs/qwerty-keyboard.md @@ -16,7 +16,7 @@ The bottom control row starts with Close keyboard, followed by Letters, Navigati Modifier keys are pressed only around each emitted shortcut and immediately released. Selecting a locked modifier does not hold that operating-system key while the scanner runs. Ordinary characters use text injection; command combinations and navigation use native key events. Native shortcuts retain the operating system's layout semantics. -The keyboard closes when the foreground target or display environment changes, or its scan session ends. Failed input clears keyboard modifiers and requires Select to resume. Stop, disconnect and application exit use the shared deterministic input cleanup. No typed text is logged, and no prediction data, context reader, protocol extension or persistent keyboard settings are added. +The keyboard closes when the foreground target or display environment changes, or its scan session ends. Failed input clears keyboard modifiers and requires Select to resume. Stop, disconnect and application exit use the shared deterministic input cleanup. Word prediction can be enabled in Scanning settings. Five suggestions appear on the Letters page. Selecting a suggestion inserts its missing suffix and a space. Prediction uses local, read-only data; no personal vocabulary is saved. See [word prediction](word-prediction.md) for context availability and validation. ## Manual validation diff --git a/docs/word-prediction.md b/docs/word-prediction.md new file mode 100644 index 00000000..987b60c2 --- /dev/null +++ b/docs/word-prediction.md @@ -0,0 +1,42 @@ +# Word prediction + +Word prediction is enabled by default in Scanning settings. Its five-position row appears only on the Letters page. Empty positions are skipped. Select the row and then a word using the existing switches. Acceptance appends the missing suffix and a space without deleting text, selecting text or using the clipboard. Shift and Caps affect completion casing; Ctrl, Alt/Option and Windows/Command suppress suggestions. + +The bundled English SQLite database works offline. Ranking backs off from the longest available three-word context to shorter contexts, then to word frequency. Prefix matching is case-insensitive. Duplicate words are removed. The data is fixed; typing is never used to train or update it. + +Windows uses UI Automation and macOS uses Accessibility range attributes. Both read at most 512 preceding characters and a short following range, reject protected fields, and hide predictions for selections or mid-word insertion. Applications that do not expose reliable text ranges may have no predictions. No compatibility claim follows from compilation alone. + +If an editable non-protected field can be identified but its text is unavailable, the fallback remembers only successful Switchify typing after a word boundary entered through Switchify. It does not reconstruct existing text. Navigation, deletion, shortcuts, external typing/clicks, focus changes and failures clear this buffer. Fallback is unavailable when its passive activity observer cannot start. The observer records only an activity counter, never external keys or text. Numbers and navigation pages do not retain fallback typing. + +A separate process performs accessibility and database work. Private bounded inherited pipes carry results to native rendering. Text and suggestions are not sent to the React UI, diagnostic history or telemetry. One request is outstanding at a time with a two-second deadline. Timeout stops predictions until the keyboard is reopened; ordinary keyboard operation remains available. Closing the keyboard, ending scanning, or exiting kills and reaps the worker. + +Before a suggestion is accepted, the worker checks the field, protection state, text/selection snapshot and external activity again. Any mismatch rejects the action. Verification and native insertion cannot be atomic across applications; a field can still change in that short interval. + +## Validation + +Automated tests use in-memory databases, fake accessibility adapters and sleeping subprocesses. They never inject desktop input. The database benchmark can be run explicitly with: + +```powershell +cargo test --manifest-path src-tauri/Cargo.toml bundled_database_benchmark -- --ignored --nocapture +``` + +This benchmark prints counts and timings only and does not read a focused field. Its ignored status keeps performance measurement separate from correctness checks. + +Native integration checks remain pending on Windows and macOS. The older standalone probe's Notepad result does not qualify this integration. Manually test synthetic text in Notepad, TextEdit, Edge/Chrome, Safari and Word where installed. Exercise typing, prediction acceptance, Shift/Caps, external edits, selection, caret movement, passwords, page changes, top/bottom docking, disconnect and app exit. On macOS use `npm run macos:run` to retain the signed Accessibility identity. + +For each app tested, collect at least 100 successful reads and report median, p95, maximum and failures. Target warm context-plus-query p95 below 50 ms; report startup and the 250-ms polling interval separately. Unsupported and untested apps must remain identified as such. + +Prepare a disposable field containing `I would like some wa`, with the caret after `wa`. From the repository root run the command below, then focus that field during its five-second countdown. It samples every 250 ms, stops after 100 supported warm samples (or 200 attempts), and prints only counts, fixture comparisons and timings. It sends read-only queries; it cannot accept a word or inject input. The first request includes worker/database startup and is excluded from warm percentiles. + +```powershell +node scripts/measure-word-prediction.mjs src-tauri/target/debug/switchify-pc.exe Notepad water +``` + +For macOS, pass the signed app's `Contents/MacOS/switchify-pc` executable after building with `npm run macos:run`. Use a fresh run for each app. Keep all focused fields synthetic throughout measurement. Repeat functional checks with an empty field, `Hello. wa`, multiline text, `café naïve wa`, a pasted sentence, deletion, a caret inside a word, a nonempty selection, selection replacement, and switching between two fields. Verify that protected fields produce no suggestions and that closing the keyboard removes the worker. Do not count empty/unsupported responses as successful latency samples. + +Measured locally on Windows in a debug build (200 synthetic database queries): median 2.59 ms, p95 10.78 ms, maximum 14.35 ms, zero query failures. Database startup took 251.13 ms. These numbers exclude accessibility and IPC and are not app compatibility results. + +| App | Integration status | +| --- | --- | +| Notepad, Edge, Chrome, Word on Windows | Untested with this integration | +| TextEdit, Safari, Chrome, Word on macOS | Untested; macOS host unavailable locally | diff --git a/scripts/measure-word-prediction.mjs b/scripts/measure-word-prediction.mjs new file mode 100644 index 00000000..466eb678 --- /dev/null +++ b/scripts/measure-word-prediction.mjs @@ -0,0 +1,97 @@ +// Manual, read-only fixture measurement. Never sends an Accept request. +import { spawn } from 'node:child_process'; +import { resolve } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { performance } from 'node:perf_hooks'; + +const [executable, app, expectedWord] = process.argv.slice(2); +if (!executable || !app || !expectedWord) { + console.error('Usage: node scripts/measure-word-prediction.mjs '); + process.exit(1); +} +console.log('Focus a synthetic test field now. Sampling starts in five seconds. No input will be injected.'); +await delay(5000); +const child = spawn(resolve(executable), [ + '--switchify-prediction-worker', + resolve('src-tauri/resources/WordData2017051601.db'), + '[]', +], { stdio: ['pipe', 'pipe', 'ignore'], windowsHide: true }); +let buffer = Buffer.alloc(0); +let pending; +let failed = false; +const fail = () => { + failed = true; + pending?.reject(new Error('Worker unavailable')); + pending = undefined; +}; +child.on('error', fail); +child.on('exit', fail); +child.stdin.on('error', fail); +child.stdout.on('data', chunk => { + buffer = Buffer.concat([buffer, chunk]); + if (buffer.length < 4) return; + const length = buffer.readUInt32LE(); + if (!length || length > 16384 || buffer.length > length + 4) return fail(); + if (buffer.length !== length + 4) return; + try { + const response = JSON.parse(buffer.subarray(4).toString('utf8')); + buffer = Buffer.alloc(0); + const request = pending; + pending = undefined; + if (!request) return fail(); + request.resolve(response); + } catch { fail(); } +}); +async function query(generation) { + if (failed) throw new Error('Worker unavailable'); + const bytes = Buffer.from(JSON.stringify({ Query: { + generation, edit: null, reset: true, shift: false, caps: false, + } })); + const header = Buffer.alloc(4); + header.writeUInt32LE(bytes.length); + let timer; + try { + return await new Promise((resolve, reject) => { + pending = { resolve, reject }; + timer = setTimeout(() => { fail(); child.kill(); }, 2000); + child.stdin.write(Buffer.concat([header, bytes])); + }); + } finally { clearTimeout(timer); } +} +const stop = () => { fail(); child.kill(); }; +process.on('SIGINT', stop); +process.on('SIGTERM', stop); +const times = []; +let unsupported = 0; +let failures = 0; +let fixtureMatches = 0; +let firstRequestMs; +try { + // Up to 200 attempts, stopping after 100 supported samples. + for (let generation = 0; generation < 200 && times.length < 100; generation++) { + const start = performance.now(); + const response = await query(generation); + const elapsed = performance.now() - start; + if (generation === 0) firstRequestMs = elapsed; + const result = response.Suggestions; + if (!result || result.generation !== generation) throw new Error('Invalid response'); + if (result.batch?.words?.length) { + if (generation !== 0) times.push(elapsed); + if (result.batch.words.some(word => word.toLowerCase() === expectedWord.toLowerCase())) fixtureMatches++; + } else unsupported++; + await delay(Math.max(0, 250 - elapsed)); + } +} catch { failures++; } +finally { + child.kill(); + process.off('SIGINT', stop); + process.off('SIGTERM', stop); +} +times.sort((a, b) => a - b); +const percentile = fraction => times.length ? +times[Math.ceil(times.length * fraction) - 1].toFixed(2) : null; +console.log(JSON.stringify({ + app, successfulSamples: times.length, unsupported, failures, fixtureMatches, + firstRequestMs: firstRequestMs === undefined ? null : +firstRequestMs.toFixed(2), + medianMs: percentile(0.5), p95Ms: percentile(0.95), maximumMs: percentile(1), +}, null, 2)); +if (failures || times.length < 100) process.exitCode = 1; diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 2acc22ac..6a379340 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -1176,6 +1176,18 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.5.0" @@ -1677,11 +1689,32 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] [[package]] name = "heck" @@ -2272,6 +2305,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -3386,6 +3430,31 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.19", +] + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags 2.13.1", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -3925,6 +3994,18 @@ dependencies = [ "system-deps", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -4006,6 +4087,7 @@ dependencies = [ "objc2-quartz-core", "rand 0.8.7", "reqwest", + "rusqlite", "serde", "serde_json", "sha2", @@ -4016,6 +4098,7 @@ dependencies = [ "tauri-plugin-updater", "tiny-skia", "tokio", + "unicode-segmentation", "uuid", "windows 0.62.2", "windows-sys 0.61.2", @@ -4455,7 +4538,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -4989,6 +5072,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version-compare" version = "0.2.1" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 2b36c36d..de474cd6 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -21,6 +21,8 @@ enigo = "=0.6.1" hmac = "0.12" keyring = "4.1.6" rand = "0.8" +rusqlite = { version = "0.40.2", features = ["bundled"] } +unicode-segmentation = "1.12" reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"] } serde = { version = "1", features = ["derive"] } serde_json = "1" @@ -60,11 +62,15 @@ windows = { version = "0.62.2", features = [ "Win32_Security", "Win32_Storage_FileSystem", "Win32_System_LibraryLoader", + "Win32_System_Com", + "Win32_System_Ole", + "Win32_System_Variant", "Win32_System_Power", "Win32_System_Registry", "Win32_System_Threading", "Win32_System_WinRT", "Win32_UI_HiDpi", + "Win32_UI_Accessibility", "Win32_UI_Input_KeyboardAndMouse", "Win32_UI_WindowsAndMessaging" ] } diff --git a/src-tauri/resources/PROVENANCE.md b/src-tauri/resources/PROVENANCE.md new file mode 100644 index 00000000..361373e2 --- /dev/null +++ b/src-tauri/resources/PROVENANCE.md @@ -0,0 +1,10 @@ +# English prediction database + +`WordData2017051601.db` was recovered from the user's private `enaboapps/sayit-ios` history, path `Assets/Databases/WordData2017051601.db`, commit `95bda9265758e50787c5da2779b16f36b53c319d`, dated May 20, 2017. Recovery took place September 18, 2026. The user authorized bundling this database with Switchify. + +- Size: 102,690,816 bytes. +- SHA-256: `dedd65d263bde8315e7e5ed7d2c8e04f17c33598a68506c9d70661a6d1f57318`. +- SQLite integrity check during recovery: `ok`. +- Rows: WORDS 165,420; BIGRAMS 976,804; TRIGRAMS 996,846; QUADGRAMS 953,630. + +This is the unchanged original SQLite file. The later Realm archive is not used. Runtime connections are read-only and ignore personal frequency columns. No captured text or user vocabulary is written into this resource. diff --git a/src-tauri/resources/WordData2017051601.db b/src-tauri/resources/WordData2017051601.db new file mode 100644 index 00000000..eb93d5d5 Binary files /dev/null and b/src-tauri/resources/WordData2017051601.db differ diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 57b7ebc1..89770a53 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -20,6 +20,7 @@ mod point_scan; mod point_scan_activation; mod point_scan_runtime; mod point_workflow; +mod prediction; mod protocol; mod remote_scan; mod scan_executor; @@ -1361,6 +1362,9 @@ fn point_scan_prepare(app: &AppHandle) -> Result<(), String> { } pub fn run() { + if prediction::worker::run_from_args() { + return; + } #[cfg(target_os = "windows")] if switch_input::run_worker_from_args() { return; diff --git a/src-tauri/src/point_scan.rs b/src-tauri/src/point_scan.rs index 88b4b1cd..32226830 100644 --- a/src-tauri/src/point_scan.rs +++ b/src-tauri/src/point_scan.rs @@ -15,6 +15,7 @@ pub enum Mode { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase", default)] pub struct Config { + pub word_prediction: bool, pub scanner_color: crate::scanning::ScannerColor, pub mode: Mode, pub automatic: bool, @@ -31,6 +32,7 @@ pub struct Config { impl Default for Config { fn default() -> Self { Self { + word_prediction: true, scanner_color: Default::default(), mode: Mode::Line, automatic: true, @@ -58,6 +60,7 @@ impl Config { } pub fn point(&self) -> PointSettings { PointSettings { + word_prediction: self.word_prediction, scanner_color: self.scanner_color, mode: self.mode, speed: self.speed, @@ -74,6 +77,7 @@ impl Config { } #[derive(Clone)] pub struct PointSettings { + pub word_prediction: bool, pub scanner_color: crate::scanning::ScannerColor, pub mode: Mode, pub speed: usize, @@ -482,6 +486,15 @@ mod tests { }) } #[test] + fn legacy_settings_enable_predictions_and_explicit_disable_roundtrips() { + let c: Config = serde_json::from_str("{}").unwrap(); + assert!(c.word_prediction); + let c: Config = serde_json::from_str(r#"{"wordPrediction":false}"#).unwrap(); + assert!(!c.word_prediction); + let restored: Config = serde_json::from_value(serde_json::to_value(c).unwrap()).unwrap(); + assert!(!restored.word_prediction); + } + #[test] fn auto_select_delays_are_bounded_and_round_trip() { for delay in [100, 500, 1000, 100_000] { let config = Config { @@ -878,6 +891,7 @@ mod tests { json["autoSelectEnabled"] = serde_json::json!(false); json["autoSelectDelayMs"] = serde_json::json!(1000); json["scannerColor"] = serde_json::json!("blue"); + json["wordPrediction"] = serde_json::json!(true); assert_eq!(serde_json::to_value(&config).unwrap(), json); assert!(!config.switches().automatic); assert_eq!(config.point().grid_size, 7); diff --git a/src-tauri/src/point_scan_runtime.rs b/src-tauri/src/point_scan_runtime.rs index a3e32724..1808542f 100644 --- a/src-tauri/src/point_scan_runtime.rs +++ b/src-tauri/src/point_scan_runtime.rs @@ -82,6 +82,14 @@ impl Adapter for PointScan { fn activate(app: &AppHandle, request: Request) -> Result, String> { crate::point_scan_ready(app)?; match request { + Request::Prediction { token, index } => { + return crate::prediction::select(token, index).map(|()| None) + } + Request::Keyboard(stroke) => { + let result = crate::scan_executor::activate(request); + crate::prediction::record(stroke, result.is_ok()); + return result.map(|()| None); + } Request::OpenKeyboard { point: Some(point) } => { let target = crate::scan_host::target_at(point)?; crate::scan_executor::activate(request)?; @@ -117,7 +125,18 @@ impl Adapter for PointScan { } Ok(true) } + fn preserve_visuals(request: &Request) -> bool { + matches!(request, Request::Keyboard(_) | Request::Prediction { .. }) + } + fn deferred(request: &Request) -> bool { + matches!(request, Request::Prediction { .. }) + } + fn poll(app: &AppHandle, technique: &mut Workflow, captured_keys: &[String]) { + let enabled = technique.prediction_enabled(); + crate::prediction::poll(app, technique.prediction_keyboard(), enabled, captured_keys); + } fn cleanup(_app: &AppHandle) -> Result<(), String> { + crate::prediction::stop(); crate::scan_executor::cleanup() } } @@ -186,6 +205,28 @@ fn validate_display(app: &AppHandle, display: Option<&Environment>) -> Result<() mod tests { use super::*; + #[test] + fn typing_preserves_visuals_but_pointer_execution_hides_them() { + assert!(PointScan::preserve_visuals(&Request::Keyboard( + crate::scan_keyboard::Stroke { + key: crate::scan_keyboard::Key::Character('a', 'A'), + modifiers: [false; 4], + caps: false, + } + ))); + assert!(PointScan::preserve_visuals(&Request::Prediction { + token: 1, + index: 0 + })); + assert!(!PointScan::preserve_visuals( + &crate::point_workflow::default_click((10, 20)) + )); + assert!(!PointScan::preserve_visuals(&Request::OpenKeyboard { + point: Some((10, 20)) + })); + assert!(!PointScan::preserve_visuals(&Request::DragStart((10, 20)))); + } + #[test] fn focus_handoff_waits_and_only_accepts_the_clicked_target() { let start = std::time::Instant::now(); diff --git a/src-tauri/src/point_workflow.rs b/src-tauri/src/point_workflow.rs index 52bd2f17..292b10bb 100644 --- a/src-tauri/src/point_workflow.rs +++ b/src-tauri/src/point_workflow.rs @@ -8,6 +8,10 @@ use serde::Serialize; pub type Point = (i32, i32); #[derive(Debug, Clone, Copy, PartialEq)] pub enum Request { + Prediction { + token: u64, + index: usize, + }, OpenKeyboard { point: Option, }, @@ -125,6 +129,18 @@ impl Workflow { self.point.start(); } } + pub fn prediction_keyboard(&mut self) -> Option<&mut crate::scan_keyboard::Keyboard> { + if self.stage == Stage::Keyboard { + self.keyboard + .enable_predictions(self.point.config.word_prediction); + Some(&mut self.keyboard) + } else { + None + } + } + pub fn prediction_enabled(&self) -> bool { + self.point.config.word_prediction + } pub fn set_keyboard_area(&mut self, area: Rect) { self.keyboard_area = area; } @@ -288,6 +304,9 @@ impl Technique for Workflow { Some(crate::scan_keyboard::Output::Stroke(stroke)) => { return Some(Request::Keyboard(stroke)) } + Some(crate::scan_keyboard::Output::Prediction { token, index }) => { + return Some(Request::Prediction { token, index }) + } Some(crate::scan_keyboard::Output::Close) => self.start(), None => {} }, diff --git a/src-tauri/src/prediction/activity.rs b/src-tauri/src/prediction/activity.rs new file mode 100644 index 00000000..d416d437 --- /dev/null +++ b/src-tauri/src/prediction/activity.rs @@ -0,0 +1,133 @@ +//! A content-free input epoch. Never suppresses or records external input. +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +static EPOCH: AtomicU64 = AtomicU64::new(0); +static HEALTHY: AtomicBool = AtomicBool::new(false); +pub fn snapshot() -> (u64, bool) { + (EPOCH.load(Ordering::SeqCst), HEALTHY.load(Ordering::SeqCst)) +} +fn unavailable() { + HEALTHY.store(false, Ordering::SeqCst); + changed(); +} +fn changed() { + EPOCH.fetch_add(1, Ordering::SeqCst); +} + +#[cfg(target_os = "windows")] +pub fn start(ignored: Vec) -> bool { + use std::cell::RefCell; + use windows_sys::Win32::{System::LibraryLoader::GetModuleHandleW, UI::WindowsAndMessaging::*}; + thread_local! { static IGNORE: RefCell> = const { RefCell::new(Vec::new()) }; } + unsafe extern "system" fn keyboard(code: i32, wp: usize, lp: isize) -> isize { + if code >= 0 && matches!(wp as u32, WM_KEYDOWN | WM_SYSKEYDOWN) { + let key = unsafe { &*(lp as *const KBDLLHOOKSTRUCT) }; + if !crate::input::own_input(key.dwExtraInfo as i64) + && !IGNORE.with(|i| i.borrow().contains(&key.vkCode)) + { + changed(); + } + } + unsafe { CallNextHookEx(std::ptr::null_mut(), code, wp, lp) } + } + unsafe extern "system" fn mouse(code: i32, wp: usize, lp: isize) -> isize { + if code >= 0 + && matches!( + wp as u32, + WM_LBUTTONDOWN | WM_RBUTTONDOWN | WM_MBUTTONDOWN | WM_MOUSEWHEEL + ) + { + let event = unsafe { &*(lp as *const MSLLHOOKSTRUCT) }; + if !crate::input::own_input(event.dwExtraInfo as i64) { + changed(); + } + } + unsafe { CallNextHookEx(std::ptr::null_mut(), code, wp, lp) } + } + let (tx, rx) = std::sync::mpsc::sync_channel(1); + std::thread::spawn(move || unsafe { + IGNORE.with(|i| *i.borrow_mut() = ignored); + let k = SetWindowsHookExW( + WH_KEYBOARD_LL, + Some(keyboard), + GetModuleHandleW(std::ptr::null()), + 0, + ); + let m = SetWindowsHookExW( + WH_MOUSE_LL, + Some(mouse), + GetModuleHandleW(std::ptr::null()), + 0, + ); + let ok = !k.is_null() && !m.is_null(); + HEALTHY.store(ok, Ordering::SeqCst); + let _ = tx.send(ok); + if ok { + let mut message = std::mem::zeroed(); + while GetMessageW(&mut message, std::ptr::null_mut(), 0, 0) > 0 { + TranslateMessage(&message); + DispatchMessageW(&message); + } + } + if !k.is_null() { + UnhookWindowsHookEx(k); + } + if !m.is_null() { + UnhookWindowsHookEx(m); + } + unavailable(); + }); + rx.recv_timeout(std::time::Duration::from_millis(500)) + .unwrap_or(false) +} +#[cfg(target_os = "macos")] +pub fn start(ignored: Vec) -> bool { + use core_graphics::event::{ + CGEventTap, CGEventTapLocation, CGEventTapOptions, CGEventTapPlacement, CGEventType, + CallbackResult, + }; + let (tx, rx) = std::sync::mpsc::sync_channel(1); + std::thread::spawn(move || { + let ready = tx.clone(); + let _ = CGEventTap::with_enabled( + CGEventTapLocation::Session, + CGEventTapPlacement::HeadInsertEventTap, + CGEventTapOptions::ListenOnly, + vec![ + CGEventType::KeyDown, + CGEventType::LeftMouseDown, + CGEventType::RightMouseDown, + CGEventType::OtherMouseDown, + CGEventType::ScrollWheel, + ], + move |_, kind, event| { + if matches!( + kind, + CGEventType::TapDisabledByTimeout | CGEventType::TapDisabledByUserInput + ) { + unavailable(); + return CallbackResult::Keep; + } + if !crate::input::own_input(event.get_integer_value_field(42)) + && (!matches!(kind, CGEventType::KeyDown) + || !ignored.contains(&(event.get_integer_value_field(9) as u32))) + { + changed(); + } + CallbackResult::Keep + }, + || { + HEALTHY.store(true, Ordering::SeqCst); + let _ = ready.send(true); + core_foundation::runloop::CFRunLoop::run_current(); + }, + ); + let _ = tx.send(false); + unavailable(); + }); + rx.recv_timeout(std::time::Duration::from_millis(500)) + .unwrap_or(false) +} +#[cfg(not(any(target_os = "windows", target_os = "macos")))] +pub fn start(_: Vec) -> bool { + false +} diff --git a/src-tauri/src/prediction/context.rs b/src-tauri/src/prediction/context.rs new file mode 100644 index 00000000..5e092491 --- /dev/null +++ b/src-tauri/src/prediction/context.rs @@ -0,0 +1,96 @@ +use unicode_segmentation::UnicodeSegmentation; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Status { + NoFocus, + Protected, + Unsupported, + AmbiguousSelection, + ProviderError, +} + +// Text-bearing types deliberately have no Debug implementation. +#[derive(Clone, PartialEq, Eq)] +pub struct RawContext { + pub before: String, + pub after: String, + pub clipped_start: bool, + pub has_selection: bool, + pub position: i64, +} +pub struct Context { + pub words: Vec, + pub prefix: String, + pub sentence_start: bool, +} +pub fn extract(text: &str, clipped: bool) -> Context { + let boundary = text.rfind(['.', '!', '?', '\n', '\r', '。', '!', '?']); + let start = boundary.map_or(0, |i| i + text[i..].chars().next().unwrap().len_utf8()); + let tail = &text[start..]; + let mut tokens: Vec<_> = tail.unicode_word_indices().collect(); + if clipped && start == 0 && tokens.first().is_some_and(|(i, _)| *i == 0) { + tokens.remove(0); + } + let prefix = if tokens + .last() + .is_some_and(|(i, w)| i + w.len() == tail.len()) + { + tokens.pop().unwrap().1.to_owned() + } else { + String::new() + }; + Context { + sentence_start: tokens.is_empty() && (!clipped || boundary.is_some()), + words: tokens + .iter() + .rev() + .take(3) + .rev() + .map(|(_, w)| (*w).to_owned()) + .collect(), + prefix, + } +} +pub fn eligible(raw: &RawContext) -> bool { + !raw.has_selection + && raw.after.chars().next().is_none_or(|c| { + c.is_whitespace() + || matches!( + c, + '.' | ',' + | '!' + | '?' + | ':' + | ';' + | ')' + | ']' + | '}' + | '"' + | '/' + | '\\' + | '-' + | '…' + ) + }) +} +pub trait Adapter { + type Target; + fn focused(&mut self) -> Result; + fn protected(&mut self, target: &Self::Target) -> Result; + fn editable(&mut self, target: &Self::Target) -> bool; + fn same(&mut self, a: &Self::Target, b: &Self::Target) -> Result; + fn read(&mut self, target: &Self::Target) -> Result; +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn unicode_boundaries_and_clipped_words() { + let c = extract("Discard this. I can’t find cafe\u{301}", false); + assert_eq!(c.words, ["I", "can’t", "find"]); + assert_eq!(c.prefix, "cafe\u{301}"); + assert_eq!(extract("agment whole pa", true).words, ["whole"]); + assert!(extract("Done! ", false).sentence_start); + } +} diff --git a/src-tauri/src/prediction/database.rs b/src-tauri/src/prediction/database.rs new file mode 100644 index 00000000..5b0ecc53 --- /dev/null +++ b/src-tauri/src/prediction/database.rs @@ -0,0 +1,129 @@ +use super::context::Context; +use rusqlite::{Connection, OpenFlags}; +use std::{ + collections::{HashMap, HashSet}, + path::Path, +}; + +pub struct Database { + connection: Connection, + words: Vec<(i64, String, String)>, + ids: HashMap, +} +impl Database { + #[cfg(test)] + pub fn fixture() -> Self { + let c = Connection::open_in_memory().unwrap(); + c.execute_batch("CREATE TABLE WORDS(ID INTEGER,WORD TEXT,BASE_FREQUENCY INTEGER); INSERT INTO WORDS VALUES(1,'water',9),(2,'walk',8),(3,'want',7),(4,'I',6),(5,'like',5); CREATE TABLE BIGRAMS(ID1 INTEGER,ID2 INTEGER,BASE_FREQUENCY INTEGER); CREATE TABLE TRIGRAMS(ID1 INTEGER,ID2 INTEGER,ID3 INTEGER,BASE_FREQUENCY INTEGER); CREATE TABLE QUADGRAMS(ID1 INTEGER,ID2 INTEGER,ID3 INTEGER,ID4 INTEGER,BASE_FREQUENCY INTEGER);").unwrap(); + Self::from_connection(c).unwrap() + } + pub fn open(path: &Path) -> Result { + let connection = Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(|_| ())?; + Self::from_connection(connection) + } + fn from_connection(connection: Connection) -> Result { + connection + .execute_batch("PRAGMA query_only=ON; PRAGMA cache_size=-8192;") + .map_err(|_| ())?; + let words = { + let mut q = connection + .prepare("SELECT ID, WORD FROM WORDS ORDER BY BASE_FREQUENCY DESC, ID ASC") + .map_err(|_| ())?; + let result = q + .query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?))) + .map_err(|_| ())? + .collect::, _>>() + .map_err(|_| ())?; + result + }; + let words: Vec<_> = words + .into_iter() + .map(|(id, word)| { + let folded = word.to_lowercase(); + (id, word, folded) + }) + .collect(); + let mut ids = HashMap::new(); + for (id, _, folded) in &words { + ids.entry(folded.clone()).or_insert(*id); + } + Ok(Self { + connection, + words, + ids, + }) + } + pub fn predict(&self, context: &Context) -> Result, ()> { + let prefix = context.prefix.to_lowercase(); + let mut result = Vec::new(); + let mut seen = HashSet::new(); + let mut push = |word: String| { + let folded = word.to_lowercase(); + if result.len() < 5 + && word.chars().count() <= 48 + && !word.chars().any(char::is_whitespace) + && folded.starts_with(&prefix) + && seen.insert(folded) + { + result.push(word); + } + result.len() == 5 + }; + for count in (1..=context.words.len().min(3)).rev() { + let ids: Option> = context.words[context.words.len() - count..] + .iter() + .map(|w| self.ids.get(&w.to_lowercase()).copied()) + .collect(); + let Some(ids) = ids else { + continue; + }; + let table = ["", "BIGRAMS", "TRIGRAMS", "QUADGRAMS"][count]; + let conditions = (1..=count) + .map(|i| format!("g.ID{i}=?{i}")) + .collect::>() + .join(" AND "); + let sql = format!("SELECT w.WORD FROM {table} g JOIN WORDS w ON w.ID=g.ID{} WHERE {conditions} ORDER BY g.BASE_FREQUENCY DESC, w.ID ASC", count+1); + let mut statement = self.connection.prepare_cached(&sql).map_err(|_| ())?; + let mut rows = statement + .query(rusqlite::params_from_iter(ids)) + .map_err(|_| ())?; + // Context indexes bound this candidate set; the entire query is also + // contained by the parent process's two-second deadline. + while let Some(row) = rows.next().map_err(|_| ())? { + if push(row.get(0).map_err(|_| ())?) { + break; + } + } + } + for (_, word, _) in self + .words + .iter() + .filter(|(_, _, folded)| folded.starts_with(&prefix)) + { + if push(word.clone()) { + break; + } + } + Ok(result) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn longest_context_wins_and_backoff_deduplicates() { + let c = Connection::open_in_memory().unwrap(); + c.execute_batch("CREATE TABLE WORDS(ID INTEGER,WORD TEXT,BASE_FREQUENCY INTEGER); INSERT INTO WORDS VALUES(1,'I',9),(2,'would',8),(3,'like',7),(4,'water',1),(5,'wine',20),(6,'walk',30); CREATE TABLE BIGRAMS(ID1 INTEGER,ID2 INTEGER,BASE_FREQUENCY INTEGER); INSERT INTO BIGRAMS VALUES(3,5,9); CREATE TABLE TRIGRAMS(ID1 INTEGER,ID2 INTEGER,ID3 INTEGER,BASE_FREQUENCY INTEGER); CREATE TABLE QUADGRAMS(ID1 INTEGER,ID2 INTEGER,ID3 INTEGER,ID4 INTEGER,BASE_FREQUENCY INTEGER); INSERT INTO QUADGRAMS VALUES(1,2,3,4,1);").unwrap(); + let db = Database::from_connection(c).unwrap(); + let result = db + .predict(&super::super::context::extract("I would like w", false)) + .unwrap(); + assert_eq!(&result[..3], ["water", "wine", "walk"]); + assert!(db.connection.execute("DELETE FROM WORDS", []).is_err()); + } +} diff --git a/src-tauri/src/prediction/macos.rs b/src-tauri/src/prediction/macos.rs new file mode 100644 index 00000000..64642983 --- /dev/null +++ b/src-tauri/src/prediction/macos.rs @@ -0,0 +1,186 @@ +use super::context::{Adapter, RawContext, Status}; +use core_foundation::{ + base::{CFEqual, CFRange, CFType, CFTypeRef, TCFType}, + string::{CFString, CFStringRef}, +}; + +#[link(name = "ApplicationServices", kind = "framework")] +unsafe extern "C" { + fn AXUIElementCreateSystemWide() -> CFTypeRef; + fn AXUIElementCopyAttributeValue(e: CFTypeRef, a: CFStringRef, out: *mut CFTypeRef) -> i32; + fn AXUIElementCopyParameterizedAttributeValue( + e: CFTypeRef, + a: CFStringRef, + p: CFTypeRef, + out: *mut CFTypeRef, + ) -> i32; + fn AXUIElementIsAttributeSettable(e: CFTypeRef, a: CFStringRef, out: *mut u8) -> i32; + fn AXUIElementSetMessagingTimeout(e: CFTypeRef, seconds: f32) -> i32; + fn AXValueCreate(kind: u32, value: *const std::ffi::c_void) -> CFTypeRef; + fn AXValueGetValue(value: CFTypeRef, kind: u32, result: *mut std::ffi::c_void) -> bool; +} +#[link(name = "Carbon", kind = "framework")] +unsafe extern "C" { + fn IsSecureEventInputEnabled() -> bool; +} +fn attribute(e: &CFType, key: &str) -> Result { + let mut out = std::ptr::null(); + let key = CFString::new(key); + if unsafe { + AXUIElementCopyAttributeValue(e.as_CFTypeRef(), key.as_concrete_TypeRef(), &mut out) + } != 0 + || out.is_null() + { + return Err(Status::Unsupported); + } + Ok(unsafe { CFType::wrap_under_create_rule(out) }) +} +fn string(value: CFType) -> Result { + if value.type_of() != CFString::type_id() { + return Err(Status::Unsupported); + } + Ok(unsafe { CFString::wrap_under_get_rule(value.as_CFTypeRef().cast()) }.to_string()) +} +pub struct MacAdapter { + system: CFType, +} +impl MacAdapter { + pub fn new() -> Result { + let system = unsafe { AXUIElementCreateSystemWide() }; + if system.is_null() { + return Err(Status::NoFocus); + } + unsafe { + AXUIElementSetMessagingTimeout(system, 0.2); + } + Ok(Self { + system: unsafe { CFType::wrap_under_create_rule(system) }, + }) + } + fn text(&self, target: &CFType, range: CFRange) -> Result { + let parameter = unsafe { AXValueCreate(4, (&range as *const CFRange).cast()) }; + if parameter.is_null() { + return Err(Status::ProviderError); + } + let parameter = unsafe { CFType::wrap_under_create_rule(parameter) }; + let key = CFString::new("AXStringForRange"); + let mut out = std::ptr::null(); + if unsafe { + AXUIElementCopyParameterizedAttributeValue( + target.as_CFTypeRef(), + key.as_concrete_TypeRef(), + parameter.as_CFTypeRef(), + &mut out, + ) + } != 0 + || out.is_null() + { + return Err(Status::Unsupported); + } + string(unsafe { CFType::wrap_under_create_rule(out) }) + } +} +impl Adapter for MacAdapter { + type Target = CFType; + fn focused(&mut self) -> Result { + attribute(&self.system, "AXFocusedUIElement") + } + fn protected(&mut self, target: &CFType) -> Result { + if unsafe { IsSecureEventInputEnabled() } { + return Ok(true); + } + let role = string(attribute(target, "AXRole")?)?; + let subrole = attribute(target, "AXSubrole") + .and_then(string) + .unwrap_or_default(); + Ok(role == "AXSecureTextField" + || subrole == "AXSecureTextField" + || !matches!(role.as_str(), "AXTextField" | "AXTextArea" | "AXComboBox")) + } + fn editable(&mut self, target: &CFType) -> bool { + let mut settable = 0; + let key = CFString::new("AXValue"); + unsafe { + AXUIElementIsAttributeSettable( + target.as_CFTypeRef(), + key.as_concrete_TypeRef(), + &mut settable, + ) == 0 + && settable != 0 + } + } + fn same(&mut self, a: &CFType, b: &CFType) -> Result { + Ok(unsafe { CFEqual(a.as_CFTypeRef(), b.as_CFTypeRef()) } != 0) + } + fn read(&mut self, target: &CFType) -> Result { + if !self.editable(target) { + return Err(Status::Unsupported); + } + if let Ok(ranges) = attribute(target, "AXSelectedTextRanges") { + use core_foundation::array::CFArray; + if ranges.type_of() != CFArray::::type_id() { + return Err(Status::Unsupported); + } + let array = + unsafe { CFArray::::wrap_under_get_rule(ranges.as_CFTypeRef().cast()) }; + if array.len() != 1 { + return Err(Status::AmbiguousSelection); + } + } + let selected = attribute(target, "AXSelectedTextRange")?; + let mut range = CFRange { + location: 0, + length: 0, + }; + if !unsafe { + AXValueGetValue( + selected.as_CFTypeRef(), + 4, + (&mut range as *mut CFRange).cast(), + ) + } || range.location < 0 + || range.length < 0 + { + return Err(Status::Unsupported); + } + let start = range.location.saturating_sub(512).max(0); + let before = self.text( + target, + CFRange { + location: start, + length: range.location - start, + }, + )?; + // AX providers may reject a range past the document end. Obtain its + // character count without reading the whole value. + use core_foundation::number::CFNumber; + let count = attribute(target, "AXNumberOfCharacters")?; + if count.type_of() != CFNumber::type_id() { + return Err(Status::Unsupported); + } + let count = unsafe { CFNumber::wrap_under_get_rule(count.as_CFTypeRef().cast()) } + .to_i64() + .ok_or(Status::Unsupported)?; + let remaining = count.saturating_sub(range.location as i64); + if remaining < 0 { + return Err(Status::Unsupported); + } + let after = self.text( + target, + CFRange { + location: range.location, + length: (remaining as isize).min(2), + }, + )?; + if before.chars().count() > 512 || after.chars().count() > 2 { + return Err(Status::Unsupported); + } + Ok(RawContext { + before, + after, + clipped_start: start > 0, + has_selection: range.length != 0, + position: range.location as i64, + }) + } +} diff --git a/src-tauri/src/prediction/mod.rs b/src-tauri/src/prediction/mod.rs new file mode 100644 index 00000000..d8836f52 --- /dev/null +++ b/src-tauri/src/prediction/mod.rs @@ -0,0 +1,395 @@ +mod activity; +mod context; +mod database; +#[cfg(target_os = "macos")] +mod macos; +#[cfg(target_os = "windows")] +mod windows; +pub mod worker; + +use crate::scan_keyboard::{Keyboard, Modifier, Page, Stroke}; +use std::{ + cell::RefCell, + path::Path, + process::{Child, ChildStdin, Command, Stdio}, + sync::mpsc, + thread::JoinHandle, + time::{Duration, Instant}, +}; +use tauri::{AppHandle, Manager}; +use worker::{Request, Response}; + +#[cfg(target_os = "windows")] +struct Job(windows_sys::Win32::Foundation::HANDLE); +#[cfg(target_os = "windows")] +impl Job { + fn contain(child: &Child) -> Result { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::System::JobObjects::*; + unsafe { + let h = CreateJobObjectW(std::ptr::null(), std::ptr::null()); + if h.is_null() { + return Err(()); + } + let job = Self(h); + let mut info: JOBOBJECT_EXTENDED_LIMIT_INFORMATION = std::mem::zeroed(); + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if SetInformationJobObject( + h, + JobObjectExtendedLimitInformation, + (&info as *const JOBOBJECT_EXTENDED_LIMIT_INFORMATION).cast(), + std::mem::size_of_val(&info) as u32, + ) == 0 + || AssignProcessToJobObject(h, child.as_raw_handle()) == 0 + { + return Err(()); + } + Ok(job) + } + } +} +#[cfg(target_os = "windows")] +impl Drop for Job { + fn drop(&mut self) { + unsafe { + windows_sys::Win32::Foundation::CloseHandle(self.0); + } + } +} +struct Client { + child: Child, + input: ChildStdin, + replies: mpsc::Receiver>, + reader: Option>, + #[cfg(target_os = "windows")] + _job: Job, +} +impl Client { + fn start(path: &Path, ignored: Vec) -> Result { + let mut command = Command::new(std::env::current_exe().map_err(|_| ())?); + command + .arg(worker::ARG) + .arg(path) + .arg(serde_json::to_string(&ignored).map_err(|_| ())?) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + Self::spawn(command) + } + fn spawn(mut command: Command) -> Result { + command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + #[cfg(target_os = "windows")] + { + use std::os::windows::process::CommandExt; + command.creation_flags(0x08000000); + } + let mut child = command.spawn().map_err(|_| ())?; + #[cfg(target_os = "windows")] + let job = match Job::contain(&child) { + Ok(j) => j, + Err(_) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(()); + } + }; + let input = child.stdin.take().ok_or(())?; + let mut output = child.stdout.take().ok_or(())?; + let (tx, replies) = mpsc::sync_channel(1); + let reader = std::thread::spawn(move || loop { + let reply = worker::receive(&mut output); + let failed = reply.is_err(); + if tx.send(reply).is_err() || failed { + break; + } + }); + Ok(Self { + child, + input, + replies, + reader: Some(reader), + #[cfg(target_os = "windows")] + _job: job, + }) + } +} +impl Client { + fn terminate(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + // Drain the bounded channel before joining, including a final EOF reply. + while let Some(reader) = self.reader.as_ref() { + if reader.is_finished() { + break; + } + let _ = self.replies.recv_timeout(Duration::from_millis(10)); + } + if let Some(reader) = self.reader.take() { + let _ = reader.join(); + } + } +} +impl Drop for Client { + fn drop(&mut self) { + self.terminate(); + } +} +#[derive(Default)] +struct Service { + client: Option, + failed: bool, + generation: u64, + outstanding: Option, + last: Option, + edit: Option, + reset: bool, + accept: Option<(u64, usize)>, + accepting: bool, + case: Option<(bool, bool, bool)>, + tracking: bool, +} +impl Service { + fn fail(&mut self, keyboard: &mut Keyboard) { + self.client = None; + self.failed = true; + self.tracking = false; + self.edit = None; + self.outstanding = None; + keyboard.predictions(None, true); + if self.accepting || self.accept.is_some() { + keyboard.failed(); + } + self.accept = None; + self.accepting = false; + } +} +thread_local! { static SERVICE: RefCell = RefCell::new(Service::default()); } +pub fn stop() { + SERVICE.with(|s| *s.borrow_mut() = Service::default()); +} +pub fn record(stroke: Stroke, success: bool) { + SERVICE.with(|s| { + let mut s = s.borrow_mut(); + s.generation = s.generation.wrapping_add(1); + s.last = None; + if success && !stroke.shortcut() && s.tracking { + if let Some(c) = stroke.character() { + if s.edit + .as_ref() + .is_some_and(|text| text.chars().count() >= 512) + { + s.edit = None; + s.reset = true; + } + s.edit.get_or_insert_with(String::new).push(c); + return; + } + } + s.edit = None; + s.reset = true; + }); +} +pub fn select(token: u64, index: usize) -> Result<(), String> { + SERVICE.with(|s| { + let mut s = s.borrow_mut(); + if s.failed || s.client.is_none() || s.accept.is_some() || s.accepting { + return Err("Prediction is unavailable.".into()); + } + s.accept = Some((token, index)); + Ok(()) + }) +} +fn resource(app: &AppHandle) -> Result { + if cfg!(debug_assertions) { + return Ok(Path::new(env!("CARGO_MANIFEST_DIR")).join("resources/WordData2017051601.db")); + } + app.path() + .resolve( + "resources/WordData2017051601.db", + tauri::path::BaseDirectory::Resource, + ) + .map_err(|_| ()) +} +pub fn poll(app: &AppHandle, keyboard: Option<&mut Keyboard>, enabled: bool, ignored: &[String]) { + let Some(keyboard) = keyboard else { + stop(); + return; + }; + if !enabled { + stop(); + keyboard.predictions(None, false); + return; + } + SERVICE.with(|slot| { + let mut s = slot.borrow_mut(); + let case = ( + keyboard.modifiers[0] != Modifier::Off, + keyboard.caps, + keyboard.modifiers[1..].iter().any(|m| *m != Modifier::Off), + ); + if s.case != Some(case) { + s.case = Some(case); + s.generation = s.generation.wrapping_add(1); + s.last = None; + keyboard.predictions(None, false); + } + if s.failed { + keyboard.predictions(None, true); + return; + } + if s.client.is_none() { + let ignored = ignored + .iter() + .filter_map(|name| crate::switch_input::prediction_key_code(name)) + .collect(); + s.client = resource(app) + .and_then(|path| Client::start(&path, ignored)) + .ok(); + if s.client.is_none() { + s.failed = true; + keyboard.predictions(None, true); + return; + } + } + if s.outstanding + .is_some_and(|t| t.elapsed() >= Duration::from_secs(2)) + { + s.fail(keyboard); + return; + } + let reply = s.client.as_ref().unwrap().replies.try_recv(); + match reply { + Ok(Ok(response)) => { + s.outstanding = None; + match response { + Response::Suggestions { + generation, + batch, + tracking, + } if generation == s.generation => { + s.tracking = tracking; + if !tracking { + s.edit = None; + s.reset = true; + } + keyboard.predictions(batch, false); + } + Response::Insert { generation, text } => { + s.accepting = false; + let result = if generation == s.generation { + text.ok_or(()).and_then(|text| { + crate::point_scan_ready(app).map_err(|_| ())?; + crate::scan_executor::prediction_text(&text).map_err(|_| ())?; + s.edit = Some(text); + s.reset = false; + Ok(()) + }) + } else { + Err(()) + }; + s.generation = s.generation.wrapping_add(1); + s.last = None; + keyboard.predictions(None, false); + if result.is_ok() { + keyboard.succeeded(); + } else { + s.reset = true; + let _ = crate::scan_executor::cleanup(); + keyboard.failed(); + } + } + _ => {} + } + } + Ok(Err(())) | Err(mpsc::TryRecvError::Disconnected) => { + s.fail(keyboard); + return; + } + Err(mpsc::TryRecvError::Empty) => {} + } + if s.outstanding.is_some() { + return; + } + let request = if let Some((token, index)) = s.accept.take() { + s.accepting = true; + Request::Accept { + generation: s.generation, + token, + index, + } + } else { + if keyboard.page != Page::Letters + || keyboard.modifiers[1..].iter().any(|m| *m != Modifier::Off) + { + keyboard.predictions(None, false); + s.tracking = false; + s.edit = None; + s.reset = true; + return; + } + if s.last + .is_some_and(|t| t.elapsed() < Duration::from_millis(250)) + { + return; + } + Request::Query { + generation: s.generation, + edit: s.edit.take(), + reset: std::mem::take(&mut s.reset), + shift: keyboard.modifiers[0] != Modifier::Off, + caps: keyboard.caps, + } + }; + let now = Instant::now(); + s.last = Some(now); + s.outstanding = Some(now); + if worker::send(&mut s.client.as_mut().unwrap().input, &request).is_err() { + s.fail(keyboard); + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn cancellation_kills_and_reaps_a_blocked_worker() { + #[cfg(target_os = "windows")] + let mut command = { + let mut c = Command::new("powershell.exe"); + c.args([ + "-NoProfile", + "-NonInteractive", + "-Command", + "Start-Sleep -Seconds 60", + ]); + c + }; + #[cfg(target_os = "macos")] + let mut command = { + let mut c = Command::new("/bin/sleep"); + c.arg("60"); + c + }; + command.stdin(Stdio::piped()); + let mut client = Client::spawn(command).expect("fake worker starts"); + let start = Instant::now(); + assert!(client + .replies + .recv_timeout(Duration::from_millis(100)) + .is_err()); + client.terminate(); + assert!(client.child.try_wait().unwrap().is_some()); + assert!(start.elapsed() < Duration::from_secs(3)); + } + #[test] + fn bounded_private_frames_reject_invalid_lengths() { + assert!( + worker::receive::(&mut std::io::Cursor::new(u32::MAX.to_le_bytes())).is_err() + ); + assert!(worker::receive::(&mut std::io::Cursor::new([0u8; 4])).is_err()); + } +} diff --git a/src-tauri/src/prediction/windows.rs b/src-tauri/src/prediction/windows.rs new file mode 100644 index 00000000..826954da --- /dev/null +++ b/src-tauri/src/prediction/windows.rs @@ -0,0 +1,249 @@ +//! Read-only UI Automation. All COM objects stay on the worker's MTA thread. +use super::context::{Adapter, RawContext, Status}; +#[derive(Clone, Copy)] +enum ApiPath { + Caret, + Selection, +} +use windows::{ + core::BOOL, + Win32::{ + System::{ + Com::{ + CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, + COINIT_MULTITHREADED, + }, + Variant::{VariantClear, VariantToBoolean, VT_BOOL}, + }, + UI::Accessibility::*, + }, +}; + +struct Apartment; +impl Drop for Apartment { + fn drop(&mut self) { + unsafe { CoUninitialize() }; + } +} + +pub struct WindowsAdapter { + automation: IUIAutomation, + _apartment: Apartment, + last_anchor: Option, + position: i64, +} + +fn provider(r: windows::core::Result) -> Result { + r.map_err(|_| Status::ProviderError) +} + +impl WindowsAdapter { + pub fn new() -> Result { + // The process entry thread has not initialized COM previously. + unsafe { + provider(CoInitializeEx(None, COINIT_MULTITHREADED).ok())?; + let apartment = Apartment; + let automation = + provider(CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER))?; + Ok(Self { + automation, + _apartment: apartment, + last_anchor: None, + position: 0, + }) + } + } + + unsafe fn text_provider( + &self, + focused: &IUIAutomationElement, + ) -> Result<(IUIAutomationElement, IUIAutomationTextPattern), Status> { + unsafe { + if let Ok(pattern) = + focused.GetCurrentPatternAs::(UIA_TextPatternId) + { + return Ok((focused.clone(), pattern)); + } + // Only follow the provider's explicit text-container relationship. A broad + // ancestor search risks capturing an unrelated document or whole window. + let child = focused + .GetCurrentPatternAs::(UIA_TextChildPatternId) + .map_err(|_| Status::Unsupported)?; + let container = provider(child.TextContainer())?; + if provider(container.CurrentIsPassword())?.as_bool() { + return Err(Status::Protected); + } + let pattern = container + .GetCurrentPatternAs::(UIA_TextPatternId) + .map_err(|_| Status::Unsupported)?; + Ok((container, pattern)) + } + } + + unsafe fn anchor( + &self, + element: &IUIAutomationElement, + pattern: &IUIAutomationTextPattern, + ) -> Result<(IUIAutomationTextRange, bool, ApiPath), Status> { + unsafe { + // A selection takes precedence over the caret, whose active end could be + // either end of the selected text. Reject multiple selections outright. + let ranges = provider(pattern.GetSelection())?; + let count = provider(ranges.Length())?; + if count > 1 { + return Err(Status::AmbiguousSelection); + } + if count < 0 { + return Err(Status::ProviderError); + } + let selection = if count == 1 { + Some(provider(ranges.GetElement(0))?) + } else { + None + }; + if let Some(range) = &selection { + if provider(range.CompareEndpoints( + TextPatternRangeEndpoint_Start, + range, + TextPatternRangeEndpoint_End, + ))? != 0 + { + return Ok((provider(range.Clone())?, true, ApiPath::Selection)); + } + } + if let Ok(pattern2) = + element.GetCurrentPatternAs::(UIA_TextPattern2Id) + { + let mut active = BOOL(0); + if let Ok(caret) = pattern2.GetCaretRange(&mut active) { + if active.as_bool() { + if provider(caret.CompareEndpoints( + TextPatternRangeEndpoint_Start, + &caret, + TextPatternRangeEndpoint_End, + ))? != 0 + { + return Err(Status::ProviderError); + } + return Ok((caret, false, ApiPath::Caret)); + } + } + } + selection + .map(|s| (s, false, ApiPath::Selection)) + .ok_or(Status::Unsupported) + } + } +} + +impl Adapter for WindowsAdapter { + type Target = IUIAutomationElement; + fn focused(&mut self) -> Result { + unsafe { + self.automation + .GetFocusedElement() + .map_err(|_| Status::NoFocus) + } + } + fn protected(&mut self, target: &Self::Target) -> Result { + unsafe { Ok(provider(target.CurrentIsPassword())?.as_bool()) } + } + fn editable(&mut self, target: &Self::Target) -> bool { + unsafe { + target + .CurrentControlType() + .is_ok_and(|c| c == UIA_EditControlTypeId) + && target.CurrentIsEnabled().is_ok_and(|v| v.as_bool()) + && target + .GetCurrentPatternAs::(UIA_ValuePatternId) + .and_then(|p| p.CurrentIsReadOnly()) + .is_ok_and(|v| !v.as_bool()) + } + } + fn same(&mut self, a: &Self::Target, b: &Self::Target) -> Result { + unsafe { Ok(provider(self.automation.CompareElements(a, b))?.as_bool()) } + } + fn read(&mut self, target: &Self::Target) -> Result { + unsafe { + let (element, pattern) = self.text_provider(target)?; + if provider(element.CurrentIsPassword())?.as_bool() { + return Err(Status::Protected); + } + let (anchor, has_selection, _path) = self.anchor(&element, &pattern)?; + let mut value = provider(anchor.GetAttributeValue(UIA_IsReadOnlyAttributeId))?; + if value.Anonymous.Anonymous.vt != VT_BOOL { + let _ = VariantClear(&mut value); + return Err(Status::Unsupported); + } + let readonly = VariantToBoolean(&value); + let _ = VariantClear(&mut value); + if readonly.map_err(|_| Status::Unsupported)?.as_bool() { + return Err(Status::Unsupported); + } + // Range manipulation is local to the UIA range object; no Select() or + // mutation API is ever used, and nothing is copied to the clipboard. + let range = provider(anchor.Clone())?; + provider(range.MoveEndpointByRange( + TextPatternRangeEndpoint_End, + &anchor, + TextPatternRangeEndpoint_Start, + ))?; + let moved = provider(range.MoveEndpointByUnit( + TextPatternRangeEndpoint_Start, + TextUnit_Character, + -512, + ))?; + if !(-512..=0).contains(&moved) { + return Err(Status::ProviderError); + } + let document = provider(pattern.DocumentRange())?; + let clipped_start = provider(range.CompareEndpoints( + TextPatternRangeEndpoint_Start, + &document, + TextPatternRangeEndpoint_Start, + ))? > 0; + // Providers differ in their definition of a Character unit. Read a + // bounded buffer and reject possible truncation rather than return + // text from the wrong end of the range. + let text = provider(range.GetText(2048))?; + if text.len() >= 2048 { + return Err(Status::Unsupported); + } + let text = String::from_utf16(&text).map_err(|_| Status::ProviderError)?; + let chars: Vec = text.chars().collect(); + let skip = chars.len().saturating_sub(512); + if self.last_anchor.as_ref().is_none_or(|old| { + anchor.CompareEndpoints( + TextPatternRangeEndpoint_Start, + old, + TextPatternRangeEndpoint_Start, + ) != Ok(0) + }) { + self.position = self.position.wrapping_add(1); + } + self.last_anchor = Some(provider(anchor.Clone())?); + let right = provider(anchor.Clone())?; + provider(right.MoveEndpointByRange( + TextPatternRangeEndpoint_End, + &anchor, + TextPatternRangeEndpoint_Start, + ))?; + provider(right.MoveEndpointByUnit( + TextPatternRangeEndpoint_End, + TextUnit_Character, + 2, + ))?; + let after = provider(right.GetText(16))?; + if after.len() >= 16 { + return Err(Status::Unsupported); + } + Ok(RawContext { + before: chars[skip..].iter().collect(), + clipped_start: clipped_start || skip > 0, + has_selection, + after: String::from_utf16(&after).map_err(|_| Status::ProviderError)?, + position: self.position, + }) + } + } +} diff --git a/src-tauri/src/prediction/worker.rs b/src-tauri/src/prediction/worker.rs new file mode 100644 index 00000000..6cd4a909 --- /dev/null +++ b/src-tauri/src/prediction/worker.rs @@ -0,0 +1,556 @@ +use super::{ + activity, + context::{self, Adapter, RawContext, Status}, + database::Database, +}; +use serde::{Deserialize, Serialize}; +use std::{ + io::{Read, Write}, + path::PathBuf, +}; +pub const ARG: &str = "--switchify-prediction-worker"; +pub const LIMIT: usize = 16384; + +// Private inherited-pipe payloads. Never log these or forward them to Tauri. +#[derive(Serialize, Deserialize)] +pub enum Request { + Query { + generation: u64, + edit: Option, + reset: bool, + shift: bool, + caps: bool, + }, + Accept { + generation: u64, + token: u64, + index: usize, + }, +} +#[derive(Clone, Serialize, Deserialize)] +pub struct Batch { + pub token: u64, + pub words: Vec, +} +#[derive(Serialize, Deserialize)] +pub enum Response { + Suggestions { + generation: u64, + batch: Option, + tracking: bool, + }, + Insert { + generation: u64, + text: Option, + }, +} +pub fn send(writer: &mut impl Write, value: &T) -> Result<(), ()> { + let bytes = serde_json::to_vec(value).map_err(|_| ())?; + if bytes.len() > LIMIT { + return Err(()); + } + writer + .write_all(&(bytes.len() as u32).to_le_bytes()) + .map_err(|_| ())?; + writer.write_all(&bytes).map_err(|_| ())?; + writer.flush().map_err(|_| ()) +} +pub fn receive(reader: &mut impl Read) -> Result { + let mut size = [0; 4]; + reader.read_exact(&mut size).map_err(|_| ())?; + let size = u32::from_le_bytes(size) as usize; + if size == 0 || size > LIMIT { + return Err(()); + } + let mut bytes = vec![0; size]; + reader.read_exact(&mut bytes).map_err(|_| ())?; + serde_json::from_slice(&bytes).map_err(|_| ()) +} +pub struct Engine { + adapter: A, + database: Database, + target: Option, + snapshot: Option, + fallback: String, + boundary: bool, + tracked: bool, + activity: u64, + observe: fn() -> (u64, bool), + batch: Option, + suffixes: Vec, + token: u64, + case: (bool, bool), +} +impl Engine { + pub fn new(adapter: A, database: Database, tracked: bool) -> Self { + Self { + adapter, + database, + tracked, + target: None, + snapshot: None, + fallback: String::new(), + boundary: false, + activity: activity::snapshot().0, + observe: activity::snapshot, + batch: None, + suffixes: Vec::new(), + token: 0, + case: (false, false), + } + } + fn clear(&mut self) { + self.fallback.clear(); + self.boundary = false; + self.snapshot = None; + self.batch = None; + self.suffixes.clear(); + } + fn target(&mut self) -> Result { + let target = self.adapter.focused()?; + if self.adapter.protected(&target)? { + return Err(Status::Protected); + } + Ok(target) + } + fn stable(&mut self, target: &A::Target, epoch: u64) -> bool { + self.target() + .is_ok_and(|current| self.adapter.same(target, ¤t).unwrap_or(false)) + && (self.observe)().0 == epoch + } + fn query( + &mut self, + edit: Option, + reset: bool, + shift: bool, + caps: bool, + ) -> Option { + let target = match self.target() { + Ok(t) => t, + Err(_) => { + self.clear(); + self.target = None; + return None; + } + }; + let (epoch, healthy) = (self.observe)(); + let same = self + .target + .as_ref() + .is_some_and(|old| self.adapter.same(old, &target).unwrap_or(false)); + let uninterrupted = epoch == self.activity; + if !same || !uninterrupted || reset || !healthy { + self.clear(); + } + self.activity = epoch; + if same && uninterrupted && !reset && self.tracked && healthy { + if let Some(text) = edit { + if text.chars().count() > 512 { + self.clear(); + } else { + for c in text.chars() { + if c.is_whitespace() || matches!(c, '.' | '!' | '?' | ',' | ';' | ':') { + self.boundary = true; + } + if self.boundary { + self.fallback.push(c); + } + } + self.fallback = self + .fallback + .chars() + .rev() + .take(512) + .collect::>() + .into_iter() + .rev() + .collect(); + } + } + } + let raw = match self.adapter.read(&target) { + Ok(raw) if context::eligible(&raw) => raw, + Err(Status::Unsupported) + if self.tracked && healthy && self.boundary && self.adapter.editable(&target) => + { + RawContext { + before: self.fallback.clone(), + after: String::new(), + clipped_start: true, + has_selection: false, + position: -1, + } + } + _ => { + self.clear(); + self.target = Some(target); + return None; + } + }; + if !self.stable(&target, epoch) { + self.clear(); + return None; + } + self.target = Some(target); + if self.snapshot.as_ref() == Some(&raw) && self.case == (shift, caps) { + return self.batch.clone(); + } + let ctx = context::extract(&raw.before, raw.clipped_start); + let words = match self.database.predict(&ctx) { + Ok(w) => w, + Err(_) => { + self.clear(); + return None; + } + }; + self.token = self.token.wrapping_add(1); + self.suffixes.clear(); + let mut labels = Vec::new(); + for word in words { + let offset = word + .char_indices() + .nth(ctx.prefix.chars().count()) + .map_or(word.len(), |(i, _)| i); + if word[..offset].to_lowercase() != ctx.prefix.to_lowercase() { + continue; + } + let mut suffix = word[offset..].to_owned(); + if caps ^ shift { + suffix = suffix.to_uppercase(); + } else if ctx.prefix.is_empty() && ctx.sentence_start { + let mut chars = suffix.chars(); + if let Some(c) = chars.next() { + suffix = c.to_uppercase().collect::() + chars.as_str(); + } + } + labels.push(ctx.prefix.clone() + &suffix); + self.suffixes.push(suffix + " "); + } + self.case = (shift, caps); + self.snapshot = Some(raw); + self.batch = Some(Batch { + token: self.token, + words: labels, + }); + self.batch.clone() + } + fn accept(&mut self, token: u64, index: usize) -> Option { + if self.batch.as_ref()?.token != token || index >= self.suffixes.len() { + return None; + } + let target = match self.target() { + Ok(t) => t, + Err(_) => { + self.clear(); + self.target = None; + return None; + } + }; + if !self + .target + .as_ref() + .is_some_and(|old| self.adapter.same(old, &target).unwrap_or(false)) + || self.activity != (self.observe)().0 + { + self.clear(); + return None; + } + let expected = self.snapshot.as_ref()?; + let valid = if expected.position == -1 { + self.tracked + && (self.observe)().1 + && self.boundary + && self.adapter.editable(&target) + && matches!(self.adapter.read(&target), Err(Status::Unsupported)) + } else { + self.adapter + .read(&target) + .is_ok_and(|raw| &raw == expected && context::eligible(&raw)) + }; + if !valid || !self.stable(&target, self.activity) { + self.clear(); + return None; + } + let suffix = self.suffixes[index].clone(); + self.batch = None; + Some(suffix) + } + pub fn respond(&mut self, request: Request) -> Response { + match request { + Request::Query { + generation, + edit, + reset, + shift, + caps, + } => { + let mut batch = self.query(edit, reset, shift, caps); + let current = self.target(); + let stable = current.as_ref().is_ok_and(|current| { + self.target + .as_ref() + .is_some_and(|old| self.adapter.same(old, current).unwrap_or(false)) + }); + if !stable { + self.clear(); + batch = None; + } + let tracking = stable + && self.tracked + && (self.observe)().1 + && current.is_ok_and(|target| self.adapter.editable(&target)); + Response::Suggestions { + generation, + batch, + tracking, + } + } + Request::Accept { + generation, + token, + index, + } => Response::Insert { + generation, + text: self.accept(token, index), + }, + } + } +} +pub fn run_from_args() -> bool { + let args: Vec<_> = std::env::args_os().collect(); + if args.get(1).is_none_or(|s| s != ARG) { + return false; + } + let work = || -> Result<(), ()> { + if args.len() != 4 { + return Err(()); + } + let path = PathBuf::from(&args[2]); + let ignored: Vec = serde_json::from_str(&args[3].to_string_lossy()).map_err(|_| ())?; + if ignored.len() > 129 { + return Err(()); + } + #[cfg(target_os = "macos")] + { + let parent = unsafe { libc::getppid() }; + std::thread::spawn(move || loop { + std::thread::sleep(std::time::Duration::from_millis(250)); + if unsafe { libc::getppid() } != parent { + std::process::exit(0); + } + }); + } + let tracked = activity::start(ignored); + #[cfg(target_os = "windows")] + let adapter = super::windows::WindowsAdapter::new().map_err(|_| ())?; + #[cfg(target_os = "macos")] + let adapter = super::macos::MacAdapter::new().map_err(|_| ())?; + #[cfg(any(target_os = "windows", target_os = "macos"))] + { + let database = Database::open(&path)?; + let mut engine = Engine::new(adapter, database, tracked); + let mut input = std::io::stdin().lock(); + let mut output = std::io::stdout().lock(); + while let Ok(request) = receive::(&mut input) { + send(&mut output, &engine.respond(request))?; + } + } + Ok(()) + }; + let _ = work(); + true +} + +#[cfg(test)] +mod tests { + use super::*; + struct Fake { + raw: RawContext, + target: usize, + protected: bool, + unsupported: bool, + reads: usize, + change_on_read: bool, + } + impl Fake { + fn new() -> Self { + Self { + raw: RawContext { + before: "I like wa".into(), + after: String::new(), + clipped_start: false, + has_selection: false, + position: 9, + }, + target: 1, + protected: false, + unsupported: false, + reads: 0, + change_on_read: false, + } + } + } + impl Adapter for Fake { + type Target = usize; + fn focused(&mut self) -> Result { + Ok(self.target) + } + fn protected(&mut self, _: &usize) -> Result { + Ok(self.protected) + } + fn editable(&mut self, _: &usize) -> bool { + true + } + fn same(&mut self, a: &usize, b: &usize) -> Result { + Ok(a == b) + } + fn read(&mut self, _: &usize) -> Result { + self.reads += 1; + if self.change_on_read { + self.target += 1; + } + if self.unsupported { + Err(Status::Unsupported) + } else { + Ok(self.raw.clone()) + } + } + } + fn engine() -> Engine { + let mut e = Engine::new(Fake::new(), Database::fixture(), true); + e.observe = || (0, true); + e.activity = 0; + e + } + #[test] + fn only_suffix_and_space_are_returned_once() { + let mut e = engine(); + let b = e.query(None, false, false, false).unwrap(); + assert_eq!(b.words[0], "water"); + assert_eq!(e.accept(b.token, 0), Some("ter ".into())); + assert!(e.accept(b.token, 0).is_none()); + } + #[test] + fn protected_fields_are_not_read_and_focus_races_discard_results() { + let mut e = engine(); + e.adapter.protected = true; + assert!(e.query(None, false, false, false).is_none()); + assert_eq!(e.adapter.reads, 0); + e.adapter.protected = false; + e.adapter.change_on_read = true; + assert!(e.query(None, false, false, false).is_none()); + } + #[test] + fn selection_middle_word_and_stale_acceptance_are_rejected() { + let mut e = engine(); + e.adapter.raw.has_selection = true; + assert!(e.query(None, false, false, false).is_none()); + e.adapter.raw.has_selection = false; + e.adapter.raw.after = "lk".into(); + assert!(e.query(None, false, false, false).is_none()); + e.adapter.raw.after.clear(); + let b = e.query(None, false, false, false).unwrap(); + e.adapter.raw.position += 1; + assert!(e.accept(b.token, 0).is_none()); + } + #[test] + fn fallback_requires_boundary_and_resets_on_edits_and_focus() { + let mut e = engine(); + e.adapter.unsupported = true; + assert!(e.query(None, false, false, false).is_none()); + assert!(e.query(Some("wa".into()), false, false, false).is_none()); + let b = e.query(Some(" wa".into()), false, false, false).unwrap(); + assert_eq!(e.accept(b.token, 0), Some("ter ".into())); + assert!(e.query(None, true, false, false).is_none()); + e.query(Some(" wa".into()), false, false, false).unwrap(); + e.adapter.target = 2; + assert!(e.query(None, false, false, false).is_none()); + } + #[test] + fn external_activity_discards_edits_queued_before_the_change() { + let mut e = engine(); + e.adapter.unsupported = true; + assert!(e.query(None, false, false, false).is_none()); + e.query(Some(" wa".into()), false, false, false).unwrap(); + e.activity = 0u64.wrapping_sub(1); + assert!(e.query(Some(" wa".into()), false, false, false).is_none()); + assert!(e.fallback.is_empty()); + assert!(!e.boundary); + } + #[test] + fn lost_observer_disables_fallback_query_and_acceptance() { + let mut e = engine(); + e.adapter.unsupported = true; + e.query(None, false, false, false); + let batch = e.query(Some(" wa".into()), false, false, false).unwrap(); + e.observe = || (0, false); + assert!(e.accept(batch.token, 0).is_none()); + assert!(e.query(Some(" wa".into()), false, false, false).is_none()); + assert!(!e.boundary); + assert!(matches!( + e.respond(Request::Query { + generation: 1, + edit: None, + reset: false, + shift: false, + caps: false, + }), + Response::Suggestions { + tracking: false, + batch: None, + .. + } + )); + } + #[test] + fn casing_and_unchanged_snapshots_keep_choice_identity() { + let mut e = engine(); + e.adapter.raw.before = "Wa".into(); + let b = e.query(None, false, false, false).unwrap(); + assert_eq!(b.words[0], "Water"); + assert_eq!(e.query(None, false, false, false).unwrap().token, b.token); + let upper = e.query(None, false, false, true).unwrap(); + assert_ne!(upper.token, b.token); + assert_eq!(upper.words[0], "WaTER"); + assert!(e.accept(b.token, 0).is_none()); + } + #[test] + fn malformed_and_oversized_private_frames_fail_closed() { + assert!(receive::(&mut &b"bad!"[..]).is_err()); + let request = Request::Query { + generation: 1, + edit: Some("x".repeat(LIMIT)), + reset: false, + shift: false, + caps: false, + }; + assert!(send(&mut Vec::new(), &request).is_err()); + } + #[test] + #[ignore = "Manual database performance measurement; no desktop input"] + fn bundled_database_benchmark() { + let start = std::time::Instant::now(); + let db = Database::open( + &std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("resources/WordData2017051601.db"), + ) + .unwrap(); + let startup = start.elapsed().as_secs_f64() * 1000.0; + let mut times = Vec::new(); + for i in 0..200 { + let text = [ + "I would like wa", + "the", + "hello ", + "I would like some ", + "zzzx", + ][i % 5]; + let ctx = context::extract(text, false); + let t = std::time::Instant::now(); + assert!(db.predict(&ctx).is_ok()); + times.push(t.elapsed().as_secs_f64() * 1000.0); + } + times.sort_by(f64::total_cmp); + println!("database startup_ms={startup:.2} samples=200 failures=0 median_ms={:.2} p95_ms={:.2} max_ms={:.2}",times[99],times[189],times[199]); + } +} diff --git a/src-tauri/src/scan_executor.rs b/src-tauri/src/scan_executor.rs index 988699ff..e93be3d5 100644 --- a/src-tauri/src/scan_executor.rs +++ b/src-tauri/src/scan_executor.rs @@ -15,6 +15,7 @@ pub fn execute( ); } match request { + Request::Prediction { .. } => Err("Prediction requires the scan controller.".into()), Request::OpenKeyboard { point } => { if input.has_active_drag() { return Err("End the active drag before typing.".into()); @@ -110,6 +111,20 @@ pub fn activate(request: Request) -> Result<(), String> { result }) } +pub fn prediction_text(text: &str) -> Result<(), String> { + if text.chars().count() > 64 || !crate::scan_host::modifiers_released() { + return Err("Prediction input is unavailable.".into()); + } + INPUT.with(|slot| { + let mut slot = slot.borrow_mut(); + let input = slot.as_mut().ok_or("Prediction input is unavailable.")?; + if input.has_active_drag() || input.has_active_switch_session() { + return Err("Prediction input is unavailable.".into()); + } + input.release_all()?; + input.type_text(text) + }) +} pub fn cleanup() -> Result<(), String> { INPUT.with(|slot| { slot.borrow_mut() diff --git a/src-tauri/src/scan_keyboard.rs b/src-tauri/src/scan_keyboard.rs index c29fbf45..4d87558f 100644 --- a/src-tauri/src/scan_keyboard.rs +++ b/src-tauri/src/scan_keyboard.rs @@ -39,6 +39,7 @@ impl Modifier { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Key { + Prediction(usize), Character(char, char), Named(&'static str), Modifier(usize), @@ -72,6 +73,7 @@ impl Stroke { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Output { + Prediction { token: u64, index: usize }, Stroke(Stroke), Close, } @@ -184,6 +186,11 @@ pub struct Keyboard { cycles: usize, forward: bool, pending: bool, + prediction_enabled: bool, + prediction_failed: bool, + predictions: Option, + queued_predictions: Option, + prefer_predictions: bool, } impl Keyboard { pub fn new(mac: bool) -> Self { @@ -203,6 +210,83 @@ impl Keyboard { cycles: 0, forward: true, pending: false, + prediction_enabled: false, + prediction_failed: false, + predictions: None, + queued_predictions: None, + prefer_predictions: true, + } + } + pub fn enable_predictions(&mut self, enabled: bool) { + if self.prediction_enabled == enabled { + return; + } + self.prediction_enabled = enabled; + self.rebuild_rows(); + } + fn rebuild_rows(&mut self) { + self.rows = rows(self.page, self.mac); + if self.prediction_enabled && self.page == Page::Letters { + self.rows.insert(0, (0..5).map(Key::Prediction).collect()); + } + self.nav = Self::navigator(&self.rows); + self.skip_disabled(); + } + fn prediction_row_active(&self) -> bool { + self.prediction_enabled + && self.page == Page::Letters + && self.nav.path().first().copied().unwrap_or(self.nav.index()) == 0 + } + pub fn predictions(&mut self, batch: Option, failed: bool) { + self.prediction_failed = failed; + if batch.is_none() { + self.predictions = None; + self.queued_predictions = None; + self.skip_disabled(); + return; + } + if self.prediction_row_active() && self.predictions.is_some() { + if self.predictions.as_ref().map(|b| b.token) != batch.as_ref().map(|b| b.token) { + self.predictions = None; + self.queued_predictions = batch; + self.skip_disabled(); + } + } else { + self.predictions = batch; + if self.prefer_predictions + && self + .predictions + .as_ref() + .is_some_and(|b| !b.words.is_empty()) + { + self.nav.reset(); + self.interval.reset(); + self.prefer_predictions = false; + } + } + } + fn disabled(&self) -> bool { + if !self.prediction_row_active() || self.nav.escaping() { + return false; + } + let count = self.predictions.as_ref().map_or(0, |b| b.words.len()); + if self.nav.path().is_empty() { + count == 0 + } else { + self.nav.index() >= count + } + } + fn skip_disabled(&mut self) { + for _ in 0..8 { + if !self.disabled() { + break; + } + self.nav.step(self.forward); + } + if !self.prediction_row_active() { + if let Some(batch) = self.queued_predictions.take() { + self.predictions = Some(batch); + } } } fn navigator(rows: &[Vec]) -> Navigator { @@ -213,21 +297,23 @@ impl Keyboard { ) } fn restart(&mut self) { + self.prefer_predictions = true; self.nav.reset(); self.interval.reset(); self.cycles = 0; self.forward = true; self.suspended = false; + self.skip_disabled(); } pub fn advance(&mut self, ms: u64, period: u64) { - if !self.pending - && !self.suspended - && self.interval.elapsed(ms, period) - && self.nav.step(self.forward) - { - self.cycles += 1; - self.suspended = self.cycles >= MAX_SCAN_CYCLES; + if !self.pending && !self.suspended && self.interval.elapsed(ms, period) { + self.prefer_predictions = false; + if self.nav.step(self.forward) { + self.cycles += 1; + self.suspended = self.cycles >= MAX_SCAN_CYCLES; + } } + self.skip_disabled(); } pub fn handle(&mut self, action: Action) -> Option { if self.pending { @@ -240,6 +326,7 @@ impl Keyboard { } return None; } + self.prefer_predictions = false; match action { Action::Select => { self.interval.reset(); @@ -263,20 +350,33 @@ impl Keyboard { } _ => {} } + self.skip_disabled(); None } fn choose(&mut self, key: Key) -> Option { match key { + Key::Prediction(index) => { + let batch = self.predictions.as_ref()?; + if index >= batch.words.len() { + return None; + } + self.pending = true; + return Some(Output::Prediction { + token: batch.token, + index, + }); + } Key::Modifier(i) => self.modifiers[i] = self.modifiers[i].next(), Key::Caps => self.caps = !self.caps, Key::Dock => self.top = !self.top, Key::Page(page) => { self.page = page; - self.rows = rows(page, self.mac); - self.nav = Self::navigator(&self.rows); + self.rebuild_rows(); } Key::Close => return Some(Output::Close), Key::Character(..) | Key::Named(_) => { + self.predictions = None; + self.queued_predictions = None; self.pending = true; return Some(Output::Stroke(Stroke { key, @@ -309,6 +409,12 @@ impl Keyboard { } fn label(&self, key: Key) -> String { match key { + Key::Prediction(i) => self + .predictions + .as_ref() + .and_then(|b| b.words.get(i)) + .cloned() + .unwrap_or_default(), Key::Character(' ', _) => "Space".into(), Key::Character(..) => Stroke { key, @@ -467,7 +573,9 @@ impl Keyboard { Page::Functions => "Navigation", Page::Numbers => "Numbers", }; - let text = if self.error { + let text = if self.prediction_failed { + "Predictions unavailable · Keyboard ready".to_owned() + } else if self.error { "Input failed · Select to try again".to_owned() } else if self.suspended { "Keyboard paused · Select to resume".to_owned() @@ -509,6 +617,62 @@ impl Keyboard { mod tests { use super::*; #[test] + fn arriving_predictions_receive_a_full_scan_interval() { + let mut keyboard = Keyboard::new(false); + keyboard.enable_predictions(true); + keyboard.advance(490, 500); + assert_eq!(keyboard.nav.index(), 1); + let batch = crate::prediction::worker::Batch { + token: 1, + words: vec!["hello".into()], + }; + keyboard.predictions(Some(batch.clone()), false); + assert_eq!(keyboard.nav.index(), 0); + keyboard.advance(10, 500); + assert_eq!(keyboard.nav.index(), 0); + keyboard.predictions(Some(batch), false); + keyboard.advance(489, 500); + assert_eq!(keyboard.nav.index(), 0); + keyboard.advance(1, 500); + assert_eq!(keyboard.nav.index(), 1); + } + + #[test] + fn predictions_skip_empty_slots_and_defer_acceptance() { + let mut k = Keyboard::new(false); + k.enable_predictions(true); + assert_eq!(k.nav.index(), 1); + k.predictions( + Some(crate::prediction::worker::Batch { + token: 7, + words: vec!["water".into(), "walk".into()], + }), + false, + ); + k.restart(); + assert_eq!(k.nav.index(), 0); + assert_eq!(k.handle(Action::Select), None); + assert_eq!( + k.handle(Action::Select), + Some(Output::Prediction { token: 7, index: 0 }) + ); + assert!(k.handle(Action::Select).is_none()); + k.succeeded(); + assert!(k.nav.path().is_empty()); + k.handle(Action::Select); + k.handle(Action::Next); + k.handle(Action::Next); + assert!(k.nav.escaping()); + k.predictions(None, false); + assert!(k.choose(Key::Prediction(0)).is_none()); + k.choose(Key::Page(Page::Numbers)); + assert!(!k + .rows + .iter() + .flatten() + .any(|k| matches!(k, Key::Prediction(_)))); + } + #[test] fn rounded_panel_has_clear_corners_and_insets_all_controls() { let frame = Keyboard::new(false).frame( Rect { diff --git a/src-tauri/src/scanning_runtime.rs b/src-tauri/src/scanning_runtime.rs index f8cf7471..6d4fa9c0 100644 --- a/src-tauri/src/scanning_runtime.rs +++ b/src-tauri/src/scanning_runtime.rs @@ -10,6 +10,13 @@ pub trait Adapter: Send + Sync + 'static { type Environment: Clone + Send; const EVENT: &'static str; const FILE: &'static str; + fn deferred(_request: &::Selection) -> bool { + false + } + fn preserve_visuals(_request: &::Selection) -> bool { + false + } + fn poll(_app: &AppHandle, _technique: &mut Self::Technique, _captured_keys: &[String]) {} fn cleanup(app: &AppHandle) -> Result<(), String>; fn validate(config: &Self::Config) -> Result<(), String>; fn switches(config: &Self::Config) -> SwitchSettings; @@ -351,6 +358,13 @@ fn ensure(app: &AppHandle) { } } } +fn prediction_captured_keys(remote: bool, settings: &Settings) -> Vec { + let mut keys = vec!["Escape".to_owned()]; + if !remote { + keys.extend(settings.bindings.iter().map(|b| b.key.clone())); + } + keys +} fn source_is_current(remote: bool, remote_current: bool, local_current: bool) -> bool { if remote { remote_current @@ -433,7 +447,10 @@ fn dispatch( if !c.enabled.load(Ordering::SeqCst) || !input_active(app, input_generation, remote) { return Err("Scan action was cancelled.".into()); } - hide_scan_visuals(); + if !A::preserve_visuals(&request) { + hide_scan_visuals(); + } + let deferred = A::deferred(&request); match A::activate(app, request) { Ok(environment) => { let mut d = c.data.lock().unwrap_or_else(|p| p.into_inner()); @@ -441,7 +458,9 @@ fn dispatch( d.display = Some(environment); } if let Some(engine) = d.engine.as_mut() { - engine.technique.execution_succeeded(); + if !deferred { + engine.technique.execution_succeeded(); + } } } Err(error) => { @@ -510,6 +529,26 @@ fn hide_scan_visuals() { }) }); } +fn update_tiles( + previous: &[crate::scanning::FrameTile], + tiles: &[crate::scanning::FrameTile], + host_count: usize, + mut present: impl FnMut(usize, Option<&crate::scanning::FrameTile>) -> Result<(), String>, +) -> Result<(), String> { + let background_changed = tiles.iter().enumerate().any(|(index, tile)| { + tile.keyboard + .is_some_and(|s| s.role == crate::scanning::KeyboardRole::Background) + && previous.get(index) != Some(tile) + }); + for index in 0..host_count { + let tile = tiles.get(index); + if !background_changed && previous.get(index) == tile { + continue; + } + present(index, tile)?; + } + Ok(()) +} fn render_tiles(tiles: &[crate::scanning::FrameTile]) -> Result<(), String> { TILES.with(|slot| { let mut slot = slot.borrow_mut(); @@ -520,20 +559,14 @@ fn render_tiles(tiles: &[crate::scanning::FrameTile]) -> Result<(), String> { slot.0.push(Host::new()?); } let (hosts, previous) = &mut *slot; - for (index, host) in hosts.iter_mut().enumerate() { - if let Some(tile) = tiles.get(index) { - if tile - .keyboard - .is_some_and(|s| s.role == crate::scanning::KeyboardRole::Background) - && previous.get(index) == Some(tile) - { - continue; - } - host.tile(tile)?; + update_tiles(previous, tiles, hosts.len(), |index, tile| { + if let Some(tile) = tile { + hosts[index].tile(tile) } else { - host.hide(); + hosts[index].hide(); + Ok(()) } - } + })?; slot.1 = tiles.to_vec(); Ok(()) }) @@ -762,7 +795,9 @@ fn tick(app: &AppHandle) { let held = d.pressed.held(); let prompt = d.pressed.prompt(now_ms); let mut request = None; + let captured_keys = prediction_captured_keys(d.remote, &d.switches); let phase_changed = if let Some(engine) = d.engine.as_mut() { + A::poll(app, &mut engine.technique, &captured_keys); let before = engine.technique.phase(); engine.tick(elapsed, held); request = engine.take_selection(); @@ -954,6 +989,57 @@ pub fn restart_point_on_display(app: &AppHandle, next: bool) -> Result<(), Strin #[cfg(test)] mod ownership_tests { + #[test] + fn keyboard_updates_preserve_unchanged_tiles_and_restore_background_order() { + let keyboard = crate::scan_keyboard::Keyboard::new(false); + let frame = keyboard.frame( + crate::scanning::Rect { + x: 0.0, + y: 0.0, + width: 1280.0, + height: 720.0, + }, + 1.0, + crate::scanning::ScannerColor::default(), + ); + let previous = frame.tiles; + let mut current = previous.clone(); + current[1].text = "updated".into(); + let mut updates = Vec::new(); + super::update_tiles(&previous, ¤t, current.len(), |index, tile| { + updates.push((index, tile.is_some())); + Ok(()) + }) + .unwrap(); + assert_eq!(updates, vec![(1, true)]); + updates.clear(); + super::update_tiles(¤t, ¤t, current.len(), |index, _| { + updates.push((index, true)); + Ok(()) + }) + .unwrap(); + assert!(updates.is_empty()); + current[0].rect.x += 10.0; + super::update_tiles(&previous, ¤t, current.len(), |index, _| { + updates.push((index, true)); + Ok(()) + }) + .unwrap(); + assert_eq!(updates.len(), current.len()); + assert_eq!(updates[0], (0, true)); + updates.clear(); + super::update_tiles(¤t, &[], current.len(), |index, tile| { + updates.push((index, tile.is_some())); + Ok(()) + }) + .unwrap(); + assert_eq!(updates.len(), current.len()); + assert!(updates.iter().all(|(_, visible)| !visible)); + assert!( + super::update_tiles(&[], ¤t, current.len(), |_, _| Err("failed".into())).is_err() + ); + } + #[test] fn failed_partial_presentations_are_hidden_even_with_empty_caches() { let mut tiles = (vec![true, true, false], Vec::::new()); @@ -969,6 +1055,22 @@ mod ownership_tests { assert_eq!(countdown, (Some(false), None)); } + #[test] + fn prediction_ignores_only_keys_captured_by_the_current_source() { + use super::{prediction_captured_keys, Action, Settings}; + let settings = Settings { + bindings: vec![crate::switches::Binding { + id: "custom".into(), + name: "Select".into(), + key: "F9".into(), + press_action: Action::Select, + hold_actions: vec![], + }], + ..Settings::default() + }; + assert_eq!(prediction_captured_keys(false, &settings), ["Escape", "F9"]); + assert_eq!(prediction_captured_keys(true, &settings), ["Escape"]); + } #[test] fn stopped_remote_input_cannot_fall_back_to_a_matching_local_generation() { assert!(!super::source_is_current(true, false, true)); diff --git a/src-tauri/src/switch_input/mod.rs b/src-tauri/src/switch_input/mod.rs index c5468ae9..96fc51ac 100644 --- a/src-tauri/src/switch_input/mod.rs +++ b/src-tauri/src/switch_input/mod.rs @@ -16,6 +16,21 @@ mod windows_worker; mod windows_worker_protocol; use anyhow::{bail, Result}; pub use keys::normalize as normalize_key; +pub fn prediction_key_code(name: &str) -> Option { + #[cfg(target_os = "windows")] + { + windows::code_for_name(name) + } + #[cfg(target_os = "macos")] + { + macos::code_for_name(name).map(u32::from) + } + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + { + let _ = name; + None + } +} #[cfg(target_os = "windows")] pub fn run_worker_from_args() -> bool { windows_worker::run_from_args() diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index a4ec8f69..e0728393 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -34,6 +34,7 @@ } }, "bundle": { + "resources": ["resources/WordData2017051601.db"], "active": true, "targets": "all", "icon": [ diff --git a/src/scanning/useScanning.ts b/src/scanning/useScanning.ts index 0fe324ab..69a59bfa 100644 --- a/src/scanning/useScanning.ts +++ b/src/scanning/useScanning.ts @@ -3,6 +3,7 @@ import { invoke } from "@tauri-apps/api/core"; import { listen } from "@tauri-apps/api/event"; export type ScannerColor = "red" | "green" | "blue" | "yellow" | "white"; export type PointScanConfig = { + wordPrediction: boolean; scannerColor: ScannerColor; mode: "line" | "grid"; automatic: boolean; @@ -26,6 +27,7 @@ export type PointScanState = { supported: boolean; }; export const defaultPointScanConfig: PointScanConfig = { + wordPrediction: true, scannerColor: "blue", mode: "line", automatic: true, diff --git a/src/settings/ScanningSection.tsx b/src/settings/ScanningSection.tsx index 4d89122e..7c077320 100644 --- a/src/settings/ScanningSection.tsx +++ b/src/settings/ScanningSection.tsx @@ -120,6 +120,7 @@ export function ScanningSection({ + update("wordPrediction", value)} /> update("autoSelectEnabled", value)} /> {config.autoSelectEnabled &&