diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13040722..d908d693 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,7 @@ jobs: cargo audit --file src-tauri/Cargo.lock cargo audit --file src-tauri/startup-launcher/Cargo.lock cargo audit --file tools/updater-signature-verifier/Cargo.lock + cargo audit --file tools/prediction-data/Cargo.lock frontend: runs-on: ubuntu-latest @@ -115,6 +116,10 @@ jobs: - run: cargo fmt --manifest-path src-tauri/Cargo.toml --check - run: cargo clippy --locked --manifest-path src-tauri/Cargo.toml --all-targets -- -D warnings - run: cargo test --locked --manifest-path src-tauri/Cargo.toml + - run: cargo fmt --manifest-path tools/prediction-data/Cargo.toml --check + - run: cargo clippy --locked --manifest-path tools/prediction-data/Cargo.toml --all-targets -- -D warnings + - run: cargo test --locked --manifest-path tools/prediction-data/Cargo.toml + - run: cargo run --release --locked --manifest-path tools/prediction-data/Cargo.toml -- --check src-tauri/resources/WordData2017051601.db src-tauri/resources/word-predictions.lookup - run: cargo fmt --manifest-path tools/updater-signature-verifier/Cargo.toml --check - run: cargo clippy --locked --manifest-path tools/updater-signature-verifier/Cargo.toml --all-targets -- -D warnings - run: cargo test --locked --manifest-path tools/updater-signature-verifier/Cargo.toml diff --git a/.gitignore b/.gitignore index 242fa8c1..5896deba 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ Thumbs.db src-tauri/icons/android/ src-tauri/icons/ios/ src-tauri/tauri.release*.generated.json +tools/prediction-data/target/ diff --git a/scripts/measure-macos-prediction.mjs b/scripts/measure-macos-prediction.mjs new file mode 100644 index 00000000..befb28d3 --- /dev/null +++ b/scripts/measure-macos-prediction.mjs @@ -0,0 +1,97 @@ +// Manual read-only fixture measurement through Launch Services. Never sends Accept. +import { spawn, execFileSync } from 'node:child_process'; +import { mkdtempSync, writeFileSync, readFileSync, copyFileSync, rmSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { performance } from 'node:perf_hooks'; +import { setTimeout as delay } from 'node:timers/promises'; + +const [bundle, resource] = process.argv.slice(2); +if (process.platform !== 'darwin' || !bundle || !resource) { + throw Error('Usage on macOS: node scripts/measure-macos-prediction.mjs '); +} +console.error('Focus a synthetic test field. Sampling starts in five seconds; no input is injected.'); +const root = mkdtempSync(join(tmpdir(), 'switchify-timing-')); +const input = join(root, 'input'); +const output = join(root, 'output'); +const lookup = join(root, 'timing.lookup'); +const owned = new Set(); +let child; +try { + copyFileSync(resource, lookup); + const frames = []; + for (let generation = 0; generation < 100; generation++) { + const body = Buffer.from(JSON.stringify({ Query: { + generation, revision: 0, edits: [], shift: false, caps: false, + } })); + const header = Buffer.alloc(4); + header.writeUInt32LE(body.length); + frames.push(header, body); + } + writeFileSync(input, Buffer.concat(frames)); + writeFileSync(output, ''); + await delay(5000); + const start = performance.now(); + child = spawn('open', ['-n', '-g', '-W', '-a', resolve(bundle), + '--stdin', input, '--stdout', output, '--stderr', '/dev/null', + '--args', '--switchify-prediction-worker', lookup, '[]']); + let done = false; + let launchError; + child.on('exit', () => { done = true; }); + child.on('error', error => { launchError = error; done = true; }); + let offset = 0; + let count = 0; + let matched = 0; + let firstResponseMs; + let firstSuggestionsMs; + let lastResponseMs; + let maxSampledRssBytes = 0; + while (performance.now() - start < 10000) { + if (launchError) throw launchError; + const data = readFileSync(output); + while (offset + 4 <= data.length) { + const size = data.readUInt32LE(offset); + if (!size || size > 16384) throw Error('Invalid frame'); + if (offset + 4 + size > data.length) break; + const response = JSON.parse(data.subarray(offset + 4, offset + 4 + size)); + offset += 4 + size; + if (response.Suggestions?.generation !== count) throw Error('Unexpected response'); + firstResponseMs ??= performance.now() - start; + if (response.Suggestions.batch?.words?.length) { + firstSuggestionsMs ??= performance.now() - start; + matched++; + } + count++; + lastResponseMs = performance.now() - start; + } + if (done) break; + try { + const rows = execFileSync('ps', ['-axo', 'pid=,rss=,command='], { encoding: 'utf8' }).split('\n'); + for (const row of rows) { + const match = row.trim().match(/^(\d+)\s+(\d+)\s+(.+)$/); + if (match && match[3].includes('--switchify-prediction-worker') && + match[3].includes(lookup) && !/^(?:\/usr\/bin\/)?open /.test(match[3])) { + owned.add(Number(match[1])); + maxSampledRssBytes = Math.max(maxSampledRssBytes, Number(match[2]) * 1024); + } + } + } catch { /* A worker may exit between samples. */ } + await delay(10); + } + console.log(JSON.stringify({ + firstResponseMs, firstSuggestionsMs, count, matched, + maxSampledRssBytes: maxSampledRssBytes || null, + warmBatchMeanObservedMs: count > 1 ? (lastResponseMs - firstResponseMs) / (count - 1) : null, + elapsedMs: performance.now() - start, + }, null, 2)); + if (count !== 100 || !matched || firstSuggestionsMs >= 2000) process.exitCode = 1; +} finally { + for (const pid of owned) { + try { + const command = execFileSync('ps', ['-p', String(pid), '-o', 'command='], { encoding: 'utf8' }); + if (command.includes(lookup)) process.kill(pid); + } catch { /* Already exited. */ } + } + child?.kill(); + rmSync(root, { recursive: true, force: true }); +} diff --git a/scripts/measure-word-prediction.mjs b/scripts/measure-word-prediction.mjs index 466eb678..f3cd278c 100644 --- a/scripts/measure-word-prediction.mjs +++ b/scripts/measure-word-prediction.mjs @@ -1,19 +1,20 @@ // Manual, read-only fixture measurement. Never sends an Accept request. -import { spawn } from 'node:child_process'; +import { spawn, execFileSync } from 'node:child_process'; import { resolve } from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; import { performance } from 'node:perf_hooks'; -const [executable, app, expectedWord] = process.argv.slice(2); +const [executable, app, expectedWord, lookupPath] = process.argv.slice(2); if (!executable || !app || !expectedWord) { - console.error('Usage: node scripts/measure-word-prediction.mjs '); + console.error('Usage: node scripts/measure-word-prediction.mjs [lookup-path]'); process.exit(1); } console.log('Focus a synthetic test field now. Sampling starts in five seconds. No input will be injected.'); await delay(5000); +const spawnedAt = performance.now(); const child = spawn(resolve(executable), [ '--switchify-prediction-worker', - resolve('src-tauri/resources/WordData2017051601.db'), + resolve(lookupPath ?? 'src-tauri/resources/word-predictions.lookup'), '[]', ], { stdio: ['pipe', 'pipe', 'ignore'], windowsHide: true }); let buffer = Buffer.alloc(0); @@ -45,7 +46,7 @@ child.stdout.on('data', chunk => { async function query(generation) { if (failed) throw new Error('Worker unavailable'); const bytes = Buffer.from(JSON.stringify({ Query: { - generation, edit: null, reset: true, shift: false, caps: false, + generation, edits: [], revision: 0, shift: false, caps: false, } })); const header = Buffer.alloc(4); header.writeUInt32LE(bytes.length); @@ -66,16 +67,24 @@ let unsupported = 0; let failures = 0; let fixtureMatches = 0; let firstRequestMs; +let firstResponseFromSpawnMs; +let firstSuggestionsFromSpawnMs; +let memoryBytes; +let memoryMetric; try { // Up to 200 attempts, stopping after 100 supported samples. for (let generation = 0; generation < 200 && times.length < 100; generation++) { const start = performance.now(); const response = await query(generation); const elapsed = performance.now() - start; - if (generation === 0) firstRequestMs = elapsed; + if (generation === 0) { + firstRequestMs = elapsed; + firstResponseFromSpawnMs = performance.now() - spawnedAt; + } const result = response.Suggestions; if (!result || result.generation !== generation) throw new Error('Invalid response'); if (result.batch?.words?.length) { + firstSuggestionsFromSpawnMs ??= performance.now() - spawnedAt; if (generation !== 0) times.push(elapsed); if (result.batch.words.some(word => word.toLowerCase() === expectedWord.toLowerCase())) fixtureMatches++; } else unsupported++; @@ -83,6 +92,15 @@ try { } } catch { failures++; } finally { + try { + if (process.platform === 'darwin') { + memoryBytes = Number(execFileSync('ps', ['-o', 'rss=', '-p', String(child.pid)], { encoding: 'utf8' }).trim()) * 1024; + memoryMetric = 'residentBytesAtEnd'; + } else if (process.platform === 'win32') { + memoryBytes = Number(execFileSync('powershell.exe', ['-NoProfile', '-Command', `(Get-Process -Id ${child.pid}).PeakWorkingSet64`], { encoding: 'utf8', windowsHide: true }).trim()); + memoryMetric = 'peakWorkingSetBytes'; + } + } catch { memoryBytes = undefined; } child.kill(); process.off('SIGINT', stop); process.off('SIGTERM', stop); @@ -90,7 +108,7 @@ finally { times.sort((a, b) => a - b); const percentile = fraction => times.length ? +times[Math.ceil(times.length * fraction) - 1].toFixed(2) : null; console.log(JSON.stringify({ - app, successfulSamples: times.length, unsupported, failures, fixtureMatches, + app, firstResponseFromSpawnMs, firstSuggestionsFromSpawnMs, memoryBytes, memoryMetric, successfulSamples: times.length, unsupported, failures, fixtureMatches, firstRequestMs: firstRequestMs === undefined ? null : +firstRequestMs.toFixed(2), medianMs: percentile(0.5), p95Ms: percentile(0.95), maximumMs: percentile(1), }, null, 2)); diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 2e2b0559..de15cae4 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -21,7 +21,6 @@ enigo = "=0.6.1" hmac = "0.12" keyring = "4.1.6" rand = "0.8" -rusqlite = { version = "0.40.2", features = ["bundled"] } unicode-segmentation = "1.12" reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"] } serde = { version = "1", features = ["derive"] } @@ -78,3 +77,6 @@ winreg = "0.55" [patch.crates-io] corebluetooth-rs = { path = "../vendor/corebluetooth-rs" } + +[dev-dependencies] +rusqlite = { version = "0.40.2", features = ["bundled"] } diff --git a/src-tauri/resources/PROVENANCE.md b/src-tauri/resources/PROVENANCE.md index 361373e2..ace5137c 100644 --- a/src-tauri/resources/PROVENANCE.md +++ b/src-tauri/resources/PROVENANCE.md @@ -7,4 +7,16 @@ - SQLite integrity check during recovery: `ok`. - Rows: WORDS 165,420; BIGRAMS 976,804; TRIGRAMS 996,846; QUADGRAMS 953,630. -This is the unchanged original SQLite file. The later Realm archive is not used. Runtime connections are read-only and ignore personal frequency columns. No captured text or user vocabulary is written into this resource. +This is the unchanged original SQLite file. The later Realm archive is not used. It is retained for conversion and parity tests, not shipped in the app. The converter ignores personal frequency columns. No captured text or user vocabulary is written into this resource. + +## Packaged dedicated lookup + +`word-predictions.lookup` is generated from the unchanged database. It contains frequency-ranked words and n-grams, a sorted prefix index, a versioned header and a SHA-256 payload checksum. The runtime validates counts, bounds, references and ordering before serving predictions. The app packages only this lookup and does not write to it. + +Regenerate with Rust 1.97.1 from the repository root: + +```sh +cargo run --release --locked --manifest-path tools/prediction-data/Cargo.toml -- src-tauri/resources/WordData2017051601.db src-tauri/resources/word-predictions.lookup +``` + +Use `-- --check` before the two paths to regenerate into a temporary file and verify byte-for-byte reproducibility. CI checks the artifact on Mac and Windows. The original SQLite implementation is retained only in tests to verify identical ordered suggestions. diff --git a/src-tauri/resources/word-predictions.lookup b/src-tauri/resources/word-predictions.lookup new file mode 100644 index 00000000..d1ba4250 Binary files /dev/null and b/src-tauri/resources/word-predictions.lookup differ diff --git a/src-tauri/src/prediction/database.rs b/src-tauri/src/prediction/database.rs index 5b0ecc53..d240ec96 100644 --- a/src-tauri/src/prediction/database.rs +++ b/src-tauri/src/prediction/database.rs @@ -1,114 +1,17 @@ -use super::context::Context; -use rusqlite::{Connection, OpenFlags}; -use std::{ - collections::{HashMap, HashSet}, - path::Path, -}; +use super::{context::Context, lookup::Lookup}; +use std::path::Path; -pub struct Database { - connection: Connection, - words: Vec<(i64, String, String)>, - ids: HashMap, -} +pub struct Database(Lookup); impl Database { - #[cfg(test)] - pub fn fixture() -> Self { - let c = Connection::open_in_memory().unwrap(); - c.execute_batch("CREATE TABLE WORDS(ID INTEGER,WORD TEXT,BASE_FREQUENCY INTEGER); INSERT INTO WORDS VALUES(1,'water',9),(2,'walk',8),(3,'want',7),(4,'I',6),(5,'like',5); CREATE TABLE BIGRAMS(ID1 INTEGER,ID2 INTEGER,BASE_FREQUENCY INTEGER); CREATE TABLE TRIGRAMS(ID1 INTEGER,ID2 INTEGER,ID3 INTEGER,BASE_FREQUENCY INTEGER); CREATE TABLE QUADGRAMS(ID1 INTEGER,ID2 INTEGER,ID3 INTEGER,ID4 INTEGER,BASE_FREQUENCY INTEGER);").unwrap(); - Self::from_connection(c).unwrap() - } pub fn open(path: &Path) -> Result { - let connection = Connection::open_with_flags( - path, - OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, - ) - .map_err(|_| ())?; - Self::from_connection(connection) - } - fn from_connection(connection: Connection) -> Result { - connection - .execute_batch("PRAGMA query_only=ON; PRAGMA cache_size=-8192;") - .map_err(|_| ())?; - let words = { - let mut q = connection - .prepare("SELECT ID, WORD FROM WORDS ORDER BY BASE_FREQUENCY DESC, ID ASC") - .map_err(|_| ())?; - let result = q - .query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?))) - .map_err(|_| ())? - .collect::, _>>() - .map_err(|_| ())?; - result - }; - let words: Vec<_> = words - .into_iter() - .map(|(id, word)| { - let folded = word.to_lowercase(); - (id, word, folded) - }) - .collect(); - let mut ids = HashMap::new(); - for (id, _, folded) in &words { - ids.entry(folded.clone()).or_insert(*id); - } - Ok(Self { - connection, - words, - ids, - }) + Lookup::open(path).map(Self) } pub fn predict(&self, context: &Context) -> Result, ()> { - let prefix = context.prefix.to_lowercase(); - let mut result = Vec::new(); - let mut seen = HashSet::new(); - let mut push = |word: String| { - let folded = word.to_lowercase(); - if result.len() < 5 - && word.chars().count() <= 48 - && !word.chars().any(char::is_whitespace) - && folded.starts_with(&prefix) - && seen.insert(folded) - { - result.push(word); - } - result.len() == 5 - }; - for count in (1..=context.words.len().min(3)).rev() { - let ids: Option> = context.words[context.words.len() - count..] - .iter() - .map(|w| self.ids.get(&w.to_lowercase()).copied()) - .collect(); - let Some(ids) = ids else { - continue; - }; - let table = ["", "BIGRAMS", "TRIGRAMS", "QUADGRAMS"][count]; - let conditions = (1..=count) - .map(|i| format!("g.ID{i}=?{i}")) - .collect::>() - .join(" AND "); - let sql = format!("SELECT w.WORD FROM {table} g JOIN WORDS w ON w.ID=g.ID{} WHERE {conditions} ORDER BY g.BASE_FREQUENCY DESC, w.ID ASC", count+1); - let mut statement = self.connection.prepare_cached(&sql).map_err(|_| ())?; - let mut rows = statement - .query(rusqlite::params_from_iter(ids)) - .map_err(|_| ())?; - // Context indexes bound this candidate set; the entire query is also - // contained by the parent process's two-second deadline. - while let Some(row) = rows.next().map_err(|_| ())? { - if push(row.get(0).map_err(|_| ())?) { - break; - } - } - } - for (_, word, _) in self - .words - .iter() - .filter(|(_, _, folded)| folded.starts_with(&prefix)) - { - if push(word.clone()) { - break; - } - } - Ok(result) + Ok(self.0.predict(&context.words, &context.prefix)) + } + #[cfg(test)] + pub fn fixture() -> Self { + Self(Lookup::fixture()) } } @@ -116,14 +19,93 @@ impl Database { mod tests { use super::*; #[test] - fn longest_context_wins_and_backoff_deduplicates() { - let c = Connection::open_in_memory().unwrap(); - c.execute_batch("CREATE TABLE WORDS(ID INTEGER,WORD TEXT,BASE_FREQUENCY INTEGER); INSERT INTO WORDS VALUES(1,'I',9),(2,'would',8),(3,'like',7),(4,'water',1),(5,'wine',20),(6,'walk',30); CREATE TABLE BIGRAMS(ID1 INTEGER,ID2 INTEGER,BASE_FREQUENCY INTEGER); INSERT INTO BIGRAMS VALUES(3,5,9); CREATE TABLE TRIGRAMS(ID1 INTEGER,ID2 INTEGER,ID3 INTEGER,BASE_FREQUENCY INTEGER); CREATE TABLE QUADGRAMS(ID1 INTEGER,ID2 INTEGER,ID3 INTEGER,ID4 INTEGER,BASE_FREQUENCY INTEGER); INSERT INTO QUADGRAMS VALUES(1,2,3,4,1);").unwrap(); - let db = Database::from_connection(c).unwrap(); - let result = db - .predict(&super::super::context::extract("I would like w", false)) - .unwrap(); - assert_eq!(&result[..3], ["water", "wine", "walk"]); - assert!(db.connection.execute("DELETE FROM WORDS", []).is_err()); + fn bundled_lookup_matches_sqlite_corpus_and_expanded_samples() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("resources"); + let source = root.join("WordData2017051601.db"); + let reference = super::super::database_reference::Database::open(&source).unwrap(); + let lookup = Database::open(&root.join("word-predictions.lookup")).unwrap(); + let c = rusqlite::Connection::open_with_flags( + source, + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .unwrap(); + let mut texts: Vec = [ + "", + "a", + "w", + "wa", + "water", + "zzzzzz", + "I ", + "I would ", + "I would like ", + "I would like wa", + "the ", + "the a", + "unknownword wa", + "café", + "cafe\u{301}", + "İ", + "你好", + "I can’t find ", + "Done! ", + "hello\nwa", + "W", + "WATER", + "🙂 ", + ] + .iter() + .map(|s| s.to_string()) + .collect(); + let ids: std::collections::HashMap = c + .prepare("SELECT ID,WORD FROM WORDS") + .unwrap() + .query_map([], |r| Ok((r.get(0)?, r.get(1)?))) + .unwrap() + .map(Result::unwrap) + .collect(); + for offset in [0, 499] { + for (n, table) in [(2, "BIGRAMS"), (3, "TRIGRAMS"), (4, "QUADGRAMS")] { + let fields = (1..=n) + .map(|i| format!("ID{i}")) + .collect::>() + .join(","); + let mut q = c + .prepare(&format!( + "SELECT {fields} FROM {table} WHERE ID%997={offset} ORDER BY ID LIMIT 80" + )) + .unwrap(); + let rows = q + .query_map([], |r| { + (0..n) + .map(|i| r.get::<_, u32>(i)) + .collect::, _>>() + }) + .unwrap(); + for row in rows { + let row = row.unwrap(); + let context = row[..n - 1] + .iter() + .map(|id| ids[id].as_str()) + .collect::>() + .join(" "); + for size in [0, 1, 3] { + texts.push(format!( + "{context} {}", + ids[&row[n - 1]].chars().take(size).collect::() + )); + } + } + } + } + assert_eq!(texts.len(), 1463); + for (i, text) in texts.iter().enumerate() { + let context = super::super::context::extract(text, i % 17 == 0); + assert_eq!( + lookup.predict(&context), + reference.predict(&context), + "case {i}" + ); + } } } diff --git a/src-tauri/src/prediction/database_reference.rs b/src-tauri/src/prediction/database_reference.rs new file mode 100644 index 00000000..78117738 --- /dev/null +++ b/src-tauri/src/prediction/database_reference.rs @@ -0,0 +1,123 @@ +use super::context::Context; +use rusqlite::{Connection, OpenFlags}; +use std::{ + collections::{HashMap, HashSet}, + path::Path, +}; + +pub struct Database { + connection: Connection, + words: Vec<(i64, String, String)>, + ids: HashMap, +} +impl Database { + pub fn open(path: &Path) -> Result { + let connection = Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(|_| ())?; + Self::from_connection(connection) + } + fn from_connection(connection: Connection) -> Result { + connection + .execute_batch("PRAGMA query_only=ON; PRAGMA cache_size=-8192;") + .map_err(|_| ())?; + let words = { + let mut q = connection + .prepare("SELECT ID, WORD FROM WORDS ORDER BY BASE_FREQUENCY DESC, ID ASC") + .map_err(|_| ())?; + let result = q + .query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?))) + .map_err(|_| ())? + .collect::, _>>() + .map_err(|_| ())?; + result + }; + let words: Vec<_> = words + .into_iter() + .map(|(id, word)| { + let folded = word.to_lowercase(); + (id, word, folded) + }) + .collect(); + let mut ids = HashMap::new(); + for (id, _, folded) in &words { + ids.entry(folded.clone()).or_insert(*id); + } + Ok(Self { + connection, + words, + ids, + }) + } + pub fn predict(&self, context: &Context) -> Result, ()> { + let prefix = context.prefix.to_lowercase(); + let mut result = Vec::new(); + let mut seen = HashSet::new(); + let mut push = |word: String| { + let folded = word.to_lowercase(); + if result.len() < 5 + && word.chars().count() <= 48 + && !word.chars().any(char::is_whitespace) + && folded.starts_with(&prefix) + && seen.insert(folded) + { + result.push(word); + } + result.len() == 5 + }; + for count in (1..=context.words.len().min(3)).rev() { + let ids: Option> = context.words[context.words.len() - count..] + .iter() + .map(|w| self.ids.get(&w.to_lowercase()).copied()) + .collect(); + let Some(ids) = ids else { + continue; + }; + let table = ["", "BIGRAMS", "TRIGRAMS", "QUADGRAMS"][count]; + let conditions = (1..=count) + .map(|i| format!("g.ID{i}=?{i}")) + .collect::>() + .join(" AND "); + let sql = format!("SELECT w.WORD FROM {table} g JOIN WORDS w ON w.ID=g.ID{} WHERE {conditions} ORDER BY g.BASE_FREQUENCY DESC, w.ID ASC", count+1); + let mut statement = self.connection.prepare_cached(&sql).map_err(|_| ())?; + let mut rows = statement + .query(rusqlite::params_from_iter(ids)) + .map_err(|_| ())?; + // Context indexes bound this candidate set; the entire query is also + // contained by the parent process's two-second deadline. + while let Some(row) = rows.next().map_err(|_| ())? { + if push(row.get(0).map_err(|_| ())?) { + break; + } + } + } + for (_, word, _) in self + .words + .iter() + .filter(|(_, _, folded)| folded.starts_with(&prefix)) + { + if push(word.clone()) { + break; + } + } + Ok(result) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn longest_context_wins_and_backoff_deduplicates() { + let c = Connection::open_in_memory().unwrap(); + c.execute_batch("CREATE TABLE WORDS(ID INTEGER,WORD TEXT,BASE_FREQUENCY INTEGER); INSERT INTO WORDS VALUES(1,'I',9),(2,'would',8),(3,'like',7),(4,'water',1),(5,'wine',20),(6,'walk',30); CREATE TABLE BIGRAMS(ID1 INTEGER,ID2 INTEGER,BASE_FREQUENCY INTEGER); INSERT INTO BIGRAMS VALUES(3,5,9); CREATE TABLE TRIGRAMS(ID1 INTEGER,ID2 INTEGER,ID3 INTEGER,BASE_FREQUENCY INTEGER); CREATE TABLE QUADGRAMS(ID1 INTEGER,ID2 INTEGER,ID3 INTEGER,ID4 INTEGER,BASE_FREQUENCY INTEGER); INSERT INTO QUADGRAMS VALUES(1,2,3,4,1);").unwrap(); + let db = Database::from_connection(c).unwrap(); + let result = db + .predict(&super::super::context::extract("I would like w", false)) + .unwrap(); + assert_eq!(&result[..3], ["water", "wine", "walk"]); + assert!(db.connection.execute("DELETE FROM WORDS", []).is_err()); + } +} diff --git a/src-tauri/src/prediction/lookup.rs b/src-tauri/src/prediction/lookup.rs new file mode 100644 index 00000000..aad8b820 --- /dev/null +++ b/src-tauri/src/prediction/lookup.rs @@ -0,0 +1,390 @@ +use sha2::{Digest, Sha256}; +use std::{ + collections::{HashMap, HashSet}, + fs::File, + io::{BufReader, Read}, + path::Path, +}; + +pub const MAGIC: &[u8; 8] = b"SWPRED02"; +const MAX_FILE: u64 = 128 * 1024 * 1024; +const MAX_WORDS: usize = 250_000; +const MAX_ROWS: usize = 1_500_000; +const MAX_WORD_BYTES: usize = 512; +const MAX_TEXT_BYTES: usize = 32 * 1024 * 1024; +const NONE: u32 = u32::MAX; + +struct Reader { + input: R, + hash: Sha256, +} +impl Reader { + fn bytes(&mut self, bytes: &mut [u8]) -> Result<(), ()> { + self.input.read_exact(bytes).map_err(|_| ())?; + self.hash.update(bytes); + Ok(()) + } + fn number(&mut self) -> Result { + let mut bytes = [0; 4]; + self.bytes(&mut bytes)?; + Ok(u32::from_le_bytes(bytes)) + } +} +struct Word { + id: u32, + text: String, + folded: String, +} +pub struct Lookup { + words: Vec, + ids: HashMap, + groups: HashMap<[u32; 3], Vec>, + prefixes: Vec, +} +impl Lookup { + pub fn open(path: &Path) -> Result { + let file = File::open(path).map_err(|_| ())?; + let metadata = file.metadata().map_err(|_| ())?; + if !metadata.is_file() || !(60..=MAX_FILE).contains(&metadata.len()) { + return Err(()); + } + Self::read(BufReader::new(file.take(MAX_FILE + 1))) + } + fn read(mut input: impl Read) -> Result { + let mut magic = [0; 8]; + input.read_exact(&mut magic).map_err(|_| ())?; + if &magic != MAGIC { + return Err(()); + } + let mut expected = [0; 32]; + input.read_exact(&mut expected).map_err(|_| ())?; + let mut r = Reader { + input, + hash: Sha256::new(), + }; + let count = r.number()? as usize; + if count == 0 || count > MAX_WORDS { + return Err(()); + } + let mut words: Vec = Vec::new(); + words.try_reserve_exact(count).map_err(|_| ())?; + let mut ids = HashMap::new(); + ids.try_reserve(count).map_err(|_| ())?; + let mut unique = HashSet::new(); + unique.try_reserve(count).map_err(|_| ())?; + let mut previous = None; + let mut total = 0usize; + for rank in 0..count { + let id = r.number()?; + let frequency = r.number()?; + if id == 0 + || !unique.insert(id) + || previous.is_some_and(|(f, i)| frequency > f || (frequency == f && id <= i)) + { + return Err(()); + } + previous = Some((frequency, id)); + let size = r.number()? as usize; + total = total.checked_add(size).ok_or(())?; + if size > MAX_WORD_BYTES || total > MAX_TEXT_BYTES { + return Err(()); + } + let mut bytes = Vec::new(); + bytes.try_reserve_exact(size).map_err(|_| ())?; + bytes.resize(size, 0); + r.bytes(&mut bytes)?; + let text = String::from_utf8(bytes).map_err(|_| ())?; + let folded = text.to_lowercase(); + ids.entry(folded.clone()).or_insert(rank as u32); + words.push(Word { id, text, folded }); + } + drop(unique); + let mut groups: HashMap<[u32; 3], Vec> = HashMap::new(); + for length in 1..=3 { + let rows = r.number()? as usize; + if rows > MAX_ROWS { + return Err(()); + } + let mut previous = None; + for _ in 0..rows { + let mut key = [NONE; 3]; + for slot in &mut key[..length] { + *slot = r.number()?; + if *slot as usize >= count { + return Err(()); + } + } + let rank = r.number()?; + let frequency = r.number()?; + let word = words.get(rank as usize).ok_or(())?; + if previous.is_some_and(|(f, i)| frequency > f || (frequency == f && word.id < i)) { + return Err(()); + } + previous = Some((frequency, word.id)); + if !groups.contains_key(&key) { + groups.try_reserve(1).map_err(|_| ())?; + } + let list = groups.entry(key).or_default(); + list.try_reserve(1).map_err(|_| ())?; + list.push(rank); + } + } + if r.number()? as usize != count { + return Err(()); + } + let mut prefixes = Vec::new(); + prefixes.try_reserve_exact(count).map_err(|_| ())?; + let mut seen = Vec::new(); + seen.try_reserve_exact(count).map_err(|_| ())?; + seen.resize(count, false); + for _ in 0..count { + let rank = r.number()?; + let word = words.get(rank as usize).ok_or(())?; + if seen[rank as usize] { + return Err(()); + } + seen[rank as usize] = true; + if let Some(&p) = prefixes.last() { + let previous: &Word = &words[p as usize]; + if (&previous.folded, p) >= (&word.folded, rank) { + return Err(()); + } + } + prefixes.push(rank); + } + let mut trailing = [0]; + if r.input.read(&mut trailing).map_err(|_| ())? != 0 + || r.hash.finalize().as_slice() != expected + { + return Err(()); + } + Ok(Self { + words, + ids, + groups, + prefixes, + }) + } + pub fn predict(&self, context: &[String], prefix: &str) -> Vec { + let prefix = prefix.to_lowercase(); + let mut result = Vec::new(); + let mut seen = HashSet::new(); + for count in (1..=context.len().min(3)).rev() { + let mut key = [NONE; 3]; + let mut known = true; + for (slot, word) in key.iter_mut().zip(&context[context.len() - count..]) { + match self.ids.get(&word.to_lowercase()) { + Some(rank) => *slot = *rank, + None => { + known = false; + break; + } + } + } + if let Some(list) = known.then(|| self.groups.get(&key)).flatten() { + for &rank in list { + self.push(rank, &prefix, &mut result, &mut seen); + if result.len() == 5 { + return result; + } + } + } + } + if prefix.is_empty() { + for rank in 0..self.words.len() { + self.push(rank as u32, &prefix, &mut result, &mut seen); + if result.len() == 5 { + break; + } + } + } else { + let start = self + .prefixes + .partition_point(|&i| self.words[i as usize].folded.as_str() < prefix.as_str()); + let mut best: Vec = Vec::new(); + for &rank in self.prefixes[start..] + .iter() + .take_while(|&&i| self.words[i as usize].folded.starts_with(&prefix)) + { + let word = &self.words[rank as usize]; + if usable(&word.text) && !seen.contains(&word.folded) { + if let Some(i) = best + .iter() + .position(|&r| self.words[r as usize].folded == word.folded) + { + best[i] = best[i].min(rank); + } else { + best.push(rank); + } + best.sort_unstable(); + best.truncate(5 - result.len()); + } + } + for rank in best { + self.push(rank, &prefix, &mut result, &mut seen); + } + } + result + } + fn push(&self, rank: u32, prefix: &str, result: &mut Vec, seen: &mut HashSet) { + let word = &self.words[rank as usize]; + if result.len() < 5 + && usable(&word.text) + && word.folded.starts_with(prefix) + && seen.insert(word.folded.clone()) + { + result.push(word.text.clone()); + } + } +} +fn usable(word: &str) -> bool { + word.chars().count() <= 48 && !word.chars().any(char::is_whitespace) +} + +#[cfg(test)] +impl Lookup { + pub fn fixture() -> Self { + Self::read(&tests::fixture_bytes()[..]).unwrap() + } +} +#[cfg(test)] +mod tests { + use super::*; + fn n(out: &mut Vec, value: u32) { + out.extend_from_slice(&value.to_le_bytes()); + } + fn seal(payload: Vec) -> Vec { + let mut out = MAGIC.to_vec(); + out.extend_from_slice(&Sha256::digest(&payload)); + out.extend(payload); + out + } + pub(super) fn fixture_bytes() -> Vec { + let mut out = Vec::new(); + n(&mut out, 5); + for (id, text, freq) in [ + (1, "water", 9), + (2, "walk", 8), + (3, "want", 7), + (4, "I", 6), + (5, "like", 5), + ] { + n(&mut out, id); + n(&mut out, freq); + n(&mut out, text.len() as u32); + out.extend_from_slice(text.as_bytes()); + } + for _ in 0..3 { + n(&mut out, 0); + } + n(&mut out, 5); + for rank in [3, 4, 1, 2, 0] { + n(&mut out, rank); + } + seal(out) + } + fn change(bytes: &[u8], offset: usize, value: u32) -> Vec { + let mut payload = bytes[40..].to_vec(); + payload[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + seal(payload) + } + #[test] + fn malformed_artifacts_fail_closed() { + let bytes = fixture_bytes(); + assert!(Lookup::read(&bytes[..]).is_ok()); + for size in 0..bytes.len() { + assert!(Lookup::read(&bytes[..size]).is_err()); + } + for i in 0..bytes.len() { + let mut bad = bytes.clone(); + bad[i] ^= 0x80; + assert!(Lookup::read(&bad[..]).is_err()); + } + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(Lookup::read(&trailing[..]).is_err()); + for (offset, value) in [ + (0, 0), + (0, u32::MAX), + (4, 0), + (8, 0), + (12, u32::MAX), + (21, 1), + ] { + assert!(Lookup::read(&change(&bytes, offset, value)[..]).is_err()); + } + let prefix_count = bytes.len() - 40 - 24; + assert!(Lookup::read(&change(&bytes, prefix_count, 4)[..]).is_err()); + assert!(Lookup::read(&change(&bytes, prefix_count + 4, 5)[..]).is_err()); + assert!(Lookup::read(&change(&bytes, prefix_count + 8, 3)[..]).is_err()); + assert!(Lookup::read(&change(&bytes, prefix_count + 4, 0)[..]).is_err()); + assert!(Lookup::read(&change(&bytes, prefix_count - 12, u32::MAX)[..]).is_err()); + } + #[test] + fn ranking_unicode_deduplication_and_context_backoff() { + let entries = [ + (1, "I", 100), + (2, "like", 90), + (3, "water", 20), + (4, "walk", 20), + (5, "Water", 19), + (6, "café", 15), + (7, "CAFÉ", 14), + (8, "café", 13), + (9, "two words", 12), + (10, "İstanbul", 11), + (11, "你好", 10), + ]; + let mut out = Vec::new(); + n(&mut out, entries.len() as u32); + for (id, text, freq) in entries { + n(&mut out, id); + n(&mut out, freq); + n(&mut out, text.len() as u32); + out.extend_from_slice(text.as_bytes()); + } + n(&mut out, 3); + for (rank, freq) in [(4, 40), (2, 30), (3, 30)] { + n(&mut out, 1); + n(&mut out, rank); + n(&mut out, freq); + } + n(&mut out, 1); + for value in [0, 1, 2, 1] { + n(&mut out, value); + } + n(&mut out, 1); + for value in [0, 0, 1, 3, 1] { + n(&mut out, value); + } + let mut prefixes: Vec<_> = (0..entries.len()).collect(); + prefixes.sort_by_key(|&i| (entries[i].1.to_lowercase(), i)); + n(&mut out, prefixes.len() as u32); + for i in prefixes { + n(&mut out, i as u32); + } + let bytes = seal(out); + let lookup = Lookup::read(&bytes[..]).unwrap(); + assert_eq!( + lookup.predict(&["I".into(), "I".into(), "like".into()], "w"), + ["walk", "water"] + ); + assert_eq!(lookup.predict(&["like".into()], "w"), ["Water", "walk"]); + assert_eq!(lookup.predict(&[], "CAF"), ["café", "café"]); + assert_eq!(lookup.predict(&[], "İ"), ["İstanbul"]); + assert_eq!(lookup.predict(&[], "你"), ["你好"]); + assert_eq!(lookup.predict(&["unknown".into()], "w"), ["water", "walk"]); + assert!(!lookup.predict(&[], "").contains(&"two words".into())); + assert!(lookup.predict(&[], "unknown").is_empty()); + } + #[test] + fn missing_and_oversized_files_fail_closed() { + let path = std::env::temp_dir().join(format!("switchify-lookup-{}", std::process::id())); + assert!(Lookup::open(&path).is_err()); + let file = File::create(&path).unwrap(); + file.set_len(MAX_FILE + 1).unwrap(); + drop(file); + assert!(Lookup::open(&path).is_err()); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/src-tauri/src/prediction/mod.rs b/src-tauri/src/prediction/mod.rs index 826a3902..f0f83f1f 100644 --- a/src-tauri/src/prediction/mod.rs +++ b/src-tauri/src/prediction/mod.rs @@ -1,6 +1,9 @@ mod activity; mod context; mod database; +#[cfg(test)] +mod database_reference; +mod lookup; #[cfg(target_os = "macos")] mod macos; #[cfg(target_os = "windows")] @@ -262,12 +265,12 @@ fn resource(app: &AppHandle) -> Result { let bundled = app .path() .resolve( - "resources/WordData2017051601.db", + "resources/word-predictions.lookup", tauri::path::BaseDirectory::Resource, ) .map_err(|_| ()); let development = cfg!(debug_assertions) - .then(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("resources/WordData2017051601.db")); + .then(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("resources/word-predictions.lookup")); database_resource(bundled, development) } diff --git a/src-tauri/src/prediction/worker.rs b/src-tauri/src/prediction/worker.rs index b33bff83..bbcd4f3d 100644 --- a/src-tauri/src/prediction/worker.rs +++ b/src-tauri/src/prediction/worker.rs @@ -1086,7 +1086,7 @@ mod tests { let start = std::time::Instant::now(); let db = Database::open( &std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("resources/WordData2017051601.db"), + .join("resources/word-predictions.lookup"), ) .unwrap(); let startup = start.elapsed().as_secs_f64() * 1000.0; diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 61d14152..f5fb34d5 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -34,7 +34,7 @@ } }, "bundle": { - "resources": ["resources/WordData2017051601.db"], + "resources": ["resources/word-predictions.lookup"], "active": true, "targets": "all", "icon": [ diff --git a/tools/prediction-data/Cargo.lock b/tools/prediction-data/Cargo.lock new file mode 100644 index 00000000..8ab0207a --- /dev/null +++ b/tools/prediction-data/Cargo.lock @@ -0,0 +1,361 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cc" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "find-msvc-tools" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror", +] + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + +[[package]] +name = "switchify-prediction-data" +version = "0.1.0" +dependencies = [ + "rusqlite", + "sha2", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] diff --git a/tools/prediction-data/Cargo.toml b/tools/prediction-data/Cargo.toml new file mode 100644 index 00000000..21aa3f39 --- /dev/null +++ b/tools/prediction-data/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "switchify-prediction-data" +version = "0.1.0" +edition = "2021" +rust-version = "1.97.1" + +[dependencies] +rusqlite = { version = "0.40.2", features = ["bundled"] } +sha2 = "0.10" diff --git a/tools/prediction-data/src/main.rs b/tools/prediction-data/src/main.rs new file mode 100644 index 00000000..2a6a68d6 --- /dev/null +++ b/tools/prediction-data/src/main.rs @@ -0,0 +1,135 @@ +#[allow(dead_code)] +#[path = "../../../src-tauri/src/prediction/lookup.rs"] +mod lookup; +use rusqlite::{Connection, OpenFlags}; +use sha2::{Digest, Sha256}; +use std::{ + collections::HashMap, + fs::{self, File}, + io::{BufWriter, Read, Seek, SeekFrom, Write}, + path::Path, +}; +type Error = Box; +fn number(out: &mut impl Write, n: u32) -> Result<(), Error> { + out.write_all(&n.to_le_bytes())?; + Ok(()) +} +fn convert(source: &Path, output: &Path) -> Result<(), Error> { + let c = Connection::open_with_flags(source, OpenFlags::SQLITE_OPEN_READ_ONLY)?; + let mut q = + c.prepare("SELECT ID,WORD,BASE_FREQUENCY FROM WORDS ORDER BY BASE_FREQUENCY DESC,ID ASC")?; + let words = q + .query_map([], |r| { + Ok(( + r.get::<_, u32>(0)?, + r.get::<_, String>(1)?, + r.get::<_, u32>(2)?, + )) + })? + .collect::, _>>()?; + let ranks: HashMap<_, _> = words + .iter() + .enumerate() + .map(|(rank, w)| (w.0, rank as u32)) + .collect(); + let temporary = output.with_extension("pending"); + let mut out = BufWriter::new(File::create(&temporary)?); + out.write_all(lookup::MAGIC)?; + out.write_all(&[0; 32])?; + number(&mut out, u32::try_from(words.len())?)?; + for (id, text, frequency) in &words { + number(&mut out, *id)?; + number(&mut out, *frequency)?; + number(&mut out, u32::try_from(text.len())?)?; + out.write_all(text.as_bytes())?; + } + for (n, table) in [(2, "BIGRAMS"), (3, "TRIGRAMS"), (4, "QUADGRAMS")] { + let count: u32 = c.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0))?; + number(&mut out, count)?; + let fields = (1..=n) + .map(|i| format!("ID{i}")) + .collect::>() + .join(","); + let context = (1..n) + .map(|i| format!("ID{i}")) + .collect::>() + .join(","); + let mut q=c.prepare(&format!("SELECT {fields},BASE_FREQUENCY FROM {table} ORDER BY BASE_FREQUENCY DESC,ID{n} ASC,{context}"))?; + let mut rows = q.query([])?; + while let Some(row) = rows.next()? { + for i in 0..n { + let id: u32 = row.get(i)?; + number(&mut out, *ranks.get(&id).ok_or("Missing word reference")?)?; + } + number(&mut out, row.get(n)?)?; + } + } + let folded: Vec<_> = words.iter().map(|w| w.1.to_lowercase()).collect(); + let mut prefixes: Vec<_> = (0..words.len()).collect(); + prefixes.sort_unstable_by(|&a, &b| folded[a].cmp(&folded[b]).then(a.cmp(&b))); + number(&mut out, u32::try_from(prefixes.len())?)?; + for rank in prefixes { + number(&mut out, u32::try_from(rank)?)?; + } + out.flush()?; + drop(out); + let mut input = File::open(&temporary)?; + input.seek(SeekFrom::Start(40))?; + let mut hash = Sha256::new(); + let mut buffer = [0; 65536]; + loop { + let n = input.read(&mut buffer)?; + if n == 0 { + break; + } + hash.update(&buffer[..n]); + } + drop(input); + let mut file = fs::OpenOptions::new().write(true).open(&temporary)?; + file.seek(SeekFrom::Start(8))?; + file.write_all(&hash.finalize())?; + file.sync_all()?; + drop(file); + lookup::Lookup::open(&temporary).map_err(|_| "Generated lookup failed validation")?; + fs::rename(&temporary, output)?; + println!("Generated {} bytes", fs::metadata(output)?.len()); + Ok(()) +} +fn checksum(path: &Path) -> Result, Error> { + let mut input = File::open(path)?; + let mut hash = Sha256::new(); + let mut buffer = [0; 65536]; + loop { + let n = input.read(&mut buffer)?; + if n == 0 { + break; + } + hash.update(&buffer[..n]); + } + Ok(hash.finalize().to_vec()) +} +fn main() -> Result<(), Error> { + let args: Vec<_> = std::env::args_os().collect(); + if args.len() == 4 && args[1] == "--check" { + let generated = std::env::temp_dir().join(format!( + "switchify-predictions-{}.lookup", + std::process::id() + )); + let result = (|| { + convert(Path::new(&args[2]), &generated)?; + if checksum(&generated)? != checksum(Path::new(&args[3]))? { + return Err("Generated lookup differs from packaged artifact".into()); + } + Ok(()) + })(); + let _ = fs::remove_file(&generated); + let _ = fs::remove_file(generated.with_extension("pending")); + return result; + } + if args.len() != 3 { + return Err( + "Usage: switchify-prediction-data [--check] ".into(), + ); + } + convert(Path::new(&args[1]), Path::new(&args[2])) +}