Use dedicated lookup for word predictions

.github/workflows/ci.yml

@@ -32,6 +32,7 @@ jobs:
           cargo audit --file src-tauri/Cargo.lock
           cargo audit --file src-tauri/startup-launcher/Cargo.lock
           cargo audit --file tools/updater-signature-verifier/Cargo.lock
+          cargo audit --file tools/prediction-data/Cargo.lock
 
   frontend:
     runs-on: ubuntu-latest
@@ -115,6 +116,10 @@ jobs:
       - run: cargo fmt --manifest-path src-tauri/Cargo.toml --check
       - run: cargo clippy --locked --manifest-path src-tauri/Cargo.toml --all-targets -- -D warnings
       - run: cargo test --locked --manifest-path src-tauri/Cargo.toml
+      - run: cargo fmt --manifest-path tools/prediction-data/Cargo.toml --check
+      - run: cargo clippy --locked --manifest-path tools/prediction-data/Cargo.toml --all-targets -- -D warnings
+      - run: cargo test --locked --manifest-path tools/prediction-data/Cargo.toml
+      - run: cargo run --release --locked --manifest-path tools/prediction-data/Cargo.toml -- --check src-tauri/resources/WordData2017051601.db src-tauri/resources/word-predictions.lookup
       - run: cargo fmt --manifest-path tools/updater-signature-verifier/Cargo.toml --check
       - run: cargo clippy --locked --manifest-path tools/updater-signature-verifier/Cargo.toml --all-targets -- -D warnings
       - run: cargo test --locked --manifest-path tools/updater-signature-verifier/Cargo.toml

.gitignore

@@ -15,3 +15,4 @@ Thumbs.db
 src-tauri/icons/android/
 src-tauri/icons/ios/
 src-tauri/tauri.release*.generated.json
+tools/prediction-data/target/

scripts/measure-macos-prediction.mjs

@@ -0,0 +1,97 @@
+// Manual read-only fixture measurement through Launch Services. Never sends Accept.
+import { spawn, execFileSync } from 'node:child_process';
+import { mkdtempSync, writeFileSync, readFileSync, copyFileSync, rmSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+import { performance } from 'node:perf_hooks';
+import { setTimeout as delay } from 'node:timers/promises';
+
+const [bundle, resource] = process.argv.slice(2);
+if (process.platform !== 'darwin' || !bundle || !resource) {
+  throw Error('Usage on macOS: node scripts/measure-macos-prediction.mjs <app-bundle> <lookup-path>');
+}
+console.error('Focus a synthetic test field. Sampling starts in five seconds; no input is injected.');
+const root = mkdtempSync(join(tmpdir(), 'switchify-timing-'));
+const input = join(root, 'input');
+const output = join(root, 'output');
+const lookup = join(root, 'timing.lookup');
+const owned = new Set();
+let child;
+try {
+  copyFileSync(resource, lookup);
+  const frames = [];
+  for (let generation = 0; generation < 100; generation++) {
+    const body = Buffer.from(JSON.stringify({ Query: {
+      generation, revision: 0, edits: [], shift: false, caps: false,
+    } }));
+    const header = Buffer.alloc(4);
+    header.writeUInt32LE(body.length);
+    frames.push(header, body);
+  }
+  writeFileSync(input, Buffer.concat(frames));
+  writeFileSync(output, '');
+  await delay(5000);
+  const start = performance.now();
+  child = spawn('open', ['-n', '-g', '-W', '-a', resolve(bundle),
+    '--stdin', input, '--stdout', output, '--stderr', '/dev/null',
+    '--args', '--switchify-prediction-worker', lookup, '[]']);
+  let done = false;
+  let launchError;
+  child.on('exit', () => { done = true; });
+  child.on('error', error => { launchError = error; done = true; });
+  let offset = 0;
+  let count = 0;
+  let matched = 0;
+  let firstResponseMs;
+  let firstSuggestionsMs;
+  let lastResponseMs;
+  let maxSampledRssBytes = 0;
+  while (performance.now() - start < 10000) {
+    if (launchError) throw launchError;
+    const data = readFileSync(output);
+    while (offset + 4 <= data.length) {
+      const size = data.readUInt32LE(offset);
+      if (!size || size > 16384) throw Error('Invalid frame');
+      if (offset + 4 + size > data.length) break;
+      const response = JSON.parse(data.subarray(offset + 4, offset + 4 + size));
+      offset += 4 + size;
+      if (response.Suggestions?.generation !== count) throw Error('Unexpected response');
+      firstResponseMs ??= performance.now() - start;
+      if (response.Suggestions.batch?.words?.length) {
+        firstSuggestionsMs ??= performance.now() - start;
+        matched++;
+      }
+      count++;
+      lastResponseMs = performance.now() - start;
+    }
+    if (done) break;
+    try {
+      const rows = execFileSync('ps', ['-axo', 'pid=,rss=,command='], { encoding: 'utf8' }).split('\n');
+      for (const row of rows) {
+        const match = row.trim().match(/^(\d+)\s+(\d+)\s+(.+)$/);
+        if (match && match[3].includes('--switchify-prediction-worker') &&
+            match[3].includes(lookup) && !/^(?:\/usr\/bin\/)?open /.test(match[3])) {
+          owned.add(Number(match[1]));
+          maxSampledRssBytes = Math.max(maxSampledRssBytes, Number(match[2]) * 1024);
+        }
+      }
+    } catch { /* A worker may exit between samples. */ }
+    await delay(10);
+  }
+  console.log(JSON.stringify({
+    firstResponseMs, firstSuggestionsMs, count, matched,
+    maxSampledRssBytes: maxSampledRssBytes || null,
+    warmBatchMeanObservedMs: count > 1 ? (lastResponseMs - firstResponseMs) / (count - 1) : null,
+    elapsedMs: performance.now() - start,
+  }, null, 2));
+  if (count !== 100 || !matched || firstSuggestionsMs >= 2000) process.exitCode = 1;
+} finally {
+  for (const pid of owned) {
+    try {
+      const command = execFileSync('ps', ['-p', String(pid), '-o', 'command='], { encoding: 'utf8' });
+      if (command.includes(lookup)) process.kill(pid);
+    } catch { /* Already exited. */ }
+  }
+  child?.kill();
+  rmSync(root, { recursive: true, force: true });
+}

scripts/measure-word-prediction.mjs

@@ -1,19 +1,20 @@
 // Manual, read-only fixture measurement. Never sends an Accept request.
-import { spawn } from 'node:child_process';
+import { spawn, execFileSync } from 'node:child_process';
 import { resolve } from 'node:path';
 import { setTimeout as delay } from 'node:timers/promises';
 import { performance } from 'node:perf_hooks';
 
-const [executable, app, expectedWord] = process.argv.slice(2);
+const [executable, app, expectedWord, lookupPath] = process.argv.slice(2);
 if (!executable || !app || !expectedWord) {
-  console.error('Usage: node scripts/measure-word-prediction.mjs <executable> <app-label> <expected-word>');
+  console.error('Usage: node scripts/measure-word-prediction.mjs <executable> <app-label> <expected-word> [lookup-path]');
   process.exit(1);
 }
 console.log('Focus a synthetic test field now. Sampling starts in five seconds. No input will be injected.');
 await delay(5000);
+const spawnedAt = performance.now();
 const child = spawn(resolve(executable), [
   '--switchify-prediction-worker',
-  resolve('src-tauri/resources/WordData2017051601.db'),
+  resolve(lookupPath ?? 'src-tauri/resources/word-predictions.lookup'),
   '[]',
 ], { stdio: ['pipe', 'pipe', 'ignore'], windowsHide: true });
 let buffer = Buffer.alloc(0);
@@ -45,7 +46,7 @@ child.stdout.on('data', chunk => {
 async function query(generation) {
   if (failed) throw new Error('Worker unavailable');
   const bytes = Buffer.from(JSON.stringify({ Query: {
-    generation, edit: null, reset: true, shift: false, caps: false,
+    generation, edits: [], revision: 0, shift: false, caps: false,
   } }));
   const header = Buffer.alloc(4);
   header.writeUInt32LE(bytes.length);
@@ -66,31 +67,48 @@ let unsupported = 0;
 let failures = 0;
 let fixtureMatches = 0;
 let firstRequestMs;
+let firstResponseFromSpawnMs;
+let firstSuggestionsFromSpawnMs;
+let memoryBytes;
+let memoryMetric;
 try {
   // Up to 200 attempts, stopping after 100 supported samples.
   for (let generation = 0; generation < 200 && times.length < 100; generation++) {
     const start = performance.now();
     const response = await query(generation);
     const elapsed = performance.now() - start;
-    if (generation === 0) firstRequestMs = elapsed;
+    if (generation === 0) {
+      firstRequestMs = elapsed;
+      firstResponseFromSpawnMs = performance.now() - spawnedAt;
+    }
     const result = response.Suggestions;
     if (!result || result.generation !== generation) throw new Error('Invalid response');
     if (result.batch?.words?.length) {
+      firstSuggestionsFromSpawnMs ??= performance.now() - spawnedAt;
       if (generation !== 0) times.push(elapsed);
       if (result.batch.words.some(word => word.toLowerCase() === expectedWord.toLowerCase())) fixtureMatches++;
     } else unsupported++;
     await delay(Math.max(0, 250 - elapsed));
   }
 } catch { failures++; }
 finally {
+  try {
+    if (process.platform === 'darwin') {
+      memoryBytes = Number(execFileSync('ps', ['-o', 'rss=', '-p', String(child.pid)], { encoding: 'utf8' }).trim()) * 1024;
+      memoryMetric = 'residentBytesAtEnd';
+    } else if (process.platform === 'win32') {
+      memoryBytes = Number(execFileSync('powershell.exe', ['-NoProfile', '-Command', `(Get-Process -Id ${child.pid}).PeakWorkingSet64`], { encoding: 'utf8', windowsHide: true }).trim());
+      memoryMetric = 'peakWorkingSetBytes';
+    }
+  } catch { memoryBytes = undefined; }
   child.kill();
   process.off('SIGINT', stop);
   process.off('SIGTERM', stop);
 }
 times.sort((a, b) => a - b);
 const percentile = fraction => times.length ? +times[Math.ceil(times.length * fraction) - 1].toFixed(2) : null;
 console.log(JSON.stringify({
-  app, successfulSamples: times.length, unsupported, failures, fixtureMatches,
+  app, firstResponseFromSpawnMs, firstSuggestionsFromSpawnMs, memoryBytes, memoryMetric, successfulSamples: times.length, unsupported, failures, fixtureMatches,
   firstRequestMs: firstRequestMs === undefined ? null : +firstRequestMs.toFixed(2),
   medianMs: percentile(0.5), p95Ms: percentile(0.95), maximumMs: percentile(1),
 }, null, 2));

src-tauri/Cargo.toml

@@ -21,7 +21,6 @@ enigo = "=0.6.1"
 hmac = "0.12"
 keyring = "4.1.6"
 rand = "0.8"
-rusqlite = { version = "0.40.2", features = ["bundled"] }
 unicode-segmentation = "1.12"
 reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"] }
 serde = { version = "1", features = ["derive"] }
@@ -78,3 +77,6 @@ winreg = "0.55"
 
 [patch.crates-io]
 corebluetooth-rs = { path = "../vendor/corebluetooth-rs" }
+
+[dev-dependencies]
+rusqlite = { version = "0.40.2", features = ["bundled"] }

src-tauri/resources/PROVENANCE.md

@@ -7,4 +7,16 @@
 - SQLite integrity check during recovery: `ok`.
 - Rows: WORDS 165,420; BIGRAMS 976,804; TRIGRAMS 996,846; QUADGRAMS 953,630.
 
-This is the unchanged original SQLite file. The later Realm archive is not used. Runtime connections are read-only and ignore personal frequency columns. No captured text or user vocabulary is written into this resource.
+This is the unchanged original SQLite file. The later Realm archive is not used. It is retained for conversion and parity tests, not shipped in the app. The converter ignores personal frequency columns. No captured text or user vocabulary is written into this resource.
+
+## Packaged dedicated lookup
+
+`word-predictions.lookup` is generated from the unchanged database. It contains frequency-ranked words and n-grams, a sorted prefix index, a versioned header and a SHA-256 payload checksum. The runtime validates counts, bounds, references and ordering before serving predictions. The app packages only this lookup and does not write to it.
+
+Regenerate with Rust 1.97.1 from the repository root:
+
+```sh
+cargo run --release --locked --manifest-path tools/prediction-data/Cargo.toml -- src-tauri/resources/WordData2017051601.db src-tauri/resources/word-predictions.lookup
+```
+
+Use `-- --check` before the two paths to regenerate into a temporary file and verify byte-for-byte reproducibility. CI checks the artifact on Mac and Windows. The original SQLite implementation is retained only in tests to verify identical ordered suggestions.