Use the verified English database for keyboard prediction

.github/workflows/ci.yml

@@ -134,3 +134,31 @@ jobs:
         run: |
           $env:SWITCHIFY_ALLOW_UNSIGNED_UIACCESS_PACKAGE = '1'
           pwsh ./scripts/Verify-WindowsUiAccessPackage.ps1
+
+      - name: Verify macOS packaged prediction resources
+        if: runner.os == 'macOS'
+        shell: bash
+        run: |
+          model=$(node scripts/check-packaged-prediction.mjs src-tauri/target/release/bundle/macos)
+          echo "SWITCHIFY_BENCHMARK_MODEL=$model" >> "$GITHUB_ENV"
+      - name: Verify Windows packaged prediction resources
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          $installer = Get-ChildItem src-tauri/target/release/bundle/nsis/*.exe | Select-Object -First 1
+          $unpacked = Join-Path $env:RUNNER_TEMP 'prediction-package'
+          & 7z x $installer.FullName "-o$unpacked" -y | Out-Null
+          if ($LASTEXITCODE -ne 0) { throw 'Could not extract the installer.' }
+          $model = node scripts/check-packaged-prediction.mjs $unpacked
+          if ($LASTEXITCODE -ne 0) { throw 'Packaged prediction resources failed verification.' }
+          "SWITCHIFY_BENCHMARK_MODEL=$model" >> $env:GITHUB_ENV
+      - name: Measure packaged prediction without desktop input
+        env:
+          SWITCHIFY_BENCHMARK_REPORT: prediction-benchmark.json
+        run: cargo test --release --lib --locked --manifest-path src-tauri/Cargo.toml bundled_prediction_benchmark -- --ignored --nocapture --test-threads=1
+      - name: Upload prediction measurements
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: prediction-benchmark-${{ runner.os }}
+          path: src-tauri/prediction-benchmark.json
+          if-no-files-found: error

.gitignore

@@ -17,3 +17,5 @@ src-tauri/icons/ios/
 src-tauri/tauri.release*.generated.json
 tools/prediction-data/target/
 src-tauri/resources/prediction-model/
+
+src-tauri/prediction-benchmark.json

package.json

@@ -10,7 +10,7 @@
     "dev": "vite",
     "build": "tsc --noEmit && vite build",
     "prediction-model": "node scripts/fetch-prediction-model.mjs",
-    "test": "vitest run && node --test scripts/create-update-feed.node-test.mjs",
+    "test": "vitest run && node --test scripts/*.node-test.mjs",
     "lint": "tsc --noEmit",
     "macos:setup-signing": "./scripts/setup-macos-dev-signing.sh",
     "macos:run": "./scripts/run-macos-signed.sh",
@@ -32,6 +32,7 @@
     "@types/react": "^19.0.0",
     "@types/react-dom": "^19.0.0",
     "@vitejs/plugin-react": "^4.3.4",
+    "fflate": "0.8.3",
     "jsdom": "^25.0.1",
     "tailwindcss": "^4.3.3",
     "typescript": "~5.8.3",

scripts/check-packaged-prediction.mjs

@@ -0,0 +1,12 @@
+// Verify shipped model/notices independently of the source checkout resources.
+import { readdir, readFile } from 'node:fs/promises';
+import { join, dirname, resolve } from 'node:path';
+import { verified } from './fetch-prediction-model.mjs';
+const root = resolve(process.argv[2]);
+const files = await readdir(root, { recursive: true });
+const models = files.filter(name => name.endsWith('english.sqlite'));
+if (models.length !== 1) throw new Error(`Expected one packaged prediction database, found ${models.length}`);
+const database = join(root, models[0]);
+const manifest = JSON.parse(await readFile(new URL('./prediction-model.json', import.meta.url),'utf8'));
+if (!(await verified(dirname(database), manifest))) throw new Error('Packaged prediction files failed verification');
+console.log(database);

scripts/fetch-prediction-model.mjs

@@ -1,58 +1,80 @@
-// Downloads the word prediction model into src-tauri/resources/prediction-model.
-// The model is too large to commit, so every file is pinned to one upstream
-// revision and verified by SHA-256. Verified files are not downloaded again.
-import { createHash } from "node:crypto";
-import { createReadStream, createWriteStream } from "node:fs";
-import { mkdir, rename, rm, stat } from "node:fs/promises";
-import { dirname, join } from "node:path";
-import { Readable } from "node:stream";
-import { pipeline } from "node:stream/promises";
-import { fileURLToPath } from "node:url";
+// Build-time acquisition only. The installed app never downloads prediction data.
+import { createHash } from 'node:crypto';
+import { readFile, writeFile, mkdir, mkdtemp, rename, rm, lstat } from 'node:fs/promises';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { unzipSync } from 'fflate';
 
-const revision = "d0ae6834f1df45e0e95b5fdae95e536f9ca7cd3f";
-const base = `https://huggingface.co/onnx-community/SmolLM2-135M-ONNX/resolve/${revision}`;
-const files = [
-  { name: "model.onnx", source: "onnx/model_int8.onnx", size: 135658354, sha256: "50ba80511ce74634d232a043b6c37775cca756b826b49d0a4a8eff958c4bbcc9" },
-  { name: "tokenizer.json", source: "tokenizer.json", size: 2053526, sha256: "139d2f4b4919b90953bdd3c0c40c94c9b23074799a766508dc3bf5eb8ab73351" },
-  { name: "config.json", source: "config.json", size: 1035, sha256: "2c5f23fddabecdf9c47d0048f555899822ec87ec4a28169393840ac7e74192c4" },
-];
-const target = join(dirname(fileURLToPath(import.meta.url)), "..", "src-tauri", "resources", "prediction-model");
+const digest = bytes => createHash('sha256').update(bytes).digest('hex');
+const MAX_ARCHIVE = 64 * 1024 * 1024;
 
-async function digest(path) {
-  const hash = createHash("sha256");
-  await pipeline(createReadStream(path), hash);
-  return hash.digest("hex");
+export async function verified(directory, manifest) {
+  try {
+    for (const [name, expected] of Object.entries(manifest.files)) {
+      const path = join(directory, name);
+      const info = await lstat(path);
+      if (!info.isFile() || info.size !== expected.size || digest(await readFile(path)) !== expected.sha256) return false;
+    }
+    return true;
+  } catch { return false; }
 }
 
-async function verified(path, file) {
-  try {
-    return (await stat(path)).size === file.size && (await digest(path)) === file.sha256;
-  } catch {
-    return false;
+export function unpack(bytes, manifest) {
+  if (bytes.length > MAX_ARCHIVE || digest(bytes) !== manifest.archive_sha256) throw new Error('Prediction archive checksum mismatch');
+  const seen = new Set();
+  const entries = unzipSync(new Uint8Array(bytes), { filter: entry => {
+    const name = entry.name;
+    if (name.startsWith('/') || name.includes('\\') || name.includes(':') || name.split('/').some(p => p === '..' || p === '.') || seen.has(name)) throw new Error('Unsafe prediction archive path');
+    seen.add(name);
+    if (!Object.hasOwn(manifest.files, name)) return false;
+    const expected = manifest.files[name];
+    if (entry.originalSize !== expected.size) throw new Error('Prediction entry size mismatch');
+    return true;
+  }});
+  for (const [name, expected] of Object.entries(manifest.files)) {
+    if (!entries[name] || entries[name].length !== expected.size || digest(entries[name]) !== expected.sha256) throw new Error('Prediction file checksum mismatch');
   }
+  return entries;
 }
 
-async function download(file) {
-  const path = join(target, file.name);
-  if (await verified(path, file)) return false;
-  const pending = `${path}.pending`;
-  for (let attempt = 1; ; attempt++) {
-    try {
-      const response = await fetch(`${base}/${file.source}`);
-      if (!response.ok) throw new Error(`HTTP ${response.status}`);
-      await pipeline(Readable.fromWeb(response.body), createWriteStream(pending));
-      if (!(await verified(pending, file))) throw new Error("size or SHA-256 mismatch");
-      await rename(pending, path);
-      return true;
-    } catch (error) {
-      await rm(pending, { force: true });
-      if (attempt === 3) throw new Error(`Could not fetch ${file.name}: ${error.message}`);
-      await new Promise((resolve) => setTimeout(resolve, attempt * 2000));
-    }
+async function download(url, fetcher) {
+  const response = await fetcher(url, { signal: AbortSignal.timeout(120000) });
+  if (!response.ok || !response.body) throw new Error(`Prediction download failed: HTTP ${response.status}`);
+  const chunks = [];
+  let size = 0;
+  for await (const chunk of response.body) {
+    size += chunk.length;
+    if (size > MAX_ARCHIVE) throw new Error('Prediction archive too large');
+    chunks.push(chunk);
   }
+  return Buffer.concat(chunks);
+}
+
+export async function acquire(target, manifest, fetcher = fetch) {
+  if (await verified(target, manifest)) return false;
+  // No destination writes until the complete archive and every selected file verify.
+  const entries = unpack(await download(manifest.url, fetcher), manifest);
+  await mkdir(dirname(target), { recursive: true });
+  const staging = await mkdtemp(`${target}.pending-`);
+  const previous = `${staging}.previous`;
+  let moved = false;
+  try {
+    for (const [name, bytes] of Object.entries(entries)) {
+      await mkdir(dirname(join(staging, name)), { recursive: true });
+      await writeFile(join(staging, name), bytes);
+    }
+    if (!(await verified(staging, manifest))) throw new Error('Prediction staging verification failed');
+    try { await rename(target, previous); moved = true; }
+    catch (error) { if (error.code !== 'ENOENT') throw error; }
+    try { await rename(staging, target); }
+    catch (error) { if (moved) await rename(previous, target); throw error; }
+    await rm(previous, { recursive: true, force: true });
+    return true;
+  } finally { await rm(staging, { recursive: true, force: true }); }
 }
 
-await mkdir(target, { recursive: true });
-for (const file of files) {
-  if (await download(file)) console.log(`Fetched ${file.name}`);
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const manifest = JSON.parse(await readFile(new URL('./prediction-model.json', import.meta.url), 'utf8'));
+  const target = fileURLToPath(new URL('../src-tauri/resources/prediction-model', import.meta.url));
+  if (await acquire(target, manifest)) console.log(`Fetched and verified ${manifest.model}`);
 }

scripts/fetch-prediction-model.node-test.mjs

@@ -0,0 +1,61 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, readFile, writeFile, rm, readdir } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createHash } from 'node:crypto';
+import { zipSync, strToU8 } from 'fflate';
+import { acquire, unpack, verified } from './fetch-prediction-model.mjs';
+const sha = b => createHash('sha256').update(b).digest('hex');
+function fixture(entries = { 'english.sqlite': strToU8('synthetic database'), 'corpus-notices/license': strToU8('test notice') }) {
+  const archive = zipSync(entries);
+  const files = Object.fromEntries(Object.entries(entries).filter(([n]) => !n.startsWith('../')).map(([n,b]) => [n, {sha256: sha(b), size:b.length}]));
+  return {archive, manifest:{url:'https://example.invalid/model', archive_sha256:sha(archive), files}};
+}
+test('verified cache works offline and corrupt cache is replaced with verified files only', async () => {
+  const dir = await mkdtemp(join(tmpdir(), 'switchify-model-'));
+  const target = join(dir, 'model');
+  const {archive,manifest} = fixture();
+  let requests = 0;
+  const fetcher = async () => { requests++; return new Response(archive); };
+  try {
+    assert.equal(await acquire(target, manifest, fetcher), true);
+    assert.equal(await verified(target, manifest), true);
+    assert.equal(await acquire(target, manifest, () => { throw Error('offline'); }), false);
+    assert.equal(requests, 1);
+    await writeFile(join(target,'english.sqlite'), 'corrupt');
+    assert.equal(await acquire(target, manifest, fetcher), true);
+    assert.equal(await verified(target, manifest), true);
+    assert.deepEqual(await readdir(dir), ['model']);
+  } finally { await rm(dir, {recursive:true, force:true}); }
+});
+test('corrupt and interrupted downloads do not replace existing resources or leave staging files', async () => {
+  const dir = await mkdtemp(join(tmpdir(), 'switchify-model-'));
+  const target = join(dir,'model');
+  const {archive,manifest} = fixture();
+  try {
+    await acquire(target,manifest,async()=>new Response(archive));
+    await writeFile(join(target,'english.sqlite'), 'old cache');
+    await assert.rejects(acquire(target,manifest,async()=>new Response('bad')), /checksum/);
+    await assert.rejects(acquire(target,manifest,async()=>new Response(new ReadableStream({start(c){c.enqueue(archive.subarray(0,8));c.error(Error('interrupted'));}}))), /interrupted/);
+    assert.equal(await readFile(join(target,'english.sqlite'),'utf8'),'old cache');
+    assert.deepEqual(await readdir(dir),['model']);
+  } finally { await rm(dir,{recursive:true,force:true}); }
+});
+test('archive paths, missing files, and per-file tampering fail closed', () => {
+  for (const path of ['../escape','/absolute','C:/absolute','folder\\escape','folder/../escape']) {
+    const {archive,manifest}=fixture({'english.sqlite':strToU8('db'),[path]:strToU8('bad')});
+    assert.throws(()=>unpack(archive,manifest), /Unsafe/);
+  }
+  const {archive,manifest}=fixture();
+  manifest.files['missing.txt']={size:1,sha256:sha(strToU8('x'))};
+  assert.throws(()=>unpack(archive,manifest), /checksum/);
+  delete manifest.files['missing.txt'];
+  manifest.files['english.sqlite'].sha256='0'.repeat(64);
+  assert.throws(()=>unpack(archive,manifest), /checksum/);
+});
+test('unlisted files are never extracted', () => {
+  const {archive,manifest}=fixture({'english.sqlite':strToU8('db'),'unused.txt':strToU8('extra')});
+  delete manifest.files['unused.txt'];
+  assert.deepEqual(Object.keys(unpack(archive,manifest)),['english.sqlite']);
+});

scripts/prediction-model.json

@@ -0,0 +1,52 @@
+{
+  "model": "en-aac-oanc-v1",
+  "revision": "c4b9d14ba8312179617ead4d2707e86ec825a2e6",
+  "url": "https://github.com/switchifyapp/switchify-prediction/releases/download/v0.1.0/switchify-english-en-aac-oanc-v1.zip",
+  "archive_sha256": "4537c70b44f553b31e618378a5cc500cd83ffa2dff940263ecabf88e18945b17",
+  "files": {
+    "english.sqlite": {
+      "sha256": "222253417d0a7a705823ffb7e599a3bcf5d5d3daf4a9d76161ac6b3e555aeaad",
+      "size": 29802496
+    },
+    "production-model.json": {
+      "sha256": "70be7dc26a372eef4e8731ef8a2ea4ef19fb9dc5d93803d14e65b02b232b7424",
+      "size": 12352
+    },
+    "source-manifest.json": {
+      "sha256": "5421600b9be1063599e8adc07ee14a4ff4bdbf91a07b6a9d4fcfc793e2aafd7d",
+      "size": 4624
+    },
+    "aac-source-manifest.json": {
+      "sha256": "4da7fef8208e9829881b8ae656d941a7db9bf22aab49a4a8e3d029576b847879",
+      "size": 3015
+    },
+    "ATTRIBUTION.md": {
+      "sha256": "3922703e6f3bb669ded05674f034d7f9ccfd7d359805d12439191496c7755266",
+      "size": 4188
+    },
+    "LICENSE": {
+      "sha256": "1e5643d5431cc5b2d04534dee7b3f716b24fc3b82493a1659534cbfa4fc8732c",
+      "size": 1079
+    },
+    "corpus-notices/aac-notice.html": {
+      "sha256": "d803db680180170cbdb8f6dafd8202ba29198ce9399f7f16b9ae802406e958d0",
+      "size": 7737
+    },
+    "corpus-notices/aac-readme.txt": {
+      "sha256": "556d1e4e136cf2f6b16296880df86301c260b669ea2eb4566ca164cdadc0f850",
+      "size": 2309
+    },
+    "corpus-notices/oanc-download.html": {
+      "sha256": "1a0e43a2434517dc58d8806b955b72ba3838ea47a70a77e7c767bdb38d185847",
+      "size": 6583
+    },
+    "corpus-notices/oanc-legacy-license.txt": {
+      "sha256": "b0dcd84d8980a7bbe5a9d82786b95376ef1c91bbd89c34969e34ef81cdc440a6",
+      "size": 12198
+    },
+    "corpus-notices/oanc-notice.html": {
+      "sha256": "d8d0e58baa168e99451586e4de0c211f801cb0a43238593a51426d98d9dea605",
+      "size": 5249
+    }
+  }
+}