From 4e0cbdafd03f08214995cbab757eab1a3157f12a Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Sun, 4 Oct 2026 16:10:21 +0100 Subject: [PATCH 1/3] Reserve three instant and three generated prediction slots --- .github/workflows/ci.yml | 4 +- docs/six-predictions.md | 36 ++ scripts/check-packaged-prediction.mjs | 2 +- scripts/fetch-prediction-neural.mjs | 28 +- scripts/fetch-prediction-neural.node-test.mjs | 26 +- scripts/measure-neural.py | 4 +- scripts/prediction-neural.json | 327 ++++++++++-------- scripts/probe-neural-worker.py | 16 +- src-tauri/Cargo.lock | 6 +- src-tauri/Cargo.toml | 4 +- src-tauri/src/point_workflow.rs | 2 +- src-tauri/src/prediction/database.rs | 13 +- src-tauri/src/prediction/mod.rs | 4 +- src-tauri/src/prediction/model.rs | 104 ++++-- src-tauri/src/prediction/worker.rs | 164 ++++++--- src-tauri/src/scan_keyboard.rs | 138 ++++++-- 16 files changed, 590 insertions(+), 288 deletions(-) create mode 100644 docs/six-predictions.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f31bede8..5844375d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,7 +112,9 @@ jobs: toolchain: 1.97.1 components: rustfmt, clippy - run: npm ci - - name: Fetch prediction model + - name: Fetch prediction model and prepared companion workers + env: + GH_TOKEN: ${{ github.token }} run: npm run prediction-model - name: Diagnose prepared ARM worker if: runner.os == 'macOS' diff --git a/docs/six-predictions.md b/docs/six-predictions.md new file mode 100644 index 00000000..c403ffff --- /dev/null +++ b/docs/six-predictions.md @@ -0,0 +1,36 @@ +# Six prediction slots + +Slots 1–3 are the statistical predictor's first three suggestions. Slots 4–6 are +additional SmolLM2 words. Empty slots stay empty and cannot be selected in either +scan direction. The entire displayed row stays stable until scanning leaves it. + +The worker retains exact displayed/latest batch tokens and validates acceptance +against the edit revision, generation, activity epoch and foreground target. +Capitalization, prefix replacement and trailing spaces use the existing path. +No focused-field capture, personal learning or new settings are introduced. + +The companion uses protocol 2. Model and tokenizer bytes are unchanged. Startup +remains bounded to 30 seconds and each reply to two seconds. Failure leaves the +instant slots available, with explicit retry or keyboard reopening required. + +## Prepared assets + +The build can consume checksum-pinned companion CI archives before publication, +using an authenticated GitHub CLI. Verified local caches work offline. These CI +artifacts expire and are not permanent release assets. Before merging/releasing +the desktop, separately authorize companion artifact publication and replace +prepared artifact sources with the permanent release URLs for the same bytes. +No runtime network access or model setup is needed. + +## Validation + +Run the repository checks and scripts/measure-neural.py --build with explicit +--model, --worker and --output paths. The integration fixture uses synthetic +context and fake input/activity adapters. It measures 1,000 warmed queries, +asserts instant-slot stability, and reports fill, latency, failures and process +tree memory. Its overall measurement bound is 40 minutes; the application reply +deadline remains two seconds. + +Use switchify-prediction scripts/generation_evaluate.py for the frozen corpus +comparison. Existing model quality regressions remain. Neither default activation +nor a successful performance run establishes unseen-data accuracy. diff --git a/scripts/check-packaged-prediction.mjs b/scripts/check-packaged-prediction.mjs index 984d1417..474870b0 100644 --- a/scripts/check-packaged-prediction.mjs +++ b/scripts/check-packaged-prediction.mjs @@ -16,7 +16,7 @@ if (!(await verified(neural, { files: bundlePins }))) throw new Error('Packaged const pin = JSON.parse(await readFile(new URL('./prediction-neural.json', import.meta.url), 'utf8')); if (JSON.stringify(JSON.parse(await readFile(join(neural, 'model-bundle.json'), 'utf8'))) !== JSON.stringify(pin.bundle)) throw new Error('Wrong packaged neural policy'); const build = JSON.parse(await readFile(join(neural, 'worker-notices/BUILD.json'), 'utf8')); -if (build.commit !== pin.revision || build.version !== '0.2.0') throw new Error('Wrong worker provenance'); +if (build.commit !== pin.revision || build.version !== '0.2.1' || build.worker_protocol !== 2) throw new Error('Wrong worker provenance'); const workerPin = pin.workers[build.target]; if (!workerPin) throw new Error('Unsupported packaged worker'); const windows = build.target.includes('windows'); diff --git a/scripts/fetch-prediction-neural.mjs b/scripts/fetch-prediction-neural.mjs index eda64403..2fbe569b 100644 --- a/scripts/fetch-prediction-neural.mjs +++ b/scripts/fetch-prediction-neural.mjs @@ -1,6 +1,6 @@ // Build-time only. Installed applications never fetch models or workers. import { createHash } from 'node:crypto'; -import { readFile, writeFile, mkdir, copyFile, chmod } from 'node:fs/promises'; +import { readFile, writeFile, mkdir, copyFile, chmod, mkdtemp, rm } from 'node:fs/promises'; import { join, resolve, basename } from 'node:path'; import { fileURLToPath } from 'node:url'; import { spawnSync } from 'node:child_process'; @@ -36,6 +36,30 @@ export async function pinnedFile(path, pin, url, fetcher = fetch) { await writeFile(path, bytes); } +// Prepared companion artifacts are review inputs, not a published release. +// Only the pinned archive and selected file hashes are trusted. A later release +// replaces this source with its permanent public asset URL without changing bytes. +export async function acquireWorker(workers, pin, cache, runner = run) { + if (!pin.prepared_artifact) return acquire(workers, pin); + if (await verified(workers, pin)) return false; + const prepared = pin.prepared_artifact; + if (!Number.isSafeInteger(prepared.run_id) || prepared.run_id <= 0 || + !/^neural-[a-z0-9-]+$/.test(prepared.name) || + basename(prepared.archive) !== prepared.archive || !prepared.archive.endsWith('.zip')) { + throw new Error('Invalid prepared worker source'); + } + await mkdir(cache, { recursive: true }); + const staging = await mkdtemp(join(cache, 'prepared-')); + try { + await runner('gh', ['run', 'download', String(prepared.run_id), '--repo', + 'switchifyapp/switchify-prediction', '--name', prepared.name, '--dir', staging], root); + const bytes = await readFile(join(staging, prepared.archive)); + return await acquire(workers, { ...pin, url: 'prepared-worker' }, async () => new Response(bytes)); + } finally { + await rm(staging, { recursive: true, force: true }); + } +} + export async function prepare() { const target = process.env.TAURI_ENV_TARGET_TRIPLE ?? (process.platform === 'win32' ? 'x86_64-pc-windows-msvc' : @@ -48,7 +72,7 @@ export async function prepare() { const binaries = join(root, 'src-tauri/binaries'); await mkdir(bundle, { recursive: true }); await mkdir(binaries, { recursive: true }); - await acquire(workers, workerPin); + await acquireWorker(workers, workerPin, cache); for (const name of Object.keys(workerPin.files)) { const file = basename(name); if (file.startsWith('switchify-smol-worker')) { diff --git a/scripts/fetch-prediction-neural.node-test.mjs b/scripts/fetch-prediction-neural.node-test.mjs index 3cfc1019..471c8066 100644 --- a/scripts/fetch-prediction-neural.node-test.mjs +++ b/scripts/fetch-prediction-neural.node-test.mjs @@ -4,7 +4,8 @@ import { createHash } from 'node:crypto'; import { mkdtemp, readFile, writeFile, rm } from 'node:fs/promises'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; -import { pinnedFile, bundlePins } from './fetch-prediction-neural.mjs'; +import { pinnedFile, bundlePins, acquireWorker } from './fetch-prediction-neural.mjs'; +import { zipSync } from 'fflate'; test('neural sources reject corrupt/oversized input and retain the previous cache', async () => { const dir = await mkdtemp(join(tmpdir(), 'neural-source-')); @@ -29,3 +30,26 @@ test('installed model pins retain the approved Q8 identity and license', () => { assert.ok(bundlePins['MODEL_LICENSE.txt']); assert.ok(bundlePins['MODEL_CARD.md']); }); + +test('prepared workers require pinned archive bytes and support verified offline reuse', async () => { + const dir = await mkdtemp(join(tmpdir(), 'neural-prepared-')); + const hash = bytes => createHash('sha256').update(bytes).digest('hex'); + const bytes = Buffer.from('worker fixture'); + const zip = zipSync({ 'bundle/worker': bytes }); + const pin = { prepared_artifact: { run_id: 1, name: 'neural-windows-latest', archive: 'workers.zip' }, + archive_sha256: hash(zip), files: { 'bundle/worker': { size: bytes.length, sha256: hash(bytes) } } }; + const target = join(dir, 'workers'); + const runner = contents => async (command, args) => { + assert.equal(command, 'gh'); + assert.equal(args[3], '--repo'); + await writeFile(join(args.at(-1), 'workers.zip'), contents); + }; + try { + await assert.rejects(acquireWorker(target, pin, dir, runner(Buffer.from('corrupt')))); + await acquireWorker(target, pin, dir, runner(zip)); + assert.equal(await readFile(join(target, 'bundle/worker'), 'utf8'), 'worker fixture'); + await acquireWorker(target, pin, dir, () => { throw new Error('offline cache was not used'); }); + await assert.rejects(acquireWorker(join(dir, 'other'), { ...pin, + prepared_artifact: { ...pin.prepared_artifact, archive: '../bad.zip' } }, dir, runner(zip))); + } finally { await rm(dir, { recursive: true, force: true }); } +}); diff --git a/scripts/measure-neural.py b/scripts/measure-neural.py index 7df6dad4..33853c28 100644 --- a/scripts/measure-neural.py +++ b/scripts/measure-neural.py @@ -58,12 +58,12 @@ peak = max(peak, rss) except psutil.NoSuchProcess: pass - if time.monotonic() - start > 1200: + if time.monotonic() - start > 2400: for child in tracked.children(recursive=True): child.kill() process.kill() process.wait() - raise RuntimeError('Integration benchmark exceeded 20 minutes') + raise RuntimeError('Integration benchmark exceeded 40 minutes') time.sleep(0.02) if report_path.is_file(): report = json.loads(report_path.read_bytes()) diff --git a/scripts/prediction-neural.json b/scripts/prediction-neural.json index c03be704..844bbfee 100644 --- a/scripts/prediction-neural.json +++ b/scripts/prediction-neural.json @@ -1,5 +1,5 @@ { - "revision": "99a3ef03cb54500006887a998547fded4ad39e0e", + "revision": "40c09c3f125403811b167d042730826ff6d451b7", "bundle": { "format_version": 1, "model_id": "smollm2-135m-q8-v1", @@ -10,7 +10,19 @@ "context_tokens": 64, "shortlist": 8, "max_results": 5, - "scoring": "whole-word log probability plus boundary probability, sequential" + "scoring": "whole-word log probability plus boundary probability, sequential", + "generation": { + "protocol_version": 2, + "beam_width": 8, + "max_tokens_per_word": 8, + "max_forward_evaluations": 64, + "max_results": 3, + "max_word_bytes": 128, + "reply_deadline_ms": 2000, + "scoring": "whole-word probability including following boundary; normalized duplicates excluded", + "minimum_boundary_probability": 0.5, + "search_cutoff_ms": 1600 + } }, "files": { "model.gguf": { @@ -63,232 +75,249 @@ }, "workers": { "aarch64-apple-darwin": { - "url": "https://github.com/switchifyapp/switchify-prediction/releases/download/v0.2.0/switchify-prediction-neural-0.2.0-aarch64-apple-darwin.zip", - "archive_sha256": "af2c2fe55e9681522bb37478d71256f9cdb5b436945eb2034eaf1a25d90b2c36", + "prepared_artifact": { + "run_id": 37211131470, + "name": "neural-macos-latest", + "archive": "switchify-prediction-neural-0.2.1-aarch64-apple-darwin.zip" + }, + "archive_sha256": "2ba7ff29b2286eb02b0435f608e44934d89d119967418d15b8f95aaf5072544c", "files": { - "switchify-prediction-neural-0.2.0-aarch64-apple-darwin/QUALIFICATION.md": { - "size": 7086, - "sha256": "f5d13a7406728515112cab4da093ee19642f68abbb9f30c8c8f511ae68ec8eb3" + "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/QUALIFICATION.md": { + "sha256": "f5d13a7406728515112cab4da093ee19642f68abbb9f30c8c8f511ae68ec8eb3", + "size": 7086 }, - "switchify-prediction-neural-0.2.0-aarch64-apple-darwin/QUALIFICATION.json": { - "size": 473, - "sha256": "55b6e9429ecb290fd194682406773a6f07102861529d5e1c0e51a6d2d4fc8c7e" + "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/QUALIFICATION.json": { + "sha256": "55b6e9429ecb290fd194682406773a6f07102861529d5e1c0e51a6d2d4fc8c7e", + "size": 473 }, - "switchify-prediction-neural-0.2.0-aarch64-apple-darwin/LICENSE": { - "size": 1079, - "sha256": "1e5643d5431cc5b2d04534dee7b3f716b24fc3b82493a1659534cbfa4fc8732c" + "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/LICENSE": { + "sha256": "1e5643d5431cc5b2d04534dee7b3f716b24fc3b82493a1659534cbfa4fc8732c", + "size": 1079 }, - "switchify-prediction-neural-0.2.0-aarch64-apple-darwin/Cargo.lock": { - "size": 46211, - "sha256": "e31dd348e4db5cce55e43fa6730302678f1cff85ae833a67fe0e7a405f64954a" + "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/Cargo.lock": { + "sha256": "a49ecb4d18fa6f26d8777701e2bbad87691fff4ec05664e5a746bf18580673b0", + "size": 46262 }, - "switchify-prediction-neural-0.2.0-aarch64-apple-darwin/THIRD_PARTY_NOTICES.md": { - "size": 1545871, - "sha256": "6bc01a7be7b3aaa59d2c04acbbdea91b20699f0f20d556debc5e674eb0d3600f" + "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/THIRD_PARTY_NOTICES.md": { + "sha256": "6bc01a7be7b3aaa59d2c04acbbdea91b20699f0f20d556debc5e674eb0d3600f", + "size": 1545871 }, - "switchify-prediction-neural-0.2.0-aarch64-apple-darwin/README.md": { - "size": 7136, - "sha256": "9db3b6561aeb463ec0988ec25200d3b32318daf96c1f11ebe39551bcb667be63" + "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/README.md": { + "sha256": "02cf02f01d6e711cf6415304ace910722cb76268e90358c5362feaec034a090b", + "size": 8913 }, - "switchify-prediction-neural-0.2.0-aarch64-apple-darwin/switchify-smol-worker": { - "size": 7059808, - "sha256": "6e1107badf39c66102fc5d3370254976218d19ca5a673ce6203ab1c047f9a7bc" + "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/switchify-smol-worker": { + "sha256": "c968323826413de70108e70a94bf1bd5554cff7db4708035590b1b99f758581e", + "size": 7322096 }, - "switchify-prediction-neural-0.2.0-aarch64-apple-darwin/SHA256SUMS": { - "size": 998, - "sha256": "917dfee26b1eb70e4aadca383d26e6a0b5f0561f9552ad639fc5682fd6066e70" + "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/SHA256SUMS": { + "sha256": "c7a37e54a371f6f16731e7bf88ddaafe4eb5934fa26696641e731876aa9fafef", + "size": 998 }, - "switchify-prediction-neural-0.2.0-aarch64-apple-darwin/RUST_LIBRARY_COPYRIGHT.html": { - "size": 279302, - "sha256": "0a65bb747c49c7bb816cbc7188319bd6e4e8d08091c1190b8a3c0971c47968ed" + "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/RUST_LIBRARY_COPYRIGHT.html": { + "sha256": "0a65bb747c49c7bb816cbc7188319bd6e4e8d08091c1190b8a3c0971c47968ed", + "size": 279302 }, - "switchify-prediction-neural-0.2.0-aarch64-apple-darwin/verify_bundle.py": { - "size": 1282, - "sha256": "69c124d037d9e25018f690c10b861c4388c01a8cd8eb3ed438d17136196c1194" + "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/verify_bundle.py": { + "sha256": "69c124d037d9e25018f690c10b861c4388c01a8cd8eb3ed438d17136196c1194", + "size": 1282 }, - "switchify-prediction-neural-0.2.0-aarch64-apple-darwin/BUILD.json": { - "size": 568, - "sha256": "0c927cae38c96e4c9067008ad70d66a894f7bb5b43fff931483284ec336549c3" + "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/BUILD.json": { + "sha256": "59cf2afc66e1bcd5c3481d241849039f02a4d2e7d0b6b7a236bfa922fb173c52", + "size": 987 }, - "switchify-prediction-neural-0.2.0-aarch64-apple-darwin/SECURITY.md": { - "size": 1973, - "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182" + "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/SECURITY.md": { + "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182", + "size": 1973 } } }, "x86_64-apple-darwin": { - "url": "https://github.com/switchifyapp/switchify-prediction/releases/download/v0.2.0/switchify-prediction-neural-0.2.0-x86_64-apple-darwin.zip", - "archive_sha256": "abe05e5edf77ea94db06a60bb9a8dd6ca30632657f9ea3fa3fb147104d74e7b5", + "prepared_artifact": { + "run_id": 37211131470, + "name": "neural-macos-15-intel", + "archive": "switchify-prediction-neural-0.2.1-x86_64-apple-darwin.zip" + }, + "archive_sha256": "e477f97866303487d2f3c4e9551fc6fe5fe6e5f38a3a453cdbf1917957f763bd", "files": { - "switchify-prediction-neural-0.2.0-x86_64-apple-darwin/QUALIFICATION.md": { + "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/QUALIFICATION.md": { "size": 7086, "sha256": "f5d13a7406728515112cab4da093ee19642f68abbb9f30c8c8f511ae68ec8eb3" }, - "switchify-prediction-neural-0.2.0-x86_64-apple-darwin/QUALIFICATION.json": { + "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/QUALIFICATION.json": { "size": 473, "sha256": "55b6e9429ecb290fd194682406773a6f07102861529d5e1c0e51a6d2d4fc8c7e" }, - "switchify-prediction-neural-0.2.0-x86_64-apple-darwin/LICENSE": { + "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/LICENSE": { "size": 1079, "sha256": "1e5643d5431cc5b2d04534dee7b3f716b24fc3b82493a1659534cbfa4fc8732c" }, - "switchify-prediction-neural-0.2.0-x86_64-apple-darwin/Cargo.lock": { - "size": 46211, - "sha256": "e31dd348e4db5cce55e43fa6730302678f1cff85ae833a67fe0e7a405f64954a" + "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/Cargo.lock": { + "size": 46262, + "sha256": "a49ecb4d18fa6f26d8777701e2bbad87691fff4ec05664e5a746bf18580673b0" }, - "switchify-prediction-neural-0.2.0-x86_64-apple-darwin/THIRD_PARTY_NOTICES.md": { + "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/THIRD_PARTY_NOTICES.md": { "size": 1545871, "sha256": "6bc01a7be7b3aaa59d2c04acbbdea91b20699f0f20d556debc5e674eb0d3600f" }, - "switchify-prediction-neural-0.2.0-x86_64-apple-darwin/README.md": { - "size": 7136, - "sha256": "9db3b6561aeb463ec0988ec25200d3b32318daf96c1f11ebe39551bcb667be63" + "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/README.md": { + "size": 8913, + "sha256": "02cf02f01d6e711cf6415304ace910722cb76268e90358c5362feaec034a090b" }, - "switchify-prediction-neural-0.2.0-x86_64-apple-darwin/switchify-smol-worker": { - "size": 7635864, - "sha256": "27e6eb09456826f5a127b3d935e20af29a90a203f91fa73cc5b68445298aa2a6" + "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/switchify-smol-worker": { + "size": 7903992, + "sha256": "7a0b9177485369195ea0af9e1578104c2efb3f721208f7fec0cb713ef8d00a9f" }, - "switchify-prediction-neural-0.2.0-x86_64-apple-darwin/switchify-smol-worker-avx2": { - "size": 7786832, - "sha256": "e5e6fefff242e27603ca1b2f2885c52d10938111958c1c674266e02f8a01c3d5" + "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/switchify-smol-worker-avx2": { + "size": 8055144, + "sha256": "7ac36b3e7f30d69be0f88b1b452b66791ac03fe4534e06ebd6c21c46c95e041a" }, - "switchify-prediction-neural-0.2.0-x86_64-apple-darwin/SHA256SUMS": { + "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/SHA256SUMS": { "size": 1091, - "sha256": "d3dbdd7df98ff6202cc55b3860eba3fb5e1f5c4a4c75d0fe1186f96d05141ebd" + "sha256": "7eac576a4042419adedb7dac5f3ab847e6bce6c3c2a8bdb23e29ffe2382ef627" }, - "switchify-prediction-neural-0.2.0-x86_64-apple-darwin/RUST_LIBRARY_COPYRIGHT.html": { + "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/RUST_LIBRARY_COPYRIGHT.html": { "size": 279302, "sha256": "0a65bb747c49c7bb816cbc7188319bd6e4e8d08091c1190b8a3c0971c47968ed" }, - "switchify-prediction-neural-0.2.0-x86_64-apple-darwin/verify_bundle.py": { + "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/verify_bundle.py": { "size": 1282, "sha256": "69c124d037d9e25018f690c10b861c4388c01a8cd8eb3ed438d17136196c1194" }, - "switchify-prediction-neural-0.2.0-x86_64-apple-darwin/BUILD.json": { - "size": 605, - "sha256": "17c25a610f304162d4b8063261d74e2cfdc986bec92ca41598301f751ae70edd" + "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/BUILD.json": { + "size": 1024, + "sha256": "1ac79006baf2fe3499c027a509357ae19130a41b1bb56fd4687e5c1b722fe9a0" }, - "switchify-prediction-neural-0.2.0-x86_64-apple-darwin/SECURITY.md": { + "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/SECURITY.md": { "size": 1973, "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182" } } }, "x86_64-pc-windows-msvc": { - "url": "https://github.com/switchifyapp/switchify-prediction/releases/download/v0.2.0/switchify-prediction-neural-0.2.0-x86_64-pc-windows-msvc.zip", - "archive_sha256": "48fb3f678fcf5788c305df40ecd3e3b1485382f186700aff6fd0ec1d9edd0a91", + "prepared_artifact": { + "run_id": 37211131470, + "name": "neural-windows-latest", + "archive": "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc.zip" + }, + "archive_sha256": "6b2a385fca62f2947add1de286440d87778c94c6d79ff6892813cc702514c45e", "files": { - "switchify-prediction-neural-0.2.0-x86_64-pc-windows-msvc/BUILD.json": { - "size": 627, - "sha256": "1358158df92945aeb66516326441e1b62066e9a901f243c899d2f02a2df87f0b" + "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/BUILD.json": { + "sha256": "ebe7986dc80c01e19b55697b740a2e42950f49314890eeedb233baaa25ddbf69", + "size": 1059 }, - "switchify-prediction-neural-0.2.0-x86_64-pc-windows-msvc/Cargo.lock": { - "size": 46211, - "sha256": "e31dd348e4db5cce55e43fa6730302678f1cff85ae833a67fe0e7a405f64954a" + "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/Cargo.lock": { + "sha256": "a49ecb4d18fa6f26d8777701e2bbad87691fff4ec05664e5a746bf18580673b0", + "size": 46262 }, - "switchify-prediction-neural-0.2.0-x86_64-pc-windows-msvc/LICENSE": { - "size": 1100, - "sha256": "38a190016c5d23b6acf37beda35bdef634a3572da76bb7c6ebfec84763648357" + "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/LICENSE": { + "sha256": "38a190016c5d23b6acf37beda35bdef634a3572da76bb7c6ebfec84763648357", + "size": 1100 }, - "switchify-prediction-neural-0.2.0-x86_64-pc-windows-msvc/QUALIFICATION.json": { - "size": 473, - "sha256": "55b6e9429ecb290fd194682406773a6f07102861529d5e1c0e51a6d2d4fc8c7e" + "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/QUALIFICATION.json": { + "sha256": "55b6e9429ecb290fd194682406773a6f07102861529d5e1c0e51a6d2d4fc8c7e", + "size": 473 }, - "switchify-prediction-neural-0.2.0-x86_64-pc-windows-msvc/QUALIFICATION.md": { - "size": 7147, - "sha256": "934945362415a5672a5c05887553911778f2032508b5efcb9a755c5e551548dc" + "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/QUALIFICATION.md": { + "sha256": "934945362415a5672a5c05887553911778f2032508b5efcb9a755c5e551548dc", + "size": 7147 }, - "switchify-prediction-neural-0.2.0-x86_64-pc-windows-msvc/README.md": { - "size": 7136, - "sha256": "9db3b6561aeb463ec0988ec25200d3b32318daf96c1f11ebe39551bcb667be63" + "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/README.md": { + "sha256": "02cf02f01d6e711cf6415304ace910722cb76268e90358c5362feaec034a090b", + "size": 8913 }, - "switchify-prediction-neural-0.2.0-x86_64-pc-windows-msvc/RUST_LIBRARY_COPYRIGHT.html": { - "size": 279302, - "sha256": "0a65bb747c49c7bb816cbc7188319bd6e4e8d08091c1190b8a3c0971c47968ed" + "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/RUST_LIBRARY_COPYRIGHT.html": { + "sha256": "0a65bb747c49c7bb816cbc7188319bd6e4e8d08091c1190b8a3c0971c47968ed", + "size": 279302 }, - "switchify-prediction-neural-0.2.0-x86_64-pc-windows-msvc/SECURITY.md": { - "size": 1973, - "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182" + "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/SECURITY.md": { + "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182", + "size": 1973 }, - "switchify-prediction-neural-0.2.0-x86_64-pc-windows-msvc/SHA256SUMS": { - "size": 1116, - "sha256": "eb5b63c21f825c8a5078e07bf3578bcf3b145b60babf18f0583a850a00383b2f" + "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/SHA256SUMS": { + "sha256": "a585397a8c2ad3f12f767f1d87a40d5bc7c10d3c087ed0183b5554ef48af9ea8", + "size": 1116 }, - "switchify-prediction-neural-0.2.0-x86_64-pc-windows-msvc/switchify-smol-worker-avx2.exe": { - "size": 8453632, - "sha256": "eeecd019b9add31d9da9c1ae598f99df4351cabf54b86b62342c1fd06facf7b1" + "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/switchify-smol-worker-avx2.exe": { + "sha256": "56884d10a431f186c369943cc335b5108b7d920d308fd465234827fba8aeb38d", + "size": 8709120 }, - "switchify-prediction-neural-0.2.0-x86_64-pc-windows-msvc/switchify-smol-worker.exe": { - "size": 8256512, - "sha256": "e0f334e7489cddff0d5978e43dcff3d79eef08c3d4f3f81b0814e816a556f524" + "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/switchify-smol-worker.exe": { + "sha256": "537028f08e65dd08e7e9353f03308e0a641ad65b8424a17af318fb8f86d8eaed", + "size": 8510464 }, - "switchify-prediction-neural-0.2.0-x86_64-pc-windows-msvc/THIRD_PARTY_NOTICES.md": { - "size": 1552635, - "sha256": "448c7f55a9ef918d5cf67f57cfb141c62f7bc6d819cc168fc4ad20f73b34c5f0" + "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/THIRD_PARTY_NOTICES.md": { + "sha256": "448c7f55a9ef918d5cf67f57cfb141c62f7bc6d819cc168fc4ad20f73b34c5f0", + "size": 1552635 }, - "switchify-prediction-neural-0.2.0-x86_64-pc-windows-msvc/verify_bundle.py": { - "size": 1315, - "sha256": "96b47d695b104c61ca65de856108d8850a6da7c00220ae72dd91b8e73cb8f6f2" + "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/verify_bundle.py": { + "sha256": "96b47d695b104c61ca65de856108d8850a6da7c00220ae72dd91b8e73cb8f6f2", + "size": 1315 } } }, "x86_64-unknown-linux-gnu": { - "url": "https://github.com/switchifyapp/switchify-prediction/releases/download/v0.2.0/switchify-prediction-neural-0.2.0-x86_64-unknown-linux-gnu.zip", - "archive_sha256": "ce3350ebd76110474f783c5b0eda352b6a2011dbc5f00f0385cb9d149f1e968e", + "prepared_artifact": { + "run_id": 37211131470, + "name": "neural-ubuntu-latest", + "archive": "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu.zip" + }, + "archive_sha256": "248b14c7220a58a9119eac6dadf63394f35d0bbc2d105d58563c55b4b701a7b2", "files": { - "switchify-prediction-neural-0.2.0-x86_64-unknown-linux-gnu/BUILD.json": { - "size": 632, - "sha256": "ae22d56dc215edec0a011b3a9f23116dc18bafdb2cab12823269ee26f5358d46" + "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/BUILD.json": { + "sha256": "1440f4897ac396ff4b3ef11baf963042c97e9113b5177fd4cac811e598e60240", + "size": 1051 }, - "switchify-prediction-neural-0.2.0-x86_64-unknown-linux-gnu/RUST_LIBRARY_COPYRIGHT.html": { - "size": 279302, - "sha256": "0a65bb747c49c7bb816cbc7188319bd6e4e8d08091c1190b8a3c0971c47968ed" + "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/RUST_LIBRARY_COPYRIGHT.html": { + "sha256": "0a65bb747c49c7bb816cbc7188319bd6e4e8d08091c1190b8a3c0971c47968ed", + "size": 279302 }, - "switchify-prediction-neural-0.2.0-x86_64-unknown-linux-gnu/switchify-smol-worker": { - "size": 9876080, - "sha256": "fb87190f587775fca62976f6f884c2ef1267a71c95921313a747ac9d667e0448" + "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/switchify-smol-worker": { + "sha256": "3dba4916ff39f2779e80906aaf8c1aac5270e746f4bc7d3ecef4e0532915977a", + "size": 10153664 }, - "switchify-prediction-neural-0.2.0-x86_64-unknown-linux-gnu/SHA256SUMS": { - "size": 1091, - "sha256": "c1ba9e8c96e6b7a1fe435e7488492ed45d8747411e208d345e17a585e1757768" + "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/SHA256SUMS": { + "sha256": "678af020c062ada94dfcc4403efb972955e574d34ccbef35760e205f012196d5", + "size": 1091 }, - "switchify-prediction-neural-0.2.0-x86_64-unknown-linux-gnu/LICENSE": { - "size": 1079, - "sha256": "1e5643d5431cc5b2d04534dee7b3f716b24fc3b82493a1659534cbfa4fc8732c" + "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/LICENSE": { + "sha256": "1e5643d5431cc5b2d04534dee7b3f716b24fc3b82493a1659534cbfa4fc8732c", + "size": 1079 }, - "switchify-prediction-neural-0.2.0-x86_64-unknown-linux-gnu/switchify-smol-worker-avx2": { - "size": 9966768, - "sha256": "7750438778be11849f11f732255104e75ad996319922c237e9f563d9ec9d15d0" + "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/switchify-smol-worker-avx2": { + "sha256": "737e8e4804e58c5b40676d4ca9237b02913682b63ce9679ed4416326276c3259", + "size": 10246512 }, - "switchify-prediction-neural-0.2.0-x86_64-unknown-linux-gnu/SECURITY.md": { - "size": 1973, - "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182" + "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/SECURITY.md": { + "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182", + "size": 1973 }, - "switchify-prediction-neural-0.2.0-x86_64-unknown-linux-gnu/verify_bundle.py": { - "size": 1282, - "sha256": "69c124d037d9e25018f690c10b861c4388c01a8cd8eb3ed438d17136196c1194" + "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/verify_bundle.py": { + "sha256": "69c124d037d9e25018f690c10b861c4388c01a8cd8eb3ed438d17136196c1194", + "size": 1282 }, - "switchify-prediction-neural-0.2.0-x86_64-unknown-linux-gnu/Cargo.lock": { - "size": 46211, - "sha256": "e31dd348e4db5cce55e43fa6730302678f1cff85ae833a67fe0e7a405f64954a" + "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/Cargo.lock": { + "sha256": "a49ecb4d18fa6f26d8777701e2bbad87691fff4ec05664e5a746bf18580673b0", + "size": 46262 }, - "switchify-prediction-neural-0.2.0-x86_64-unknown-linux-gnu/README.md": { - "size": 7136, - "sha256": "9db3b6561aeb463ec0988ec25200d3b32318daf96c1f11ebe39551bcb667be63" + "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/README.md": { + "sha256": "02cf02f01d6e711cf6415304ace910722cb76268e90358c5362feaec034a090b", + "size": 8913 }, - "switchify-prediction-neural-0.2.0-x86_64-unknown-linux-gnu/THIRD_PARTY_NOTICES.md": { - "size": 1535176, - "sha256": "93cad9c9282edec579015f6d822901ab48596ff21e9f74bb3f07dc2ae8b7314c" + "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/THIRD_PARTY_NOTICES.md": { + "sha256": "93cad9c9282edec579015f6d822901ab48596ff21e9f74bb3f07dc2ae8b7314c", + "size": 1535176 }, - "switchify-prediction-neural-0.2.0-x86_64-unknown-linux-gnu/QUALIFICATION.json": { - "size": 473, - "sha256": "55b6e9429ecb290fd194682406773a6f07102861529d5e1c0e51a6d2d4fc8c7e" + "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/QUALIFICATION.json": { + "sha256": "55b6e9429ecb290fd194682406773a6f07102861529d5e1c0e51a6d2d4fc8c7e", + "size": 473 }, - "switchify-prediction-neural-0.2.0-x86_64-unknown-linux-gnu/QUALIFICATION.md": { - "size": 7086, - "sha256": "f5d13a7406728515112cab4da093ee19642f68abbb9f30c8c8f511ae68ec8eb3" + "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/QUALIFICATION.md": { + "sha256": "f5d13a7406728515112cab4da093ee19642f68abbb9f30c8c8f511ae68ec8eb3", + "size": 7086 } } } - } + }, + "library_revision": "70fc4a869ffd22a405527e4d81d25c149aebc418" } diff --git a/scripts/probe-neural-worker.py b/scripts/probe-neural-worker.py index 6ddcee83..e7a0bb67 100644 --- a/scripts/probe-neural-worker.py +++ b/scripts/probe-neural-worker.py @@ -1,7 +1,7 @@ """Text-free packaged-worker diagnostic with a fixed synthetic query. The five-second diagnostic bound measures slow workers without changing the -application's 500 ms inference deadline. No desktop input is injected. +application's two-second inference deadline. No desktop input is injected. """ import json import os @@ -39,7 +39,7 @@ def receive(process, timeout): def main(): worker = Path(os.environ['SWITCHIFY_BENCHMARK_WORKER']) bundle = Path(os.environ['SWITCHIFY_BENCHMARK_MODEL']).parent.parent / 'prediction-neural' - report = {'ready': False, 'ranked': False} + report = {'ready': False, 'generated': False} start = time.monotonic() process = subprocess.Popen([str(worker), str(bundle)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, @@ -48,19 +48,17 @@ def main(): try: ready = receive(process, 30) report['ready_ms'] = (time.monotonic() - start) * 1000 - report['ready'] = isinstance(ready, dict) and 'Ready' in ready + report['ready'] = isinstance(ready, dict) and ready.get('Ready', {}).get('version') == 2 if report['ready']: - frame = json.dumps({'Predict': {'id': 1, 'session': 0, - 'before': 'please send the', - 'candidates': ['message', 'email', 'file', 'letter', - 'report', 'information', 'details', 'document'], - 'limit': 5}}).encode() + frame = json.dumps({'Generate': {'id': 1, 'session': 0, + 'before': 'please send the', 'prefix': '', + 'exclude': ['receipt', 'order', 'tickets'], 'limit': 3}}).encode() start = time.monotonic() process.stdin.write(struct.pack(' Result { std::thread::sleep(self.1); self.0.clone().map(|w| Prediction { - words: w.iter().map(|w| (*w).to_owned()).collect(), + words: super::super::model::slots(w), }) } } @@ -206,7 +207,7 @@ mod tests { Duration::ZERO, ))); let prediction = db.predict(&context()).1; - assert_eq!(prediction.words.len(), 5); + assert_eq!(prediction.words.len(), 6); db.state = State::Ready(Box::new(Fake(Err(()), Duration::ZERO))); assert_eq!( db.predict(&context()), @@ -227,7 +228,7 @@ mod tests { let (status, prediction) = db.predict(&context()); assert_eq!( (status, prediction.words), - (Status::Ready, vec!["late".to_owned()]) + (Status::Ready, super::super::model::slots(["late"])) ); } db.state = prompt(); diff --git a/src-tauri/src/prediction/mod.rs b/src-tauri/src/prediction/mod.rs index 1131e870..164fdf95 100644 --- a/src-tauri/src/prediction/mod.rs +++ b/src-tauri/src/prediction/mod.rs @@ -2,6 +2,8 @@ mod activity; mod context; mod database; mod model; +#[cfg(test)] +pub(crate) use model::slots; pub mod worker; use crate::scan_keyboard::{Key, Keyboard, Modifier, Page, Stroke}; @@ -983,7 +985,7 @@ mod tests { let batch = worker::Batch { refined: false, token: 1, - words: vec!["water".into()], + words: crate::prediction::slots(vec!["water".to_owned()]), }; service.received_suggestions( &mut keyboard, diff --git a/src-tauri/src/prediction/model.rs b/src-tauri/src/prediction/model.rs index ccbee6ca..1bb1fbc7 100644 --- a/src-tauri/src/prediction/model.rs +++ b/src-tauri/src/prediction/model.rs @@ -7,7 +7,13 @@ const DATABASE_SHA256: &str = "222253417d0a7a705823ffb7e599a3bcf5d5d3daf4a9d7616 #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct Prediction { - pub words: Vec, + pub words: [Option; 6], +} + +#[cfg(test)] +pub fn slots(words: impl IntoIterator>) -> [Option; 6] { + let mut words = words.into_iter(); + std::array::from_fn(|_| words.next().map(Into::into)) } pub trait Predict: Send { @@ -29,6 +35,7 @@ pub struct Model { predictor: Predictor, neural: Option, request: Option, + instant: [Option; 3], config: Option, } @@ -95,6 +102,7 @@ impl Model { neural: None, config, request: None, + instant: Default::default(), }) } } @@ -117,28 +125,23 @@ impl Predict for Model { self.neural = switchify_prediction_neural::Refiner::new(config).ok(); } let options = Options { - limit: 5, + limit: 3, min_chars: 0, unigram_only: false, }; - if let Some(neural) = &mut self.neural { - if let Ok(immediate) = neural.submit(&self.predictor, before, prefix, options, 0) { - self.request = immediate - .refinement_requested - .then_some(immediate.request_id); - return Ok(Prediction { - words: immediate.words.into_iter().map(display_word).collect(), - }); - } - } - self.request = None; + let words: Vec = self + .predictor + .predict(before, prefix, options) + .into_iter() + .map(|s| s.word) + .collect(); + self.instant = std::array::from_fn(|i| words.get(i).cloned().map(display_word)); + self.request = self + .neural + .as_mut() + .and_then(|n| n.generate(before, prefix, 0, &words, 3).ok().flatten()); Ok(Prediction { - words: self - .predictor - .predict(before, prefix, options) - .into_iter() - .map(|s| display_word(s.word)) - .collect(), + words: std::array::from_fn(|i| self.instant.get(i).cloned().flatten()), }) } fn poll(&mut self) -> Option { @@ -148,7 +151,13 @@ impl Predict for Model { } self.request = None; Some(Prediction { - words: result.words.into_iter().map(display_word).collect(), + words: std::array::from_fn(|i| { + if i < 3 { + self.instant[i].clone() + } else { + result.words.get(i - 3).cloned().map(display_word) + } + }), }) } fn pending(&self) -> bool { @@ -163,6 +172,7 @@ impl Predict for Model { } fn reset(&mut self) { self.request = None; + self.instant = Default::default(); if let Some(n) = &mut self.neural { n.reset(); } @@ -200,6 +210,7 @@ mod tests { neural: None, config: None, request: None, + instant: Default::default(), }; model.config = Some(switchify_prediction_neural::Config { bundle: dir.0.join("missing"), @@ -207,19 +218,49 @@ mod tests { accelerated_worker: None, threads: 4, }); - assert_eq!(model.predict("I need ", "h").unwrap().words[0], "help"); + assert_eq!( + model.predict("I need ", "h").unwrap().words[0] + .as_deref() + .unwrap(), + "help" + ); assert!(!model.pending()); assert!(model.config.is_none()); assert!(model.poll().is_none()); - assert_eq!(model.predict("I drink ", "").unwrap().words[0], "water"); - assert!(model.predict("", "zyzzy").unwrap().words.is_empty()); + assert_eq!( + model.predict("I drink ", "").unwrap().words[0] + .as_deref() + .unwrap(), + "water" + ); + assert!(model + .predict("", "zyzzy") + .unwrap() + .words + .iter() + .all(Option::is_none)); assert_eq!( model.predict("", "cafe\u{301}").unwrap().words, - vec!["café"] + crate::prediction::slots(["café"]) + ); + assert_eq!( + model.predict("", "can’").unwrap().words, + crate::prediction::slots(["can't"]) + ); + assert_eq!( + model.predict("", "i’").unwrap().words, + crate::prediction::slots(["I'm"]) + ); + assert_eq!( + model + .predict("", "h") + .unwrap() + .words + .iter() + .flatten() + .count(), + 3 ); - assert_eq!(model.predict("", "can’").unwrap().words, vec!["can't"]); - assert_eq!(model.predict("", "i’").unwrap().words, vec!["I'm"]); - assert_eq!(model.predict("", "h").unwrap().words.len(), 5); assert_eq!( model.predict("I need.", "h").unwrap(), model.predict("", "h").unwrap() @@ -287,13 +328,18 @@ mod tests { ][i % 4]; let start = Instant::now(); let suggestions = model.predict(context, prefix).unwrap(); - assert!(!suggestions.words.is_empty()); + assert!(!suggestions.words.iter().all(Option::is_none)); if i >= 20 { samples.push(start.elapsed().as_secs_f64() * 1000.0); } } samples.sort_by(f64::total_cmp); - assert_eq!(model.predict("I need ", "he").unwrap().words[0], "help"); + assert_eq!( + model.predict("I need ", "he").unwrap().words[0] + .as_deref() + .unwrap(), + "help" + ); #[cfg(target_os = "macos")] let cpu = std::process::Command::new("sysctl") .args(["-n", "machdep.cpu.brand_string"]) diff --git a/src-tauri/src/prediction/worker.rs b/src-tauri/src/prediction/worker.rs index 4ed7f32f..755b6a46 100644 --- a/src-tauri/src/prediction/worker.rs +++ b/src-tauri/src/prediction/worker.rs @@ -62,7 +62,7 @@ pub enum Request { pub struct Batch { pub token: u64, pub refined: bool, - pub words: Vec, + pub words: [Option; 6], } #[derive(Serialize, Deserialize)] pub enum Response { @@ -105,6 +105,8 @@ pub fn receive(reader: &mut impl Read) -> Result reader.read_exact(&mut bytes).map_err(|_| ())?; serde_json::from_slice(&bytes).map_err(|_| ()) } +type Inserts = [Option<(usize, String)>; 6]; + pub struct Engine { database: Database, status: Status, @@ -120,10 +122,10 @@ pub struct Engine { snapshot: Option, batch: Option, /// Per suggestion: how many typed characters to delete, then the text. - inserts: Vec<(usize, String)>, + inserts: Inserts, token: u64, case: (Shift, bool, bool), - displayed: Option<(Batch, Vec<(usize, String)>)>, + displayed: Option<(Batch, Inserts)>, generation: u64, } impl Engine { @@ -142,7 +144,7 @@ impl Engine { revision: 0, snapshot: None, batch: None, - inserts: Vec::new(), + inserts: Default::default(), token: 0, case: (Shift::Off, false, false), displayed: None, @@ -156,7 +158,7 @@ impl Engine { self.clipped = false; self.snapshot = None; self.batch = None; - self.inserts.clear(); + self.inserts = Default::default(); } fn stable(&self, target: usize, epoch: u64) -> bool { self.tracked && (self.observe)() == (epoch, true) && (self.foreground)() == Ok(target) @@ -250,9 +252,12 @@ impl Engine { (prediction.words, false) }; self.token = self.token.wrapping_add(1); - self.inserts.clear(); - let mut labels = Vec::new(); - for word in words { + self.inserts = Default::default(); + let mut labels = std::array::from_fn(|_| None); + for (index, word) in words.into_iter().enumerate() { + let Some(word) = word else { + continue; + }; let Some(offset) = prefix_offset(&word, &ctx.prefix) else { continue; }; @@ -275,15 +280,15 @@ impl Engine { }; match head { Some(head) => { - labels.push(format!("{head}{suffix}")); - self.inserts.push(( + labels[index] = Some(format!("{head}{suffix}")); + self.inserts[index] = Some(( ctx.prefix.graphemes(true).count(), format!("{head}{suffix} "), )); } None => { - labels.push(ctx.prefix.clone() + &suffix); - self.inserts.push((0, suffix + " ")); + labels[index] = Some(ctx.prefix.clone() + &suffix); + self.inserts[index] = Some((0, suffix + " ")); } } } @@ -317,11 +322,11 @@ impl Engine { } else { return None; }; - let insert = inserts.get(index)?.clone(); + let insert = inserts.get(index)?.clone()?; self.database.reset(); self.batch = None; self.displayed = None; - self.inserts.clear(); + self.inserts = Default::default(); self.snapshot = None; Some(insert) } @@ -576,12 +581,19 @@ mod tests { fn predict(&mut self, _: &str, _: &str) -> Result { self.pending = true; Ok(super::super::model::Prediction { - words: vec!["water".into(), "walk".into()], + words: crate::prediction::slots(["water", "walk", "waffle"]), }) } fn poll(&mut self) -> Option { std::mem::take(&mut self.pending).then(|| super::super::model::Prediction { - words: vec!["walk".into(), "water".into()], + words: [ + Some("water".into()), + Some("walk".into()), + Some("waffle".into()), + Some("walnut".into()), + None, + Some("wavy".into()), + ], }) } fn pending(&self) -> bool { @@ -621,6 +633,7 @@ mod tests { let mut immediate = Vec::new(); let mut refined = Vec::new(); let mut failures = 0; + let mut generated_words = 0; let mut session_reports = Vec::new(); // A timeout deliberately disables refinement for the keyboard session. // Simulate at most five explicit reopenings to collect 1,000 successes, @@ -652,6 +665,7 @@ mod tests { revision, None, ))); + assert!(first.words[3..].iter().all(Option::is_none)); let first_ms = start.elapsed().as_secs_f64() * 1000.; let mut success = false; while e.database.pending() { @@ -660,6 +674,10 @@ mod tests { let next = response_batch(e.respond(request(vec![], revision, Some(first.token)))); if next.refined { + assert_eq!(&first.words[..3], &next.words[..3]); + if i >= 20 { + generated_words += next.words[3..].iter().flatten().count(); + } success = true; break; } @@ -707,6 +725,9 @@ mod tests { "sessions":session_reports,"maximum_sessions":5, "retry_policy":"Benchmark simulates explicit keyboard reopenings after failure; production never retries automatically", "scope":"Production Engine and model adapter, real neural child IPC, 20ms polling, fake input/activity; excludes outer desktop pipe and rendering", + "policy":"three instant plus three generated optional slots", + "generated_words":generated_words, + "neural_slot_fill_rate":generated_words as f64 / (refined.len().max(1) * 3) as f64, "production_qualified":false}); println!("{report}"); if let Some(path) = std::env::var_os("SWITCHIFY_NEURAL_REPORT") { @@ -727,14 +748,16 @@ mod tests { assert!(!immediate.refined); let refined = response_batch(e.respond(request(vec![], 1, Some(immediate.token)))); assert!(refined.refined); - assert_eq!(refined.words, vec!["walk", "water"]); + assert_eq!(&refined.words[..3], &immediate.words[..3]); + assert_eq!(refined.words[3].as_deref(), Some("walnut")); + assert!(e.accept(refined.token, 4).is_none()); if accept_old { assert_eq!(e.accept(immediate.token, 0), Some((0, "ter ".into()))); assert!(e.accept(refined.token, 0).is_none()); } else { e.respond(request(vec![], 1, Some(refined.token))); assert!(e.accept(immediate.token, 0).is_none()); - assert_eq!(e.accept(refined.token, 0), Some((0, "lk ".into()))); + assert_eq!(e.accept(refined.token, 5), Some((0, "vy ".into()))); } } } @@ -821,7 +844,7 @@ mod tests { let b = e .query(vec![append(typed)], 1, Shift::Off, false, false) .unwrap(); - assert_eq!(b.words, vec![expected]); + assert_eq!(b.words, crate::prediction::slots([expected])); assert_eq!(e.accept(b.token, 0), Some((deletion, insertion.to_owned()))); } let mut e = engine(); @@ -841,11 +864,14 @@ mod tests { let waiting = e .query(vec![append("wa")], 1, Shift::Off, false, false) .unwrap(); - assert!(waiting.words.is_empty()); + assert!(waiting.words.iter().all(Option::is_none)); assert_eq!(e.status, Status::Loading); e.database = Database::fixture(); let ready = e.query(vec![], 1, Shift::Off, false, false).unwrap(); - assert_eq!(ready.words, vec!["water", "waffle", "walk"]); + assert_eq!( + ready.words, + crate::prediction::slots(["water", "waffle", "walk"]) + ); assert_ne!(ready.token, waiting.token); // Once ready, an unchanged query retains the scan choice identity. assert_eq!( @@ -855,15 +881,18 @@ mod tests { } #[test] - fn five_slots_contain_ranked_words_only() { + fn six_slots_preserve_populated_positions() { let mut e = engine(); let b = e .query(vec![append("wa")], 1, Shift::Off, false, false) .unwrap(); - assert_eq!(b.words, vec!["water", "waffle", "walk"]); + assert_eq!( + b.words, + crate::prediction::slots(["water", "waffle", "walk"]) + ); assert_eq!(e.accept(b.token, 1).unwrap(), (0, "ffle ".to_owned())); let upper = e.query(vec![], 1, Shift::Off, true, false).unwrap(); - assert_eq!(upper.words[1], "waFFLE"); + assert_eq!(upper.words[1].as_deref().unwrap(), "waFFLE"); } #[test] fn first_letter_and_completion_chain_use_only_buffer() { @@ -871,11 +900,11 @@ mod tests { let first = e .query(vec![append("w")], 1, Shift::Off, false, false) .unwrap(); - assert!(first.words.iter().any(|w| w == "water")); + assert!(first.words.iter().flatten().any(|w| w == "water")); let b = e .query(vec![append("a")], 2, Shift::Off, false, false) .unwrap(); - assert_eq!(b.words[0], "water"); + assert_eq!(b.words[0].as_deref().unwrap(), "water"); let (deleted, suffix) = e.accept(b.token, 0).unwrap(); assert_eq!(deleted, 0); assert_eq!(suffix, "ter "); @@ -988,18 +1017,17 @@ mod tests { let b = e .query(vec![append("Wa")], 1, Shift::Off, false, false) .unwrap(); - assert_eq!(b.words[0], "Water"); + assert_eq!(b.words[0].as_deref().unwrap(), "Water"); assert_eq!( e.query(vec![], 1, Shift::Off, false, false).unwrap().token, b.token ); let upper = e.query(vec![], 1, Shift::Off, true, false).unwrap(); - assert_eq!(upper.words[0], "WaTER"); + assert_eq!(upper.words[0].as_deref().unwrap(), "WaTER"); assert_ne!(upper.token, b.token); } #[test] fn accepting_restores_a_capital_the_typed_prefix_lacks_and_nothing_else() { - let w = |s: &[&str]| s.iter().map(|s| (*s).to_owned()).collect::>(); assert_eq!(restored("lon", "Lon").as_deref(), Some("Lon")); assert_eq!(restored("i’", "I’").as_deref(), Some("I’")); assert_eq!(restored("whats", "Whats").as_deref(), Some("Whats")); @@ -1011,7 +1039,7 @@ mod tests { let b = e .query(vec![append("wh")], 1, Shift::Off, false, false) .unwrap(); - assert_eq!(b.words, w(&["WhatsApp"])); + assert_eq!(b.words, crate::prediction::slots(["WhatsApp"])); assert_eq!(e.accept(b.token, 0).unwrap(), (2, "WhatsApp ".to_owned())); // The edits the service queues afterwards leave the buffer reading // as the screen does. @@ -1037,7 +1065,7 @@ mod tests { false, ) .unwrap(); - assert_eq!(b.words, w(&["WhatsApp"])); + assert_eq!(b.words, crate::prediction::slots(["WhatsApp"])); assert_eq!(e.accept(b.token, 0).unwrap(), (0, "atsApp ".to_owned())); // A word in capitals is never retyped. let b = e @@ -1070,16 +1098,20 @@ mod tests { e.query(vec![append("WA")], 1, Shift::Off, false, false) .unwrap(); let row = e.query(vec![], 1, Shift::Off, false, false).unwrap(); - assert_eq!(row.words[0], "WATER"); - assert_eq!(row.words[1], "WAFFLE"); + assert_eq!(row.words[0].as_deref().unwrap(), "WATER"); + assert_eq!(row.words[1].as_deref().unwrap(), "WAFFLE"); assert_eq!( e.query(vec![], 1, Shift::Locked, false, false) .unwrap() - .words[0], + .words[0] + .as_deref() + .unwrap(), "WATER" ); assert_eq!( - e.query(vec![], 1, Shift::Off, true, false).unwrap().words[0], + e.query(vec![], 1, Shift::Off, true, false).unwrap().words[0] + .as_deref() + .unwrap(), "WATER" ); // One capital is a capitalised word, not a word in capitals. @@ -1092,7 +1124,9 @@ mod tests { ) .unwrap(); assert_eq!( - e.query(vec![], 2, Shift::Off, false, false).unwrap().words[0], + e.query(vec![], 2, Shift::Off, false, false).unwrap().words[0] + .as_deref() + .unwrap(), "Water" ); } @@ -1105,26 +1139,32 @@ mod tests { let buffer_only = e .query(vec![append("Done! ")], 1, Shift::Off, false, false) .unwrap(); - assert_eq!(buffer_only.words[0], "water"); + assert_eq!(buffer_only.words[0].as_deref().unwrap(), "water"); let suggestions = e.query(vec![], 1, Shift::Off, false, true).unwrap(); - assert_eq!(suggestions.words[0], "Water"); + assert_eq!(suggestions.words[0].as_deref().unwrap(), "Water"); let locked = e.query(vec![], 1, Shift::Locked, false, true).unwrap(); - assert_eq!(locked.words[0], "WATER"); + assert_eq!(locked.words[0].as_deref().unwrap(), "WATER"); let once = e.query(vec![], 1, Shift::Once, false, true).unwrap(); - assert_eq!(once.words[0], "Water"); + assert_eq!(once.words[0].as_deref().unwrap(), "Water"); // Modifiers still mirror typing at a sentence start: Caps with Shift // locked cancel to lowercase letters, and Shift once under Caps // lowers only the first one, exactly as the keys would type them. assert_eq!( - e.query(vec![], 1, Shift::Off, true, true).unwrap().words[0], + e.query(vec![], 1, Shift::Off, true, true).unwrap().words[0] + .as_deref() + .unwrap(), "WATER" ); assert_eq!( - e.query(vec![], 1, Shift::Locked, true, true).unwrap().words[0], + e.query(vec![], 1, Shift::Locked, true, true).unwrap().words[0] + .as_deref() + .unwrap(), "Water" ); assert_eq!( - e.query(vec![], 1, Shift::Once, true, true).unwrap().words[0], + e.query(vec![], 1, Shift::Once, true, true).unwrap().words[0] + .as_deref() + .unwrap(), "wATER" ); } @@ -1134,47 +1174,65 @@ mod tests { e.query(vec![append("Send ")], 1, Shift::Off, false, false) .unwrap(); assert_eq!( - e.query(vec![], 1, Shift::Off, false, false).unwrap().words[0], + e.query(vec![], 1, Shift::Off, false, false).unwrap().words[0] + .as_deref() + .unwrap(), "water" ); assert_eq!( - e.query(vec![], 1, Shift::Once, false, false).unwrap().words[0], + e.query(vec![], 1, Shift::Once, false, false).unwrap().words[0] + .as_deref() + .unwrap(), "Water" ); assert_eq!( e.query(vec![], 1, Shift::Locked, false, false) .unwrap() - .words[0], + .words[0] + .as_deref() + .unwrap(), "WATER" ); assert_eq!( - e.query(vec![], 1, Shift::Off, true, false).unwrap().words[0], + e.query(vec![], 1, Shift::Off, true, false).unwrap().words[0] + .as_deref() + .unwrap(), "WATER" ); assert_eq!( e.query(vec![], 1, Shift::Locked, true, false) .unwrap() - .words[0], + .words[0] + .as_deref() + .unwrap(), "water" ); assert_eq!( - e.query(vec![], 1, Shift::Once, true, false).unwrap().words[0], + e.query(vec![], 1, Shift::Once, true, false).unwrap().words[0] + .as_deref() + .unwrap(), "wATER" ); assert_eq!( - e.query(vec![], 1, Shift::Off, false, false).unwrap().words[3], + e.query(vec![], 1, Shift::Off, false, false).unwrap().words[3] + .as_deref() + .unwrap(), "WhatsApp" ); e.query(vec![append("wa")], 2, Shift::Off, false, false) .unwrap(); assert_eq!( - e.query(vec![], 2, Shift::Once, false, false).unwrap().words[0], + e.query(vec![], 2, Shift::Once, false, false).unwrap().words[0] + .as_deref() + .unwrap(), "water" ); assert_eq!( e.query(vec![], 2, Shift::Locked, false, false) .unwrap() - .words[0], + .words[0] + .as_deref() + .unwrap(), "waTER" ); } @@ -1257,7 +1315,7 @@ mod tests { let row = e .query(vec![], revision, shift, caps, sentence_start) .unwrap_or_else(|| panic!("no suggestions for {case}")); - assert_eq!(row.words[0], label, "label for {case}"); + assert_eq!(row.words[0].as_deref().unwrap(), label, "label for {case}"); assert_eq!( e.accept(row.token, 0), Some((deleted, text.to_owned())), diff --git a/src-tauri/src/scan_keyboard.rs b/src-tauri/src/scan_keyboard.rs index 21766e60..c3ce64e2 100644 --- a/src-tauri/src/scan_keyboard.rs +++ b/src-tauri/src/scan_keyboard.rs @@ -391,7 +391,7 @@ impl Keyboard { rows_for(self.page, self.mac, self.layout) }; if self.prediction_enabled && self.page == Page::Letters && !self.positioning { - self.rows.insert(0, (0..5).map(Key::Prediction).collect()); + self.rows.insert(0, (0..6).map(Key::Prediction).collect()); } if self.prediction_enabled && self.prediction_failed && !self.positioning { self.rows.last_mut().unwrap().push(Key::RetryPrediction); @@ -451,7 +451,7 @@ impl Keyboard { && self .predictions .as_ref() - .is_some_and(|b| !b.words.is_empty()) + .is_some_and(|b| b.words.iter().any(Option::is_some)) { self.scan.nav.reset(); self.scan.reset_clock(); @@ -478,9 +478,9 @@ impl Keyboard { "Suggestions updating".into() } else { match self.predictions.as_ref() { - Some(batch) if batch.words.len() == 1 => "1 suggestion".into(), - Some(batch) if !batch.words.is_empty() => { - format!("{} suggestions", batch.words.len()) + Some(batch) if batch.words.iter().flatten().count() == 1 => "1 suggestion".into(), + Some(batch) if batch.words.iter().any(Option::is_some) => { + format!("{} suggestions", batch.words.iter().flatten().count()) } Some(_) => "No suggestions".into(), None => "Type for suggestions".into(), @@ -499,11 +499,15 @@ impl Keyboard { if !self.prediction_row_active() || self.scan.nav.escaping() { return false; } - let count = self.predictions.as_ref().map_or(0, |b| b.words.len()); - self.scan - .position(&self.rows) - .1 - .map_or(count == 0, |column| column >= count) + let words = self.predictions.as_ref().map(|b| &b.words); + self.scan.position(&self.rows).1.map_or_else( + || words.is_none_or(|w| w.iter().all(Option::is_none)), + |column| { + words + .and_then(|w| w.get(column)) + .is_none_or(Option::is_none) + }, + ) } fn skip_disabled(&mut self, automatic: bool) { for _ in 0..self.rows.iter().map(Vec::len).sum::() + 1 { @@ -666,7 +670,7 @@ impl Keyboard { Key::RetryPrediction => return None, Key::Prediction(index) => { let batch = self.predictions.as_ref()?; - if index >= batch.words.len() { + if batch.words.get(index).is_none_or(Option::is_none) { return None; } self.activation = self.scan.begin_activation(); @@ -868,6 +872,7 @@ impl Keyboard { .predictions .as_ref() .and_then(|b| b.words.get(i)) + .and_then(Option::as_ref) .cloned() .unwrap_or_default(), Key::RetryPrediction => "Retry predictions".into(), @@ -1045,7 +1050,7 @@ mod tests { Some(crate::prediction::worker::Batch { token: 1, refined: false, - words: vec!["water".into(), "walk".into()], + words: crate::prediction::slots(vec!["water".to_owned(), "walk".to_owned()]), }), false, ); @@ -1056,15 +1061,86 @@ mod tests { Some(crate::prediction::worker::Batch { token: 2, refined: true, - words: vec!["walk".into(), "water".into()], + words: [ + Some("water".into()), + Some("walk".into()), + None, + Some("walnut".into()), + None, + Some("wavy".into()), + ], }), false, ); assert_eq!(k.displayed_prediction(), Some(1)); - assert_eq!(k.predictions.as_ref().unwrap().words[0], "water"); + assert_eq!( + k.predictions.as_ref().unwrap().words[0].as_deref().unwrap(), + "water" + ); assert_eq!(k.queued_predictions.as_ref().unwrap().token, 2); assert_eq!(k.scan.position(&k.rows), position); } + #[test] + fn six_slots_skip_internal_gaps_in_both_directions() { + use crate::scan_preferences::{Direction, Pattern, Resolved}; + for direction in [Direction::Forward, Direction::Reverse] { + let mut k = Keyboard::configured( + false, + Resolved { + direction, + pattern: Pattern::Linear, + pass_limit: 0, + ..Default::default() + }, + ); + k.enable_predictions(true); + k.predictions( + Some(crate::prediction::worker::Batch { + token: 19, + refined: true, + words: [ + Some("one".into()), + None, + None, + Some("four".into()), + None, + Some("six".into()), + ], + }), + false, + ); + k.restart(); + assert_eq!(k.rows[0].len(), 6); + let mut seen = std::collections::BTreeSet::new(); + for _ in 0..k.rows.iter().map(Vec::len).sum::() * 3 { + if let (0, Some(column)) = k.scan.position(&k.rows) { + assert!([0, 3, 5].contains(&column)); + seen.insert(column); + } + k.handle(Action::Next); + } + assert_eq!(seen, [0, 3, 5].into_iter().collect()); + assert!(k.choose(Key::Prediction(1)).is_none()); + assert!(k.choose(Key::Prediction(4)).is_none()); + let frame = k.frame( + Rect { + x: 0., + y: 0., + width: 1920., + height: 1080., + }, + 1., + ScannerColor::default(), + ); + let tiles = &frame.tiles[1..7]; + assert!(tiles + .iter() + .all(|t| (t.rect.width - tiles[0].rect.width).abs() < 0.01)); + assert_eq!(tiles[1].text, ""); + assert_eq!(tiles[3].text, "four"); + } + } + #[test] fn prediction_badge_explains_empty_slots_without_replacing_scan_prompt() { let mut keyboard = Keyboard::new(false); @@ -1091,7 +1167,7 @@ mod tests { Some(crate::prediction::worker::Batch { refined: false, token: 1, - words: vec![], + words: Default::default(), }), false, ); @@ -1100,7 +1176,7 @@ mod tests { Some(crate::prediction::worker::Batch { refined: false, token: 2, - words: vec!["water".into()], + words: crate::prediction::slots(vec!["water".to_owned()]), }), false, ); @@ -1430,7 +1506,7 @@ mod tests { Some(crate::prediction::worker::Batch { refined: false, token: 1, - words: vec!["hello".into()], + words: crate::prediction::slots(vec!["hello".to_owned()]), }), false, ); @@ -1456,7 +1532,7 @@ mod tests { Some(crate::prediction::worker::Batch { refined: false, token: 2, - words: vec!["hello".into()], + words: crate::prediction::slots(vec!["hello".to_owned()]), }), false, ); @@ -1527,7 +1603,7 @@ mod tests { Some(crate::prediction::worker::Batch { refined: false, token: 1, - words: vec!["hello".into()], + words: crate::prediction::slots(vec!["hello".to_owned()]), }), false, ); @@ -1541,7 +1617,7 @@ mod tests { Some(crate::prediction::worker::Batch { refined: false, token, - words: vec!["world".into()], + words: crate::prediction::slots(vec!["world".to_owned()]), }), false, ); @@ -1615,7 +1691,7 @@ mod tests { let batch = crate::prediction::worker::Batch { refined: false, token: 1, - words: vec!["hello".into()], + words: crate::prediction::slots(vec!["hello".to_owned()]), }; keyboard.predictions(Some(batch.clone()), false); assert_eq!(keyboard.scan.nav.index(), 0); @@ -1645,7 +1721,10 @@ mod tests { let batch = crate::prediction::worker::Batch { refined: false, token: 7, - words: vec!["water".into(), "walk".into()], + words: crate::prediction::slots(vec![ + "water".to_owned(), + "walk".to_owned(), + ]), }; for k in [&mut baseline, &mut polled] { k.enable_predictions(true); @@ -1685,7 +1764,7 @@ mod tests { let batch = crate::prediction::worker::Batch { refined: false, token: 7, - words: vec!["water".into(), "walk".into()], + words: crate::prediction::slots(vec!["water".to_owned(), "walk".to_owned()]), }; k.predictions(Some(batch.clone()), false); k.handle(Action::Select); @@ -1710,7 +1789,7 @@ mod tests { Some(crate::prediction::worker::Batch { refined: false, token: 7, - words: vec!["water".into(), "walk".into()], + words: crate::prediction::slots(vec!["water".to_owned(), "walk".to_owned()]), }), false, ); @@ -2018,7 +2097,7 @@ mod tests { Some(crate::prediction::worker::Batch { refined: false, token: 9, - words: vec!["hello".into()], + words: crate::prediction::slots(vec!["hello".to_owned()]), }), false, ); @@ -2345,7 +2424,7 @@ mod tests { Some(crate::prediction::worker::Batch { refined: false, token: 1, - words: vec!["hello".into(), "world".into()], + words: crate::prediction::slots(vec!["hello".to_owned(), "world".to_owned()]), }), false, ); @@ -2387,7 +2466,10 @@ mod tests { Some(crate::prediction::worker::Batch { refined: false, token: 1, - words: vec!["hello".into(), "world".into()], + words: crate::prediction::slots(vec![ + "hello".to_owned(), + "world".to_owned(), + ]), }), false, ); @@ -2405,7 +2487,7 @@ mod tests { let next = replacement.then(|| crate::prediction::worker::Batch { refined: false, token: 2, - words: vec!["new".into()], + words: crate::prediction::slots(vec!["new".to_owned()]), }); for _ in 0..3 { k.predictions(next.clone(), false); From 217c32ffab09eac92414863125b66b3fa889f328 Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Sun, 4 Oct 2026 16:34:44 +0100 Subject: [PATCH 2/3] Pin final generation companion artifacts and record measurements --- .../six-predictions/generation-macos.json | 1622 +++++++++++++++++ .../six-predictions/generation-windows.json | 1620 ++++++++++++++++ .../six-predictions/integration-windows.json | 36 + docs/six-predictions.md | 58 + scripts/prediction-neural.json | 194 +- src-tauri/Cargo.lock | 6 +- src-tauri/Cargo.toml | 4 +- 7 files changed, 3438 insertions(+), 102 deletions(-) create mode 100644 docs/measurements/six-predictions/generation-macos.json create mode 100644 docs/measurements/six-predictions/generation-windows.json create mode 100644 docs/measurements/six-predictions/integration-windows.json diff --git a/docs/measurements/six-predictions/generation-macos.json b/docs/measurements/six-predictions/generation-macos.json new file mode 100644 index 00000000..607fcbd2 --- /dev/null +++ b/docs/measurements/six-predictions/generation-macos.json @@ -0,0 +1,1622 @@ +{ + "schema": 1, + "policy": { + "context_tokens": 64, + "shortlist": 8, + "max_results": 5, + "scoring": "whole-word log probability plus boundary probability, sequential", + "generation": { + "protocol_version": 2, + "beam_width": 8, + "max_tokens_per_word": 8, + "max_forward_evaluations": 64, + "max_results": 3, + "max_word_bytes": 128, + "reply_deadline_ms": 2000, + "scoring": "whole-word probability including following boundary; normalized duplicates excluded", + "minimum_boundary_probability": 0.5, + "search_cutoff_ms": 1600 + } + }, + "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", + "queries": 1000, + "warmup_queries": 20, + "warmup_failures": 0, + "failures": { + "rerank": 1 + }, + "latency": { + "immediate_with_ipc": { + "samples": 1000, + "median_ms": 0.3935830000045826, + "p95_ms": 7.174999999961074, + "max_ms": 13.297833000024184 + }, + "generation_with_ipc": { + "samples": 1000, + "median_ms": 349.16329199995744, + "p95_ms": 767.1424170000591, + "max_ms": 1701.419875000056 + } + }, + "cold_load_ms": 3774.631999999997, + "peak_process_tree_rss_bytes": 904331264, + "capabilities": { + "accelerated": false, + "context_tokens": 64, + "deadline_ms": 2000, + "shortlist": 8, + "threads": 4 + }, + "cells": { + "regression/documents/0": { + "queries": 37, + "generated_words": 111, + "filled_neural_queries": 37, + "generated_oov_words": 3, + "oov_targets": 5, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 6, + "current_top6_hits": 6, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 4, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 11, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 8, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.16216216216216217, + "current_top6_accuracy": 0.16216216216216217, + "statistical_six_top3_accuracy": 0.02702702702702703, + "statistical_six_top6_accuracy": 0.10810810810810811, + "six_slots_top3_accuracy": 0.02702702702702703, + "six_slots_top6_accuracy": 0.2972972972972973, + "neural_slot_fill_rate": 1.0 + }, + "regression/documents/1": { + "queries": 36, + "generated_words": 108, + "filled_neural_queries": 36, + "generated_oov_words": 7, + "oov_targets": 5, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 6, + "current_top6_hits": 6, + "statistical_six_top3_hits": 5, + "statistical_six_top6_hits": 5, + "six_slots_top3_hits": 5, + "six_slots_top6_hits": 19, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 14, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.16666666666666666, + "current_top6_accuracy": 0.16666666666666666, + "statistical_six_top3_accuracy": 0.1388888888888889, + "statistical_six_top6_accuracy": 0.1388888888888889, + "six_slots_top3_accuracy": 0.1388888888888889, + "six_slots_top6_accuracy": 0.5277777777777778, + "neural_slot_fill_rate": 1.0 + }, + "regression/documents/2": { + "queries": 37, + "generated_words": 111, + "filled_neural_queries": 37, + "generated_oov_words": 23, + "oov_targets": 3, + "oov_hits": 1, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 17, + "current_top6_hits": 17, + "statistical_six_top3_hits": 12, + "statistical_six_top6_hits": 14, + "six_slots_top3_hits": 12, + "six_slots_top6_hits": 30, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 16, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.4594594594594595, + "current_top6_accuracy": 0.4594594594594595, + "statistical_six_top3_accuracy": 0.32432432432432434, + "statistical_six_top6_accuracy": 0.3783783783783784, + "six_slots_top3_accuracy": 0.32432432432432434, + "six_slots_top6_accuracy": 0.8108108108108109, + "neural_slot_fill_rate": 1.0 + }, + "regression/documents/3": { + "queries": 36, + "generated_words": 98, + "filled_neural_queries": 34, + "generated_oov_words": 47, + "oov_targets": 5, + "oov_hits": 4, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 21, + "current_top6_hits": 21, + "statistical_six_top3_hits": 15, + "statistical_six_top6_hits": 20, + "six_slots_top3_hits": 15, + "six_slots_top6_hits": 32, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 13, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.5833333333333334, + "current_top6_accuracy": 0.5833333333333334, + "statistical_six_top3_accuracy": 0.4166666666666667, + "statistical_six_top6_accuracy": 0.5555555555555556, + "six_slots_top3_accuracy": 0.4166666666666667, + "six_slots_top6_accuracy": 0.8888888888888888, + "neural_slot_fill_rate": 0.9074074074074074 + }, + "regression/documents/4": { + "queries": 36, + "generated_words": 101, + "filled_neural_queries": 36, + "generated_oov_words": 56, + "oov_targets": 4, + "oov_hits": 4, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 28, + "current_top6_hits": 28, + "statistical_six_top3_hits": 21, + "statistical_six_top6_hits": 27, + "six_slots_top3_hits": 21, + "six_slots_top6_hits": 36, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 9, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.7777777777777778, + "current_top6_accuracy": 0.7777777777777778, + "statistical_six_top3_accuracy": 0.5833333333333334, + "statistical_six_top6_accuracy": 0.75, + "six_slots_top3_accuracy": 0.5833333333333334, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.9351851851851852 + }, + "regression/email/0": { + "queries": 37, + "generated_words": 111, + "filled_neural_queries": 37, + "generated_oov_words": 2, + "oov_targets": 5, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 9, + "current_top6_hits": 10, + "statistical_six_top3_hits": 6, + "statistical_six_top6_hits": 7, + "six_slots_top3_hits": 6, + "six_slots_top6_hits": 17, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 11, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.24324324324324326, + "current_top6_accuracy": 0.2702702702702703, + "statistical_six_top3_accuracy": 0.16216216216216217, + "statistical_six_top6_accuracy": 0.1891891891891892, + "six_slots_top3_accuracy": 0.16216216216216217, + "six_slots_top6_accuracy": 0.4594594594594595, + "neural_slot_fill_rate": 1.0 + }, + "regression/email/1": { + "queries": 36, + "generated_words": 108, + "filled_neural_queries": 36, + "generated_oov_words": 5, + "oov_targets": 1, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 18, + "current_top6_hits": 18, + "statistical_six_top3_hits": 14, + "statistical_six_top6_hits": 16, + "six_slots_top3_hits": 14, + "six_slots_top6_hits": 26, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 10, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.5, + "current_top6_accuracy": 0.5, + "statistical_six_top3_accuracy": 0.3888888888888889, + "statistical_six_top6_accuracy": 0.4444444444444444, + "six_slots_top3_accuracy": 0.3888888888888889, + "six_slots_top6_accuracy": 0.7222222222222222, + "neural_slot_fill_rate": 1.0 + }, + "regression/email/2": { + "queries": 36, + "generated_words": 107, + "filled_neural_queries": 36, + "generated_oov_words": 24, + "oov_targets": 1, + "oov_hits": 1, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 22, + "current_top6_hits": 23, + "statistical_six_top3_hits": 20, + "statistical_six_top6_hits": 23, + "six_slots_top3_hits": 20, + "six_slots_top6_hits": 30, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 7, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.6111111111111112, + "current_top6_accuracy": 0.6388888888888888, + "statistical_six_top3_accuracy": 0.5555555555555556, + "statistical_six_top6_accuracy": 0.6388888888888888, + "six_slots_top3_accuracy": 0.5555555555555556, + "six_slots_top6_accuracy": 0.8333333333333334, + "neural_slot_fill_rate": 0.9907407407407407 + }, + "regression/email/3": { + "queries": 37, + "generated_words": 99, + "filled_neural_queries": 35, + "generated_oov_words": 42, + "oov_targets": 5, + "oov_hits": 4, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 25, + "current_top6_hits": 25, + "statistical_six_top3_hits": 20, + "statistical_six_top6_hits": 24, + "six_slots_top3_hits": 20, + "six_slots_top6_hits": 35, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 12, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.6756756756756757, + "current_top6_accuracy": 0.6756756756756757, + "statistical_six_top3_accuracy": 0.5405405405405406, + "statistical_six_top6_accuracy": 0.6486486486486487, + "six_slots_top3_accuracy": 0.5405405405405406, + "six_slots_top6_accuracy": 0.9459459459459459, + "neural_slot_fill_rate": 0.8918918918918919 + }, + "regression/email/4": { + "queries": 36, + "generated_words": 79, + "filled_neural_queries": 30, + "generated_oov_words": 52, + "oov_targets": 4, + "oov_hits": 4, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 29, + "current_top6_hits": 29, + "statistical_six_top3_hits": 19, + "statistical_six_top6_hits": 24, + "six_slots_top3_hits": 19, + "six_slots_top6_hits": 35, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 11, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.8055555555555556, + "current_top6_accuracy": 0.8055555555555556, + "statistical_six_top3_accuracy": 0.5277777777777778, + "statistical_six_top6_accuracy": 0.6666666666666666, + "six_slots_top3_accuracy": 0.5277777777777778, + "six_slots_top6_accuracy": 0.9722222222222222, + "neural_slot_fill_rate": 0.7314814814814815 + }, + "regression/messages/0": { + "queries": 36, + "generated_words": 108, + "filled_neural_queries": 36, + "generated_oov_words": 0, + "oov_targets": 1, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 8, + "current_top6_hits": 11, + "statistical_six_top3_hits": 7, + "statistical_six_top6_hits": 8, + "six_slots_top3_hits": 7, + "six_slots_top6_hits": 14, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 7, + "regressions_vs_current": 2, + "current_top3_accuracy": 0.2222222222222222, + "current_top6_accuracy": 0.3055555555555556, + "statistical_six_top3_accuracy": 0.19444444444444445, + "statistical_six_top6_accuracy": 0.2222222222222222, + "six_slots_top3_accuracy": 0.19444444444444445, + "six_slots_top6_accuracy": 0.3888888888888889, + "neural_slot_fill_rate": 1.0 + }, + "regression/messages/1": { + "queries": 37, + "generated_words": 111, + "filled_neural_queries": 37, + "generated_oov_words": 3, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 23, + "current_top6_hits": 23, + "statistical_six_top3_hits": 21, + "statistical_six_top6_hits": 25, + "six_slots_top3_hits": 21, + "six_slots_top6_hits": 28, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 4, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.6216216216216216, + "current_top6_accuracy": 0.6216216216216216, + "statistical_six_top3_accuracy": 0.5675675675675675, + "statistical_six_top6_accuracy": 0.6756756756756757, + "six_slots_top3_accuracy": 0.5675675675675675, + "six_slots_top6_accuracy": 0.7567567567567568, + "neural_slot_fill_rate": 1.0 + }, + "regression/messages/2": { + "queries": 36, + "generated_words": 107, + "filled_neural_queries": 36, + "generated_oov_words": 7, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 32, + "current_top6_hits": 33, + "statistical_six_top3_hits": 27, + "statistical_six_top6_hits": 32, + "six_slots_top3_hits": 27, + "six_slots_top6_hits": 34, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.8888888888888888, + "current_top6_accuracy": 0.9166666666666666, + "statistical_six_top3_accuracy": 0.75, + "statistical_six_top6_accuracy": 0.8888888888888888, + "six_slots_top3_accuracy": 0.75, + "six_slots_top6_accuracy": 0.9444444444444444, + "neural_slot_fill_rate": 0.9907407407407407 + }, + "regression/messages/3": { + "queries": 37, + "generated_words": 99, + "filled_neural_queries": 34, + "generated_oov_words": 46, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 32, + "current_top6_hits": 33, + "statistical_six_top3_hits": 30, + "statistical_six_top6_hits": 33, + "six_slots_top3_hits": 30, + "six_slots_top6_hits": 36, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.8648648648648649, + "current_top6_accuracy": 0.8918918918918919, + "statistical_six_top3_accuracy": 0.8108108108108109, + "statistical_six_top6_accuracy": 0.8918918918918919, + "six_slots_top3_accuracy": 0.8108108108108109, + "six_slots_top6_accuracy": 0.972972972972973, + "neural_slot_fill_rate": 0.8918918918918919 + }, + "regression/messages/4": { + "queries": 36, + "generated_words": 89, + "filled_neural_queries": 31, + "generated_oov_words": 71, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 35, + "current_top6_hits": 36, + "statistical_six_top3_hits": 33, + "statistical_six_top6_hits": 36, + "six_slots_top3_hits": 33, + "six_slots_top6_hits": 35, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.9722222222222222, + "current_top6_accuracy": 1.0, + "statistical_six_top3_accuracy": 0.9166666666666666, + "statistical_six_top6_accuracy": 1.0, + "six_slots_top3_accuracy": 0.9166666666666666, + "six_slots_top6_accuracy": 0.9722222222222222, + "neural_slot_fill_rate": 0.8240740740740741 + }, + "regression/search/0": { + "queries": 36, + "generated_words": 108, + "filled_neural_queries": 36, + "generated_oov_words": 1, + "oov_targets": 3, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 3, + "current_top6_hits": 3, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 2, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 8, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 6, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.08333333333333333, + "current_top6_accuracy": 0.08333333333333333, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.05555555555555555, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.2222222222222222, + "neural_slot_fill_rate": 1.0 + }, + "regression/search/1": { + "queries": 37, + "generated_words": 111, + "filled_neural_queries": 37, + "generated_oov_words": 7, + "oov_targets": 2, + "oov_hits": 1, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 6, + "current_top6_hits": 6, + "statistical_six_top3_hits": 5, + "statistical_six_top6_hits": 5, + "six_slots_top3_hits": 5, + "six_slots_top6_hits": 22, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 17, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.16216216216216217, + "current_top6_accuracy": 0.16216216216216217, + "statistical_six_top3_accuracy": 0.13513513513513514, + "statistical_six_top6_accuracy": 0.13513513513513514, + "six_slots_top3_accuracy": 0.13513513513513514, + "six_slots_top6_accuracy": 0.5945945945945946, + "neural_slot_fill_rate": 1.0 + }, + "regression/search/2": { + "queries": 36, + "generated_words": 108, + "filled_neural_queries": 36, + "generated_oov_words": 25, + "oov_targets": 6, + "oov_hits": 3, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 11, + "current_top6_hits": 11, + "statistical_six_top3_hits": 9, + "statistical_six_top6_hits": 11, + "six_slots_top3_hits": 9, + "six_slots_top6_hits": 28, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 18, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.3055555555555556, + "current_top6_accuracy": 0.3055555555555556, + "statistical_six_top3_accuracy": 0.25, + "statistical_six_top6_accuracy": 0.3055555555555556, + "six_slots_top3_accuracy": 0.25, + "six_slots_top6_accuracy": 0.7777777777777778, + "neural_slot_fill_rate": 1.0 + }, + "regression/search/3": { + "queries": 36, + "generated_words": 101, + "filled_neural_queries": 34, + "generated_oov_words": 53, + "oov_targets": 6, + "oov_hits": 3, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 22, + "current_top6_hits": 23, + "statistical_six_top3_hits": 17, + "statistical_six_top6_hits": 21, + "six_slots_top3_hits": 17, + "six_slots_top6_hits": 33, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 12, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.6111111111111112, + "current_top6_accuracy": 0.6388888888888888, + "statistical_six_top3_accuracy": 0.4722222222222222, + "statistical_six_top6_accuracy": 0.5833333333333334, + "six_slots_top3_accuracy": 0.4722222222222222, + "six_slots_top6_accuracy": 0.9166666666666666, + "neural_slot_fill_rate": 0.9351851851851852 + }, + "regression/search/4": { + "queries": 37, + "generated_words": 80, + "filled_neural_queries": 28, + "generated_oov_words": 56, + "oov_targets": 4, + "oov_hits": 3, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 30, + "current_top6_hits": 30, + "statistical_six_top3_hits": 23, + "statistical_six_top6_hits": 28, + "six_slots_top3_hits": 23, + "six_slots_top6_hits": 33, + "regressions_vs_statistical_six": 2, + "gains_vs_statistical_six": 7, + "regressions_vs_current": 2, + "current_top3_accuracy": 0.8108108108108109, + "current_top6_accuracy": 0.8108108108108109, + "statistical_six_top3_accuracy": 0.6216216216216216, + "statistical_six_top6_accuracy": 0.7567567567567568, + "six_slots_top3_accuracy": 0.6216216216216216, + "six_slots_top6_accuracy": 0.8918918918918919, + "neural_slot_fill_rate": 0.7207207207207207 + }, + "development/documents/1": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 0, + "current_top6_hits": 0, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 2, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.0, + "current_top6_accuracy": 0.0, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.2, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.4, + "neural_slot_fill_rate": 1.0 + }, + "development/documents/3": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 8, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 2, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 2, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 5, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.2, + "current_top6_accuracy": 0.4, + "statistical_six_top3_accuracy": 0.2, + "statistical_six_top6_accuracy": 0.4, + "six_slots_top3_accuracy": 0.2, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 1.0 + }, + "development/documents/4": { + "queries": 4, + "generated_words": 9, + "filled_neural_queries": 3, + "generated_oov_words": 3, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 2, + "current_top6_hits": 2, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.5, + "current_top6_accuracy": 0.5, + "statistical_six_top3_accuracy": 0.25, + "statistical_six_top6_accuracy": 0.25, + "six_slots_top3_accuracy": 0.25, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.75 + }, + "development/documents/0": { + "queries": 4, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 1, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 0, + "current_top6_hits": 0, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 0, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 0, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.0, + "current_top6_accuracy": 0.0, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.0, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.0, + "neural_slot_fill_rate": 1.0 + }, + "development/documents/2": { + "queries": 4, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 1, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 3, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.25, + "current_top6_accuracy": 0.25, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.25, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.75, + "neural_slot_fill_rate": 1.0 + }, + "development/email/0": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 1, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.2, + "current_top6_accuracy": 0.2, + "statistical_six_top3_accuracy": 0.2, + "statistical_six_top6_accuracy": 0.2, + "six_slots_top3_accuracy": 0.2, + "six_slots_top6_accuracy": 0.2, + "neural_slot_fill_rate": 1.0 + }, + "development/email/1": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 0, + "current_top6_hits": 0, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 0, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 1, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.0, + "current_top6_accuracy": 0.0, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.0, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.2, + "neural_slot_fill_rate": 1.0 + }, + "development/email/3": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 6, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 4, + "current_top6_hits": 4, + "statistical_six_top3_hits": 2, + "statistical_six_top6_hits": 3, + "six_slots_top3_hits": 2, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.8, + "current_top6_accuracy": 0.8, + "statistical_six_top3_accuracy": 0.4, + "statistical_six_top6_accuracy": 0.6, + "six_slots_top3_accuracy": 0.4, + "six_slots_top6_accuracy": 0.8, + "neural_slot_fill_rate": 1.0 + }, + "development/email/2": { + "queries": 4, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 1, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 3, + "current_top6_hits": 3, + "statistical_six_top3_hits": 2, + "statistical_six_top6_hits": 2, + "six_slots_top3_hits": 2, + "six_slots_top6_hits": 3, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.75, + "current_top6_accuracy": 0.75, + "statistical_six_top3_accuracy": 0.5, + "statistical_six_top6_accuracy": 0.5, + "six_slots_top3_accuracy": 0.5, + "six_slots_top6_accuracy": 0.75, + "neural_slot_fill_rate": 1.0 + }, + "development/email/4": { + "queries": 4, + "generated_words": 7, + "filled_neural_queries": 3, + "generated_oov_words": 7, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 4, + "current_top6_hits": 4, + "statistical_six_top3_hits": 4, + "statistical_six_top6_hits": 4, + "six_slots_top3_hits": 4, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 1.0, + "current_top6_accuracy": 1.0, + "statistical_six_top3_accuracy": 1.0, + "statistical_six_top6_accuracy": 1.0, + "six_slots_top3_accuracy": 1.0, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.5833333333333334 + }, + "development/messages/0": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 2, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 1, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.2, + "current_top6_accuracy": 0.2, + "statistical_six_top3_accuracy": 0.2, + "statistical_six_top6_accuracy": 0.4, + "six_slots_top3_accuracy": 0.2, + "six_slots_top6_accuracy": 0.2, + "neural_slot_fill_rate": 1.0 + }, + "development/messages/2": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 2, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 2, + "current_top6_hits": 2, + "statistical_six_top3_hits": 2, + "statistical_six_top6_hits": 2, + "six_slots_top3_hits": 2, + "six_slots_top6_hits": 3, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.4, + "current_top6_accuracy": 0.4, + "statistical_six_top3_accuracy": 0.4, + "statistical_six_top6_accuracy": 0.4, + "six_slots_top3_accuracy": 0.4, + "six_slots_top6_accuracy": 0.6, + "neural_slot_fill_rate": 1.0 + }, + "development/messages/4": { + "queries": 5, + "generated_words": 8, + "filled_neural_queries": 3, + "generated_oov_words": 7, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 5, + "current_top6_hits": 5, + "statistical_six_top3_hits": 5, + "statistical_six_top6_hits": 5, + "six_slots_top3_hits": 5, + "six_slots_top6_hits": 5, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 1.0, + "current_top6_accuracy": 1.0, + "statistical_six_top3_accuracy": 1.0, + "statistical_six_top6_accuracy": 1.0, + "six_slots_top3_accuracy": 1.0, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.5333333333333333 + }, + "development/messages/1": { + "queries": 4, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 2, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.25, + "current_top6_accuracy": 0.25, + "statistical_six_top3_accuracy": 0.25, + "statistical_six_top6_accuracy": 0.25, + "six_slots_top3_accuracy": 0.25, + "six_slots_top6_accuracy": 0.5, + "neural_slot_fill_rate": 1.0 + }, + "development/messages/3": { + "queries": 4, + "generated_words": 6, + "filled_neural_queries": 2, + "generated_oov_words": 2, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 3, + "current_top6_hits": 3, + "statistical_six_top3_hits": 3, + "statistical_six_top6_hits": 3, + "six_slots_top3_hits": 3, + "six_slots_top6_hits": 3, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.75, + "current_top6_accuracy": 0.75, + "statistical_six_top3_accuracy": 0.75, + "statistical_six_top6_accuracy": 0.75, + "six_slots_top3_accuracy": 0.75, + "six_slots_top6_accuracy": 0.75, + "neural_slot_fill_rate": 0.5 + }, + "development/search/0": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 1, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 0, + "current_top6_hits": 0, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 0, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 2, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.0, + "current_top6_accuracy": 0.0, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.0, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.4, + "neural_slot_fill_rate": 1.0 + }, + "development/search/2": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 4, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.2, + "current_top6_accuracy": 0.2, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.2, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.8, + "neural_slot_fill_rate": 1.0 + }, + "development/search/4": { + "queries": 5, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 11, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 5, + "current_top6_hits": 5, + "statistical_six_top3_hits": 5, + "statistical_six_top6_hits": 5, + "six_slots_top3_hits": 5, + "six_slots_top6_hits": 5, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 1.0, + "current_top6_accuracy": 1.0, + "statistical_six_top3_accuracy": 1.0, + "statistical_six_top6_accuracy": 1.0, + "six_slots_top3_accuracy": 1.0, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.8 + }, + "development/search/1": { + "queries": 4, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 2, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.25, + "current_top6_accuracy": 0.25, + "statistical_six_top3_accuracy": 0.25, + "statistical_six_top6_accuracy": 0.25, + "six_slots_top3_accuracy": 0.25, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 1.0 + }, + "development/search/3": { + "queries": 4, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 6, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 3, + "current_top6_hits": 3, + "statistical_six_top3_hits": 3, + "statistical_six_top6_hits": 3, + "six_slots_top3_hits": 3, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.75, + "current_top6_accuracy": 0.75, + "statistical_six_top3_accuracy": 0.75, + "statistical_six_top6_accuracy": 0.75, + "six_slots_top3_accuracy": 0.75, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 1.0 + }, + "test/documents/1": { + "queries": 10, + "generated_words": 30, + "filled_neural_queries": 10, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 0, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 4, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.1, + "current_top6_accuracy": 0.1, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.0, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.4, + "neural_slot_fill_rate": 1.0 + }, + "test/documents/3": { + "queries": 11, + "generated_words": 30, + "filled_neural_queries": 10, + "generated_oov_words": 3, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 7, + "current_top6_hits": 7, + "statistical_six_top3_hits": 4, + "statistical_six_top6_hits": 5, + "six_slots_top3_hits": 4, + "six_slots_top6_hits": 11, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 6, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.6363636363636364, + "current_top6_accuracy": 0.6363636363636364, + "statistical_six_top3_accuracy": 0.36363636363636365, + "statistical_six_top6_accuracy": 0.45454545454545453, + "six_slots_top3_accuracy": 0.36363636363636365, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.9090909090909091 + }, + "test/documents/4": { + "queries": 8, + "generated_words": 19, + "filled_neural_queries": 7, + "generated_oov_words": 13, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 6, + "current_top6_hits": 6, + "statistical_six_top3_hits": 5, + "statistical_six_top6_hits": 6, + "six_slots_top3_hits": 5, + "six_slots_top6_hits": 7, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.75, + "current_top6_accuracy": 0.75, + "statistical_six_top3_accuracy": 0.625, + "statistical_six_top6_accuracy": 0.75, + "six_slots_top3_accuracy": 0.625, + "six_slots_top6_accuracy": 0.875, + "neural_slot_fill_rate": 0.7916666666666666 + }, + "test/documents/0": { + "queries": 8, + "generated_words": 24, + "filled_neural_queries": 8, + "generated_oov_words": 2, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 0, + "current_top6_hits": 0, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 0, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 2, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.0, + "current_top6_accuracy": 0.0, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.0, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.25, + "neural_slot_fill_rate": 1.0 + }, + "test/documents/2": { + "queries": 8, + "generated_words": 24, + "filled_neural_queries": 8, + "generated_oov_words": 8, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 2, + "current_top6_hits": 2, + "statistical_six_top3_hits": 2, + "statistical_six_top6_hits": 2, + "six_slots_top3_hits": 2, + "six_slots_top6_hits": 7, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 5, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.25, + "current_top6_accuracy": 0.25, + "statistical_six_top3_accuracy": 0.25, + "statistical_six_top6_accuracy": 0.25, + "six_slots_top3_accuracy": 0.25, + "six_slots_top6_accuracy": 0.875, + "neural_slot_fill_rate": 1.0 + }, + "test/email/0": { + "queries": 9, + "generated_words": 27, + "filled_neural_queries": 9, + "generated_oov_words": 0, + "oov_targets": 1, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 0, + "current_top6_hits": 0, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 0, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 0, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.0, + "current_top6_accuracy": 0.0, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.0, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.0, + "neural_slot_fill_rate": 1.0 + }, + "test/email/2": { + "queries": 10, + "generated_words": 30, + "filled_neural_queries": 10, + "generated_oov_words": 3, + "oov_targets": 1, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 5, + "current_top6_hits": 6, + "statistical_six_top3_hits": 4, + "statistical_six_top6_hits": 5, + "six_slots_top3_hits": 4, + "six_slots_top6_hits": 6, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 2, + "current_top3_accuracy": 0.5, + "current_top6_accuracy": 0.6, + "statistical_six_top3_accuracy": 0.4, + "statistical_six_top6_accuracy": 0.5, + "six_slots_top3_accuracy": 0.4, + "six_slots_top6_accuracy": 0.6, + "neural_slot_fill_rate": 1.0 + }, + "test/email/4": { + "queries": 11, + "generated_words": 23, + "filled_neural_queries": 9, + "generated_oov_words": 15, + "oov_targets": 1, + "oov_hits": 1, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 10, + "current_top6_hits": 10, + "statistical_six_top3_hits": 8, + "statistical_six_top6_hits": 9, + "six_slots_top3_hits": 8, + "six_slots_top6_hits": 11, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.9090909090909091, + "current_top6_accuracy": 0.9090909090909091, + "statistical_six_top3_accuracy": 0.7272727272727273, + "statistical_six_top6_accuracy": 0.8181818181818182, + "six_slots_top3_accuracy": 0.7272727272727273, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.696969696969697 + }, + "test/email/1": { + "queries": 9, + "generated_words": 27, + "filled_neural_queries": 9, + "generated_oov_words": 1, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 2, + "current_top6_hits": 3, + "statistical_six_top3_hits": 2, + "statistical_six_top6_hits": 3, + "six_slots_top3_hits": 2, + "six_slots_top6_hits": 6, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.2222222222222222, + "current_top6_accuracy": 0.3333333333333333, + "statistical_six_top3_accuracy": 0.2222222222222222, + "statistical_six_top6_accuracy": 0.3333333333333333, + "six_slots_top3_accuracy": 0.2222222222222222, + "six_slots_top6_accuracy": 0.6666666666666666, + "neural_slot_fill_rate": 1.0 + }, + "test/email/3": { + "queries": 7, + "generated_words": 19, + "filled_neural_queries": 7, + "generated_oov_words": 7, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 6, + "current_top6_hits": 6, + "statistical_six_top3_hits": 3, + "statistical_six_top6_hits": 6, + "six_slots_top3_hits": 3, + "six_slots_top6_hits": 7, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.8571428571428571, + "current_top6_accuracy": 0.8571428571428571, + "statistical_six_top3_accuracy": 0.42857142857142855, + "statistical_six_top6_accuracy": 0.8571428571428571, + "six_slots_top3_accuracy": 0.42857142857142855, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.9047619047619048 + }, + "test/messages/1": { + "queries": 8, + "generated_words": 24, + "filled_neural_queries": 8, + "generated_oov_words": 1, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 2, + "current_top6_hits": 3, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 2, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 5, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.25, + "current_top6_accuracy": 0.375, + "statistical_six_top3_accuracy": 0.125, + "statistical_six_top6_accuracy": 0.25, + "six_slots_top3_accuracy": 0.125, + "six_slots_top6_accuracy": 0.625, + "neural_slot_fill_rate": 1.0 + }, + "test/messages/3": { + "queries": 10, + "generated_words": 27, + "filled_neural_queries": 9, + "generated_oov_words": 10, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 10, + "current_top6_hits": 10, + "statistical_six_top3_hits": 7, + "statistical_six_top6_hits": 9, + "six_slots_top3_hits": 7, + "six_slots_top6_hits": 10, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 1.0, + "current_top6_accuracy": 1.0, + "statistical_six_top3_accuracy": 0.7, + "statistical_six_top6_accuracy": 0.9, + "six_slots_top3_accuracy": 0.7, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.9 + }, + "test/messages/0": { + "queries": 10, + "generated_words": 30, + "filled_neural_queries": 10, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 0, + "current_top6_hits": 1, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 0, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.0, + "current_top6_accuracy": 0.1, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.1, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.0, + "neural_slot_fill_rate": 1.0 + }, + "test/messages/2": { + "queries": 9, + "generated_words": 27, + "filled_neural_queries": 9, + "generated_oov_words": 3, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 4, + "current_top6_hits": 6, + "statistical_six_top3_hits": 3, + "statistical_six_top6_hits": 5, + "six_slots_top3_hits": 3, + "six_slots_top6_hits": 5, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 2, + "current_top3_accuracy": 0.4444444444444444, + "current_top6_accuracy": 0.6666666666666666, + "statistical_six_top3_accuracy": 0.3333333333333333, + "statistical_six_top6_accuracy": 0.5555555555555556, + "six_slots_top3_accuracy": 0.3333333333333333, + "six_slots_top6_accuracy": 0.5555555555555556, + "neural_slot_fill_rate": 1.0 + }, + "test/messages/4": { + "queries": 8, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 5, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 7, + "current_top6_hits": 8, + "statistical_six_top3_hits": 8, + "statistical_six_top6_hits": 8, + "six_slots_top3_hits": 8, + "six_slots_top6_hits": 8, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.875, + "current_top6_accuracy": 1.0, + "statistical_six_top3_accuracy": 1.0, + "statistical_six_top6_accuracy": 1.0, + "six_slots_top3_accuracy": 1.0, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.5 + }, + "test/search/0": { + "queries": 8, + "generated_words": 24, + "filled_neural_queries": 8, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.125, + "current_top6_accuracy": 0.125, + "statistical_six_top3_accuracy": 0.125, + "statistical_six_top6_accuracy": 0.125, + "six_slots_top3_accuracy": 0.125, + "six_slots_top6_accuracy": 0.5, + "neural_slot_fill_rate": 1.0 + }, + "test/search/2": { + "queries": 9, + "generated_words": 27, + "filled_neural_queries": 9, + "generated_oov_words": 9, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 5, + "current_top6_hits": 5, + "statistical_six_top3_hits": 2, + "statistical_six_top6_hits": 3, + "six_slots_top3_hits": 2, + "six_slots_top6_hits": 8, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 5, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.5555555555555556, + "current_top6_accuracy": 0.5555555555555556, + "statistical_six_top3_accuracy": 0.2222222222222222, + "statistical_six_top6_accuracy": 0.3333333333333333, + "six_slots_top3_accuracy": 0.2222222222222222, + "six_slots_top6_accuracy": 0.8888888888888888, + "neural_slot_fill_rate": 1.0 + }, + "test/search/4": { + "queries": 9, + "generated_words": 21, + "filled_neural_queries": 7, + "generated_oov_words": 12, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 7, + "current_top6_hits": 7, + "statistical_six_top3_hits": 6, + "statistical_six_top6_hits": 7, + "six_slots_top3_hits": 6, + "six_slots_top6_hits": 9, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.7777777777777778, + "current_top6_accuracy": 0.7777777777777778, + "statistical_six_top3_accuracy": 0.6666666666666666, + "statistical_six_top6_accuracy": 0.7777777777777778, + "six_slots_top3_accuracy": 0.6666666666666666, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.7777777777777778 + }, + "test/search/1": { + "queries": 10, + "generated_words": 30, + "filled_neural_queries": 10, + "generated_oov_words": 4, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 6, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 5, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.1, + "current_top6_accuracy": 0.1, + "statistical_six_top3_accuracy": 0.1, + "statistical_six_top6_accuracy": 0.1, + "six_slots_top3_accuracy": 0.1, + "six_slots_top6_accuracy": 0.6, + "neural_slot_fill_rate": 1.0 + }, + "test/search/3": { + "queries": 9, + "generated_words": 24, + "filled_neural_queries": 8, + "generated_oov_words": 13, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 6, + "current_top6_hits": 6, + "statistical_six_top3_hits": 5, + "statistical_six_top6_hits": 6, + "six_slots_top3_hits": 5, + "six_slots_top6_hits": 9, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.6666666666666666, + "current_top6_accuracy": 0.6666666666666666, + "statistical_six_top3_accuracy": 0.5555555555555556, + "statistical_six_top6_accuracy": 0.6666666666666666, + "six_slots_top3_accuracy": 0.5555555555555556, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.8888888888888888 + } + }, + "baseline_sha256": "222253417d0a7a705823ffb7e599a3bcf5d5d3daf4a9d76161ac6b3e555aeaad", + "worker_sha256": "c968323826413de70108e70a94bf1bd5554cff7db4708035590b1b99f758581e", + "accelerated_worker_sha256": null, + "fixture_hashes": { + "regression.json": "5e7fe93fc126862da5895d6c7304414c2a49a48303688648349afb3db956fe76", + "general-writing.json": "315cb0dacf6026d752c60482915d2dae17ff8842a77290325c6b5f780769fee9" + }, + "note": "Regression comparison only; fixture overlap with model pretraining is unknown. Current mode returns at most five words. Explicit benchmark retries counted; production never retries automatically.", + "production_qualified": false +} diff --git a/docs/measurements/six-predictions/generation-windows.json b/docs/measurements/six-predictions/generation-windows.json new file mode 100644 index 00000000..eb8a56e1 --- /dev/null +++ b/docs/measurements/six-predictions/generation-windows.json @@ -0,0 +1,1620 @@ +{ + "schema": 1, + "policy": { + "context_tokens": 64, + "shortlist": 8, + "max_results": 5, + "scoring": "whole-word log probability plus boundary probability, sequential", + "generation": { + "protocol_version": 2, + "beam_width": 8, + "max_tokens_per_word": 8, + "max_forward_evaluations": 64, + "max_results": 3, + "max_word_bytes": 128, + "reply_deadline_ms": 2000, + "scoring": "whole-word probability including following boundary; normalized duplicates excluded", + "minimum_boundary_probability": 0.5, + "search_cutoff_ms": 1600 + } + }, + "platform": "Windows-11-10.0.26200-SP0", + "queries": 1000, + "warmup_queries": 20, + "warmup_failures": 0, + "failures": {}, + "latency": { + "immediate_with_ipc": { + "samples": 1000, + "median_ms": 0.7227000023704022, + "p95_ms": 8.238699985668063, + "max_ms": 15.401700016809627 + }, + "generation_with_ipc": { + "samples": 1000, + "median_ms": 1152.2917000111192, + "p95_ms": 1388.2117999892216, + "max_ms": 1671.0320000129286 + } + }, + "cold_load_ms": 4366.459100012435, + "peak_process_tree_rss_bytes": 690114560, + "capabilities": { + "accelerated": false, + "context_tokens": 64, + "deadline_ms": 2000, + "shortlist": 8, + "threads": 4 + }, + "cells": { + "regression/documents/0": { + "queries": 37, + "generated_words": 111, + "filled_neural_queries": 37, + "generated_oov_words": 4, + "oov_targets": 5, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 6, + "current_top6_hits": 6, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 4, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 11, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 8, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.16216216216216217, + "current_top6_accuracy": 0.16216216216216217, + "statistical_six_top3_accuracy": 0.02702702702702703, + "statistical_six_top6_accuracy": 0.10810810810810811, + "six_slots_top3_accuracy": 0.02702702702702703, + "six_slots_top6_accuracy": 0.2972972972972973, + "neural_slot_fill_rate": 1.0 + }, + "regression/documents/1": { + "queries": 36, + "generated_words": 108, + "filled_neural_queries": 36, + "generated_oov_words": 7, + "oov_targets": 5, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 6, + "current_top6_hits": 6, + "statistical_six_top3_hits": 5, + "statistical_six_top6_hits": 5, + "six_slots_top3_hits": 5, + "six_slots_top6_hits": 19, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 14, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.16666666666666666, + "current_top6_accuracy": 0.16666666666666666, + "statistical_six_top3_accuracy": 0.1388888888888889, + "statistical_six_top6_accuracy": 0.1388888888888889, + "six_slots_top3_accuracy": 0.1388888888888889, + "six_slots_top6_accuracy": 0.5277777777777778, + "neural_slot_fill_rate": 1.0 + }, + "regression/documents/2": { + "queries": 37, + "generated_words": 111, + "filled_neural_queries": 37, + "generated_oov_words": 24, + "oov_targets": 3, + "oov_hits": 1, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 17, + "current_top6_hits": 17, + "statistical_six_top3_hits": 12, + "statistical_six_top6_hits": 14, + "six_slots_top3_hits": 12, + "six_slots_top6_hits": 30, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 16, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.4594594594594595, + "current_top6_accuracy": 0.4594594594594595, + "statistical_six_top3_accuracy": 0.32432432432432434, + "statistical_six_top6_accuracy": 0.3783783783783784, + "six_slots_top3_accuracy": 0.32432432432432434, + "six_slots_top6_accuracy": 0.8108108108108109, + "neural_slot_fill_rate": 1.0 + }, + "regression/documents/3": { + "queries": 36, + "generated_words": 97, + "filled_neural_queries": 34, + "generated_oov_words": 46, + "oov_targets": 5, + "oov_hits": 5, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 21, + "current_top6_hits": 21, + "statistical_six_top3_hits": 15, + "statistical_six_top6_hits": 20, + "six_slots_top3_hits": 15, + "six_slots_top6_hits": 34, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 15, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.5833333333333334, + "current_top6_accuracy": 0.5833333333333334, + "statistical_six_top3_accuracy": 0.4166666666666667, + "statistical_six_top6_accuracy": 0.5555555555555556, + "six_slots_top3_accuracy": 0.4166666666666667, + "six_slots_top6_accuracy": 0.9444444444444444, + "neural_slot_fill_rate": 0.8981481481481481 + }, + "regression/documents/4": { + "queries": 36, + "generated_words": 100, + "filled_neural_queries": 36, + "generated_oov_words": 54, + "oov_targets": 4, + "oov_hits": 4, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 28, + "current_top6_hits": 28, + "statistical_six_top3_hits": 21, + "statistical_six_top6_hits": 27, + "six_slots_top3_hits": 21, + "six_slots_top6_hits": 36, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 9, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.7777777777777778, + "current_top6_accuracy": 0.7777777777777778, + "statistical_six_top3_accuracy": 0.5833333333333334, + "statistical_six_top6_accuracy": 0.75, + "six_slots_top3_accuracy": 0.5833333333333334, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.9259259259259259 + }, + "regression/email/0": { + "queries": 37, + "generated_words": 111, + "filled_neural_queries": 37, + "generated_oov_words": 1, + "oov_targets": 5, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 9, + "current_top6_hits": 10, + "statistical_six_top3_hits": 6, + "statistical_six_top6_hits": 7, + "six_slots_top3_hits": 6, + "six_slots_top6_hits": 17, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 11, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.24324324324324326, + "current_top6_accuracy": 0.2702702702702703, + "statistical_six_top3_accuracy": 0.16216216216216217, + "statistical_six_top6_accuracy": 0.1891891891891892, + "six_slots_top3_accuracy": 0.16216216216216217, + "six_slots_top6_accuracy": 0.4594594594594595, + "neural_slot_fill_rate": 1.0 + }, + "regression/email/1": { + "queries": 36, + "generated_words": 108, + "filled_neural_queries": 36, + "generated_oov_words": 5, + "oov_targets": 1, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 18, + "current_top6_hits": 18, + "statistical_six_top3_hits": 14, + "statistical_six_top6_hits": 16, + "six_slots_top3_hits": 14, + "six_slots_top6_hits": 26, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 10, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.5, + "current_top6_accuracy": 0.5, + "statistical_six_top3_accuracy": 0.3888888888888889, + "statistical_six_top6_accuracy": 0.4444444444444444, + "six_slots_top3_accuracy": 0.3888888888888889, + "six_slots_top6_accuracy": 0.7222222222222222, + "neural_slot_fill_rate": 1.0 + }, + "regression/email/2": { + "queries": 36, + "generated_words": 107, + "filled_neural_queries": 36, + "generated_oov_words": 25, + "oov_targets": 1, + "oov_hits": 1, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 23, + "current_top6_hits": 23, + "statistical_six_top3_hits": 20, + "statistical_six_top6_hits": 23, + "six_slots_top3_hits": 20, + "six_slots_top6_hits": 30, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 7, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.6388888888888888, + "current_top6_accuracy": 0.6388888888888888, + "statistical_six_top3_accuracy": 0.5555555555555556, + "statistical_six_top6_accuracy": 0.6388888888888888, + "six_slots_top3_accuracy": 0.5555555555555556, + "six_slots_top6_accuracy": 0.8333333333333334, + "neural_slot_fill_rate": 0.9907407407407407 + }, + "regression/email/3": { + "queries": 37, + "generated_words": 101, + "filled_neural_queries": 35, + "generated_oov_words": 41, + "oov_targets": 5, + "oov_hits": 3, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 25, + "current_top6_hits": 25, + "statistical_six_top3_hits": 20, + "statistical_six_top6_hits": 24, + "six_slots_top3_hits": 20, + "six_slots_top6_hits": 34, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 11, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.6756756756756757, + "current_top6_accuracy": 0.6756756756756757, + "statistical_six_top3_accuracy": 0.5405405405405406, + "statistical_six_top6_accuracy": 0.6486486486486487, + "six_slots_top3_accuracy": 0.5405405405405406, + "six_slots_top6_accuracy": 0.918918918918919, + "neural_slot_fill_rate": 0.9099099099099099 + }, + "regression/email/4": { + "queries": 36, + "generated_words": 82, + "filled_neural_queries": 30, + "generated_oov_words": 55, + "oov_targets": 4, + "oov_hits": 4, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 28, + "current_top6_hits": 29, + "statistical_six_top3_hits": 19, + "statistical_six_top6_hits": 24, + "six_slots_top3_hits": 19, + "six_slots_top6_hits": 35, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 11, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.7777777777777778, + "current_top6_accuracy": 0.8055555555555556, + "statistical_six_top3_accuracy": 0.5277777777777778, + "statistical_six_top6_accuracy": 0.6666666666666666, + "six_slots_top3_accuracy": 0.5277777777777778, + "six_slots_top6_accuracy": 0.9722222222222222, + "neural_slot_fill_rate": 0.7592592592592593 + }, + "regression/messages/0": { + "queries": 36, + "generated_words": 108, + "filled_neural_queries": 36, + "generated_oov_words": 0, + "oov_targets": 1, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 8, + "current_top6_hits": 10, + "statistical_six_top3_hits": 7, + "statistical_six_top6_hits": 8, + "six_slots_top3_hits": 7, + "six_slots_top6_hits": 14, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 7, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.2222222222222222, + "current_top6_accuracy": 0.2777777777777778, + "statistical_six_top3_accuracy": 0.19444444444444445, + "statistical_six_top6_accuracy": 0.2222222222222222, + "six_slots_top3_accuracy": 0.19444444444444445, + "six_slots_top6_accuracy": 0.3888888888888889, + "neural_slot_fill_rate": 1.0 + }, + "regression/messages/1": { + "queries": 37, + "generated_words": 111, + "filled_neural_queries": 37, + "generated_oov_words": 3, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 23, + "current_top6_hits": 23, + "statistical_six_top3_hits": 21, + "statistical_six_top6_hits": 25, + "six_slots_top3_hits": 21, + "six_slots_top6_hits": 28, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 4, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.6216216216216216, + "current_top6_accuracy": 0.6216216216216216, + "statistical_six_top3_accuracy": 0.5675675675675675, + "statistical_six_top6_accuracy": 0.6756756756756757, + "six_slots_top3_accuracy": 0.5675675675675675, + "six_slots_top6_accuracy": 0.7567567567567568, + "neural_slot_fill_rate": 1.0 + }, + "regression/messages/2": { + "queries": 36, + "generated_words": 107, + "filled_neural_queries": 36, + "generated_oov_words": 7, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 32, + "current_top6_hits": 33, + "statistical_six_top3_hits": 27, + "statistical_six_top6_hits": 32, + "six_slots_top3_hits": 27, + "six_slots_top6_hits": 33, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.8888888888888888, + "current_top6_accuracy": 0.9166666666666666, + "statistical_six_top3_accuracy": 0.75, + "statistical_six_top6_accuracy": 0.8888888888888888, + "six_slots_top3_accuracy": 0.75, + "six_slots_top6_accuracy": 0.9166666666666666, + "neural_slot_fill_rate": 0.9907407407407407 + }, + "regression/messages/3": { + "queries": 37, + "generated_words": 99, + "filled_neural_queries": 34, + "generated_oov_words": 46, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 32, + "current_top6_hits": 33, + "statistical_six_top3_hits": 30, + "statistical_six_top6_hits": 33, + "six_slots_top3_hits": 30, + "six_slots_top6_hits": 36, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.8648648648648649, + "current_top6_accuracy": 0.8918918918918919, + "statistical_six_top3_accuracy": 0.8108108108108109, + "statistical_six_top6_accuracy": 0.8918918918918919, + "six_slots_top3_accuracy": 0.8108108108108109, + "six_slots_top6_accuracy": 0.972972972972973, + "neural_slot_fill_rate": 0.8918918918918919 + }, + "regression/messages/4": { + "queries": 36, + "generated_words": 90, + "filled_neural_queries": 31, + "generated_oov_words": 73, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 35, + "current_top6_hits": 36, + "statistical_six_top3_hits": 33, + "statistical_six_top6_hits": 36, + "six_slots_top3_hits": 33, + "six_slots_top6_hits": 34, + "regressions_vs_statistical_six": 2, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 2, + "current_top3_accuracy": 0.9722222222222222, + "current_top6_accuracy": 1.0, + "statistical_six_top3_accuracy": 0.9166666666666666, + "statistical_six_top6_accuracy": 1.0, + "six_slots_top3_accuracy": 0.9166666666666666, + "six_slots_top6_accuracy": 0.9444444444444444, + "neural_slot_fill_rate": 0.8333333333333334 + }, + "regression/search/0": { + "queries": 36, + "generated_words": 108, + "filled_neural_queries": 36, + "generated_oov_words": 1, + "oov_targets": 3, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 3, + "current_top6_hits": 3, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 2, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 7, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 5, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.08333333333333333, + "current_top6_accuracy": 0.08333333333333333, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.05555555555555555, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.19444444444444445, + "neural_slot_fill_rate": 1.0 + }, + "regression/search/1": { + "queries": 37, + "generated_words": 111, + "filled_neural_queries": 37, + "generated_oov_words": 6, + "oov_targets": 2, + "oov_hits": 1, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 6, + "current_top6_hits": 6, + "statistical_six_top3_hits": 5, + "statistical_six_top6_hits": 5, + "six_slots_top3_hits": 5, + "six_slots_top6_hits": 22, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 17, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.16216216216216217, + "current_top6_accuracy": 0.16216216216216217, + "statistical_six_top3_accuracy": 0.13513513513513514, + "statistical_six_top6_accuracy": 0.13513513513513514, + "six_slots_top3_accuracy": 0.13513513513513514, + "six_slots_top6_accuracy": 0.5945945945945946, + "neural_slot_fill_rate": 1.0 + }, + "regression/search/2": { + "queries": 36, + "generated_words": 107, + "filled_neural_queries": 36, + "generated_oov_words": 25, + "oov_targets": 6, + "oov_hits": 3, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 11, + "current_top6_hits": 11, + "statistical_six_top3_hits": 9, + "statistical_six_top6_hits": 11, + "six_slots_top3_hits": 9, + "six_slots_top6_hits": 28, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 18, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.3055555555555556, + "current_top6_accuracy": 0.3055555555555556, + "statistical_six_top3_accuracy": 0.25, + "statistical_six_top6_accuracy": 0.3055555555555556, + "six_slots_top3_accuracy": 0.25, + "six_slots_top6_accuracy": 0.7777777777777778, + "neural_slot_fill_rate": 0.9907407407407407 + }, + "regression/search/3": { + "queries": 36, + "generated_words": 99, + "filled_neural_queries": 34, + "generated_oov_words": 53, + "oov_targets": 6, + "oov_hits": 3, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 22, + "current_top6_hits": 23, + "statistical_six_top3_hits": 17, + "statistical_six_top6_hits": 21, + "six_slots_top3_hits": 17, + "six_slots_top6_hits": 33, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 12, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.6111111111111112, + "current_top6_accuracy": 0.6388888888888888, + "statistical_six_top3_accuracy": 0.4722222222222222, + "statistical_six_top6_accuracy": 0.5833333333333334, + "six_slots_top3_accuracy": 0.4722222222222222, + "six_slots_top6_accuracy": 0.9166666666666666, + "neural_slot_fill_rate": 0.9166666666666666 + }, + "regression/search/4": { + "queries": 37, + "generated_words": 80, + "filled_neural_queries": 28, + "generated_oov_words": 56, + "oov_targets": 4, + "oov_hits": 3, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 30, + "current_top6_hits": 30, + "statistical_six_top3_hits": 23, + "statistical_six_top6_hits": 28, + "six_slots_top3_hits": 23, + "six_slots_top6_hits": 33, + "regressions_vs_statistical_six": 2, + "gains_vs_statistical_six": 7, + "regressions_vs_current": 2, + "current_top3_accuracy": 0.8108108108108109, + "current_top6_accuracy": 0.8108108108108109, + "statistical_six_top3_accuracy": 0.6216216216216216, + "statistical_six_top6_accuracy": 0.7567567567567568, + "six_slots_top3_accuracy": 0.6216216216216216, + "six_slots_top6_accuracy": 0.8918918918918919, + "neural_slot_fill_rate": 0.7207207207207207 + }, + "development/documents/1": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 0, + "current_top6_hits": 0, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 2, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.0, + "current_top6_accuracy": 0.0, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.2, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.4, + "neural_slot_fill_rate": 1.0 + }, + "development/documents/3": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 8, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 2, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 2, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 5, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.2, + "current_top6_accuracy": 0.4, + "statistical_six_top3_accuracy": 0.2, + "statistical_six_top6_accuracy": 0.4, + "six_slots_top3_accuracy": 0.2, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 1.0 + }, + "development/documents/4": { + "queries": 4, + "generated_words": 9, + "filled_neural_queries": 3, + "generated_oov_words": 3, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 2, + "current_top6_hits": 2, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.5, + "current_top6_accuracy": 0.5, + "statistical_six_top3_accuracy": 0.25, + "statistical_six_top6_accuracy": 0.25, + "six_slots_top3_accuracy": 0.25, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.75 + }, + "development/documents/0": { + "queries": 4, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 1, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 0, + "current_top6_hits": 0, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 0, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 0, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.0, + "current_top6_accuracy": 0.0, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.0, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.0, + "neural_slot_fill_rate": 1.0 + }, + "development/documents/2": { + "queries": 4, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 1, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 3, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.25, + "current_top6_accuracy": 0.25, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.25, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.75, + "neural_slot_fill_rate": 1.0 + }, + "development/email/0": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 1, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.2, + "current_top6_accuracy": 0.2, + "statistical_six_top3_accuracy": 0.2, + "statistical_six_top6_accuracy": 0.2, + "six_slots_top3_accuracy": 0.2, + "six_slots_top6_accuracy": 0.2, + "neural_slot_fill_rate": 1.0 + }, + "development/email/1": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 0, + "current_top6_hits": 0, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 0, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 1, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.0, + "current_top6_accuracy": 0.0, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.0, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.2, + "neural_slot_fill_rate": 1.0 + }, + "development/email/3": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 6, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 4, + "current_top6_hits": 4, + "statistical_six_top3_hits": 2, + "statistical_six_top6_hits": 3, + "six_slots_top3_hits": 2, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.8, + "current_top6_accuracy": 0.8, + "statistical_six_top3_accuracy": 0.4, + "statistical_six_top6_accuracy": 0.6, + "six_slots_top3_accuracy": 0.4, + "six_slots_top6_accuracy": 0.8, + "neural_slot_fill_rate": 1.0 + }, + "development/email/2": { + "queries": 4, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 1, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 3, + "current_top6_hits": 3, + "statistical_six_top3_hits": 2, + "statistical_six_top6_hits": 2, + "six_slots_top3_hits": 2, + "six_slots_top6_hits": 3, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.75, + "current_top6_accuracy": 0.75, + "statistical_six_top3_accuracy": 0.5, + "statistical_six_top6_accuracy": 0.5, + "six_slots_top3_accuracy": 0.5, + "six_slots_top6_accuracy": 0.75, + "neural_slot_fill_rate": 1.0 + }, + "development/email/4": { + "queries": 4, + "generated_words": 11, + "filled_neural_queries": 4, + "generated_oov_words": 10, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 4, + "current_top6_hits": 4, + "statistical_six_top3_hits": 4, + "statistical_six_top6_hits": 4, + "six_slots_top3_hits": 4, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 1.0, + "current_top6_accuracy": 1.0, + "statistical_six_top3_accuracy": 1.0, + "statistical_six_top6_accuracy": 1.0, + "six_slots_top3_accuracy": 1.0, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.9166666666666666 + }, + "development/messages/0": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 2, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 1, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.2, + "current_top6_accuracy": 0.2, + "statistical_six_top3_accuracy": 0.2, + "statistical_six_top6_accuracy": 0.4, + "six_slots_top3_accuracy": 0.2, + "six_slots_top6_accuracy": 0.2, + "neural_slot_fill_rate": 1.0 + }, + "development/messages/2": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 2, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 2, + "statistical_six_top3_hits": 2, + "statistical_six_top6_hits": 2, + "six_slots_top3_hits": 2, + "six_slots_top6_hits": 3, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.2, + "current_top6_accuracy": 0.4, + "statistical_six_top3_accuracy": 0.4, + "statistical_six_top6_accuracy": 0.4, + "six_slots_top3_accuracy": 0.4, + "six_slots_top6_accuracy": 0.6, + "neural_slot_fill_rate": 1.0 + }, + "development/messages/4": { + "queries": 5, + "generated_words": 8, + "filled_neural_queries": 3, + "generated_oov_words": 7, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 5, + "current_top6_hits": 5, + "statistical_six_top3_hits": 5, + "statistical_six_top6_hits": 5, + "six_slots_top3_hits": 5, + "six_slots_top6_hits": 5, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 1.0, + "current_top6_accuracy": 1.0, + "statistical_six_top3_accuracy": 1.0, + "statistical_six_top6_accuracy": 1.0, + "six_slots_top3_accuracy": 1.0, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.5333333333333333 + }, + "development/messages/1": { + "queries": 4, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 2, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.25, + "current_top6_accuracy": 0.25, + "statistical_six_top3_accuracy": 0.25, + "statistical_six_top6_accuracy": 0.25, + "six_slots_top3_accuracy": 0.25, + "six_slots_top6_accuracy": 0.5, + "neural_slot_fill_rate": 1.0 + }, + "development/messages/3": { + "queries": 4, + "generated_words": 6, + "filled_neural_queries": 2, + "generated_oov_words": 2, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 3, + "current_top6_hits": 3, + "statistical_six_top3_hits": 3, + "statistical_six_top6_hits": 3, + "six_slots_top3_hits": 3, + "six_slots_top6_hits": 3, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.75, + "current_top6_accuracy": 0.75, + "statistical_six_top3_accuracy": 0.75, + "statistical_six_top6_accuracy": 0.75, + "six_slots_top3_accuracy": 0.75, + "six_slots_top6_accuracy": 0.75, + "neural_slot_fill_rate": 0.5 + }, + "development/search/0": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 1, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 0, + "current_top6_hits": 0, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 0, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 2, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.0, + "current_top6_accuracy": 0.0, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.0, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.4, + "neural_slot_fill_rate": 1.0 + }, + "development/search/2": { + "queries": 5, + "generated_words": 15, + "filled_neural_queries": 5, + "generated_oov_words": 4, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.2, + "current_top6_accuracy": 0.2, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.2, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.8, + "neural_slot_fill_rate": 1.0 + }, + "development/search/4": { + "queries": 5, + "generated_words": 11, + "filled_neural_queries": 4, + "generated_oov_words": 10, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 5, + "current_top6_hits": 5, + "statistical_six_top3_hits": 5, + "statistical_six_top6_hits": 5, + "six_slots_top3_hits": 5, + "six_slots_top6_hits": 5, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 1.0, + "current_top6_accuracy": 1.0, + "statistical_six_top3_accuracy": 1.0, + "statistical_six_top6_accuracy": 1.0, + "six_slots_top3_accuracy": 1.0, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.7333333333333333 + }, + "development/search/1": { + "queries": 4, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 2, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.25, + "current_top6_accuracy": 0.25, + "statistical_six_top3_accuracy": 0.25, + "statistical_six_top6_accuracy": 0.25, + "six_slots_top3_accuracy": 0.25, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 1.0 + }, + "development/search/3": { + "queries": 4, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 6, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 3, + "current_top6_hits": 3, + "statistical_six_top3_hits": 3, + "statistical_six_top6_hits": 3, + "six_slots_top3_hits": 3, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.75, + "current_top6_accuracy": 0.75, + "statistical_six_top3_accuracy": 0.75, + "statistical_six_top6_accuracy": 0.75, + "six_slots_top3_accuracy": 0.75, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 1.0 + }, + "test/documents/1": { + "queries": 10, + "generated_words": 30, + "filled_neural_queries": 10, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 0, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 4, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.1, + "current_top6_accuracy": 0.1, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.0, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.4, + "neural_slot_fill_rate": 1.0 + }, + "test/documents/3": { + "queries": 11, + "generated_words": 30, + "filled_neural_queries": 10, + "generated_oov_words": 4, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 7, + "current_top6_hits": 7, + "statistical_six_top3_hits": 4, + "statistical_six_top6_hits": 5, + "six_slots_top3_hits": 4, + "six_slots_top6_hits": 11, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 6, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.6363636363636364, + "current_top6_accuracy": 0.6363636363636364, + "statistical_six_top3_accuracy": 0.36363636363636365, + "statistical_six_top6_accuracy": 0.45454545454545453, + "six_slots_top3_accuracy": 0.36363636363636365, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.9090909090909091 + }, + "test/documents/4": { + "queries": 8, + "generated_words": 19, + "filled_neural_queries": 7, + "generated_oov_words": 13, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 6, + "current_top6_hits": 6, + "statistical_six_top3_hits": 5, + "statistical_six_top6_hits": 6, + "six_slots_top3_hits": 5, + "six_slots_top6_hits": 7, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.75, + "current_top6_accuracy": 0.75, + "statistical_six_top3_accuracy": 0.625, + "statistical_six_top6_accuracy": 0.75, + "six_slots_top3_accuracy": 0.625, + "six_slots_top6_accuracy": 0.875, + "neural_slot_fill_rate": 0.7916666666666666 + }, + "test/documents/0": { + "queries": 8, + "generated_words": 24, + "filled_neural_queries": 8, + "generated_oov_words": 2, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 0, + "current_top6_hits": 0, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 0, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 2, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.0, + "current_top6_accuracy": 0.0, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.0, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.25, + "neural_slot_fill_rate": 1.0 + }, + "test/documents/2": { + "queries": 8, + "generated_words": 24, + "filled_neural_queries": 8, + "generated_oov_words": 8, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 2, + "current_top6_hits": 2, + "statistical_six_top3_hits": 2, + "statistical_six_top6_hits": 2, + "six_slots_top3_hits": 2, + "six_slots_top6_hits": 7, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 5, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.25, + "current_top6_accuracy": 0.25, + "statistical_six_top3_accuracy": 0.25, + "statistical_six_top6_accuracy": 0.25, + "six_slots_top3_accuracy": 0.25, + "six_slots_top6_accuracy": 0.875, + "neural_slot_fill_rate": 1.0 + }, + "test/email/0": { + "queries": 9, + "generated_words": 27, + "filled_neural_queries": 9, + "generated_oov_words": 0, + "oov_targets": 1, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 0, + "current_top6_hits": 0, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 0, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 0, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.0, + "current_top6_accuracy": 0.0, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.0, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.0, + "neural_slot_fill_rate": 1.0 + }, + "test/email/2": { + "queries": 10, + "generated_words": 30, + "filled_neural_queries": 10, + "generated_oov_words": 2, + "oov_targets": 1, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 5, + "current_top6_hits": 5, + "statistical_six_top3_hits": 4, + "statistical_six_top6_hits": 5, + "six_slots_top3_hits": 4, + "six_slots_top6_hits": 7, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.5, + "current_top6_accuracy": 0.5, + "statistical_six_top3_accuracy": 0.4, + "statistical_six_top6_accuracy": 0.5, + "six_slots_top3_accuracy": 0.4, + "six_slots_top6_accuracy": 0.7, + "neural_slot_fill_rate": 1.0 + }, + "test/email/4": { + "queries": 11, + "generated_words": 23, + "filled_neural_queries": 9, + "generated_oov_words": 15, + "oov_targets": 1, + "oov_hits": 1, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 10, + "current_top6_hits": 10, + "statistical_six_top3_hits": 8, + "statistical_six_top6_hits": 9, + "six_slots_top3_hits": 8, + "six_slots_top6_hits": 11, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.9090909090909091, + "current_top6_accuracy": 0.9090909090909091, + "statistical_six_top3_accuracy": 0.7272727272727273, + "statistical_six_top6_accuracy": 0.8181818181818182, + "six_slots_top3_accuracy": 0.7272727272727273, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.696969696969697 + }, + "test/email/1": { + "queries": 9, + "generated_words": 27, + "filled_neural_queries": 9, + "generated_oov_words": 1, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 2, + "current_top6_hits": 3, + "statistical_six_top3_hits": 2, + "statistical_six_top6_hits": 3, + "six_slots_top3_hits": 2, + "six_slots_top6_hits": 6, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.2222222222222222, + "current_top6_accuracy": 0.3333333333333333, + "statistical_six_top3_accuracy": 0.2222222222222222, + "statistical_six_top6_accuracy": 0.3333333333333333, + "six_slots_top3_accuracy": 0.2222222222222222, + "six_slots_top6_accuracy": 0.6666666666666666, + "neural_slot_fill_rate": 1.0 + }, + "test/email/3": { + "queries": 7, + "generated_words": 19, + "filled_neural_queries": 7, + "generated_oov_words": 7, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 5, + "current_top6_hits": 6, + "statistical_six_top3_hits": 3, + "statistical_six_top6_hits": 6, + "six_slots_top3_hits": 3, + "six_slots_top6_hits": 7, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.7142857142857143, + "current_top6_accuracy": 0.8571428571428571, + "statistical_six_top3_accuracy": 0.42857142857142855, + "statistical_six_top6_accuracy": 0.8571428571428571, + "six_slots_top3_accuracy": 0.42857142857142855, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.9047619047619048 + }, + "test/messages/1": { + "queries": 8, + "generated_words": 24, + "filled_neural_queries": 8, + "generated_oov_words": 1, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 2, + "current_top6_hits": 3, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 2, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 5, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.25, + "current_top6_accuracy": 0.375, + "statistical_six_top3_accuracy": 0.125, + "statistical_six_top6_accuracy": 0.25, + "six_slots_top3_accuracy": 0.125, + "six_slots_top6_accuracy": 0.625, + "neural_slot_fill_rate": 1.0 + }, + "test/messages/3": { + "queries": 10, + "generated_words": 27, + "filled_neural_queries": 9, + "generated_oov_words": 10, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 10, + "current_top6_hits": 10, + "statistical_six_top3_hits": 7, + "statistical_six_top6_hits": 9, + "six_slots_top3_hits": 7, + "six_slots_top6_hits": 10, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 0, + "current_top3_accuracy": 1.0, + "current_top6_accuracy": 1.0, + "statistical_six_top3_accuracy": 0.7, + "statistical_six_top6_accuracy": 0.9, + "six_slots_top3_accuracy": 0.7, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.9 + }, + "test/messages/0": { + "queries": 10, + "generated_words": 30, + "filled_neural_queries": 10, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 0, + "current_top6_hits": 1, + "statistical_six_top3_hits": 0, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 0, + "six_slots_top6_hits": 0, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 1, + "current_top3_accuracy": 0.0, + "current_top6_accuracy": 0.1, + "statistical_six_top3_accuracy": 0.0, + "statistical_six_top6_accuracy": 0.1, + "six_slots_top3_accuracy": 0.0, + "six_slots_top6_accuracy": 0.0, + "neural_slot_fill_rate": 1.0 + }, + "test/messages/2": { + "queries": 9, + "generated_words": 27, + "filled_neural_queries": 9, + "generated_oov_words": 3, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 4, + "current_top6_hits": 6, + "statistical_six_top3_hits": 3, + "statistical_six_top6_hits": 5, + "six_slots_top3_hits": 3, + "six_slots_top6_hits": 5, + "regressions_vs_statistical_six": 1, + "gains_vs_statistical_six": 1, + "regressions_vs_current": 2, + "current_top3_accuracy": 0.4444444444444444, + "current_top6_accuracy": 0.6666666666666666, + "statistical_six_top3_accuracy": 0.3333333333333333, + "statistical_six_top6_accuracy": 0.5555555555555556, + "six_slots_top3_accuracy": 0.3333333333333333, + "six_slots_top6_accuracy": 0.5555555555555556, + "neural_slot_fill_rate": 1.0 + }, + "test/messages/4": { + "queries": 8, + "generated_words": 12, + "filled_neural_queries": 4, + "generated_oov_words": 5, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 7, + "current_top6_hits": 8, + "statistical_six_top3_hits": 8, + "statistical_six_top6_hits": 8, + "six_slots_top3_hits": 8, + "six_slots_top6_hits": 8, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 0, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.875, + "current_top6_accuracy": 1.0, + "statistical_six_top3_accuracy": 1.0, + "statistical_six_top6_accuracy": 1.0, + "six_slots_top3_accuracy": 1.0, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.5 + }, + "test/search/0": { + "queries": 8, + "generated_words": 24, + "filled_neural_queries": 8, + "generated_oov_words": 0, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 4, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.125, + "current_top6_accuracy": 0.125, + "statistical_six_top3_accuracy": 0.125, + "statistical_six_top6_accuracy": 0.125, + "six_slots_top3_accuracy": 0.125, + "six_slots_top6_accuracy": 0.5, + "neural_slot_fill_rate": 1.0 + }, + "test/search/2": { + "queries": 9, + "generated_words": 27, + "filled_neural_queries": 9, + "generated_oov_words": 9, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 5, + "current_top6_hits": 5, + "statistical_six_top3_hits": 2, + "statistical_six_top6_hits": 3, + "six_slots_top3_hits": 2, + "six_slots_top6_hits": 8, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 5, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.5555555555555556, + "current_top6_accuracy": 0.5555555555555556, + "statistical_six_top3_accuracy": 0.2222222222222222, + "statistical_six_top6_accuracy": 0.3333333333333333, + "six_slots_top3_accuracy": 0.2222222222222222, + "six_slots_top6_accuracy": 0.8888888888888888, + "neural_slot_fill_rate": 1.0 + }, + "test/search/4": { + "queries": 9, + "generated_words": 18, + "filled_neural_queries": 6, + "generated_oov_words": 10, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 7, + "current_top6_hits": 7, + "statistical_six_top3_hits": 6, + "statistical_six_top6_hits": 7, + "six_slots_top3_hits": 6, + "six_slots_top6_hits": 9, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 2, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.7777777777777778, + "current_top6_accuracy": 0.7777777777777778, + "statistical_six_top3_accuracy": 0.6666666666666666, + "statistical_six_top6_accuracy": 0.7777777777777778, + "six_slots_top3_accuracy": 0.6666666666666666, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.6666666666666666 + }, + "test/search/1": { + "queries": 10, + "generated_words": 30, + "filled_neural_queries": 10, + "generated_oov_words": 3, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 1, + "current_top6_hits": 1, + "statistical_six_top3_hits": 1, + "statistical_six_top6_hits": 1, + "six_slots_top3_hits": 1, + "six_slots_top6_hits": 6, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 5, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.1, + "current_top6_accuracy": 0.1, + "statistical_six_top3_accuracy": 0.1, + "statistical_six_top6_accuracy": 0.1, + "six_slots_top3_accuracy": 0.1, + "six_slots_top6_accuracy": 0.6, + "neural_slot_fill_rate": 1.0 + }, + "test/search/3": { + "queries": 9, + "generated_words": 24, + "filled_neural_queries": 8, + "generated_oov_words": 13, + "oov_targets": 0, + "oov_hits": 0, + "generation_failures": 0, + "generation_context_hits": 0, + "current_top3_hits": 6, + "current_top6_hits": 6, + "statistical_six_top3_hits": 5, + "statistical_six_top6_hits": 6, + "six_slots_top3_hits": 5, + "six_slots_top6_hits": 9, + "regressions_vs_statistical_six": 0, + "gains_vs_statistical_six": 3, + "regressions_vs_current": 0, + "current_top3_accuracy": 0.6666666666666666, + "current_top6_accuracy": 0.6666666666666666, + "statistical_six_top3_accuracy": 0.5555555555555556, + "statistical_six_top6_accuracy": 0.6666666666666666, + "six_slots_top3_accuracy": 0.5555555555555556, + "six_slots_top6_accuracy": 1.0, + "neural_slot_fill_rate": 0.8888888888888888 + } + }, + "baseline_sha256": "222253417d0a7a705823ffb7e599a3bcf5d5d3daf4a9d76161ac6b3e555aeaad", + "worker_sha256": "cbd444279144c8fd79a3cef6f519fa5840a6672febc37effa3022db41c7f5c03", + "accelerated_worker_sha256": null, + "fixture_hashes": { + "regression.json": "5e7fe93fc126862da5895d6c7304414c2a49a48303688648349afb3db956fe76", + "general-writing.json": "315cb0dacf6026d752c60482915d2dae17ff8842a77290325c6b5f780769fee9" + }, + "note": "Regression comparison only; fixture overlap with model pretraining is unknown. Current mode returns at most five words. Explicit benchmark retries counted; production never retries automatically.", + "production_qualified": false +} diff --git a/docs/measurements/six-predictions/integration-windows.json b/docs/measurements/six-predictions/integration-windows.json new file mode 100644 index 00000000..bd8d7f1d --- /dev/null +++ b/docs/measurements/six-predictions/integration-windows.json @@ -0,0 +1,36 @@ +{ + "arch": "x86_64", + "failures_including_warmup": 0, + "generated_words": 3000, + "immediate": { + "max_ms": 15.2479, + "median_ms": 8.3391, + "p95_ms": 9.3506, + "samples": 1000 + }, + "maximum_sessions": 5, + "neural_slot_fill_rate": 1.0, + "os": "windows", + "policy": "three instant plus three generated optional slots", + "production_qualified": false, + "queries": 1000, + "refinement": { + "max_ms": 1034.8055, + "median_ms": 496.2131, + "p95_ms": 536.6225, + "samples": 1000 + }, + "retry_policy": "Benchmark simulates explicit keyboard reopenings after failure; production never retries automatically", + "scope": "Production Engine and model adapter, real neural child IPC, 20ms polling, fake input/activity; excludes outer desktop pipe and rendering", + "sessions": [ + { + "failed_during_warmup": false, + "neural_status": "Ready", + "session": 1, + "successful_refinements": 1000, + "warmup_completed": 20 + } + ], + "process_tree_peak_rss_bytes": 695799808, + "memory_note": "20ms sampled sum of test process and child RSS; shared pages may be counted twice and short peaks missed." +} diff --git a/docs/six-predictions.md b/docs/six-predictions.md index c403ffff..17f0faf8 100644 --- a/docs/six-predictions.md +++ b/docs/six-predictions.md @@ -34,3 +34,61 @@ deadline remains two seconds. Use switchify-prediction scripts/generation_evaluate.py for the frozen corpus comparison. Existing model quality regressions remain. Neither default activation nor a successful performance run establishes unseen-data accuracy. + +## Initial comparison + +A deterministic 1,000-query sample of the 1,760-query frozen development/test +workload produced the following results. These runs measured implementation +70fc4a8; final companion c25f068 changes release versions and documentation only. +Generation starts with a fresh context cache for each measured query. + +| Metric | Windows portable | macOS ARM | +| --- | ---: | ---: | +| Existing reranker top-three | 50.1% | 50.3% | +| Existing reranker top-six, at most five returned | 51.8% | 52.0% | +| Six statistical top-three | 39.9% | 39.9% | +| Six statistical top-six | 48.2% | 48.2% | +| Six-slot top-three | 39.9% | 39.9% | +| Six-slot top-six | 72.6% | 72.7% | +| Neural slot fill | 93.5% | 93.4% | +| Correct OOV query instances | 29/63 | 29/63 | +| Gains / regressions against six statistical | 260 / 16 | 260 / 15 | +| Generation median / p95 | 1152 / 1388 ms | 349 / 767 ms | +| Generation maximum | 1671 ms | 1701 ms | +| Peak process-tree RSS | 658 MiB | 862 MiB | + +Both runs completed 1,000 generation queries without generation failures. The +macOS comparison reranker failed once and was explicitly retried. Windows had +no failures. Local Windows development checks overlapped part of measurement; +final CI repeats qualification on dedicated runners. The comparison covers a +sample of the frozen workload, not the whole corpus. Generated OOV occurrences +were 702 and 701 respectively, counting repeated query occurrences. + +Keeping the statistical first three sacrifices the previous reranker's +first-three accuracy. Total top-six accuracy improves on this sample, but some +queries regress. The existing quality qualification failures remain; unknown +training overlap prevents claims about unseen-data accuracy. + +Reproduce in the companion checkout with Python dependencies psutil==7.0.0 and +regex==2025.11.3, using explicit paths to the prepared offline assets: + +```text +python scripts/generation_evaluate.py --cli --baseline --bundle --worker --samples 1000 --output +``` + +Omit --samples to evaluate all 1,760 queries. The companion CI uploads platform +JSON reports as generation-windows-latest and generation-macos-latest. Desktop +CI separately checks packaged resources and measures the production integration +with fake input adapters. Those measurements are distinct from CLI generation. + +The extracted Windows installer also completed 1,000 production-engine queries +with its optimized worker and no failures or retries. Immediate median/p95/max +were 8.34/9.35/15.25 ms; generation was 496/537/1035 ms. All 3,000 neural slots +filled on this synthetic fixture, and peak process-tree RSS was 664 MiB. This +measurement includes the engine, model adapter and child IPC with 20 ms polling. +It excludes the outer desktop pipe and rendering and does not establish corpus +accuracy. Installer contents and offline loading were verified separately. + +The measured JSON reports are checked in under docs/measurements/six-predictions. +Their worker and corpus hashes identify the exact inputs. Final-head CI reports +are additional evidence and may vary with runner hardware and timing cutoffs. diff --git a/scripts/prediction-neural.json b/scripts/prediction-neural.json index 844bbfee..61fe36bd 100644 --- a/scripts/prediction-neural.json +++ b/scripts/prediction-neural.json @@ -1,5 +1,5 @@ { - "revision": "40c09c3f125403811b167d042730826ff6d451b7", + "revision": "9ec55f767ca6e7e97a775df0bf0dc8d4be42c709", "bundle": { "format_version": 1, "model_id": "smollm2-135m-q8-v1", @@ -76,77 +76,77 @@ "workers": { "aarch64-apple-darwin": { "prepared_artifact": { - "run_id": 37211131470, + "run_id": 37212539426, "name": "neural-macos-latest", "archive": "switchify-prediction-neural-0.2.1-aarch64-apple-darwin.zip" }, - "archive_sha256": "2ba7ff29b2286eb02b0435f608e44934d89d119967418d15b8f95aaf5072544c", + "archive_sha256": "3976ac1274488b23438b9af299cfca64a9feffecfd1d4548a3ec0dc1b8681f8c", "files": { "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/QUALIFICATION.md": { - "sha256": "f5d13a7406728515112cab4da093ee19642f68abbb9f30c8c8f511ae68ec8eb3", - "size": 7086 + "size": 7086, + "sha256": "f5d13a7406728515112cab4da093ee19642f68abbb9f30c8c8f511ae68ec8eb3" }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/QUALIFICATION.json": { - "sha256": "55b6e9429ecb290fd194682406773a6f07102861529d5e1c0e51a6d2d4fc8c7e", - "size": 473 + "size": 714, + "sha256": "636ff7eabd1eec3cbfcf3d554d1908175cbae42d65330b7d8cf80bc4f0cbc509" }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/LICENSE": { - "sha256": "1e5643d5431cc5b2d04534dee7b3f716b24fc3b82493a1659534cbfa4fc8732c", - "size": 1079 + "size": 1079, + "sha256": "1e5643d5431cc5b2d04534dee7b3f716b24fc3b82493a1659534cbfa4fc8732c" }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/Cargo.lock": { - "sha256": "a49ecb4d18fa6f26d8777701e2bbad87691fff4ec05664e5a746bf18580673b0", - "size": 46262 + "size": 46262, + "sha256": "1ca440eb3b6acbe7a5b40ebc39bf2db2fb761ab5592f6610fd1a61adf04f2ffe" }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/THIRD_PARTY_NOTICES.md": { - "sha256": "6bc01a7be7b3aaa59d2c04acbbdea91b20699f0f20d556debc5e674eb0d3600f", - "size": 1545871 + "size": 1545871, + "sha256": "6bc01a7be7b3aaa59d2c04acbbdea91b20699f0f20d556debc5e674eb0d3600f" }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/README.md": { - "sha256": "02cf02f01d6e711cf6415304ace910722cb76268e90358c5362feaec034a090b", - "size": 8913 + "size": 8954, + "sha256": "c1ffe05faa70725dfe43e54657b2088baec45fcd1356e5cc802ec89a4965c39b" }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/switchify-smol-worker": { - "sha256": "c968323826413de70108e70a94bf1bd5554cff7db4708035590b1b99f758581e", - "size": 7322096 + "size": 7321632, + "sha256": "acb543af042d1dc40b928edbfd4d5dfb88270d05f19efe98b8b322002a7469d3" }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/SHA256SUMS": { - "sha256": "c7a37e54a371f6f16731e7bf88ddaafe4eb5934fa26696641e731876aa9fafef", - "size": 998 + "size": 998, + "sha256": "a95f16357fb6e6747ab2e50df2ef7cdbf09a2749019d7b762e592272089a1abd" }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/RUST_LIBRARY_COPYRIGHT.html": { - "sha256": "0a65bb747c49c7bb816cbc7188319bd6e4e8d08091c1190b8a3c0971c47968ed", - "size": 279302 + "size": 279302, + "sha256": "0a65bb747c49c7bb816cbc7188319bd6e4e8d08091c1190b8a3c0971c47968ed" }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/verify_bundle.py": { - "sha256": "69c124d037d9e25018f690c10b861c4388c01a8cd8eb3ed438d17136196c1194", - "size": 1282 + "size": 1282, + "sha256": "69c124d037d9e25018f690c10b861c4388c01a8cd8eb3ed438d17136196c1194" }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/BUILD.json": { - "sha256": "59cf2afc66e1bcd5c3481d241849039f02a4d2e7d0b6b7a236bfa922fb173c52", - "size": 987 + "size": 987, + "sha256": "701bed567579160606ebf4ad203ebb5560a7a32093ed68194fa66bbad5de06d9" }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/SECURITY.md": { - "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182", - "size": 1973 + "size": 1973, + "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182" } } }, "x86_64-apple-darwin": { "prepared_artifact": { - "run_id": 37211131470, + "run_id": 37212539426, "name": "neural-macos-15-intel", "archive": "switchify-prediction-neural-0.2.1-x86_64-apple-darwin.zip" }, - "archive_sha256": "e477f97866303487d2f3c4e9551fc6fe5fe6e5f38a3a453cdbf1917957f763bd", + "archive_sha256": "c5df5cc3ff7732fa0ca4bd53b7dd5c9f5bcfb9531a721e727d9ecd3bb4cac78a", "files": { "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/QUALIFICATION.md": { "size": 7086, "sha256": "f5d13a7406728515112cab4da093ee19642f68abbb9f30c8c8f511ae68ec8eb3" }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/QUALIFICATION.json": { - "size": 473, - "sha256": "55b6e9429ecb290fd194682406773a6f07102861529d5e1c0e51a6d2d4fc8c7e" + "size": 714, + "sha256": "636ff7eabd1eec3cbfcf3d554d1908175cbae42d65330b7d8cf80bc4f0cbc509" }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/LICENSE": { "size": 1079, @@ -154,27 +154,27 @@ }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/Cargo.lock": { "size": 46262, - "sha256": "a49ecb4d18fa6f26d8777701e2bbad87691fff4ec05664e5a746bf18580673b0" + "sha256": "1ca440eb3b6acbe7a5b40ebc39bf2db2fb761ab5592f6610fd1a61adf04f2ffe" }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/THIRD_PARTY_NOTICES.md": { "size": 1545871, "sha256": "6bc01a7be7b3aaa59d2c04acbbdea91b20699f0f20d556debc5e674eb0d3600f" }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/README.md": { - "size": 8913, - "sha256": "02cf02f01d6e711cf6415304ace910722cb76268e90358c5362feaec034a090b" + "size": 8954, + "sha256": "c1ffe05faa70725dfe43e54657b2088baec45fcd1356e5cc802ec89a4965c39b" }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/switchify-smol-worker": { - "size": 7903992, - "sha256": "7a0b9177485369195ea0af9e1578104c2efb3f721208f7fec0cb713ef8d00a9f" + "size": 7903704, + "sha256": "5a518794a91d7ad9edb6f537c5f9e12ec765cfeffa2160a1a46ecf1e3eb35790" }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/switchify-smol-worker-avx2": { - "size": 8055144, - "sha256": "7ac36b3e7f30d69be0f88b1b452b66791ac03fe4534e06ebd6c21c46c95e041a" + "size": 8054416, + "sha256": "6151ef19e3848ce7375a3748b844ae9ef2d5f52037c60eebd49b7df09004a2ba" }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/SHA256SUMS": { "size": 1091, - "sha256": "7eac576a4042419adedb7dac5f3ab847e6bce6c3c2a8bdb23e29ffe2382ef627" + "sha256": "13b66da70c11dafffd1dbcad71aa9e803ad21ccf36d4f51b913e4fca9a361182" }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/RUST_LIBRARY_COPYRIGHT.html": { "size": 279302, @@ -186,7 +186,7 @@ }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/BUILD.json": { "size": 1024, - "sha256": "1ac79006baf2fe3499c027a509357ae19130a41b1bb56fd4687e5c1b722fe9a0" + "sha256": "e49098008cbcb1a968d6c2abb67ad5245fbb3891a32a768f8d3e5de3c4d2abf6" }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/SECURITY.md": { "size": 1973, @@ -196,128 +196,128 @@ }, "x86_64-pc-windows-msvc": { "prepared_artifact": { - "run_id": 37211131470, + "run_id": 37212539426, "name": "neural-windows-latest", "archive": "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc.zip" }, - "archive_sha256": "6b2a385fca62f2947add1de286440d87778c94c6d79ff6892813cc702514c45e", + "archive_sha256": "175f3f9a8c021078c8bd1fefa1d6389afbdaec3f383cecdd14810ac8f6acba1a", "files": { "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/BUILD.json": { - "sha256": "ebe7986dc80c01e19b55697b740a2e42950f49314890eeedb233baaa25ddbf69", - "size": 1059 + "size": 1059, + "sha256": "5a90cebe110f2711f9d262d8b0542dfebf56b407f019a6dc75bb0395fe2efd96" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/Cargo.lock": { - "sha256": "a49ecb4d18fa6f26d8777701e2bbad87691fff4ec05664e5a746bf18580673b0", - "size": 46262 + "size": 46262, + "sha256": "1ca440eb3b6acbe7a5b40ebc39bf2db2fb761ab5592f6610fd1a61adf04f2ffe" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/LICENSE": { - "sha256": "38a190016c5d23b6acf37beda35bdef634a3572da76bb7c6ebfec84763648357", - "size": 1100 + "size": 1100, + "sha256": "38a190016c5d23b6acf37beda35bdef634a3572da76bb7c6ebfec84763648357" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/QUALIFICATION.json": { - "sha256": "55b6e9429ecb290fd194682406773a6f07102861529d5e1c0e51a6d2d4fc8c7e", - "size": 473 + "size": 714, + "sha256": "636ff7eabd1eec3cbfcf3d554d1908175cbae42d65330b7d8cf80bc4f0cbc509" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/QUALIFICATION.md": { - "sha256": "934945362415a5672a5c05887553911778f2032508b5efcb9a755c5e551548dc", - "size": 7147 + "size": 7147, + "sha256": "934945362415a5672a5c05887553911778f2032508b5efcb9a755c5e551548dc" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/README.md": { - "sha256": "02cf02f01d6e711cf6415304ace910722cb76268e90358c5362feaec034a090b", - "size": 8913 + "size": 8954, + "sha256": "c1ffe05faa70725dfe43e54657b2088baec45fcd1356e5cc802ec89a4965c39b" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/RUST_LIBRARY_COPYRIGHT.html": { - "sha256": "0a65bb747c49c7bb816cbc7188319bd6e4e8d08091c1190b8a3c0971c47968ed", - "size": 279302 + "size": 279302, + "sha256": "0a65bb747c49c7bb816cbc7188319bd6e4e8d08091c1190b8a3c0971c47968ed" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/SECURITY.md": { - "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182", - "size": 1973 + "size": 1973, + "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/SHA256SUMS": { - "sha256": "a585397a8c2ad3f12f767f1d87a40d5bc7c10d3c087ed0183b5554ef48af9ea8", - "size": 1116 + "size": 1116, + "sha256": "31d12eb51fce3223f4a3ed141cf388f186c592731127ba231149bab8d9008a1c" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/switchify-smol-worker-avx2.exe": { - "sha256": "56884d10a431f186c369943cc335b5108b7d920d308fd465234827fba8aeb38d", - "size": 8709120 + "size": 8709120, + "sha256": "8e9efe60e59816f2cc5d4127b3e9ed47a8d5ca18675ea5c3fc527106471d0c32" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/switchify-smol-worker.exe": { - "sha256": "537028f08e65dd08e7e9353f03308e0a641ad65b8424a17af318fb8f86d8eaed", - "size": 8510464 + "size": 8510464, + "sha256": "47123b4cc20002f7c2d6fa369a9ce994cf1b1a1939d03318d1c1088ed1865783" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/THIRD_PARTY_NOTICES.md": { - "sha256": "448c7f55a9ef918d5cf67f57cfb141c62f7bc6d819cc168fc4ad20f73b34c5f0", - "size": 1552635 + "size": 1552635, + "sha256": "448c7f55a9ef918d5cf67f57cfb141c62f7bc6d819cc168fc4ad20f73b34c5f0" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/verify_bundle.py": { - "sha256": "96b47d695b104c61ca65de856108d8850a6da7c00220ae72dd91b8e73cb8f6f2", - "size": 1315 + "size": 1315, + "sha256": "96b47d695b104c61ca65de856108d8850a6da7c00220ae72dd91b8e73cb8f6f2" } } }, "x86_64-unknown-linux-gnu": { "prepared_artifact": { - "run_id": 37211131470, + "run_id": 37212539426, "name": "neural-ubuntu-latest", "archive": "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu.zip" }, - "archive_sha256": "248b14c7220a58a9119eac6dadf63394f35d0bbc2d105d58563c55b4b701a7b2", + "archive_sha256": "c8fb614e53c4656bd1eeb808552a7f51c9da09b45ba77687b7f07e7775b2f479", "files": { "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/BUILD.json": { - "sha256": "1440f4897ac396ff4b3ef11baf963042c97e9113b5177fd4cac811e598e60240", - "size": 1051 + "size": 1051, + "sha256": "29b25c63447c59ada1b7ed8549905fb9735540b2d698e6a5d4c3ea9857fdeb3d" }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/RUST_LIBRARY_COPYRIGHT.html": { - "sha256": "0a65bb747c49c7bb816cbc7188319bd6e4e8d08091c1190b8a3c0971c47968ed", - "size": 279302 + "size": 279302, + "sha256": "0a65bb747c49c7bb816cbc7188319bd6e4e8d08091c1190b8a3c0971c47968ed" }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/switchify-smol-worker": { - "sha256": "3dba4916ff39f2779e80906aaf8c1aac5270e746f4bc7d3ecef4e0532915977a", - "size": 10153664 + "size": 10154408, + "sha256": "d537aff0606ac07289ae2dae69dab503a090be9d5d2248919e0fb91bf7c29df3" }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/SHA256SUMS": { - "sha256": "678af020c062ada94dfcc4403efb972955e574d34ccbef35760e205f012196d5", - "size": 1091 + "size": 1091, + "sha256": "2d6d80a8edb6bb0b5c03787b3218c4e76cb5bf963a3a8f636f0a3a26076d576c" }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/LICENSE": { - "sha256": "1e5643d5431cc5b2d04534dee7b3f716b24fc3b82493a1659534cbfa4fc8732c", - "size": 1079 + "size": 1079, + "sha256": "1e5643d5431cc5b2d04534dee7b3f716b24fc3b82493a1659534cbfa4fc8732c" }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/switchify-smol-worker-avx2": { - "sha256": "737e8e4804e58c5b40676d4ca9237b02913682b63ce9679ed4416326276c3259", - "size": 10246512 + "size": 10245944, + "sha256": "2a75322f581d8bfdad17094d36fd1e529effcdf03d482a6a7d876eaf9c4cd91e" }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/SECURITY.md": { - "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182", - "size": 1973 + "size": 1973, + "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182" }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/verify_bundle.py": { - "sha256": "69c124d037d9e25018f690c10b861c4388c01a8cd8eb3ed438d17136196c1194", - "size": 1282 + "size": 1282, + "sha256": "69c124d037d9e25018f690c10b861c4388c01a8cd8eb3ed438d17136196c1194" }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/Cargo.lock": { - "sha256": "a49ecb4d18fa6f26d8777701e2bbad87691fff4ec05664e5a746bf18580673b0", - "size": 46262 + "size": 46262, + "sha256": "1ca440eb3b6acbe7a5b40ebc39bf2db2fb761ab5592f6610fd1a61adf04f2ffe" }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/README.md": { - "sha256": "02cf02f01d6e711cf6415304ace910722cb76268e90358c5362feaec034a090b", - "size": 8913 + "size": 8954, + "sha256": "c1ffe05faa70725dfe43e54657b2088baec45fcd1356e5cc802ec89a4965c39b" }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/THIRD_PARTY_NOTICES.md": { - "sha256": "93cad9c9282edec579015f6d822901ab48596ff21e9f74bb3f07dc2ae8b7314c", - "size": 1535176 + "size": 1535176, + "sha256": "93cad9c9282edec579015f6d822901ab48596ff21e9f74bb3f07dc2ae8b7314c" }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/QUALIFICATION.json": { - "sha256": "55b6e9429ecb290fd194682406773a6f07102861529d5e1c0e51a6d2d4fc8c7e", - "size": 473 + "size": 714, + "sha256": "636ff7eabd1eec3cbfcf3d554d1908175cbae42d65330b7d8cf80bc4f0cbc509" }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/QUALIFICATION.md": { - "sha256": "f5d13a7406728515112cab4da093ee19642f68abbb9f30c8c8f511ae68ec8eb3", - "size": 7086 + "size": 7086, + "sha256": "f5d13a7406728515112cab4da093ee19642f68abbb9f30c8c8f511ae68ec8eb3" } } } }, - "library_revision": "70fc4a869ffd22a405527e4d81d25c149aebc418" + "library_revision": "c25f068dacef8fbf65dde6b98c745ad4cd56d555" } diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index df67f9b3..b896925c 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -4216,8 +4216,8 @@ dependencies = [ [[package]] name = "switchify-prediction" -version = "0.2.0" -source = "git+https://github.com/switchifyapp/switchify-prediction?rev=70fc4a869ffd22a405527e4d81d25c149aebc418#70fc4a869ffd22a405527e4d81d25c149aebc418" +version = "0.2.1" +source = "git+https://github.com/switchifyapp/switchify-prediction?rev=c25f068dacef8fbf65dde6b98c745ad4cd56d555#c25f068dacef8fbf65dde6b98c745ad4cd56d555" dependencies = [ "clap", "rusqlite", @@ -4233,7 +4233,7 @@ dependencies = [ [[package]] name = "switchify-prediction-neural" version = "0.2.1" -source = "git+https://github.com/switchifyapp/switchify-prediction?rev=70fc4a869ffd22a405527e4d81d25c149aebc418#70fc4a869ffd22a405527e4d81d25c149aebc418" +source = "git+https://github.com/switchifyapp/switchify-prediction?rev=c25f068dacef8fbf65dde6b98c745ad4cd56d555#c25f068dacef8fbf65dde6b98c745ad4cd56d555" dependencies = [ "clap", "serde", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 75d4cb79..9f93c2ec 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -14,8 +14,8 @@ crate-type = ["staticlib", "cdylib", "rlib"] tauri-build = { version = "2", features = [] } [dependencies] -switchify-prediction = { git = "https://github.com/switchifyapp/switchify-prediction", rev = "70fc4a869ffd22a405527e4d81d25c149aebc418" } -switchify-prediction-neural = { git = "https://github.com/switchifyapp/switchify-prediction", rev = "70fc4a869ffd22a405527e4d81d25c149aebc418" } +switchify-prediction = { git = "https://github.com/switchifyapp/switchify-prediction", rev = "c25f068dacef8fbf65dde6b98c745ad4cd56d555" } +switchify-prediction-neural = { git = "https://github.com/switchifyapp/switchify-prediction", rev = "c25f068dacef8fbf65dde6b98c745ad4cd56d555" } anyhow = "1" base64 = "0.22" directories = "6.0.0" From 07ff11ed71425cfce3e6b6a8e3422686e6888c3a Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Sun, 4 Oct 2026 17:42:53 +0100 Subject: [PATCH 3/3] Pin published v0.2.1 companion assets --- .github/workflows/ci.yml | 6 ++-- docs/six-predictions.md | 15 ++++---- scripts/prediction-neural.json | 64 +++++++++++++--------------------- src-tauri/Cargo.lock | 4 +-- src-tauri/Cargo.toml | 4 +-- 5 files changed, 37 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5844375d..acba3e4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,11 +112,9 @@ jobs: toolchain: 1.97.1 components: rustfmt, clippy - run: npm ci - - name: Fetch prediction model and prepared companion workers - env: - GH_TOKEN: ${{ github.token }} + - name: Fetch prediction model and released companion workers run: npm run prediction-model - - name: Diagnose prepared ARM worker + - name: Diagnose released ARM worker if: runner.os == 'macOS' env: SWITCHIFY_BENCHMARK_MODEL: src-tauri/resources/prediction-model/english.sqlite diff --git a/docs/six-predictions.md b/docs/six-predictions.md index 17f0faf8..d895c76a 100644 --- a/docs/six-predictions.md +++ b/docs/six-predictions.md @@ -13,14 +13,13 @@ The companion uses protocol 2. Model and tokenizer bytes are unchanged. Startup remains bounded to 30 seconds and each reply to two seconds. Failure leaves the instant slots available, with explicit retry or keyboard reopening required. -## Prepared assets - -The build can consume checksum-pinned companion CI archives before publication, -using an authenticated GitHub CLI. Verified local caches work offline. These CI -artifacts expire and are not permanent release assets. Before merging/releasing -the desktop, separately authorize companion artifact publication and replace -prepared artifact sources with the permanent release URLs for the same bytes. -No runtime network access or model setup is needed. +## Release assets + +The build downloads checksum-pinned v0.2.1 companion release archives from +GitHub. Both libraries and worker provenance are pinned to the release commit. +Verified local caches work offline, and installed applications need no network +access or model setup. Model, tokenizer and statistical database bytes remain +unchanged. ## Validation diff --git a/scripts/prediction-neural.json b/scripts/prediction-neural.json index 61fe36bd..fc73199f 100644 --- a/scripts/prediction-neural.json +++ b/scripts/prediction-neural.json @@ -1,5 +1,5 @@ { - "revision": "9ec55f767ca6e7e97a775df0bf0dc8d4be42c709", + "revision": "2a2ed0c2b0013663ffa9317dba3389ec96b6f14b", "bundle": { "format_version": 1, "model_id": "smollm2-135m-q8-v1", @@ -75,12 +75,7 @@ }, "workers": { "aarch64-apple-darwin": { - "prepared_artifact": { - "run_id": 37212539426, - "name": "neural-macos-latest", - "archive": "switchify-prediction-neural-0.2.1-aarch64-apple-darwin.zip" - }, - "archive_sha256": "3976ac1274488b23438b9af299cfca64a9feffecfd1d4548a3ec0dc1b8681f8c", + "archive_sha256": "aaa633461090006311557e7b2a4b9876c225af29933501feb65afc4c8c048d49", "files": { "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/QUALIFICATION.md": { "size": 7086, @@ -112,7 +107,7 @@ }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/SHA256SUMS": { "size": 998, - "sha256": "a95f16357fb6e6747ab2e50df2ef7cdbf09a2749019d7b762e592272089a1abd" + "sha256": "ba76f5b2bd7aa375164d1886548f913c0faf83cde9edb8f463c89877b59a3aea" }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/RUST_LIBRARY_COPYRIGHT.html": { "size": 279302, @@ -124,21 +119,17 @@ }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/BUILD.json": { "size": 987, - "sha256": "701bed567579160606ebf4ad203ebb5560a7a32093ed68194fa66bbad5de06d9" + "sha256": "efe9910c9f3de2f33846eb08cae659f5f01b12cadddd3e4121903239c4ff97e3" }, "switchify-prediction-neural-0.2.1-aarch64-apple-darwin/SECURITY.md": { "size": 1973, "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182" } - } + }, + "url": "https://github.com/switchifyapp/switchify-prediction/releases/download/v0.2.1/switchify-prediction-neural-0.2.1-aarch64-apple-darwin.zip" }, "x86_64-apple-darwin": { - "prepared_artifact": { - "run_id": 37212539426, - "name": "neural-macos-15-intel", - "archive": "switchify-prediction-neural-0.2.1-x86_64-apple-darwin.zip" - }, - "archive_sha256": "c5df5cc3ff7732fa0ca4bd53b7dd5c9f5bcfb9531a721e727d9ecd3bb4cac78a", + "archive_sha256": "6882754360e1f854c96052d816cccbf1f93598ffc7b81f711ba2150cddbbd465", "files": { "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/QUALIFICATION.md": { "size": 7086, @@ -174,7 +165,7 @@ }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/SHA256SUMS": { "size": 1091, - "sha256": "13b66da70c11dafffd1dbcad71aa9e803ad21ccf36d4f51b913e4fca9a361182" + "sha256": "d1b7a00190a833867cdf2924fc5835e2865c9c365b294130dab136535a71892d" }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/RUST_LIBRARY_COPYRIGHT.html": { "size": 279302, @@ -186,25 +177,21 @@ }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/BUILD.json": { "size": 1024, - "sha256": "e49098008cbcb1a968d6c2abb67ad5245fbb3891a32a768f8d3e5de3c4d2abf6" + "sha256": "d910e349135956cd1341383b1d1bd2a8b76fe0f5ea957776d54c8a94d2ec83ac" }, "switchify-prediction-neural-0.2.1-x86_64-apple-darwin/SECURITY.md": { "size": 1973, "sha256": "d79bf51500d3927bc47fad3e61b987be10764c7003b82df0f9d1f752f72a5182" } - } + }, + "url": "https://github.com/switchifyapp/switchify-prediction/releases/download/v0.2.1/switchify-prediction-neural-0.2.1-x86_64-apple-darwin.zip" }, "x86_64-pc-windows-msvc": { - "prepared_artifact": { - "run_id": 37212539426, - "name": "neural-windows-latest", - "archive": "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc.zip" - }, - "archive_sha256": "175f3f9a8c021078c8bd1fefa1d6389afbdaec3f383cecdd14810ac8f6acba1a", + "archive_sha256": "5580543007f7e0cc8601685be195be6fb68215a6521bd5eb8864ac58590bad1c", "files": { "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/BUILD.json": { "size": 1059, - "sha256": "5a90cebe110f2711f9d262d8b0542dfebf56b407f019a6dc75bb0395fe2efd96" + "sha256": "b168a91658032969073ff350bec657113039905d9c5b64a5e0c6c7b442530bce" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/Cargo.lock": { "size": 46262, @@ -236,15 +223,15 @@ }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/SHA256SUMS": { "size": 1116, - "sha256": "31d12eb51fce3223f4a3ed141cf388f186c592731127ba231149bab8d9008a1c" + "sha256": "117a918e874ac6fe4d473fd6e449e472205d327f5c7588abe12802d11b7e234d" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/switchify-smol-worker-avx2.exe": { "size": 8709120, - "sha256": "8e9efe60e59816f2cc5d4127b3e9ed47a8d5ca18675ea5c3fc527106471d0c32" + "sha256": "46d05730c283e7d6ca6d325611410c0b2196d0431e6cfe8b973cd61c24011d2b" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/switchify-smol-worker.exe": { "size": 8510464, - "sha256": "47123b4cc20002f7c2d6fa369a9ce994cf1b1a1939d03318d1c1088ed1865783" + "sha256": "d6087919a6704a673ab6f4609e87a113e3f59a4bba692a9be4cd1f7bcdc2dc69" }, "switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc/THIRD_PARTY_NOTICES.md": { "size": 1552635, @@ -254,19 +241,15 @@ "size": 1315, "sha256": "96b47d695b104c61ca65de856108d8850a6da7c00220ae72dd91b8e73cb8f6f2" } - } + }, + "url": "https://github.com/switchifyapp/switchify-prediction/releases/download/v0.2.1/switchify-prediction-neural-0.2.1-x86_64-pc-windows-msvc.zip" }, "x86_64-unknown-linux-gnu": { - "prepared_artifact": { - "run_id": 37212539426, - "name": "neural-ubuntu-latest", - "archive": "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu.zip" - }, - "archive_sha256": "c8fb614e53c4656bd1eeb808552a7f51c9da09b45ba77687b7f07e7775b2f479", + "archive_sha256": "7411a6773589f7e28975e6754a6a48e893808b5c3953252a9251bdec792474bd", "files": { "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/BUILD.json": { "size": 1051, - "sha256": "29b25c63447c59ada1b7ed8549905fb9735540b2d698e6a5d4c3ea9857fdeb3d" + "sha256": "aad2210a773e9979ceae63e668f91c10db49d7906f90f274eed474d6124c0c35" }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/RUST_LIBRARY_COPYRIGHT.html": { "size": 279302, @@ -278,7 +261,7 @@ }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/SHA256SUMS": { "size": 1091, - "sha256": "2d6d80a8edb6bb0b5c03787b3218c4e76cb5bf963a3a8f636f0a3a26076d576c" + "sha256": "589c2588753eb078508c447dd374dd1da4a34a0d62d6e824a9adffc1c01cda69" }, "switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu/LICENSE": { "size": 1079, @@ -316,8 +299,9 @@ "size": 7086, "sha256": "f5d13a7406728515112cab4da093ee19642f68abbb9f30c8c8f511ae68ec8eb3" } - } + }, + "url": "https://github.com/switchifyapp/switchify-prediction/releases/download/v0.2.1/switchify-prediction-neural-0.2.1-x86_64-unknown-linux-gnu.zip" } }, - "library_revision": "c25f068dacef8fbf65dde6b98c745ad4cd56d555" + "library_revision": "2a2ed0c2b0013663ffa9317dba3389ec96b6f14b" } diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index b896925c..96d3c084 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -4217,7 +4217,7 @@ dependencies = [ [[package]] name = "switchify-prediction" version = "0.2.1" -source = "git+https://github.com/switchifyapp/switchify-prediction?rev=c25f068dacef8fbf65dde6b98c745ad4cd56d555#c25f068dacef8fbf65dde6b98c745ad4cd56d555" +source = "git+https://github.com/switchifyapp/switchify-prediction?rev=2a2ed0c2b0013663ffa9317dba3389ec96b6f14b#2a2ed0c2b0013663ffa9317dba3389ec96b6f14b" dependencies = [ "clap", "rusqlite", @@ -4233,7 +4233,7 @@ dependencies = [ [[package]] name = "switchify-prediction-neural" version = "0.2.1" -source = "git+https://github.com/switchifyapp/switchify-prediction?rev=c25f068dacef8fbf65dde6b98c745ad4cd56d555#c25f068dacef8fbf65dde6b98c745ad4cd56d555" +source = "git+https://github.com/switchifyapp/switchify-prediction?rev=2a2ed0c2b0013663ffa9317dba3389ec96b6f14b#2a2ed0c2b0013663ffa9317dba3389ec96b6f14b" dependencies = [ "clap", "serde", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 9f93c2ec..f37b5cbe 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -14,8 +14,8 @@ crate-type = ["staticlib", "cdylib", "rlib"] tauri-build = { version = "2", features = [] } [dependencies] -switchify-prediction = { git = "https://github.com/switchifyapp/switchify-prediction", rev = "c25f068dacef8fbf65dde6b98c745ad4cd56d555" } -switchify-prediction-neural = { git = "https://github.com/switchifyapp/switchify-prediction", rev = "c25f068dacef8fbf65dde6b98c745ad4cd56d555" } +switchify-prediction = { git = "https://github.com/switchifyapp/switchify-prediction", rev = "2a2ed0c2b0013663ffa9317dba3389ec96b6f14b" } +switchify-prediction-neural = { git = "https://github.com/switchifyapp/switchify-prediction", rev = "2a2ed0c2b0013663ffa9317dba3389ec96b6f14b" } anyhow = "1" base64 = "0.22" directories = "6.0.0"