From 273914860d2a38c54f241c44facb5a5a21c8032c Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Sat, 3 Oct 2026 15:47:59 +0100 Subject: [PATCH 1/5] Add general-purpose offline neural prediction experiment --- .github/workflows/neural-experiment.yml | 36 + README.md | 3 + docs/futo-investigation.md | 38 + docs/general-neural-protocol.md | 93 ++ experiments/neural/Cargo.lock | 1845 +++++++++++++++++++++ experiments/neural/Cargo.toml | 20 + experiments/neural/fixtures.json | 106 ++ experiments/neural/src/main.rs | 476 ++++++ general-neural-source-manifest.json | 26 + scripts/general_neural_experiment.py | 114 ++ scripts/test_general_neural_experiment.py | 28 + 11 files changed, 2785 insertions(+) create mode 100644 .github/workflows/neural-experiment.yml create mode 100644 docs/futo-investigation.md create mode 100644 docs/general-neural-protocol.md create mode 100644 experiments/neural/Cargo.lock create mode 100644 experiments/neural/Cargo.toml create mode 100644 experiments/neural/fixtures.json create mode 100644 experiments/neural/src/main.rs create mode 100644 general-neural-source-manifest.json create mode 100644 scripts/general_neural_experiment.py create mode 100644 scripts/test_general_neural_experiment.py diff --git a/.github/workflows/neural-experiment.yml b/.github/workflows/neural-experiment.yml new file mode 100644 index 0000000..76f5be1 --- /dev/null +++ b/.github/workflows/neural-experiment.yml @@ -0,0 +1,36 @@ +name: Neural experiment +on: + pull_request: + workflow_dispatch: +permissions: + contents: read +concurrency: + group: neural-experiment-${{ github.ref }} + cancel-in-progress: true +jobs: + neural-checks: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: dtolnay/rust-toolchain@4716b85f2fac3e324e64fa2810f6b5c3905760a5 + with: + components: rustfmt, clippy + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 + with: + workspaces: experiments/neural -> ../../target/neural + - run: cargo fmt --manifest-path experiments/neural/Cargo.toml -- --check + - run: cargo clippy --locked --manifest-path experiments/neural/Cargo.toml --target-dir target/neural --all-targets -- -D warnings + - run: cargo test --locked --manifest-path experiments/neural/Cargo.toml --target-dir target/neural + neural-audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: dtolnay/rust-toolchain@4716b85f2fac3e324e64fa2810f6b5c3905760a5 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 + - run: cargo install cargo-audit --locked --version 0.22.0 + # Candle transitively uses this unmaintained macro crate; see the protocol. + - run: cargo audit --file experiments/neural/Cargo.lock --deny warnings --ignore RUSTSEC-2024-0436 diff --git a/README.md b/README.md index 9c0633f..7e66610 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,9 @@ remain. English alone is supported. The [AAC experiment protocol](docs/aac-experiment.md) and earlier [conversational results](docs/quality-results.md) document model-selection history. +Switchify is a general-purpose typing app. The separate +[neural experiment](docs/general-neural-protocol.md) tests messages, email, +documents and search-style text equally, without changing the production model. The `prepare` and `evaluate` commands retain the earlier comparison workflows; `prepare` alone does not prepare the promoted production corpus. Use the commands above for production. A new model-selection exercise needs a new protocol; these diff --git a/docs/futo-investigation.md b/docs/futo-investigation.md new file mode 100644 index 0000000..82cce67 --- /dev/null +++ b/docs/futo-investigation.md @@ -0,0 +1,38 @@ +# FUTO keyboard model investigation + +Inspected on 2026-10-03. FUTO is a relevant general-purpose keyboard reference. +It has not been benchmarked or integrated in this experiment. + +The official source at commit +`70a5d390c505a6bbcc4e14966e5628e43ca3f1fc` selects the English `ml4_q6_k` model. +The resource is available from the official `keyboard-large-resources` submodule +at commit `d87d9dbdf3966bbe18413be375dab2f6c7bbdfdd`, path +`raw/ml4_q6_k.gguf`. GitLab's file metadata reports 30,662,880 bytes and content +SHA-256 `6545c1c9ef2d76e9bfb87ad4fcf2061889513af84fcf30d907412be7fcdedb7b`. +These are publisher metadata, not a locally verified download hash. Its name +identifies the Q6_K model; no comparison with the F32 SmolLM2 timings is implied. + +The model is not an arbitrary chat GGUF. The Android loader requires KeyboardLM +metadata and recognized feature flags. The native prediction path includes +special word-boundary behavior, candidate search, capitalization filters and +``, ``, `` / character tokens for supported correction modes. +Loading weights with a generic GGUF runtime alone would not reproduce FUTO. + +A standalone Windows/macOS benchmark appears feasible by isolating the native +prediction implementation and its matching runtime behind a Rust adapter. +That is an engineering assessment from the source, not a tested build. A useful +next step is to reproduce its next-word and partial-word outputs on fixed +fixtures before comparing latency or accuracy. Its correction mode must not +silently violate Switchify's exact-prefix completion contract. + +The repository identifies its code licence as FUTO Source First 1.1. Model terms +also need checking before any redistribution; this investigation neither copies +FUTO code into the MIT library nor distributes its model. + +## Sources + +- [Pinned model loader](https://github.com/futo-org/android-keyboard/blob/70a5d390c505a6bbcc4e14966e5628e43ca3f1fc/java/src/org/futo/inputmethod/latin/xlm/ModelPaths.kt) +- [Pinned native prediction path](https://github.com/futo-org/android-keyboard/blob/70a5d390c505a6bbcc4e14966e5628e43ca3f1fc/native/jni/org_futo_inputmethod_latin_xlm_LanguageModel.cpp) +- [Pinned resource submodule declarations](https://github.com/futo-org/android-keyboard/blob/70a5d390c505a6bbcc4e14966e5628e43ca3f1fc/.gitmodules) +- [Official model resource](https://gitlab.futo.org/keyboard/keyboard-large-resources/-/blob/d87d9dbdf3966bbe18413be375dab2f6c7bbdfdd/raw/ml4_q6_k.gguf) +- [Pinned code licence](https://github.com/futo-org/android-keyboard/blob/70a5d390c505a6bbcc4e14966e5628e43ca3f1fc/LICENSE.md) diff --git a/docs/general-neural-protocol.md b/docs/general-neural-protocol.md new file mode 100644 index 0000000..062a0a4 --- /dev/null +++ b/docs/general-neural-protocol.md @@ -0,0 +1,93 @@ +# General-purpose neural experiment + +Frozen 2026-10-03 before scoring. Switchify is a general-purpose typing app. +Messages, email, documents and search-style text receive equal weight. AAC is +not a selection objective. The existing released model is a control, including +its historical training mixture; this experiment does not endorse that mixture. + +## Fixed comparison + +Compare the released predictor with a SmolLM2-135M base-model reranker using +Candle 0.11.0, CPU F32, four Rayon threads and no GPU, personal learning or +network access during inference. This initial implementation is unquantized. +Weights, tokenizer and configuration are pinned by SHA-256 and upstream commit. +The experiment is a separate, unpublished Rust crate. Production dependencies, +APIs and model formats stay unchanged. + +Both modes request the existing engine's top eight prefix-matching words. The +control returns its first five. The neural mode scores each entire candidate's +token sequence, followed by the probability of a word boundary, and returns +the five highest scores. It uses BOS plus the last 63 tokens of preceding text +within the current sentence. It does not score only the first subword, normalize +by word length, mix frequency scales or learn from targets. Ties retain baseline +order. Each query starts with an empty KV cache; the context cache is shared only +between candidate branches within that query. Prefixes filter candidates before +neural scoring. This is a bounded reranker, not unrestricted neural generation; +report top-eight candidate coverage as its accuracy ceiling. + +Boundary probability sums the next-token probabilities for EOS and tokens whose +decoded text starts with whitespace, a number, or one of `. ! ? ; : , ( ) [ ] { } +" - /`. Apostrophe continuations remain inside a word. This is an explicit +approximation, not an exact tokenizer-independent word probability. + +## Workload and decision + +The checked-in fixtures contain 24 newly authored synthetic examples in each +of four domains. They are diagnostic fixtures, not a representative user corpus +or independent evidence of real-world accuracy. Normalize with the existing +Unicode/sentence tokenizer. Reject exact full-sentence overlap with the pinned +production training file. Unknown overlap with neural pretraining remains. + +For each domain and each prefix length 0 through 4 Unicode graphemes, hash-sort +eligible target positions and take 64 distinct queries, excluding words already +complete at that prefix. The fixed workload has 1,280 queries per mode. Duplicate +preceding-text/prefix/target triples within a cell are removed. All words before +the target are supplied identically; each predictor uses its supported context. +Lowercasing and removing punctuation limit conclusions about realistic writing. + +Report top-one, top-five and candidate coverage per domain/prefix, cold load, +warm median, p95, maximum, failures, model file size and process peak memory. +Prime once without timing. Abort on inference errors instead of silently falling +back. A smaller `--per-cell` run is calibration only, not a result used to tune +the model, shortlist or ranking rule. + +A promising result requires higher mean top-five accuracy over prefixes 0,1,2 +and all four domains, no domain/prefix loss above one percentage point there, +and warm p95 below 20 ms. Do not tune after seeing fixture scores or promote a +model automatically. Describe tradeoffs even if the criterion fails. Windows +is the local benchmark platform; macOS build/tests do not establish macOS speed. + +Investigate FUTO's official model integration separately. Do not equate generic +GGUF loading with implementing its custom keyboard prompts and tokenizer rules. +No FUTO accuracy claim without running its actual prediction path. + +## Reproduction + +Use Rust 1.97.1 and Python 3. Prepare the unchanged production partitions with +`python scripts/aac_experiment.py --prepare-only` if they are not cached. Extract +`english.sqlite` from the model bundle on the +[v0.1.0 release](https://github.com/switchifyapp/switchify-prediction/releases/tag/v0.1.0). +Then run from the repository root: + +```sh +cargo build --release --locked --manifest-path experiments/neural/Cargo.toml --target-dir target/neural +python scripts/general_neural_experiment.py --baseline /path/to/english.sqlite --output artifacts/general-neural +``` + +The Python command downloads and verifies the pinned model before launching the +offline Rust executables. Use a fresh output directory for each full run. +It records aggregate results and Windows process peak working set. Other +platforms report memory as unavailable, not zero. No captured user text is used. + +Run formatting, Clippy and tests with the separate crate's manifest in addition +to the repository checks. Its lockfile contains Candle's transitive `paste` +1.0.15 dependency, which has the unmaintained-package advisory RUSTSEC-2024-0436. +The production dependency audit remains unchanged. For this unpublished +experiment only, audit the separate lockfile with that single advisory exception: + +```sh +cargo audit --file experiments/neural/Cargo.lock --deny warnings --ignore RUSTSEC-2024-0436 +``` + +This exception does not assert that the dependency is maintained. Reassess the +dependency before considering production integration. diff --git a/experiments/neural/Cargo.lock b/experiments/neural/Cargo.lock new file mode 100644 index 0000000..d9a9103 --- /dev/null +++ b/experiments/neural/Cargo.lock @@ -0,0 +1,1845 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "candle-core" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ecb245093b0f791b89d3420c3df9c6d49c60ab63ba54db896bf8a3baf486706" +dependencies = [ + "byteorder", + "float8", + "gemm", + "half", + "libc", + "libm", + "memmap2", + "num-traits", + "num_cpus", + "rand", + "rand_distr", + "rayon", + "safetensors", + "thiserror 2.0.21", + "tokenizers", + "yoke", + "zerocopy", + "zip", +] + +[[package]] +name = "candle-nn" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaa10b6ccc365b33210ce404fbf45e60d3e0bdac1004463cf1052e6ee1c1739a" +dependencies = [ + "candle-core", + "half", + "libc", + "num-traits", + "rayon", + "safetensors", + "serde", + "thiserror 2.0.21", +] + +[[package]] +name = "candle-transformers" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bcbbf7ff00ff6fe2af22b93600195917fe90e90ff48424a140d1a926c44b1c1" +dependencies = [ + "byteorder", + "candle-core", + "candle-nn", + "fancy-regex 0.18.0", + "num-traits", + "rand", + "rayon", + "serde", + "serde_json", + "serde_plain", + "tracing", +] + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dyn-stack" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c4713e43e2886ba72b8271aa66c93d722116acf7a75555cce11dcde84388fe8" +dependencies = [ + "bytemuck", + "dyn-stack-macros", +] + +[[package]] +name = "dyn-stack-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9" + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fancy-regex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "fancy-regex" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "float8" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d1f04709a8ac06e8e8042875a3c466cc4832d3c1a18dbcb9dba3c6e83046bc" +dependencies = [ + "half", + "num-traits", + "rand", + "rand_distr", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "gemm" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa0673db364b12263d103b68337a68fbecc541d6f6b61ba72fe438654709eacb" +dependencies = [ + "dyn-stack", + "gemm-c32", + "gemm-c64", + "gemm-common", + "gemm-f16", + "gemm-f32", + "gemm-f64", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "086936dbdcb99e37aad81d320f98f670e53c1e55a98bee70573e83f95beb128c" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20c8aeeeec425959bda4d9827664029ba1501a90a0d1e6228e48bef741db3a3f" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-common" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88027625910cc9b1085aaaa1c4bc46bb3a36aad323452b33c25b5e4e7c8e2a3e" +dependencies = [ + "bytemuck", + "dyn-stack", + "half", + "libm", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp", + "raw-cpuid", + "rayon", + "seq-macro", + "sysctl", +] + +[[package]] +name = "gemm-f16" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3df7a55202e6cd6739d82ae3399c8e0c7e1402859b30e4cb780e61525d9486e" +dependencies = [ + "dyn-stack", + "gemm-common", + "gemm-f32", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "gemm-f32" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e0b8c9da1fbec6e3e3ab2ce6bc259ef18eb5f6f0d3e4edf54b75f9fd41a81c" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "056131e8f2a521bfab322f804ccd652520c79700d81209e9d9275bbdecaadc6a" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "bytemuck", + "cfg-if", + "crunchy", + "num-traits", + "rand", + "rand_distr", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", + "serde", + "serde_core", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" +dependencies = [ + "macro_rules_attribute-proc_macro", + "pastey", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", + "stable_deref_trait", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "bytemuck", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pulp" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a" +dependencies = [ + "bytemuck", + "cfg-if", + "libm", + "num-complex", + "paste", + "pulp-wasm-simd-flag", + "raw-cpuid", + "reborrow", + "version_check", +] + +[[package]] +name = "pulp-wasm-simd-flag" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740" + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.21", +] + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "safetensors" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b079b829cb27a1c3c374341345ed2e8b2c0c839034522cee576c140bd7f846" +dependencies = [ + "hashbrown 0.16.1", + "libc", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_plain" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "smallvec" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64", + "nom", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "switchify-neural-experiment" +version = "0.1.0" +dependencies = [ + "anyhow", + "candle-core", + "candle-nn", + "candle-transformers", + "clap", + "serde", + "serde_json", + "sha2", + "switchify-prediction", + "tokenizers", + "unicode-segmentation", +] + +[[package]] +name = "switchify-prediction" +version = "0.1.0" +dependencies = [ + "clap", + "rusqlite", + "serde", + "serde_json", + "sha2", + "tempfile", + "thiserror 2.0.21", + "unicode-normalization", + "unicode-segmentation", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "sysctl" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" +dependencies = [ + "bitflags", + "byteorder", + "enum-as-inner", + "libc", + "thiserror 1.0.69", + "walkdir", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl 2.0.21", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokenizers" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str", + "dary_heap", + "derive_builder", + "esaxx-rs", + "fancy-regex 0.14.0", + "getrandom 0.3.4", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.21", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.6", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", + "synstructure", +] + +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "indexmap", + "memchr", + "typed-path", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/experiments/neural/Cargo.toml b/experiments/neural/Cargo.toml new file mode 100644 index 0000000..b3e2399 --- /dev/null +++ b/experiments/neural/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "switchify-neural-experiment" +version = "0.1.0" +edition = "2024" +publish = false + +[dependencies] +anyhow = "1" +candle-core = "=0.11.0" +candle-nn = "=0.11.0" +candle-transformers = "=0.11.0" +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +switchify-prediction = { path = "../.." } +tokenizers = { version = "0.22", default-features = false, features = ["fancy-regex"] } +unicode-segmentation = "1" + +[workspace] diff --git a/experiments/neural/fixtures.json b/experiments/neural/fixtures.json new file mode 100644 index 0000000..fcf5924 --- /dev/null +++ b/experiments/neural/fixtures.json @@ -0,0 +1,106 @@ +{ + "messages": [ + "I'm running a little late because the train stopped outside the station.", + "Could you bring my blue jacket when you come over this evening?", + "We should book a table somewhere near the cinema before the weekend.", + "The delivery arrived this morning and I left your parcel beside the stairs.", + "Let me know when you finish work and we can arrange dinner.", + "I thought the meeting was tomorrow but the calendar says Thursday afternoon.", + "My phone battery is nearly empty so I'll call you after lunch.", + "Thanks for helping me move the furniture into the spare bedroom yesterday.", + "Do you remember which supermarket sells the coffee we bought last month?", + "I'll pick up some vegetables and bread on the way back home.", + "The weather looks better tomorrow so we could walk along the beach.", + "Your photographs from the holiday look amazing especially the mountain views.", + "Please send me the address again because I can't find the message.", + "We have enough chairs but somebody needs to bring another folding table.", + "I haven't watched the final episode yet so please don't spoil anything.", + "The tickets are cheaper if we travel before the morning rush hour.", + "Can you check whether the front door is locked before you leave?", + "I found your headphones underneath the sofa while I was cleaning today.", + "Let's choose another restaurant because the place we wanted is fully booked.", + "The children are staying with their grandparents until late on Sunday evening.", + "I've ordered a replacement charger and it should arrive early next week.", + "Would you prefer to meet outside the library or beside the station?", + "There is some leftover pasta in the fridge if you want lunch.", + "We enjoyed the concert although finding somewhere to park was quite difficult." + ], + "email": [ + "Thank you for sending the revised proposal and the updated delivery schedule.", + "Please find the requested documents attached for your review before our meeting.", + "Could you confirm whether the replacement parts will arrive by Friday afternoon?", + "I would appreciate a breakdown of the additional charges on this invoice.", + "Following our discussion yesterday we have adjusted the project milestones accordingly.", + "The finance team needs your approval before processing the outstanding supplier payment.", + "Unfortunately I am unavailable on Tuesday but Wednesday morning would suit me.", + "Please include the purchase reference when replying so we can locate your order.", + "We are reviewing the applications and expect to contact shortlisted candidates shortly.", + "Your subscription renewal has been processed and the receipt is attached below.", + "I noticed that the contact details in the shared document are outdated.", + "Would it be possible to extend the deadline until the following Monday?", + "The customer reported that the replacement device still displays the same error.", + "We have reserved the conference room for the training session next week.", + "Please let us know if you require any further information about registration.", + "I am writing to request a quotation for the equipment listed below.", + "The updated contract reflects the changes agreed during our last telephone conversation.", + "Thank you for your patience while our technical team investigates this issue.", + "We will send the final agenda once all speakers have confirmed attendance.", + "Please ensure that each expense claim includes a copy of the original receipt.", + "I have forwarded your enquiry to the colleague responsible for international deliveries.", + "The maintenance work is scheduled outside normal office hours to minimise disruption.", + "Could everyone review the attached spreadsheet and correct any missing information?", + "We look forward to welcoming your team to our offices next month." + ], + "documents": [ + "The experiment compared three methods under identical conditions to reduce measurement bias.", + "Participants completed a short questionnaire before beginning the practical assessment session.", + "The proposed timetable allows additional preparation time between the morning presentations.", + "A reliable backup process should preserve recent changes without interrupting normal work.", + "The building includes a shared kitchen and several meeting rooms on each floor.", + "This section explains how to replace the filter and clean the surrounding housing.", + "The committee recommended further consultation before making a decision about the redevelopment.", + "Changes in customer demand affected both production planning and warehouse storage requirements.", + "The first chapter describes the historical background and introduces the main research questions.", + "Employees should report damaged equipment to their supervisor before attempting any repairs.", + "The new route connects the residential neighbourhood with the hospital and central station.", + "Each observation was recorded separately and checked against the original laboratory notes.", + "The installation process creates a shortcut and copies the required files automatically.", + "A small increase in temperature caused the material to expand along its length.", + "The report identifies several opportunities to improve communication between regional offices.", + "Successful applicants will receive written confirmation together with instructions for the next stage.", + "The museum collection includes paintings and photographs donated by local residents.", + "Regular maintenance helps prevent unexpected failures and extends the useful life of equipment.", + "The survey results suggest that visitors value clear information and convenient opening hours.", + "A detailed inventory should include the location and condition of every stored item.", + "The software displays a warning when the available storage falls below the recommended level.", + "The final design includes wider entrances and improved lighting throughout the public areas.", + "Researchers compared the original measurements with results collected during the second trial.", + "This guide provides practical examples of common problems and explains possible solutions." + ], + "search": [ + "best lightweight waterproof jackets for walking during rainy weather", + "how to replace a damaged charging cable on a laptop", + "restaurants serving vegetarian meals near the central railway station", + "compare electricity tariffs for apartments with electric heating systems", + "train timetables between the airport and the city centre", + "instructions for cleaning reusable coffee filters without harsh chemicals", + "comfortable office chairs with adjustable armrests and lumbar support", + "weekend activities for families visiting the natural history museum", + "troubleshooting wireless headphones that disconnect during telephone conversations", + "simple recipes using roasted vegetables and leftover cooked chicken", + "opening hours for the public swimming pool during school holidays", + "how to recover an accidentally deleted folder from a backup", + "affordable accommodation within walking distance of the conference centre", + "cycling routes suitable for beginners around the coastal villages", + "portable monitors compatible with laptops using standard display connections", + "average rainfall during autumn in popular European holiday destinations", + "where to recycle broken household appliances and electronic equipment", + "beginner photography courses covering lighting composition and camera settings", + "quiet washing machines with efficient programmes for delicate clothing", + "requirements for renewing an expired passport before travelling abroad", + "comparison of cloud storage services offering automatic photograph backups", + "how to transfer calendar appointments between different email accounts", + "replacement cushions for outdoor furniture exposed to winter weather", + "local evening classes teaching conversational Spanish to complete beginners" + ] +} diff --git a/experiments/neural/src/main.rs b/experiments/neural/src/main.rs new file mode 100644 index 0000000..ac2f5af --- /dev/null +++ b/experiments/neural/src/main.rs @@ -0,0 +1,476 @@ +use anyhow::{Context, Result, bail, ensure}; +use candle_core::{DType, Device, Tensor}; +use candle_nn::VarBuilder; +use candle_transformers::models::llama::{Cache, Llama, LlamaConfig}; +use clap::{Parser, ValueEnum}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + io::Read, + path::{Path, PathBuf}, + time::Instant, +}; +use switchify_prediction::{Options, Predictor, sentences}; +use tokenizers::Tokenizer; +use unicode_segmentation::UnicodeSegmentation; + +const BASELINE_SHA: &str = "222253417d0a7a705823ffb7e599a3bcf5d5d3daf4a9d76161ac6b3e555aeaad"; +const TRAIN_SHA: &str = "ff724de5ab3b6e609ed77b699c51287fc82f79d026fdecfa4892939abfbb2834"; +const SHORTLIST: usize = 8; +const CONTEXT_TOKENS: usize = 64; + +#[derive(Clone, Copy, Debug, ValueEnum, Serialize)] +enum Mode { + Baseline, + Neural, +} + +#[derive(Parser)] +struct Args { + #[arg(long)] + baseline: PathBuf, + #[arg(long)] + model: PathBuf, + #[arg(long)] + manifest: PathBuf, + #[arg(long)] + fixtures: PathBuf, + #[arg(long)] + training: PathBuf, + #[arg(long)] + output: PathBuf, + #[arg(long, value_enum)] + mode: Mode, + /// Use 64 for the frozen experiment; smaller values are calibration only. + #[arg(long, default_value_t = 64)] + per_cell: usize, +} + +fn sha(path: &Path) -> Result { + let mut file = fs::File::open(path)?; + let mut h = Sha256::new(); + let mut buffer = [0; 65536]; + loop { + let n = file.read(&mut buffer)?; + if n == 0 { + break; + } + h.update(&buffer[..n]); + } + Ok(format!("{:x}", h.finalize())) +} + +#[derive(Deserialize)] +struct Source { + sha256: String, +} +#[derive(Deserialize)] +struct Manifest { + files: BTreeMap, +} + +fn verify_model(model: &Path, manifest: &Path) -> Result<()> { + let data: Manifest = serde_json::from_slice(&fs::read(manifest)?)?; + for name in ["config.json", "tokenizer.json", "model.safetensors"] { + let expected = data.files.get(name).context("Missing model checksum")?; + ensure!( + sha(&model.join(name))? == expected.sha256, + "Model checksum mismatch: {name}" + ); + } + Ok(()) +} + +fn boundary(text: &str) -> bool { + text.chars() + .next() + .is_some_and(|c| c.is_whitespace() || c.is_numeric() || ".!?;:,()[]{}\"-/".contains(c)) +} + +fn log_prob(logits: &[f32], token: usize) -> f64 { + let maximum = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max) as f64; + logits[token] as f64 + - maximum + - logits + .iter() + .map(|x| (*x as f64 - maximum).exp()) + .sum::() + .ln() +} + +fn log_boundary(logits: &[f32], ids: &[usize]) -> f64 { + let maximum = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max) as f64; + let all: f64 = logits.iter().map(|x| (*x as f64 - maximum).exp()).sum(); + let end: f64 = ids + .iter() + .map(|&i| (logits[i] as f64 - maximum).exp()) + .sum(); + (end / all).ln() +} + +fn word_score( + tokens: &[u32], + first_logits: &[f32], + cache: &S, + position: usize, + boundary_ids: &[usize], + mut forward: impl FnMut(u32, usize, &mut S) -> Result>, +) -> Result { + ensure!(!tokens.is_empty(), "Empty candidate tokenization"); + let mut branch = cache.clone(); + let mut logits = first_logits.to_vec(); + let mut score = 0.; + for (index, &token) in tokens.iter().enumerate() { + score += log_prob(&logits, token as usize); + logits = forward(token, position + index, &mut branch)?; + } + score += log_boundary(&logits, boundary_ids); + ensure!(score.is_finite(), "Non-finite neural score"); + Ok(score) +} + +struct Neural { + model: Llama, + tokenizer: Tokenizer, + empty_cache: Cache, + boundary_ids: Vec, +} + +impl Neural { + fn load(path: &Path) -> Result { + let tokenizer = + Tokenizer::from_file(path.join("tokenizer.json")).map_err(anyhow::Error::msg)?; + let cfg: LlamaConfig = serde_json::from_slice(&fs::read(path.join("config.json"))?)?; + let config = cfg.into_config(false); + ensure!( + config.bos_token_id == Some(0), + "Expected pinned SmolLM2 BOS token" + ); + let empty_cache = Cache::new(true, DType::F32, &config, &Device::Cpu)?; + // Safe loader owns its bytes; no mmap lifetime or externally mutable mapping. + let weights = fs::read(path.join("model.safetensors"))?; + let vb = VarBuilder::from_buffered_safetensors(weights, DType::F32, &Device::Cpu)?; + let model = Llama::load(vb, &config)?; + let mut boundary_ids = vec![0]; // Pinned EOS. + for id in 1..config.vocab_size { + let decoded = tokenizer + .decode(&[id as u32], false) + .map_err(anyhow::Error::msg)?; + if boundary(&decoded) { + boundary_ids.push(id); + } + } + Ok(Self { + model, + tokenizer, + empty_cache, + boundary_ids, + }) + } + + fn forward(&self, ids: &[u32], position: usize, cache: &mut Cache) -> Result> { + let input = Tensor::new(ids, &Device::Cpu)?.unsqueeze(0)?; + Ok(self + .model + .forward(&input, position, cache)? + .squeeze(0)? + .to_vec1()?) + } + + fn rank(&self, before: &str, candidates: &[String]) -> Result> { + if candidates.is_empty() { + return Ok(Vec::new()); + } + let encoded = self + .tokenizer + .encode(before, false) + .map_err(anyhow::Error::msg)?; + let ids = encoded.get_ids(); + let mut context = vec![0]; + context.extend_from_slice(&ids[ids.len().saturating_sub(CONTEXT_TOKENS - 1)..]); + let mut cache = self.empty_cache.clone(); + let first_logits = self.forward(&context, 0, &mut cache)?; + let mut scored = Vec::new(); + for (rank, word) in candidates.iter().enumerate() { + let text = if before.is_empty() { + word.clone() + } else { + format!(" {word}") + }; + let encoded = self + .tokenizer + .encode(text, false) + .map_err(anyhow::Error::msg)?; + let tokens = encoded.get_ids(); + let score = word_score( + tokens, + &first_logits, + &cache, + context.len(), + &self.boundary_ids, + |token, position, branch| self.forward(&[token], position, branch), + )?; + scored.push((score, rank, word.clone())); + } + scored.sort_by(|a, b| b.0.total_cmp(&a.0).then(a.1.cmp(&b.1))); + Ok(scored.into_iter().take(5).map(|x| x.2).collect()) + } +} + +#[derive(Clone, Debug)] +struct Query { + domain: String, + prefix_chars: usize, + before: String, + prefix: String, + target: String, + key: String, +} + +fn queries( + fixtures: &BTreeMap>, + training: &BTreeSet, + per_cell: usize, +) -> Result> { + ensure!(per_cell > 0, "per-cell must be positive"); + let mut result = Vec::new(); + for (domain, texts) in fixtures { + let unique: BTreeSet = texts + .iter() + .flat_map(|t| sentences(t)) + .map(|s| s.join(" ")) + .collect(); + ensure!( + unique.is_disjoint(training), + "Fixture/training overlap in {domain}" + ); + for prefix_chars in 0..=4 { + let mut cell = Vec::new(); + let mut seen = BTreeSet::new(); + for sentence in &unique { + let words: Vec<_> = sentence.split_whitespace().collect(); + for (index, target) in words.iter().enumerate() { + let graphemes: Vec<_> = target.graphemes(true).collect(); + if graphemes.len() <= prefix_chars { + continue; + } + let before = words[..index].join(" "); + let prefix = graphemes[..prefix_chars].concat(); + if !seen.insert((before.clone(), prefix.clone(), target.to_string())) { + continue; + } + let key = switchify_prediction::digest( + format!("{domain}\n{sentence}\n{index}\n{prefix_chars}").as_bytes(), + ); + cell.push(Query { + domain: domain.clone(), + prefix_chars, + before, + prefix, + target: target.to_string(), + key, + }); + } + } + cell.sort_by(|a, b| a.key.cmp(&b.key)); + ensure!( + cell.len() >= per_cell, + "Not enough queries in {domain}/{prefix_chars}: {}", + cell.len() + ); + result.extend(cell.into_iter().take(per_cell)); + } + } + Ok(result) +} + +#[derive(Default, Serialize)] +struct Cell { + queries: usize, + top1_hits: usize, + top5_hits: usize, + shortlist_hits: usize, +} +#[derive(Serialize)] +struct Report { + mode: Mode, + fixture_sha256: String, + baseline_sha256: String, + model_manifest_sha256: String, + per_cell: usize, + shortlist: usize, + context_tokens: usize, + cold_load_ms: f64, + query_count: usize, + warm_median_ms: f64, + warm_p95_ms: f64, + warm_max_ms: f64, + failed_queries: usize, + model_file_bytes: u64, + baseline_payload_bytes: usize, + cells: BTreeMap>, +} + +fn percentile(times: &[f64], percent: usize) -> f64 { + times[(times.len() * percent).div_ceil(100).saturating_sub(1)] +} + +fn main() -> Result<()> { + let args = Args::parse(); + ensure!(!args.output.exists(), "Output already exists"); + ensure!( + sha(&args.baseline)? == BASELINE_SHA, + "Baseline checksum mismatch" + ); + ensure!( + sha(&args.training)? == TRAIN_SHA, + "Training checksum mismatch" + ); + let fixtures: BTreeMap> = + serde_json::from_slice(&fs::read(&args.fixtures)?)?; + let training = fs::read_to_string(&args.training)? + .lines() + .map(str::to_string) + .collect(); + let cases = queries(&fixtures, &training, args.per_cell)?; + if cases.is_empty() { + bail!("Empty workload"); + } + verify_model(&args.model, &args.manifest)?; + let start = Instant::now(); + let baseline = Predictor::open(&args.baseline, None)?; + let neural = match args.mode { + Mode::Baseline => None, + Mode::Neural => Some(Neural::load(&args.model)?), + }; + let cold_load_ms = start.elapsed().as_secs_f64() * 1000.; + let predict = |q: &Query| -> Result<(Vec, Vec)> { + let candidates: Vec<_> = baseline + .predict( + &q.before, + &q.prefix, + Options { + limit: SHORTLIST, + min_chars: 0, + ..Options::default() + }, + ) + .into_iter() + .map(|s| s.word) + .collect(); + let words = match &neural { + Some(n) => n.rank(&q.before, &candidates)?, + None => candidates.iter().take(5).cloned().collect(), + }; + ensure!( + words.iter().all(|w| w.starts_with(&q.prefix)), + "Invalid prefix completion" + ); + Ok((words, candidates)) + }; + // Untimed prime. Each measured request starts with a fresh neural context cache. + predict(&cases[0])?; + let mut times = Vec::new(); + let mut cells: BTreeMap> = BTreeMap::new(); + for (index, q) in cases.iter().enumerate() { + let start = Instant::now(); + let (words, candidates) = predict(q)?; + times.push(start.elapsed().as_secs_f64() * 1000.); + let cell = cells + .entry(q.domain.clone()) + .or_default() + .entry(q.prefix_chars) + .or_default(); + cell.queries += 1; + cell.top1_hits += usize::from(words.first() == Some(&q.target)); + cell.top5_hits += usize::from(words.contains(&q.target)); + cell.shortlist_hits += usize::from(candidates.contains(&q.target)); + if index % 50 == 0 { + eprintln!("Scored {}/{}", index + 1, cases.len()); + } + } + times.sort_by(f64::total_cmp); + let report = Report { + mode: args.mode, + fixture_sha256: sha(&args.fixtures)?, + baseline_sha256: sha(&args.baseline)?, + model_manifest_sha256: sha(&args.manifest)?, + per_cell: args.per_cell, + shortlist: SHORTLIST, + context_tokens: CONTEXT_TOKENS, + cold_load_ms, + query_count: times.len(), + warm_median_ms: percentile(×, 50), + warm_p95_ms: percentile(×, 95), + warm_max_ms: *times.last().unwrap(), + failed_queries: 0, + model_file_bytes: fs::metadata(args.model.join("model.safetensors"))?.len(), + baseline_payload_bytes: baseline.model_payload_bytes(), + cells, + }; + fs::write(&args.output, serde_json::to_vec_pretty(&report)?)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn whole_word_scoring_uses_every_token_and_isolates_branches() { + let context = vec![99]; + let forward = |token: u32, position: usize, branch: &mut Vec| { + assert_eq!(position, branch.len()); + branch.push(token); + Ok(vec![0., 0., 0., 0.]) + }; + let one = word_score(&[1], &[0.; 4], &context, 1, &[0, 3], forward).unwrap(); + let two = word_score(&[1, 2], &[0.; 4], &context, 1, &[0, 3], forward).unwrap(); + assert!((one.exp() - 0.125).abs() < 1e-9); + assert!((two.exp() - 0.03125).abs() < 1e-9); + assert_eq!(context, vec![99]); + assert!(word_score(&[], &[0.; 4], &context, 1, &[0], forward).is_err()); + assert!( + word_score(&[1], &[0.; 4], &context, 1, &[0], |_, _, _| bail!( + "fake inference failure" + )) + .is_err() + ); + } + #[test] + fn unicode_boundaries_and_sentence_context() { + let fixtures = BTreeMap::from([( + "test".into(), + vec![ + "Earlier sentence. Café’s résumé includes naïveté and multilingual communication." + .into(), + ], + )]); + let cases = queries(&fixtures, &BTreeSet::new(), 1).unwrap(); + assert_eq!(cases.len(), 5); + assert!(cases.iter().all(|q| !q.before.contains("sentence café"))); + assert_eq!( + queries(&fixtures, &BTreeSet::new(), 1) + .unwrap() + .iter() + .map(|q| &q.key) + .collect::>(), + cases.iter().map(|q| &q.key).collect::>() + ); + assert!(queries(&fixtures, &BTreeSet::from(["earlier sentence".into()]), 1).is_err()); + assert!(queries(&fixtures, &BTreeSet::new(), 0).is_err()); + assert!(boundary(" next")); + assert!(boundary(".")); + assert!(!boundary("ing")); + assert!(!boundary("'s")); + assert!(!boundary("")); + } + #[test] + fn probabilities_and_percentiles() { + let logits = [0., 0., 0., 0.]; + assert!((log_prob(&logits, 1).exp() - 0.25).abs() < 1e-9); + assert!((log_boundary(&logits, &[0, 2]).exp() - 0.5).abs() < 1e-9); + assert_eq!(percentile(&[1., 2., 3., 4.], 95), 4.); + } +} diff --git a/general-neural-source-manifest.json b/general-neural-source-manifest.json new file mode 100644 index 0000000..c6c54d6 --- /dev/null +++ b/general-neural-source-manifest.json @@ -0,0 +1,26 @@ +{ + "model": "HuggingFaceTB/SmolLM2-135M", + "revision": "93efa2f097d58c2a74874c7e644dbc9b0cee75a2", + "files": { + "config.json": { + "url": "https://huggingface.co/HuggingFaceTB/SmolLM2-135M/resolve/93efa2f097d58c2a74874c7e644dbc9b0cee75a2/config.json", + "sha256": "1d556eab73b69c7f11f64c557a2f9c6f440bd4c6b89bb2584a6b498c92603843", + "bytes": 704 + }, + "tokenizer.json": { + "url": "https://huggingface.co/HuggingFaceTB/SmolLM2-135M/resolve/93efa2f097d58c2a74874c7e644dbc9b0cee75a2/tokenizer.json", + "sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c", + "bytes": 2104556 + }, + "model.safetensors": { + "url": "https://huggingface.co/HuggingFaceTB/SmolLM2-135M/resolve/93efa2f097d58c2a74874c7e644dbc9b0cee75a2/model.safetensors", + "sha256": "80521b40281d6ce74e35c9282c22539e75aa0ac8578892b2a59955ef78d55da1", + "bytes": 269060552 + }, + "README.md": { + "url": "https://huggingface.co/HuggingFaceTB/SmolLM2-135M/resolve/93efa2f097d58c2a74874c7e644dbc9b0cee75a2/README.md", + "sha256": "d1ba68cae64a89b6b434b11526e6e2271ee5ffd2c914ec35ed515f9d84c6085c", + "bytes": 6340 + } + } +} diff --git a/scripts/general_neural_experiment.py b/scripts/general_neural_experiment.py new file mode 100644 index 0000000..d665c16 --- /dev/null +++ b/scripts/general_neural_experiment.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Fetch pinned model files, then benchmark two offline Rust processes.""" +import argparse +import ctypes +import json +import os +from pathlib import Path +import platform +import subprocess +import tempfile +import time + +from aac_experiment import sha + +ROOT = Path(__file__).resolve().parent.parent +MANIFEST = ROOT / 'general-neural-source-manifest.json' + + +def fetch(directory): + directory.mkdir(parents=True, exist_ok=True) + sources = json.loads(MANIFEST.read_text(encoding='utf-8')) + for name, source in sources['files'].items(): + path = directory / name + if not path.exists(): + with tempfile.TemporaryDirectory(dir=directory) as temporary: + downloaded = Path(temporary) / 'source' + subprocess.run(['curl', '-fsSL', '--retry', '3', '--max-time', '600', + '-o', str(downloaded), source['url']], check=True) + if sha(downloaded) != source['sha256']: + raise ValueError(f'Corrupt download: {name}') + downloaded.replace(path) + if sha(path) != source['sha256']: + raise ValueError(f'Corrupt cached source: {name}') + + +def windows_peak(process): + class Counters(ctypes.Structure): + _fields_ = [('cb', ctypes.c_ulong), ('faults', ctypes.c_ulong)] + [ + (name, ctypes.c_size_t) for name in ('peak_working_set', 'working_set', + 'peak_paged', 'paged', 'peak_nonpaged', 'nonpaged', 'pagefile', 'peak_pagefile')] + counters = Counters() + counters.cb = ctypes.sizeof(counters) + function = ctypes.WinDLL('psapi', use_last_error=True).GetProcessMemoryInfo + function.argtypes = [ctypes.c_void_p, ctypes.POINTER(Counters), ctypes.c_ulong] + function.restype = ctypes.c_int + if not function(int(process._handle), ctypes.byref(counters), counters.cb): + return None + return counters.peak_working_set + + +def run(command, env): + peak = None + process = subprocess.Popen(command, env=env, cwd=ROOT) + while process.poll() is None: + if os.name == 'nt': + current = windows_peak(process) + if current is not None: + peak = max(peak or 0, current) + time.sleep(0.05) + if process.returncode: + raise subprocess.CalledProcessError(process.returncode, command) + return peak + + +def decision(reports): + gains = [] + for domain, cells in reports['baseline']['cells'].items(): + for prefix in ('0', '1', '2'): + base, neural = cells[prefix], reports['neural']['cells'][domain][prefix] + if base['queries'] != neural['queries']: + raise ValueError('Mismatched query counts') + gains.append((neural['top5_hits'] - base['top5_hits']) / base['queries']) + mean = sum(gains) / len(gains) + return {'mean_early_top5_gain': mean, 'worst_early_cell_gain': min(gains), + 'accuracy_passed': mean > 0 and min(gains) >= -0.01, + 'latency_passed': reports['neural']['warm_p95_ms'] < 20, + 'promote': False} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--baseline', required=True, type=Path) + parser.add_argument('--output', type=Path, default=ROOT / 'artifacts/general-neural') + parser.add_argument('--fetch-only', action='store_true') + args = parser.parse_args() + out = args.output.resolve() + if any((out / f'{name}.json').exists() for name in ('baseline', 'neural', 'results')): + raise ValueError('Previous results exist; use a fresh output directory') + fetch(out / 'model') + if args.fetch_only: + return + exe = ROOT / 'target/neural/release' / ('switchify-neural-experiment.exe' if os.name == 'nt' else 'switchify-neural-experiment') + env = dict(os.environ, RAYON_NUM_THREADS='4') + report = {'protocol': 'general-neural-v1', 'hardware': f'{platform.platform()}; {platform.machine()}; {os.cpu_count()} logical CPUs', + 'rayon_threads': 4, 'dtype': 'F32', 'device': 'CPU', 'executable_sha256': sha(exe), + 'fixture_sha256': sha(ROOT / 'experiments/neural/fixtures.json'), + 'reports': {}, 'memory_method': 'Windows peak working set sampled from OS high-water counter; unavailable on other platforms. Includes loading and scoring, not model payload alone.'} + for mode in ('baseline', 'neural'): + dest = out / f'{mode}.json' + command = [str(exe), '--baseline', str(args.baseline.resolve()), '--model', str(out / 'model'), + '--manifest', str(MANIFEST), '--fixtures', str(ROOT / 'experiments/neural/fixtures.json'), + '--training', str(ROOT / 'data/aac-oanc/prepared/candidate.txt'), + '--mode', mode, '--output', str(dest)] + peak = run(command, env) + result = json.loads(dest.read_text(encoding='utf-8')) + result['peak_process_working_set_bytes'] = peak + report['reports'][mode] = result + report['decision'] = decision(report['reports']) + (out / 'results.json').write_bytes((json.dumps(report, indent=2) + '\n').encode()) + print(json.dumps(report['decision'], indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/test_general_neural_experiment.py b/scripts/test_general_neural_experiment.py new file mode 100644 index 0000000..a265bab --- /dev/null +++ b/scripts/test_general_neural_experiment.py @@ -0,0 +1,28 @@ +import copy +import unittest + +from general_neural_experiment import decision + + +class GeneralNeuralTests(unittest.TestCase): + def test_general_domain_gates(self): + base = {'cells': {d: {str(p): {'queries': 100, 'top5_hits': 50} for p in range(5)} + for d in ['messages', 'email', 'documents', 'search']}, 'warm_p95_ms': 5} + neural = copy.deepcopy(base) + for cells in neural['cells'].values(): + for cell in cells.values(): + cell['top5_hits'] = 52 + reports = {'baseline': base, 'neural': neural} + self.assertTrue(decision(reports)['accuracy_passed']) + neural['cells']['email']['2']['top5_hits'] = 48 + self.assertFalse(decision(reports)['accuracy_passed']) + neural['warm_p95_ms'] = 20 + self.assertFalse(decision(reports)['latency_passed']) + self.assertFalse(decision(reports)['promote']) + neural['cells']['email']['2']['queries'] = 99 + with self.assertRaises(ValueError): + decision(reports) + + +if __name__ == '__main__': + unittest.main() From a5ee35a681c8264cafd07d80ac34a8ebea7bf260 Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Sat, 3 Oct 2026 15:52:48 +0100 Subject: [PATCH 2/5] Record general-purpose neural accuracy and latency results --- README.md | 2 + docs/general-neural-results.json | 325 +++++++++++++++++++++++++++++++ docs/general-neural-results.md | 102 ++++++++++ 3 files changed, 429 insertions(+) create mode 100644 docs/general-neural-results.json create mode 100644 docs/general-neural-results.md diff --git a/README.md b/README.md index 7e66610..aee1546 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,8 @@ The [AAC experiment protocol](docs/aac-experiment.md) and earlier Switchify is a general-purpose typing app. The separate [neural experiment](docs/general-neural-protocol.md) tests messages, email, documents and search-style text equally, without changing the production model. +Its [results](docs/general-neural-results.md) show the measured accuracy gain +and CPU latency cost on synthetic fixtures. The `prepare` and `evaluate` commands retain the earlier comparison workflows; `prepare` alone does not prepare the promoted production corpus. Use the commands above for production. A new model-selection exercise needs a new protocol; these diff --git a/docs/general-neural-results.json b/docs/general-neural-results.json new file mode 100644 index 0000000..76db8eb --- /dev/null +++ b/docs/general-neural-results.json @@ -0,0 +1,325 @@ +{ + "protocol": "general-neural-v1", + "hardware": "Windows-11-10.0.26200-SP0; AMD64; 24 logical CPUs", + "rayon_threads": 4, + "dtype": "F32", + "device": "CPU", + "executable_sha256": "67cc3ad32dd7abd22c699b8bd1973c6110cc084b2bf3cf16d4da3d393c8c72b2", + "fixture_sha256": "5e7fe93fc126862da5895d6c7304414c2a49a48303688648349afb3db956fe76", + "reports": { + "baseline": { + "mode": "Baseline", + "fixture_sha256": "5e7fe93fc126862da5895d6c7304414c2a49a48303688648349afb3db956fe76", + "baseline_sha256": "222253417d0a7a705823ffb7e599a3bcf5d5d3daf4a9d76161ac6b3e555aeaad", + "model_manifest_sha256": "5099892d4ea02608c0710c55358c4fd38e5890c9fd061fa3291b89b21cacf140", + "per_cell": 64, + "shortlist": 8, + "context_tokens": 64, + "cold_load_ms": 2543.1276000000003, + "query_count": 1280, + "warm_median_ms": 0.0792, + "warm_p95_ms": 6.0288, + "warm_max_ms": 8.2803, + "failed_queries": 0, + "model_file_bytes": 269060552, + "baseline_payload_bytes": 16821218, + "cells": { + "documents": { + "0": { + "queries": 64, + "top1_hits": 2, + "top5_hits": 6, + "shortlist_hits": 10 + }, + "1": { + "queries": 64, + "top1_hits": 7, + "top5_hits": 9, + "shortlist_hits": 10 + }, + "2": { + "queries": 64, + "top1_hits": 10, + "top5_hits": 19, + "shortlist_hits": 23 + }, + "3": { + "queries": 64, + "top1_hits": 12, + "top5_hits": 34, + "shortlist_hits": 41 + }, + "4": { + "queries": 64, + "top1_hits": 22, + "top5_hits": 46, + "shortlist_hits": 53 + } + }, + "email": { + "0": { + "queries": 64, + "top1_hits": 3, + "top5_hits": 12, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 16, + "top5_hits": 29, + "shortlist_hits": 31 + }, + "2": { + "queries": 64, + "top1_hits": 27, + "top5_hits": 42, + "shortlist_hits": 44 + }, + "3": { + "queries": 64, + "top1_hits": 26, + "top5_hits": 42, + "shortlist_hits": 47 + }, + "4": { + "queries": 64, + "top1_hits": 26, + "top5_hits": 47, + "shortlist_hits": 52 + } + }, + "messages": { + "0": { + "queries": 64, + "top1_hits": 10, + "top5_hits": 13, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 23, + "top5_hits": 34, + "shortlist_hits": 41 + }, + "2": { + "queries": 64, + "top1_hits": 32, + "top5_hits": 55, + "shortlist_hits": 57 + }, + "3": { + "queries": 64, + "top1_hits": 38, + "top5_hits": 59, + "shortlist_hits": 59 + }, + "4": { + "queries": 64, + "top1_hits": 42, + "top5_hits": 62, + "shortlist_hits": 63 + } + }, + "search": { + "0": { + "queries": 64, + "top1_hits": 2, + "top5_hits": 5, + "shortlist_hits": 7 + }, + "1": { + "queries": 64, + "top1_hits": 6, + "top5_hits": 10, + "shortlist_hits": 12 + }, + "2": { + "queries": 64, + "top1_hits": 10, + "top5_hits": 19, + "shortlist_hits": 22 + }, + "3": { + "queries": 64, + "top1_hits": 15, + "top5_hits": 36, + "shortlist_hits": 42 + }, + "4": { + "queries": 64, + "top1_hits": 24, + "top5_hits": 49, + "shortlist_hits": 53 + } + } + }, + "peak_process_working_set_bytes": 389099520 + }, + "neural": { + "mode": "Neural", + "fixture_sha256": "5e7fe93fc126862da5895d6c7304414c2a49a48303688648349afb3db956fe76", + "baseline_sha256": "222253417d0a7a705823ffb7e599a3bcf5d5d3daf4a9d76161ac6b3e555aeaad", + "model_manifest_sha256": "5099892d4ea02608c0710c55358c4fd38e5890c9fd061fa3291b89b21cacf140", + "per_cell": 64, + "shortlist": 8, + "context_tokens": 64, + "cold_load_ms": 2818.6385, + "query_count": 1280, + "warm_median_ms": 165.7749, + "warm_p95_ms": 229.35410000000002, + "warm_max_ms": 396.5521, + "failed_queries": 0, + "model_file_bytes": 269060552, + "baseline_payload_bytes": 16821218, + "cells": { + "documents": { + "0": { + "queries": 64, + "top1_hits": 6, + "top5_hits": 10, + "shortlist_hits": 10 + }, + "1": { + "queries": 64, + "top1_hits": 9, + "top5_hits": 10, + "shortlist_hits": 10 + }, + "2": { + "queries": 64, + "top1_hits": 21, + "top5_hits": 23, + "shortlist_hits": 23 + }, + "3": { + "queries": 64, + "top1_hits": 38, + "top5_hits": 41, + "shortlist_hits": 41 + }, + "4": { + "queries": 64, + "top1_hits": 47, + "top5_hits": 53, + "shortlist_hits": 53 + } + }, + "email": { + "0": { + "queries": 64, + "top1_hits": 11, + "top5_hits": 18, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 24, + "top5_hits": 30, + "shortlist_hits": 31 + }, + "2": { + "queries": 64, + "top1_hits": 32, + "top5_hits": 44, + "shortlist_hits": 44 + }, + "3": { + "queries": 64, + "top1_hits": 37, + "top5_hits": 47, + "shortlist_hits": 47 + }, + "4": { + "queries": 64, + "top1_hits": 43, + "top5_hits": 52, + "shortlist_hits": 52 + } + }, + "messages": { + "0": { + "queries": 64, + "top1_hits": 13, + "top5_hits": 18, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 25, + "top5_hits": 38, + "shortlist_hits": 41 + }, + "2": { + "queries": 64, + "top1_hits": 42, + "top5_hits": 56, + "shortlist_hits": 57 + }, + "3": { + "queries": 64, + "top1_hits": 50, + "top5_hits": 59, + "shortlist_hits": 59 + }, + "4": { + "queries": 64, + "top1_hits": 49, + "top5_hits": 63, + "shortlist_hits": 63 + } + }, + "search": { + "0": { + "queries": 64, + "top1_hits": 5, + "top5_hits": 7, + "shortlist_hits": 7 + }, + "1": { + "queries": 64, + "top1_hits": 11, + "top5_hits": 12, + "shortlist_hits": 12 + }, + "2": { + "queries": 64, + "top1_hits": 18, + "top5_hits": 22, + "shortlist_hits": 22 + }, + "3": { + "queries": 64, + "top1_hits": 32, + "top5_hits": 41, + "shortlist_hits": 42 + }, + "4": { + "queries": 64, + "top1_hits": 35, + "top5_hits": 53, + "shortlist_hits": 53 + } + } + }, + "peak_process_working_set_bytes": 1087586304 + } + }, + "memory_method": "Windows peak working set sampled from OS high-water counter; unavailable on other platforms. Includes loading and scoring, not model payload alone.", + "decision": { + "mean_early_top5_gain": 0.045572916666666664, + "worst_early_cell_gain": 0.015625, + "accuracy_passed": true, + "latency_passed": false, + "promote": false + }, + "cpu": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M", + "date": "2026-10-03", + "experiment_source_commit": "f9f42dc26076e6d18173d8dc390d2729e29379c7", + "limitations": [ + "Synthetic diagnostic fixtures, not representative user text or evidence of unseen-user accuracy.", + "A bounded top-eight reranker, not unrestricted neural prediction.", + "Unquantized CPU F32 without cross-query KV caching; no inference network access.", + "Windows measured; macOS performance not measured.", + "Peak process working set includes validation, loading and scoring, not just model payload." + ] +} diff --git a/docs/general-neural-results.md b/docs/general-neural-results.md new file mode 100644 index 0000000..b348167 --- /dev/null +++ b/docs/general-neural-results.md @@ -0,0 +1,102 @@ +# General-purpose neural prediction results + +The SmolLM2 reranker improved early top-five accuracy by **4.56 percentage +points** on these synthetic fixtures, but missed the 20 ms p95 target. It is +worth optimizing and evaluating on independent everyday writing before any +production decision. The shipping model and Switchify PC are unchanged. + +Measured 2026-10-03 on Windows 11, AMD Ryzen AI 9 HX 370, 24 logical CPUs. +Candle 0.11.0 used CPU F32 with four Rayon threads. Each mode completed 1,280 +queries, 64 per domain/prefix cell, with zero inference failures. + +## Accuracy + +These are 96 newly authored diagnostic fixtures, 24 each for messages, email, +documents and search-style text. They are not real-user observations. AAC was +not included in the selection objective. The existing control retains its +historical AAC-weighted training mixture; reranking does not fix its candidate +vocabulary or recover words absent from its top-eight shortlist. + +Early accuracy averages prefixes 0, 1 and 2 with equal domain/cell weights. + +| Domain | Current top-one | Neural top-one | Current top-five | Neural top-five | Top-eight ceiling | +|---|---:|---:|---:|---:|---:| +| documents | 9.90% | 18.75% | 17.71% | 22.40% | 22.40% | +| email | 23.96% | 34.90% | 43.23% | 47.92% | 48.96% | +| messages | 33.85% | 41.67% | 53.12% | 58.33% | 60.94% | +| search | 9.38% | 17.71% | 17.71% | 21.35% | 21.35% | + +| Typed graphemes | Current top-five | Neural top-five | Change | +|---|---:|---:|---:| +| 0 | 14.06% | 20.70% | +6.64 pp | +| 1 | 32.03% | 35.16% | +3.12 pp | +| 2 | 52.73% | 56.64% | +3.91 pp | +| 3 | 66.80% | 73.44% | +6.64 pp | +| 4 | 79.69% | 86.33% | +6.64 pp | + +The mean early top-five score rose from 32.94% to 37.50%. Every one of the +twelve early domain/prefix cells improved, with the smallest gain one hit out +of 64. The accuracy criterion passed. Prefix rows use independently hash-selected +target positions, so they do not measure actual keystrokes saved for a fixed +set of words. Per-domain/per-prefix raw hit counts and candidate coverage are +available in the JSON report. + +## Speed and memory + +| Metric | Current engine | Neural reranker | +|---|---:|---:| +| Cold load ms | 2543.128 | 2818.639 | +| Warm median ms | 0.079 | 165.775 | +| Warm p95 ms | 6.029 | 229.354 | +| Warm maximum ms | 8.280 | 396.552 | +| Peak process working set MiB | 371.07 | 1037.20 | +| Inference failures | 0 | 0 | + +The baseline database is 28.42 MiB. The neural weights add 256.60 MiB, plus +about 2 MiB for the tokenizer and small configuration/notice files. F32 tensors +and runtime allocations make memory use larger than the stored weights. + +Both modes load once and predict entirely in memory. The timed neural path +includes baseline shortlist generation and whole-word scoring of up to eight +candidates. Context computation is shared within a query, with no reuse across +keystrokes. Each candidate has its own cloned context cache. Cold load starts +after checksum verification and therefore does not represent an OS-cold disk. +Peak working set includes validation, model loading and scoring. It is not +comparable to a model-payload estimate. + +No local builds or other benchmark processes ran during the full scoring run. +The 20-query calibration checked execution and timing before the full run; it +did not change the shortlist, model, fixtures, selection rule or scoring method. + +## Decision and next work + +Do not promote this implementation. Its accuracy result is encouraging, but +229 ms p95 is well above 20 ms. Quantization, batching and reuse of context across +keystrokes are optimization candidates, not measured improvements here. Any +optimized version must repeat accuracy checks and be tested on Windows and macOS. + +A stronger evaluation should use independent, appropriately licensed everyday +writing with realistic capitalization and punctuation. The current fixtures +were authored for this experiment, exact training-sentence overlap was rejected, +and unknown overlap with neural pretraining cannot be excluded. Candidate +shortlisting also limits how much longer-context reasoning can help. + +The [FUTO investigation](futo-investigation.md) located the official English +model and its custom prediction path. It has not been run here. It remains a +separate engineering candidate; these measurements do not rank FUTO against +SmolLM2. + +## Validation and reproduction + +Production formatting, Clippy with warnings denied, 22 Rust tests and 12 Python +tests passed locally. The separate Rust experiment passed formatting, Clippy +and three fake/pure tests covering full-token scores, cache isolation, sentence +boundaries, Unicode queries and inference errors. Automated tests inject no input. + +The production dependency audit passed. The separate experiment documents one +unmaintained-package exception for Candle's transitive `paste` dependency, +`RUSTSEC-2024-0436`. It is not a production dependency change. + +See the [frozen protocol and reproduction command](general-neural-protocol.md) +and [aggregate machine-readable results](general-neural-results.json). +No experimental weights or user text are committed or distributed. From 381ec0945a52f2e712ac26a0a69eb91b3e93989d Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Sat, 3 Oct 2026 18:01:50 +0100 Subject: [PATCH 3/5] Add cached batched and quantized prediction experiments --- .github/workflows/neural-experiment.yml | 19 ++ docs/futo-probe-protocol.md | 53 ++++ docs/neural-latency-protocol.md | 68 +++++ experiments/futo/CMakeLists.txt | 27 ++ experiments/futo/probe.cpp | 63 +++++ experiments/neural/src/bin/quantize.rs | 143 ++++++++++ experiments/neural/src/main.rs | 349 ++++++++++++++++++++++-- scripts/futo_experiment.py | 121 ++++++++ scripts/neural_latency_experiment.py | 70 +++++ scripts/prepare_futo_probe.py | 53 ++++ scripts/test_futo_experiment.py | 25 ++ 11 files changed, 966 insertions(+), 25 deletions(-) create mode 100644 docs/futo-probe-protocol.md create mode 100644 docs/neural-latency-protocol.md create mode 100644 experiments/futo/CMakeLists.txt create mode 100644 experiments/futo/probe.cpp create mode 100644 experiments/neural/src/bin/quantize.rs create mode 100644 scripts/futo_experiment.py create mode 100644 scripts/neural_latency_experiment.py create mode 100644 scripts/prepare_futo_probe.py create mode 100644 scripts/test_futo_experiment.py diff --git a/.github/workflows/neural-experiment.yml b/.github/workflows/neural-experiment.yml index 76f5be1..0bd5ce2 100644 --- a/.github/workflows/neural-experiment.yml +++ b/.github/workflows/neural-experiment.yml @@ -34,3 +34,22 @@ jobs: - run: cargo install cargo-audit --locked --version 0.22.0 # Candle transitively uses this unmaintained macro crate; see the protocol. - run: cargo audit --file experiments/neural/Cargo.lock --deny warnings --ignore RUSTSEC-2024-0436 + futo-windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - name: Prepare pinned native source + shell: pwsh + run: | + New-Item -ItemType Directory -Force artifacts | Out-Null + curl.exe -fsSL --retry 3 https://codeload.github.com/futo-org/android-keyboard/zip/70a5d390c505a6bbcc4e14966e5628e43ca3f1fc -o artifacts/futo-source.zip + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + python scripts/prepare_futo_probe.py --archive artifacts/futo-source.zip --output artifacts/futo-probe + - name: Build isolated native probe + shell: pwsh + run: | + $sourcePath = (Resolve-Path artifacts/futo-probe/source).Path.Replace('\', '/') + $preparedPath = (Resolve-Path artifacts/futo-probe).Path.Replace('\', '/') + cmake -S experiments/futo -B artifacts/futo-build "-DFUTO_SOURCE=$sourcePath" "-DFUTO_PREPARED=$preparedPath" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + cmake --build artifacts/futo-build --config Release --target futo-probe --parallel 2 diff --git a/docs/futo-probe-protocol.md b/docs/futo-probe-protocol.md new file mode 100644 index 0000000..ae60d94 --- /dev/null +++ b/docs/futo-probe-protocol.md @@ -0,0 +1,53 @@ +# FUTO native probe + +This is a local research harness, not a production Rust adapter or an Android +equivalence certification. Upstream code and weights stay in ignored artifacts. +Do not distribute either through the MIT prediction library. + +Use the pinned source and model from `futo-investigation.md`. The preparation +script verifies the source archive SHA-256 and extracts its native code locally. +The model SHA-256 is checked before execution. Keep upstream licence notices +with the downloaded source. Redistribution terms remain a separate review. + +The harness keeps upstream `LanguageModelState`, `PredictNextWord`, +`PredictCorrection`, tokenization, logit transforms, sampling and three-result +limit. Typed ASCII prefixes use upstream's no-coordinate fallback with one +character weight set to one; no pointer coordinates are synthesized or injected. +Non-ASCII/nonletter prefixes are reported as unsupported, not silently dropped. +Corrections may violate the typed prefix. Report native top-one/top-three and +exact-prefix-filtered top-one/top-three separately, without backfill. Do not call +these five-slot results or silently compare them as equivalent to the reranker. + +Use the same 1,280 Rust-exported synthetic queries and one initial warm-up query. +Keep upstream's single-thread setting. Record load time, internal query latency, +model bytes, peak working set and failures. A 30-minute process timeout terminates +a stuck native run. No keyboard/pointer input, learning or network inference. + +Mechanical local source changes remove JNI includes/entry points, replace one +variable-length C++ array with `std::vector`, and guard empty-string trimming. +Two bundled Abseil include paths are adjusted to the repository's layout. +The harness retains the JNI wrapper's exact-match preference before sorting. +No weights, sampling thresholds or ranking are tuned against the fixtures. +The native state maintains its own context reuse across the fixed query order. +These timings are separate from the explicit hit/miss Candle measurements. + +Preparation and build: + +```sh +curl -fL https://codeload.github.com/futo-org/android-keyboard/zip/70a5d390c505a6bbcc4e14966e5628e43ca3f1fc -o futo-source.zip +python scripts/prepare_futo_probe.py --archive futo-source.zip --output artifacts/futo-probe +cmake -S experiments/futo -B artifacts/futo-build -DFUTO_SOURCE=/absolute/path/artifacts/futo-probe/source -DFUTO_PREPARED=/absolute/path/artifacts/futo-probe +cmake --build artifacts/futo-build --config Release --target futo-probe +``` + +Export queries using the neural executable's normal pinned baseline/model/ +manifest/fixtures/training arguments, plus `--mode baseline --export-queries +--output artifacts/futo-queries.json`. Then run: + +```sh +python scripts/futo_experiment.py --executable /path/to/futo-probe --model /path/to/ml4_q6_k.gguf --queries artifacts/futo-queries.json --output artifacts/futo-results +``` + +Use an empty output directory. Raw fixture outputs remain local; commit only +aggregate measurements and provenance. A failed native build/run must be +reported as such, with no invented accuracy or latency. diff --git a/docs/neural-latency-protocol.md b/docs/neural-latency-protocol.md new file mode 100644 index 0000000..d85b7a4 --- /dev/null +++ b/docs/neural-latency-protocol.md @@ -0,0 +1,68 @@ +# Neural latency follow-up + +Frozen before scoring on 2026-10-03. Follow-up to the general-purpose experiment, +using exactly its fixtures, hash-selected queries, normalization, top-eight +shortlist, whole-word scoring and four-thread CPU setting. No personal learning. + +Compare the current statistical baseline, original F32 neural inference, and +Q8_0 neural inference. Convert the pinned SmolLM2 weights locally with Candle +0.11.0. Quantize all matrices including tied embeddings/output to Q8_0; keep +normalization vectors F32. The GGUF runtime dequantizes embeddings in memory. +Record file hashes and sizes. No weights are committed or redistributed. + +First convert without quantization and check all logits on four fixed contexts +and one continuation token each against the original implementation. Maximum +absolute error must be below 0.002. This catches tensor-name and RoPE-layout +conversion mistakes before Q8 results are accepted. + +Each of 1,280 frozen queries starts with no saved context. For each neural mode, +repeat that same request once with its context KV cache and first-token logits +retained. Recompute every candidate score, and require identical ordered results +for the pair. Report miss and hit latency separately, each with 1,280 samples. +The single-entry cache replaces itself when the effective context tokens differ. +No candidate results or text are persisted by the cache. + +This measures best-case context reuse, not a realistic typing trace or hit rate. +Do not average hit and miss timings into an apparent real-world number. Cache +reuse across a changed prefix is valid because the completed-word context stays +fixed, but the changed shortlist must still be scored. Sentence normalization +and the 64-token context limit remain unchanged. Desktop field identity and +password checks are outside this isolated benchmark. + +Report accuracy by domain/prefix, whole-output digest, cold load time after +checksum verification, warm median/p95/maximum, peak process memory and failures. +The initial p95 target remains 20 ms. Do not tune the model or shortlist from +these results, and do not promote automatically. Synthetic fixtures are +diagnostic comparisons, not proof of representative or unseen-user accuracy. + +Additional optimization fixed before its runs: batch candidate branches for both +F32 and Q8. Prefill identical context in each batch lane, score all next tokens +together, and add each word's boundary probability at its own final token. +Finished lanes receive padding that never contributes to their score. The cache +key includes batch width. This trades duplicated context work and memory for +fewer model calls. Keep the same queries and report both misses and hits. Compare +ordered-output digests and cell accuracy with the sequential implementation; +floating-point kernel differences may affect close rankings. `--only-batched` +allows reproducing this follow-up separately from the initial three runs. + +Inspecting Candle's pinned Q8 implementation also identified compile-time AVX2 +gates. The default portable Rust build does not enable these. Run a separate +host-native build with `RUSTFLAGS="-C target-cpu=native"`, a separate target +directory, and the same Q8/Q8-batched workloads. Label these results explicitly +and record executable hashes. This is a hardware-specific experiment, not a +portable shipping binary. FUTO's Windows harness likewise uses `/arch:AVX2`. +No accuracy policy changes are allowed for this build comparison. + +Pass `--binary-dir target/neural-native/release --build-label host-native +--modes q8 q8-batched` to the runner for the separately built executable. + +Reproduce after preparing the existing training data: + +```sh +cargo build --release --locked --manifest-path experiments/neural/Cargo.toml --target-dir target/neural +python scripts/neural_latency_experiment.py --baseline /path/to/english.sqlite --model artifacts/general-neural/model --output artifacts/neural-latency-run +``` + +Use an empty output directory. The existing pinned model cache can be reused. +The experiment retains the previously documented isolated `paste` audit warning +exception. Production dependencies and its strict audit remain unchanged. diff --git a/experiments/futo/CMakeLists.txt b/experiments/futo/CMakeLists.txt new file mode 100644 index 0000000..5152bd1 --- /dev/null +++ b/experiments/futo/CMakeLists.txt @@ -0,0 +1,27 @@ +cmake_minimum_required(VERSION 3.22) +project(futo_probe LANGUAGES C CXX) +include(GNUInstallDirs) +set(CMAKE_INSTALL_INCDIR ${CMAKE_INSTALL_INCLUDEDIR}) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_C_STANDARD 11) +if(NOT DEFINED FUTO_SOURCE OR NOT DEFINED FUTO_PREPARED) + message(FATAL_ERROR "Set FUTO_SOURCE and FUTO_PREPARED to locally prepared sources") +endif() +set(SPM_USE_BUILTIN_PROTOBUF ON) +set(SPM_ENABLE_SHARED OFF) +set(SPM_ENABLE_NFKC_COMPILE OFF) +add_subdirectory(${FUTO_SOURCE}/src/sentencepiece sentencepiece EXCLUDE_FROM_ALL) +target_include_directories(sentencepiece-static PRIVATE ${FUTO_SOURCE}/src ${FUTO_SOURCE}/src/sentencepiece) +set(G ${FUTO_SOURCE}/src/ggml) +add_executable(futo-probe probe.cpp + ${G}/ggml.c ${G}/ggml-alloc.c ${G}/ggml-backend.c ${G}/ggml-quants.c + ${G}/llama.cpp ${G}/LanguageModel.cpp ${G}/ModelMeta.cpp ${G}/context.cpp) +target_include_directories(futo-probe PRIVATE ${FUTO_PREPARED} ${FUTO_SOURCE}/src ${G} + ${FUTO_SOURCE}/src/sentencepiece ${FUTO_SOURCE}/src/third_party) +target_link_libraries(futo-probe PRIVATE sentencepiece-static) +if(MSVC) + target_compile_options(futo-probe PRIVATE /utf-8 /arch:AVX2) + target_compile_definitions(futo-probe PRIVATE _CRT_SECURE_NO_WARNINGS NOMINMAX) +else() + target_compile_options(futo-probe PRIVATE -O3) +endif() diff --git a/experiments/futo/probe.cpp b/experiments/futo/probe.cpp new file mode 100644 index 0000000..5eb9d01 --- /dev/null +++ b/experiments/futo/probe.cpp @@ -0,0 +1,63 @@ +// Harness only. Upstream decoder is downloaded and prepared outside the source tree. +#include +#include +#include +#include +#include +#include "decoder.inc" + +int main(int argc, char **argv) { + if (argc != 3) return 2; + llama_backend_init(false); + auto start = std::chrono::steady_clock::now(); + LanguageModelState state; + if (!state.Initialize(argv[1])) return 3; + auto elapsed = [](auto start) { + return std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + }; + std::cout << "load\t" << elapsed(start) << '\n'; + std::ifstream input(argv[2]); + if (!input) return 4; + std::string line; + size_t index = 0; + while (std::getline(input, line)) { + auto split = line.find('\t'); + if (split == std::string::npos) return 5; + std::string context = line.substr(0, split), prefix = line.substr(split + 1); + start = std::chrono::steady_clock::now(); + std::vector> words; + if (prefix.empty()) { + words = state.PredictNextWord(context, {}); + } else { + std::vector mixes; + // Use upstream's no-coordinate fallback. Non-ASCII prefixes are unsupported. + bool supported = true; + for (unsigned char c : prefix) { + if (c < 'a' || c > 'z') { supported = false; break; } + TokenMix mix{}; + mix.x = mix.y = -1.0f; + for (auto &part : mix.mixes) part.token = state.specialTokens.LETTERS_TO_IDS[c - 'a']; + mix.mixes[0].weight = 1.0f; + mixes.push_back(mix); + } + if (supported) words = state.PredictCorrection(context, mixes, false, WordCapitalizeMode::IgnoredCapitals, {}); + // Preserve the JNI wrapper's exact-match preference before sorting. + bool exact = false; + for (const auto &word : words) exact |= isExactMatch(word.second, prefix); + if (exact) { + for (auto &word : words) { + if (!isExactMatch(word.second, prefix)) word.first -= 1.0f; + } + } + } + sortProbabilityPairVectorDescending(words); + const auto ms = elapsed(start); + std::cout << index++ << '\t' << std::setprecision(12) << ms; + for (const auto &word : words) { + if (word.second.find_first_of("\r\n\t") != std::string::npos) return 6; + std::cout << '\t' << word.second; + } + std::cout << '\n'; + } + return 0; +} diff --git a/experiments/neural/src/bin/quantize.rs b/experiments/neural/src/bin/quantize.rs new file mode 100644 index 0000000..f0764b7 --- /dev/null +++ b/experiments/neural/src/bin/quantize.rs @@ -0,0 +1,143 @@ +//! Local conversion of the pinned SmolLM2 weights. No downloaded GGUF is trusted. +use anyhow::{Result, ensure}; +use candle_core::{ + DType, Device, Tensor, + quantized::{ + GgmlDType, QTensor, + gguf_file::{self, Value}, + }, +}; +use candle_transformers::models::llama::LlamaConfig; +use std::{collections::BTreeMap, fs, path::PathBuf}; + +fn interleave_rope(tensor: Tensor, heads: usize) -> Result { + let (rows, cols) = tensor.dims2()?; + ensure!(heads > 0 && rows % (heads * 2) == 0, "Invalid rotary shape"); + Ok(tensor + .reshape((heads, 2, rows / heads / 2, cols))? + .transpose(1, 2)? + .contiguous()? + .reshape((rows, cols))?) +} + +fn main() -> Result<()> { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + ensure!( + args.len() == 3, + "Usage: quantize MODEL_DIR OUTPUT.gguf f32|q8" + ); + let root = PathBuf::from(&args[0]); + let output = PathBuf::from(&args[1]); + ensure!(!output.exists(), "Output already exists"); + let dtype = match args[2].to_str() { + Some("f32") => GgmlDType::F32, + Some("q8") => GgmlDType::Q8_0, + _ => anyhow::bail!("Expected f32 or q8"), + }; + let cfg: LlamaConfig = serde_json::from_slice(&fs::read(root.join("config.json"))?)?; + ensure!( + cfg.hidden_size == 576 + && cfg.num_hidden_layers == 30 + && cfg.tie_word_embeddings == Some(true) + && cfg.rope_scaling.is_none(), + "Expected pinned SmolLM2 configuration" + ); + let source = candle_core::safetensors::load(root.join("model.safetensors"), &Device::Cpu)?; + let mut tensors = BTreeMap::new(); + let mut add = |target: String, name: &str, heads: Option| -> Result<()> { + let mut tensor: Tensor = source + .get(name) + .ok_or_else(|| anyhow::anyhow!("Missing {name}"))? + .to_dtype(DType::F32)?; + // HF RoPE pairs the two halves; GGUF llama pairs adjacent coordinates. + if let Some(heads) = heads { + tensor = interleave_rope(tensor, heads)?; + } + let kind = if tensor.rank() == 2 { + dtype + } else { + GgmlDType::F32 + }; + tensors.insert(target, QTensor::quantize(&tensor, kind)?); + Ok(()) + }; + add( + "token_embd.weight".into(), + "model.embed_tokens.weight", + None, + )?; + add("output_norm.weight".into(), "model.norm.weight", None)?; + for i in 0..cfg.num_hidden_layers { + for (target, name, heads) in [ + ("attn_q", "self_attn.q_proj", Some(cfg.num_attention_heads)), + ( + "attn_k", + "self_attn.k_proj", + Some(cfg.num_key_value_heads()), + ), + ("attn_v", "self_attn.v_proj", None), + ("attn_output", "self_attn.o_proj", None), + ("ffn_gate", "mlp.gate_proj", None), + ("ffn_down", "mlp.down_proj", None), + ("ffn_up", "mlp.up_proj", None), + ("attn_norm", "input_layernorm", None), + ("ffn_norm", "post_attention_layernorm", None), + ] { + add( + format!("blk.{i}.{target}.weight"), + &format!("model.layers.{i}.{name}.weight"), + heads, + )?; + } + } + let metadata = [ + ("general.architecture", Value::String("llama".into())), + ( + "llama.attention.head_count", + Value::U32(cfg.num_attention_heads as u32), + ), + ( + "llama.attention.head_count_kv", + Value::U32(cfg.num_key_value_heads() as u32), + ), + ( + "llama.block_count", + Value::U32(cfg.num_hidden_layers as u32), + ), + ("llama.embedding_length", Value::U32(cfg.hidden_size as u32)), + ( + "llama.rope.dimension_count", + Value::U32((cfg.hidden_size / cfg.num_attention_heads) as u32), + ), + ( + "llama.attention.layer_norm_rms_epsilon", + Value::F32(cfg.rms_norm_eps as f32), + ), + ("llama.rope.freq_base", Value::F32(cfg.rope_theta)), + ]; + let metadata: Vec<_> = metadata.iter().map(|(k, v)| (*k, v)).collect(); + let tensors: Vec<_> = tensors.iter().map(|(k, v)| (k.as_str(), v)).collect(); + gguf_file::write(&mut fs::File::create_new(output)?, &metadata, &tensors)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn rotary_permutation_stays_within_each_attention_head() { + let tensor = Tensor::arange(0_f32, 16_f32, &Device::Cpu) + .unwrap() + .reshape((8, 2)) + .unwrap(); + let result = interleave_rope(tensor.clone(), 2) + .unwrap() + .to_vec2::() + .unwrap(); + assert_eq!( + result.iter().map(|row| row[0]).collect::>(), + [0., 4., 2., 6., 8., 12., 10., 14.] + ); + assert!(interleave_rope(tensor, 3).is_err()); + } +} diff --git a/experiments/neural/src/main.rs b/experiments/neural/src/main.rs index ac2f5af..ae83782 100644 --- a/experiments/neural/src/main.rs +++ b/experiments/neural/src/main.rs @@ -2,6 +2,7 @@ use anyhow::{Context, Result, bail, ensure}; use candle_core::{DType, Device, Tensor}; use candle_nn::VarBuilder; use candle_transformers::models::llama::{Cache, Llama, LlamaConfig}; +use candle_transformers::models::quantized_llama::ModelWeights; use clap::{Parser, ValueEnum}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -46,6 +47,23 @@ struct Args { /// Use 64 for the frozen experiment; smaller values are calibration only. #[arg(long, default_value_t = 64)] per_cell: usize, + /// Explicit locally converted GGUF; its digest is mandatory. + #[arg(long, requires = "gguf_sha256")] + gguf: Option, + #[arg(long, requires = "gguf")] + gguf_sha256: Option, + /// Measure a second request with the same context, retaining only one KV entry. + #[arg(long)] + cache_probe: bool, + /// Validate the F32 GGUF conversion against the original loader before scoring. + #[arg(long)] + validate_conversion: bool, + /// Export only the frozen synthetic workload for external decoder comparisons. + #[arg(long)] + export_queries: bool, + /// Evaluate candidate branches together, padding finished lanes only. + #[arg(long)] + batch_candidates: bool, } fn sha(path: &Path) -> Result { @@ -131,15 +149,89 @@ fn word_score( Ok(score) } +fn batch_word_scores( + tokens: &[Vec], + first_logits: &[Vec], + cache: &S, + position: usize, + boundary_ids: &[usize], + mut forward: impl FnMut(&[u32], usize, &mut S) -> Result>>, +) -> Result> { + ensure!( + !tokens.is_empty() && tokens.iter().all(|t| !t.is_empty()), + "Empty candidate tokenization" + ); + ensure!(tokens.len() == first_logits.len(), "Invalid batch width"); + let mut branch = cache.clone(); + let mut logits = first_logits.to_vec(); + let mut scores = vec![0.; tokens.len()]; + for index in 0..tokens.iter().map(Vec::len).max().unwrap() { + let input: Vec<_> = tokens + .iter() + .map(|t| t.get(index).copied().unwrap_or(0)) + .collect(); + for (lane, t) in tokens.iter().enumerate() { + if index < t.len() { + scores[lane] += log_prob(&logits[lane], t[index] as usize); + } + } + logits = forward(&input, position + index, &mut branch)?; + ensure!(logits.len() == tokens.len(), "Invalid output batch width"); + for (lane, t) in tokens.iter().enumerate() { + if index + 1 == t.len() { + scores[lane] += log_boundary(&logits[lane], boundary_ids); + } + } + } + ensure!( + scores.iter().all(|s| s.is_finite()), + "Non-finite batched score" + ); + Ok(scores) +} + +enum Engine { + Float(Llama), + Quantized, +} + +#[derive(Clone)] +enum State { + Float(Cache), + Quantized(ModelWeights), +} + +#[derive(Clone)] +struct Prepared { + ids: Vec, + lanes: usize, + state: State, + logits: Vec>, +} + +fn refresh_context( + entry: &mut Option, + matches: impl FnOnce(&T) -> bool, + build: impl FnOnce() -> Result, +) -> Result<()> { + if !entry.as_ref().is_some_and(matches) { + *entry = None; + *entry = Some(build()?); + } + Ok(()) +} + struct Neural { - model: Llama, + model: Engine, tokenizer: Tokenizer, - empty_cache: Cache, + empty_cache: State, boundary_ids: Vec, + prepared: Option, + batch_candidates: bool, } impl Neural { - fn load(path: &Path) -> Result { + fn load(path: &Path, gguf: Option<&Path>) -> Result { let tokenizer = Tokenizer::from_file(path.join("tokenizer.json")).map_err(anyhow::Error::msg)?; let cfg: LlamaConfig = serde_json::from_slice(&fs::read(path.join("config.json"))?)?; @@ -148,11 +240,21 @@ impl Neural { config.bos_token_id == Some(0), "Expected pinned SmolLM2 BOS token" ); - let empty_cache = Cache::new(true, DType::F32, &config, &Device::Cpu)?; - // Safe loader owns its bytes; no mmap lifetime or externally mutable mapping. - let weights = fs::read(path.join("model.safetensors"))?; - let vb = VarBuilder::from_buffered_safetensors(weights, DType::F32, &Device::Cpu)?; - let model = Llama::load(vb, &config)?; + let (model, empty_cache) = if let Some(path) = gguf { + let mut reader = fs::File::open(path)?; + let content = candle_core::quantized::gguf_file::Content::read(&mut reader)?; + let model = ModelWeights::from_gguf(content, &mut reader, &Device::Cpu)?; + (Engine::Quantized, State::Quantized(model)) + } else { + let cache = Cache::new(true, DType::F32, &config, &Device::Cpu)?; + // Safe loader owns its bytes; no mmap lifetime or externally mutable mapping. + let weights = fs::read(path.join("model.safetensors"))?; + let vb = VarBuilder::from_buffered_safetensors(weights, DType::F32, &Device::Cpu)?; + ( + Engine::Float(Llama::load(vb, &config)?), + State::Float(cache), + ) + }; let mut boundary_ids = vec![0]; // Pinned EOS. for id in 1..config.vocab_size { let decoded = tokenizer @@ -167,19 +269,29 @@ impl Neural { tokenizer, empty_cache, boundary_ids, + prepared: None, + batch_candidates: false, }) } - fn forward(&self, ids: &[u32], position: usize, cache: &mut Cache) -> Result> { + fn forward(&self, ids: &[u32], position: usize, cache: &mut State) -> Result> { let input = Tensor::new(ids, &Device::Cpu)?.unsqueeze(0)?; Ok(self - .model - .forward(&input, position, cache)? + .forward_tensor(&input, position, cache)? .squeeze(0)? .to_vec1()?) } - fn rank(&self, before: &str, candidates: &[String]) -> Result> { + fn forward_tensor(&self, input: &Tensor, position: usize, cache: &mut State) -> Result { + let logits = match (&self.model, cache) { + (Engine::Float(model), State::Float(cache)) => model.forward(input, position, cache)?, + (Engine::Quantized, State::Quantized(model)) => model.forward(input, position)?, + _ => bail!("Mismatched inference state"), + }; + Ok(logits) + } + + fn rank(&mut self, before: &str, candidates: &[String]) -> Result> { if candidates.is_empty() { return Ok(Vec::new()); } @@ -190,9 +302,35 @@ impl Neural { let ids = encoded.get_ids(); let mut context = vec![0]; context.extend_from_slice(&ids[ids.len().saturating_sub(CONTEXT_TOKENS - 1)..]); - let mut cache = self.empty_cache.clone(); - let first_logits = self.forward(&context, 0, &mut cache)?; + let lanes = if self.batch_candidates { + candidates.len() + } else { + 1 + }; + let mut prepared = self.prepared.take(); + refresh_context( + &mut prepared, + |p| p.ids == context && p.lanes == lanes, + || { + let mut state = self.empty_cache.clone(); + let input = Tensor::new(context.as_slice(), &Device::Cpu)? + .unsqueeze(0)? + .repeat((lanes, 1))?; + let logits = self.forward_tensor(&input, 0, &mut state)?.to_vec2()?; + Ok(Prepared { + ids: context.clone(), + lanes, + state, + logits, + }) + }, + )?; + self.prepared = prepared; + let prepared = self.prepared.as_ref().context("Missing context state")?; + let cache = &prepared.state; + let first_logits = &prepared.logits; let mut scored = Vec::new(); + let mut all_tokens = Vec::new(); for (rank, word) in candidates.iter().enumerate() { let text = if before.is_empty() { word.clone() @@ -204,22 +342,46 @@ impl Neural { .encode(text, false) .map_err(anyhow::Error::msg)?; let tokens = encoded.get_ids(); + if self.batch_candidates { + all_tokens.push(tokens.to_vec()); + continue; + } let score = word_score( tokens, - &first_logits, - &cache, + &first_logits[0], + cache, context.len(), &self.boundary_ids, |token, position, branch| self.forward(&[token], position, branch), )?; scored.push((score, rank, word.clone())); } + if self.batch_candidates { + let scores = batch_word_scores( + &all_tokens, + first_logits, + cache, + context.len(), + &self.boundary_ids, + |ids, position, state| { + let input = Tensor::new(ids, &Device::Cpu)?.unsqueeze(1)?; + Ok(self.forward_tensor(&input, position, state)?.to_vec2()?) + }, + )?; + scored.extend( + scores + .into_iter() + .zip(candidates) + .enumerate() + .map(|(rank, (score, word))| (score, rank, word.clone())), + ); + } scored.sort_by(|a, b| b.0.total_cmp(&a.0).then(a.1.cmp(&b.1))); Ok(scored.into_iter().take(5).map(|x| x.2).collect()) } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize)] struct Query { domain: String, prefix_chars: usize, @@ -311,6 +473,53 @@ struct Report { model_file_bytes: u64, baseline_payload_bytes: usize, cells: BTreeMap>, + gguf_sha256: Option, + cache_hit: Option, + prediction_sha256: String, + batch_candidates: bool, +} + +#[derive(Serialize)] +struct Timing { + queries: usize, + median_ms: f64, + p95_ms: f64, + max_ms: f64, +} + +fn validate_conversion(path: &Path, converted: &Neural) -> Result<()> { + let original = Neural::load(path, None)?; + for text in [ + "", + "I would like to", + "Please send the documents tomorrow", + "Café résumé", + ] { + let encoded = original + .tokenizer + .encode(text, false) + .map_err(anyhow::Error::msg)?; + let mut ids = vec![0]; + ids.extend_from_slice(encoded.get_ids()); + let mut left = original.empty_cache.clone(); + let mut right = converted.empty_cache.clone(); + // Check prefill and a continuation, exercising positional rotation and KV caches. + for (tokens, position) in [(ids.as_slice(), 0), (&[42_u32][..], ids.len())] { + let a = original.forward(tokens, position, &mut left)?; + let b = converted.forward(tokens, position, &mut right)?; + let error = a + .iter() + .zip(&b) + .map(|(a, b)| (a - b).abs()) + .fold(0_f32, f32::max); + ensure!( + error < 0.002, + "F32 conversion logit error {error} exceeds tolerance" + ); + } + } + eprintln!("F32 conversion prefill and continuation parity passed"); + Ok(()) } fn percentile(times: &[f64], percent: usize) -> f64 { @@ -338,15 +547,39 @@ fn main() -> Result<()> { if cases.is_empty() { bail!("Empty workload"); } + if args.export_queries { + fs::write(&args.output, serde_json::to_vec_pretty(&cases)?)?; + return Ok(()); + } verify_model(&args.model, &args.manifest)?; + if let Some(path) = &args.gguf { + ensure!( + Some(sha(path)?) == args.gguf_sha256, + "GGUF checksum mismatch" + ); + ensure!( + matches!(args.mode, Mode::Neural), + "GGUF requires neural mode" + ); + } let start = Instant::now(); let baseline = Predictor::open(&args.baseline, None)?; - let neural = match args.mode { + let mut neural = match args.mode { Mode::Baseline => None, - Mode::Neural => Some(Neural::load(&args.model)?), + Mode::Neural => Some(Neural::load(&args.model, args.gguf.as_deref())?), }; let cold_load_ms = start.elapsed().as_secs_f64() * 1000.; - let predict = |q: &Query| -> Result<(Vec, Vec)> { + if let Some(neural) = &mut neural { + neural.batch_candidates = args.batch_candidates; + } + if args.validate_conversion { + ensure!(args.gguf.is_some(), "Conversion validation requires GGUF"); + validate_conversion( + &args.model, + neural.as_ref().context("Requires neural mode")?, + )?; + } + let mut predict = |q: &Query, clear: bool| -> Result<(Vec, Vec)> { let candidates: Vec<_> = baseline .predict( &q.before, @@ -360,8 +593,13 @@ fn main() -> Result<()> { .into_iter() .map(|s| s.word) .collect(); - let words = match &neural { - Some(n) => n.rank(&q.before, &candidates)?, + let words = match &mut neural { + Some(n) => { + if clear { + n.prepared = None; + } + n.rank(&q.before, &candidates)? + } None => candidates.iter().take(5).cloned().collect(), }; ensure!( @@ -371,13 +609,22 @@ fn main() -> Result<()> { Ok((words, candidates)) }; // Untimed prime. Each measured request starts with a fresh neural context cache. - predict(&cases[0])?; + predict(&cases[0], true)?; let mut times = Vec::new(); + let mut hits = Vec::new(); + let mut predictions = Sha256::new(); let mut cells: BTreeMap> = BTreeMap::new(); for (index, q) in cases.iter().enumerate() { let start = Instant::now(); - let (words, candidates) = predict(q)?; + let (words, candidates) = predict(q, true)?; times.push(start.elapsed().as_secs_f64() * 1000.); + predictions.update(serde_json::to_vec(&(&q.key, &words))?); + if args.cache_probe { + let start = Instant::now(); + let (cached, _) = predict(q, false)?; + hits.push(start.elapsed().as_secs_f64() * 1000.); + ensure!(cached == words, "Cached inference changed results"); + } let cell = cells .entry(q.domain.clone()) .or_default() @@ -392,6 +639,7 @@ fn main() -> Result<()> { } } times.sort_by(f64::total_cmp); + hits.sort_by(f64::total_cmp); let report = Report { mode: args.mode, fixture_sha256: sha(&args.fixtures)?, @@ -406,9 +654,23 @@ fn main() -> Result<()> { warm_p95_ms: percentile(×, 95), warm_max_ms: *times.last().unwrap(), failed_queries: 0, - model_file_bytes: fs::metadata(args.model.join("model.safetensors"))?.len(), + model_file_bytes: fs::metadata( + args.gguf + .clone() + .unwrap_or_else(|| args.model.join("model.safetensors")), + )? + .len(), baseline_payload_bytes: baseline.model_payload_bytes(), cells, + gguf_sha256: args.gguf_sha256, + cache_hit: (!hits.is_empty()).then(|| Timing { + queries: hits.len(), + median_ms: percentile(&hits, 50), + p95_ms: percentile(&hits, 95), + max_ms: *hits.last().unwrap(), + }), + prediction_sha256: format!("{:x}", predictions.finalize()), + batch_candidates: args.batch_candidates, }; fs::write(&args.output, serde_json::to_vec_pretty(&report)?)?; Ok(()) @@ -418,6 +680,43 @@ fn main() -> Result<()> { mod tests { use super::*; #[test] + fn batched_scoring_ignores_padding_after_each_word_boundary() { + let cache = vec![99]; + let scores = batch_word_scores( + &[vec![1], vec![1, 2]], + &[vec![0.; 4], vec![0.; 4]], + &cache, + 1, + &[0, 3], + |ids, position, state| { + assert_eq!(state.len(), position); + if position == 1 { + assert_eq!(ids, [1, 1]); + } else { + assert_eq!(ids, [0, 2]); + } + state.push(0); + Ok(vec![vec![0.; 4]; 2]) + }, + ) + .unwrap(); + assert!((scores[0].exp() - 0.125).abs() < 1e-9); + assert!((scores[1].exp() - 0.03125).abs() < 1e-9); + assert_eq!(cache, [99]); + } + #[test] + fn context_changes_and_failed_rebuilds_discard_stale_state() { + let mut entry = Some((vec![0, 1], 7)); + refresh_context(&mut entry, |p| p.0 == [0, 1], || bail!("must reuse")).unwrap(); + assert_eq!(entry.as_ref().unwrap().1, 7); + refresh_context(&mut entry, |p| p.0 == [0, 2], || Ok((vec![0, 2], 9))).unwrap(); + assert_eq!(entry.as_ref().unwrap().1, 9); + assert!(refresh_context(&mut entry, |p| p.0 == [0, 3], || bail!("fake error")).is_err()); + assert!(entry.is_none()); + refresh_context(&mut entry, |_| false, || Ok((vec![0], 1))).unwrap(); + assert_eq!(entry.unwrap(), (vec![0], 1)); + } + #[test] fn whole_word_scoring_uses_every_token_and_isolates_branches() { let context = vec![99]; let forward = |token: u32, position: usize, branch: &mut Vec| { diff --git a/scripts/futo_experiment.py b/scripts/futo_experiment.py new file mode 100644 index 0000000..421da0b --- /dev/null +++ b/scripts/futo_experiment.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Score the pinned native FUTO three-suggestion decoder on exported synthetic queries.""" +import argparse +import json +import math +import os +from pathlib import Path +import platform +import subprocess +import time +import unicodedata + +from aac_experiment import sha +from general_neural_experiment import windows_peak + +MODEL_SHA = '6545c1c9ef2d76e9bfb87ad4fcf2061889513af84fcf30d907412be7fcdedb7b' + + +def normalize_word(text): + word = unicodedata.normalize('NFC', text.strip().replace('’', "'").lower()) + if not word: + return None + for i, c in enumerate(word): + if c.isalpha() or (unicodedata.category(c).startswith('M') and i > 0): + continue + if c == "'" and i > 0 and i + 1 < len(word) and word[i + 1].isalpha(): + continue + return None + return word + + +def summarize(queries, lines): + if len(lines) != len(queries) + 2 or not lines[0].startswith('load\t'): + raise ValueError('Incomplete native output') + cells, latencies = {}, [] + filtered, unsupported = 0, 0 + # First query is an untimed prime, then each frozen query is measured once. + for index, (q, line) in enumerate(zip(queries, lines[2:])): + parts = line.split('\t') + if int(parts[0]) != index + 1: + raise ValueError('Native query order mismatch') + ms = float(parts[1]) + if not math.isfinite(ms) or ms < 0: + raise ValueError('Invalid timing') + latencies.append(ms) + native, exact = [], [] + for raw in parts[2:]: + word = normalize_word(raw) + if word and word not in native: + native.append(word) + if word.startswith(q['prefix']): + exact.append(word) + else: + filtered += 1 + if any(c < 'a' or c > 'z' for c in q['prefix']): + unsupported += 1 + cell = cells.setdefault(q['domain'], {}).setdefault(str(q['prefix_chars']), + dict(queries=0, native_top1_hits=0, native_top3_hits=0, + exact_prefix_top1_hits=0, exact_prefix_top3_hits=0, empty_exact_results=0)) + cell['queries'] += 1 + cell['native_top1_hits'] += int(bool(native) and native[0] == q['target']) + cell['native_top3_hits'] += int(q['target'] in native[:3]) + cell['exact_prefix_top1_hits'] += int(bool(exact) and exact[0] == q['target']) + cell['exact_prefix_top3_hits'] += int(q['target'] in exact[:3]) + cell['empty_exact_results'] += int(not exact) + latencies.sort() + percentile = lambda p: latencies[math.ceil(len(latencies) * p) - 1] + return dict(query_count=len(queries), cold_load_ms=float(lines[0].split('\t')[1]), + warm_median_ms=percentile(.5), warm_p95_ms=percentile(.95), warm_max_ms=latencies[-1], + filtered_non_prefix_suggestions=filtered, unsupported_prefix_queries=unsupported, + failed_queries=0, cells=cells) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ('executable', 'model', 'queries', 'output'): + parser.add_argument('--' + name, type=Path, required=True) + args = parser.parse_args() + if sha(args.model) != MODEL_SHA: + raise ValueError('FUTO model checksum mismatch') + out = args.output.resolve() + out.mkdir(parents=True, exist_ok=True) + if any(out.iterdir()): + raise ValueError('Use an empty output directory') + queries = json.loads(args.queries.read_text(encoding='utf-8')) + if not queries: + raise ValueError('Empty workload') + for q in queries: + if any(c in q['before'] + q['prefix'] for c in '\r\n\t'): + raise ValueError('Unsafe fixture transport') + fixture = out / 'queries.tsv' + fixture.write_bytes(''.join(q['before'] + '\t' + q['prefix'] + '\n' + for q in [queries[0]] + queries).encode()) + peak = None + with (out / 'native.tsv').open('wb') as stdout, (out / 'native.log').open('wb') as stderr: + process = subprocess.Popen([str(args.executable.resolve()), str(args.model.resolve()), str(fixture)], + stdout=stdout, stderr=stderr) + deadline = time.monotonic() + 1800 + while process.poll() is None: + if time.monotonic() > deadline: + process.kill() + process.wait() + raise TimeoutError('FUTO benchmark exceeded 30 minutes') + if os.name == 'nt': + current = windows_peak(process) + if current is not None: + peak = max(peak or 0, current) + time.sleep(.05) + if process.returncode: + raise RuntimeError(f'Native FUTO exited {process.returncode}; inspect native.log') + report = summarize(queries, (out / 'native.tsv').read_text(encoding='utf-8').splitlines()) + report.update(protocol='futo-native-v1', model_sha256=MODEL_SHA, + model_file_bytes=args.model.stat().st_size, executable_sha256=sha(args.executable), + queries_sha256=sha(args.queries), hardware=platform.platform(), + peak_process_working_set_bytes=peak, threads=1, results_limit=3, + promote=False, note='Native no-coordinate correction path, then exact-prefix filtering. No backfill. Not a five-slot reranker; not validated against an Android device.') + (out / 'results.json').write_bytes((json.dumps(report, indent=2) + '\n').encode()) + + +if __name__ == '__main__': + main() diff --git a/scripts/neural_latency_experiment.py b/scripts/neural_latency_experiment.py new file mode 100644 index 0000000..56de36f --- /dev/null +++ b/scripts/neural_latency_experiment.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Run frozen F32/Q8 context-cache experiments without changing production.""" +import argparse +import json +import os +from pathlib import Path +import platform +import subprocess + +from aac_experiment import sha +from general_neural_experiment import ROOT, MANIFEST, fetch, run + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--baseline', required=True, type=Path) + parser.add_argument('--model', required=True, type=Path) + parser.add_argument('--output', required=True, type=Path) + parser.add_argument('--only-batched', action='store_true', help='Run the batched follow-up modes only') + parser.add_argument('--binary-dir', type=Path, default=ROOT / 'target/neural/release') + parser.add_argument('--modes', nargs='+', choices=('baseline', 'f32', 'q8', 'f32-batched', 'q8-batched')) + parser.add_argument('--build-label', default='portable-default') + args = parser.parse_args() + out = args.output.resolve() + out.mkdir(parents=True, exist_ok=True) + if any(out.iterdir()): + raise ValueError('Use an empty output directory') + model = args.model.resolve() + fetch(model) # Verify all pinned source bytes before local conversion. + suffix = '.exe' if os.name == 'nt' else '' + binary = args.binary_dir.resolve() / ('switchify-neural-experiment' + suffix) + converter = binary.with_name('quantize' + suffix) + env = dict(os.environ, RAYON_NUM_THREADS='4') + for kind in ('f32', 'q8'): + subprocess.run([str(converter), str(model), str(out / f'{kind}.gguf'), kind], env=env, check=True) + common = [str(binary), '--baseline', str(args.baseline.resolve()), '--model', str(model), + '--manifest', str(MANIFEST), '--fixtures', str(ROOT / 'experiments/neural/fixtures.json'), + '--training', str(ROOT / 'data/aac-oanc/prepared/candidate.txt')] + # F32 conversion must reproduce original logits before any quantized score is trusted. + control = out / 'f32.gguf' + subprocess.run(common + ['--mode', 'neural', '--gguf', str(control), '--gguf-sha256', sha(control), + '--validate-conversion', '--per-cell', '1', '--output', str(out / 'parity.json')], + env=env, check=True) + result = {'protocol': 'neural-latency-v1', 'hardware': platform.platform(), + 'logical_cpus': os.cpu_count(), 'rayon_threads': 4, + 'executable_sha256': sha(binary), 'converter_sha256': sha(converter), + 'build_label': args.build_label, + 'reports': {}, 'promote': False, + 'cache_workload': 'Each frozen query starts with no cached context and is then repeated once. Only context KV/logits are cached, never candidate scores. This is a best-case cache-hit probe, not a typing trace.', + 'memory_method': 'Windows peak process working set, including loading; unavailable on other platforms.'} + modes = args.modes or (('f32-batched', 'q8-batched') if args.only_batched else ('baseline', 'f32', 'q8', 'f32-batched', 'q8-batched')) + for mode in modes: + dest = out / f'{mode}.json' + command = common + ['--mode', 'baseline' if mode == 'baseline' else 'neural', '--output', str(dest)] + if mode != 'baseline': + command += ['--cache-probe'] + if mode.endswith('-batched'): + command += ['--batch-candidates'] + if mode.startswith('q8'): + quantized = out / 'q8.gguf' + command += ['--gguf', str(quantized), '--gguf-sha256', sha(quantized)] + peak = run(command, env) + report = json.loads(dest.read_text(encoding='utf-8')) + report['peak_process_working_set_bytes'] = peak + result['reports'][mode] = report + (out / 'results.json').write_bytes((json.dumps(result, indent=2) + '\n').encode()) + + +if __name__ == '__main__': + main() diff --git a/scripts/prepare_futo_probe.py b/scripts/prepare_futo_probe.py new file mode 100644 index 0000000..44b6f83 --- /dev/null +++ b/scripts/prepare_futo_probe.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Prepare pinned FUTO sources locally; never vendor or redistribute the decoder/model.""" +import argparse +from pathlib import Path +import zipfile + +from aac_experiment import sha + +ARCHIVE_SHA = '465677ea63dcd53ed7ba58da9353c0d1dadf744d55a9e4b7978ccf6da2c79b6d' +COMMIT = '70a5d390c505a6bbcc4e14966e5628e43ca3f1fc' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--archive', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + if sha(args.archive) != ARCHIVE_SHA: + raise ValueError('FUTO archive checksum mismatch') + root = args.output.resolve() + root.mkdir(parents=True, exist_ok=True) + if any(root.iterdir()): + raise ValueError('Use an empty output directory') + prefix = f'android-keyboard-{COMMIT}/native/jni/' + with zipfile.ZipFile(args.archive) as archive: + (root / 'UPSTREAM-LICENSE.md').write_bytes(archive.read(f'android-keyboard-{COMMIT}/LICENSE.md')) + for name in archive.namelist(): + if not name.startswith(prefix) or name.endswith('/'): + continue + relative = Path(name[len(prefix):]) + if relative.is_absolute() or '..' in relative.parts: + raise ValueError('Unsafe archive path') + dest = root / 'source' / relative + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(archive.read(name)) + source = root / 'source' + flags = source / 'src/third_party/absl/flags/flag.cc' + flag_text = flags.read_text(encoding='utf-8').replace('"src/common.h"', '"sentencepiece/common.h"').replace('"src/util.h"', '"sentencepiece/util.h"') + flags.write_bytes(flag_text.encode()) + text = (source / 'org_futo_inputmethod_latin_xlm_LanguageModel.cpp').read_text(encoding='utf-8') + # Keep the native class and sampling implementation, omit Android/JNI entry points. + text = text[:text.index('struct SuggestionItemToRescore')] + text = '\n'.join(line for line in text.splitlines() if not line.startswith('#include')) + # Mechanical portability/safety fixes only, no ranking or sampling changes. + text = text.replace('int seq_id_use_count[n_results];', 'std::vector seq_id_use_count(n_results);') + text = text.replace('auto start = s.begin();', 'if (s.empty()) return {};\n auto start = s.begin();') + text = '#include \n#include \n#include "ggml/LanguageModel.h"\nusing std::isnan;\n' + text + (root / 'decoder.inc').write_bytes(text.encode()) + print(source) + + +if __name__ == '__main__': + main() diff --git a/scripts/test_futo_experiment.py b/scripts/test_futo_experiment.py new file mode 100644 index 0000000..cbbf355 --- /dev/null +++ b/scripts/test_futo_experiment.py @@ -0,0 +1,25 @@ +import unittest +from futo_experiment import normalize_word, summarize + + +class FutoTests(unittest.TestCase): + def test_words_and_apostrophes(self): + self.assertEqual(normalize_word(' CAFÉ’S '), "café's") + self.assertEqual(normalize_word('cafe\u0301'), 'café') + for word in ('', 'two words', 'word.', '123', "'bad", "bad'", '\u0301bad'): + self.assertIsNone(normalize_word(word)) + + def test_three_slot_correction_and_exact_prefix_scores_are_separate(self): + queries = [dict(domain='messages', prefix_chars=1, prefix='w', target='word')] + report = summarize(queries, ['load\t10', '0\t1\tprime', '1\t2\tthe\tword\tWorld']) + cell = report['cells']['messages']['1'] + self.assertEqual(cell['native_top1_hits'], 0) + self.assertEqual(cell['exact_prefix_top1_hits'], 1) + self.assertEqual(report['filtered_non_prefix_suggestions'], 1) + self.assertEqual(report['warm_p95_ms'], 2) + with self.assertRaises(ValueError): + summarize(queries, ['load\t10']) + + +if __name__ == '__main__': + unittest.main() From 33d1a133b76d9a095f9fde0754872a2e88df4119 Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Sat, 3 Oct 2026 18:08:06 +0100 Subject: [PATCH 4/5] Fail closed on native decoder errors and handle quantized batched prefill --- docs/futo-probe-protocol.md | 4 ++++ experiments/futo/probe.cpp | 12 ++++++++++-- experiments/neural/src/main.rs | 21 +++++++++++++++++---- scripts/futo_experiment.py | 12 +++++++++--- scripts/prepare_futo_probe.py | 21 ++++++++++++++++++++- scripts/test_futo_experiment.py | 6 +++++- 6 files changed, 65 insertions(+), 11 deletions(-) diff --git a/docs/futo-probe-protocol.md b/docs/futo-probe-protocol.md index ae60d94..b6763c5 100644 --- a/docs/futo-probe-protocol.md +++ b/docs/futo-probe-protocol.md @@ -27,6 +27,10 @@ Mechanical local source changes remove JNI includes/entry points, replace one variable-length C++ array with `std::vector`, and guard empty-string trimming. Two bundled Abseil include paths are adjusted to the repository's layout. The harness retains the JNI wrapper's exact-match preference before sorting. +All four decode calls are checked, and upstream empty/default error returns +raise a native error. Each completed query emits an explicit success marker; +the runner rejects missing/error markers and non-finite timings. These checks +change failure handling only, never successful rankings. No weights, sampling thresholds or ranking are tuned against the fixtures. The native state maintains its own context reuse across the fixed query order. These timings are separate from the explicit hit/miss Candle measurements. diff --git a/experiments/futo/probe.cpp b/experiments/futo/probe.cpp index 5eb9d01..0e11c7f 100644 --- a/experiments/futo/probe.cpp +++ b/experiments/futo/probe.cpp @@ -6,7 +6,7 @@ #include #include "decoder.inc" -int main(int argc, char **argv) { +int probe_main(int argc, char **argv) { if (argc != 3) return 2; llama_backend_init(false); auto start = std::chrono::steady_clock::now(); @@ -52,7 +52,7 @@ int main(int argc, char **argv) { } sortProbabilityPairVectorDescending(words); const auto ms = elapsed(start); - std::cout << index++ << '\t' << std::setprecision(12) << ms; + std::cout << index++ << "\tok\t" << std::setprecision(12) << ms; for (const auto &word : words) { if (word.second.find_first_of("\r\n\t") != std::string::npos) return 6; std::cout << '\t' << word.second; @@ -61,3 +61,11 @@ int main(int argc, char **argv) { } return 0; } + +int main(int argc, char **argv) { + try { return probe_main(argc, argv); } + catch (const std::exception &error) { + std::cerr << error.what() << '\n'; + return 7; + } +} diff --git a/experiments/neural/src/main.rs b/experiments/neural/src/main.rs index ae83782..e3d1294 100644 --- a/experiments/neural/src/main.rs +++ b/experiments/neural/src/main.rs @@ -313,10 +313,23 @@ impl Neural { |p| p.ids == context && p.lanes == lanes, || { let mut state = self.empty_cache.clone(); - let input = Tensor::new(context.as_slice(), &Device::Cpu)? - .unsqueeze(0)? - .repeat((lanes, 1))?; - let logits = self.forward_tensor(&input, 0, &mut state)?.to_vec2()?; + let logits = if lanes > 1 && matches!(self.model, Engine::Quantized) { + // Candle 0.11 GGUF's output slice is non-contiguous for B>1, + // T>1. Single-token prefill keeps its public API contiguous. + let mut last = None; + for (position, token) in context.iter().enumerate() { + let input = Tensor::new(&[*token], &Device::Cpu)? + .unsqueeze(0)? + .repeat((lanes, 1))?; + last = Some(self.forward_tensor(&input, position, &mut state)?); + } + last.context("Empty context")?.to_vec2()? + } else { + let input = Tensor::new(context.as_slice(), &Device::Cpu)? + .unsqueeze(0)? + .repeat((lanes, 1))?; + self.forward_tensor(&input, 0, &mut state)?.to_vec2()? + }; Ok(Prepared { ids: context.clone(), lanes, diff --git a/scripts/futo_experiment.py b/scripts/futo_experiment.py index 421da0b..70a5177 100644 --- a/scripts/futo_experiment.py +++ b/scripts/futo_experiment.py @@ -32,19 +32,25 @@ def normalize_word(text): def summarize(queries, lines): if len(lines) != len(queries) + 2 or not lines[0].startswith('load\t'): raise ValueError('Incomplete native output') + cold_load = float(lines[0].split('\t')[1]) + prime = lines[1].split('\t') + if not math.isfinite(cold_load) or cold_load < 0 or len(prime) < 3 or prime[:2] != ['0', 'ok']: + raise ValueError('Invalid load or warm-up result') cells, latencies = {}, [] filtered, unsupported = 0, 0 # First query is an untimed prime, then each frozen query is measured once. for index, (q, line) in enumerate(zip(queries, lines[2:])): parts = line.split('\t') + if len(parts) < 3 or parts[1] != 'ok': + raise ValueError('Native inference did not report success') if int(parts[0]) != index + 1: raise ValueError('Native query order mismatch') - ms = float(parts[1]) + ms = float(parts[2]) if not math.isfinite(ms) or ms < 0: raise ValueError('Invalid timing') latencies.append(ms) native, exact = [], [] - for raw in parts[2:]: + for raw in parts[3:]: word = normalize_word(raw) if word and word not in native: native.append(word) @@ -65,7 +71,7 @@ def summarize(queries, lines): cell['empty_exact_results'] += int(not exact) latencies.sort() percentile = lambda p: latencies[math.ceil(len(latencies) * p) - 1] - return dict(query_count=len(queries), cold_load_ms=float(lines[0].split('\t')[1]), + return dict(query_count=len(queries), successful_inference_queries=len(queries) - unsupported, cold_load_ms=cold_load, warm_median_ms=percentile(.5), warm_p95_ms=percentile(.95), warm_max_ms=latencies[-1], filtered_non_prefix_suggestions=filtered, unsupported_prefix_queries=unsupported, failed_queries=0, cells=cells) diff --git a/scripts/prepare_futo_probe.py b/scripts/prepare_futo_probe.py index 44b6f83..87c0a68 100644 --- a/scripts/prepare_futo_probe.py +++ b/scripts/prepare_futo_probe.py @@ -3,6 +3,7 @@ import argparse from pathlib import Path import zipfile +import re from aac_experiment import sha @@ -44,7 +45,25 @@ def main(): # Mechanical portability/safety fixes only, no ranking or sampling changes. text = text.replace('int seq_id_use_count[n_results];', 'std::vector seq_id_use_count(n_results);') text = text.replace('auto start = s.begin();', 'if (s.empty()) return {};\n auto start = s.begin();') - text = '#include \n#include \n#include "ggml/LanguageModel.h"\nusing std::isnan;\n' + text + # Upstream sometimes returns default/empty state on errors, and one decode + # call is unchecked. A benchmark must not count those as successful queries. + head, state = text.split('struct LanguageModelState {', 1) + state = re.sub(r'return\s*\{\s*\};', 'throw std::runtime_error("FUTO decoder failure or unsupported model");', state) + if state.count('llama_decode(ctx,') != 4: + raise ValueError('Pinned decode call layout changed') + state = state.replace('llama_decode(ctx,', 'checked_llama_decode(ctx,') + header = '''#include +#include +#include +#include "ggml/LanguageModel.h" +using std::isnan; +static int checked_llama_decode(llama_context *ctx, llama_batch batch) { + const int result = llama_decode(ctx, batch); + if (result != 0) throw std::runtime_error("FUTO llama_decode failed"); + return result; +} +''' + text = header + head + 'struct LanguageModelState {' + state (root / 'decoder.inc').write_bytes(text.encode()) print(source) diff --git a/scripts/test_futo_experiment.py b/scripts/test_futo_experiment.py index cbbf355..e4949fa 100644 --- a/scripts/test_futo_experiment.py +++ b/scripts/test_futo_experiment.py @@ -11,7 +11,7 @@ def test_words_and_apostrophes(self): def test_three_slot_correction_and_exact_prefix_scores_are_separate(self): queries = [dict(domain='messages', prefix_chars=1, prefix='w', target='word')] - report = summarize(queries, ['load\t10', '0\t1\tprime', '1\t2\tthe\tword\tWorld']) + report = summarize(queries, ['load\t10', '0\tok\t1\tprime', '1\tok\t2\tthe\tword\tWorld']) cell = report['cells']['messages']['1'] self.assertEqual(cell['native_top1_hits'], 0) self.assertEqual(cell['exact_prefix_top1_hits'], 1) @@ -19,6 +19,10 @@ def test_three_slot_correction_and_exact_prefix_scores_are_separate(self): self.assertEqual(report['warm_p95_ms'], 2) with self.assertRaises(ValueError): summarize(queries, ['load\t10']) + with self.assertRaises(ValueError): + summarize(queries, ['load\t10', '0\tok\t1\tprime', '1\tfailed\t0']) + with self.assertRaises(ValueError): + summarize(queries, ['load\t10', '0\tok\t1\tprime', '1\tok\tnan']) if __name__ == '__main__': From 9fcd9eb661864104aab79af2786b88f7ba477705 Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Sat, 3 Oct 2026 18:24:07 +0100 Subject: [PATCH 5/5] Record optimized neural and FUTO prediction measurements --- README.md | 4 + docs/futo-investigation.md | 5 +- docs/futo-probe-protocol.md | 8 +- docs/neural-latency-protocol.md | 7 + docs/neural-latency-results.json | 1373 ++++++++++++++++++++++++++ docs/neural-latency-results.md | 149 +++ experiments/futo/probe.cpp | 5 +- scripts/futo_experiment.py | 12 +- scripts/neural_latency_experiment.py | 12 +- scripts/prepare_futo_probe.py | 2 + 10 files changed, 1566 insertions(+), 11 deletions(-) create mode 100644 docs/neural-latency-results.json create mode 100644 docs/neural-latency-results.md diff --git a/README.md b/README.md index aee1546..d4dec8a 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,10 @@ Switchify is a general-purpose typing app. The separate documents and search-style text equally, without changing the production model. Its [results](docs/general-neural-results.md) show the measured accuracy gain and CPU latency cost on synthetic fixtures. +The [speed follow-up](docs/neural-latency-results.md) measures caching, batching, +Q8/SIMD builds and a separate FUTO native decoder. FUTO met the initial speed +target with four threads but still needs completion-specific evaluation; +none of these experiments changes the production predictor. The `prepare` and `evaluate` commands retain the earlier comparison workflows; `prepare` alone does not prepare the promoted production corpus. Use the commands above for production. A new model-selection exercise needs a new protocol; these diff --git a/docs/futo-investigation.md b/docs/futo-investigation.md index 82cce67..474d585 100644 --- a/docs/futo-investigation.md +++ b/docs/futo-investigation.md @@ -1,7 +1,8 @@ # FUTO keyboard model investigation Inspected on 2026-10-03. FUTO is a relevant general-purpose keyboard reference. -It has not been benchmarked or integrated in this experiment. +The subsequent [native speed probe](neural-latency-results.md) benchmarks it on +Windows. It remains separate from production and has not been integrated. The official source at commit `70a5d390c505a6bbcc4e14966e5628e43ca3f1fc` selects the English `ml4_q6_k` model. @@ -9,7 +10,7 @@ The resource is available from the official `keyboard-large-resources` submodule at commit `d87d9dbdf3966bbe18413be375dab2f6c7bbdfdd`, path `raw/ml4_q6_k.gguf`. GitLab's file metadata reports 30,662,880 bytes and content SHA-256 `6545c1c9ef2d76e9bfb87ad4fcf2061889513af84fcf30d907412be7fcdedb7b`. -These are publisher metadata, not a locally verified download hash. Its name +The subsequent probe downloaded the model and verified this hash locally. Its name identifies the Q6_K model; no comparison with the F32 SmolLM2 timings is implied. The model is not an arbitrary chat GGUF. The Android loader requires KeyboardLM diff --git a/docs/futo-probe-protocol.md b/docs/futo-probe-protocol.md index b6763c5..fa8c2fb 100644 --- a/docs/futo-probe-protocol.md +++ b/docs/futo-probe-protocol.md @@ -24,7 +24,8 @@ model bytes, peak working set and failures. A 30-minute process timeout terminat a stuck native run. No keyboard/pointer input, learning or network inference. Mechanical local source changes remove JNI includes/entry points, replace one -variable-length C++ array with `std::vector`, and guard empty-string trimming. +variable-length C++ array with `std::vector`, and guard empty/whitespace trimming +and unsigned character classification. Two bundled Abseil include paths are adjusted to the repository's layout. The harness retains the JNI wrapper's exact-match preference before sorting. All four decode calls are checked, and upstream empty/default error returns @@ -35,6 +36,11 @@ No weights, sampling thresholds or ranking are tuned against the fixtures. The native state maintains its own context reuse across the fixed query order. These timings are separate from the explicit hit/miss Candle measurements. +Also run a fixed four-thread comparison via `llama_set_n_threads`, matching the +Candle thread budget. Use `--threads 4` with a new output directory. This changes +execution parallelism only; compare ordered-output digests with the one-thread +run. Neither threading choice changes the native three-result policy. + Preparation and build: ```sh diff --git a/docs/neural-latency-protocol.md b/docs/neural-latency-protocol.md index d85b7a4..31bb70b 100644 --- a/docs/neural-latency-protocol.md +++ b/docs/neural-latency-protocol.md @@ -53,6 +53,13 @@ and record executable hashes. This is a hardware-specific experiment, not a portable shipping binary. FUTO's Windows harness likewise uses `/arch:AVX2`. No accuracy policy changes are allowed for this build comparison. +The initial portable Q8 batched attempt failed on Candle's non-contiguous output +slice when both batch and prefill lengths exceeded one. The adapter now prefills +Q8 batched contexts one token at a time, keeping the public API's input/output +layout valid. This preserves causal attention semantics but changes prefill cost; +report it explicitly when comparing the subsequent host-native batched run. +Completed per-mode results are saved even if a later mode fails. + Pass `--binary-dir target/neural-native/release --build-label host-native --modes q8 q8-batched` to the runner for the separately built executable. diff --git a/docs/neural-latency-results.json b/docs/neural-latency-results.json new file mode 100644 index 0000000..3ca4fc4 --- /dev/null +++ b/docs/neural-latency-results.json @@ -0,0 +1,1373 @@ +{ + "protocol": "neural-latency-followup-v1", + "date": "2026-10-03", + "hardware": "Windows 11, AMD Ryzen AI 9 HX 370, 24 logical CPUs", + "scope": "Synthetic general-purpose fixtures; no personal learning or production changes.", + "portable_sequential": { + "protocol": "neural-latency-v1", + "hardware": "Windows-11-10.0.26200-SP0", + "logical_cpus": 24, + "rayon_threads": 4, + "executable_sha256": "ed625a3a2c48b6a9caeabca303a9f0a62f2a3ac7b7d3d5ff553542812d083275", + "converter_sha256": "54540399ec91b5c3e53abbade20a141e0cb6e72415c519229d9643218a49fcea", + "reports": { + "baseline": { + "mode": "Baseline", + "fixture_sha256": "5e7fe93fc126862da5895d6c7304414c2a49a48303688648349afb3db956fe76", + "baseline_sha256": "222253417d0a7a705823ffb7e599a3bcf5d5d3daf4a9d76161ac6b3e555aeaad", + "model_manifest_sha256": "5099892d4ea02608c0710c55358c4fd38e5890c9fd061fa3291b89b21cacf140", + "per_cell": 64, + "shortlist": 8, + "context_tokens": 64, + "cold_load_ms": 2595.6348, + "query_count": 1280, + "warm_median_ms": 0.0829, + "warm_p95_ms": 6.1265, + "warm_max_ms": 8.575, + "failed_queries": 0, + "model_file_bytes": 269060552, + "baseline_payload_bytes": 16821218, + "cells": { + "documents": { + "0": { + "queries": 64, + "top1_hits": 2, + "top5_hits": 6, + "shortlist_hits": 10 + }, + "1": { + "queries": 64, + "top1_hits": 7, + "top5_hits": 9, + "shortlist_hits": 10 + }, + "2": { + "queries": 64, + "top1_hits": 10, + "top5_hits": 19, + "shortlist_hits": 23 + }, + "3": { + "queries": 64, + "top1_hits": 12, + "top5_hits": 34, + "shortlist_hits": 41 + }, + "4": { + "queries": 64, + "top1_hits": 22, + "top5_hits": 46, + "shortlist_hits": 53 + } + }, + "email": { + "0": { + "queries": 64, + "top1_hits": 3, + "top5_hits": 12, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 16, + "top5_hits": 29, + "shortlist_hits": 31 + }, + "2": { + "queries": 64, + "top1_hits": 27, + "top5_hits": 42, + "shortlist_hits": 44 + }, + "3": { + "queries": 64, + "top1_hits": 26, + "top5_hits": 42, + "shortlist_hits": 47 + }, + "4": { + "queries": 64, + "top1_hits": 26, + "top5_hits": 47, + "shortlist_hits": 52 + } + }, + "messages": { + "0": { + "queries": 64, + "top1_hits": 10, + "top5_hits": 13, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 23, + "top5_hits": 34, + "shortlist_hits": 41 + }, + "2": { + "queries": 64, + "top1_hits": 32, + "top5_hits": 55, + "shortlist_hits": 57 + }, + "3": { + "queries": 64, + "top1_hits": 38, + "top5_hits": 59, + "shortlist_hits": 59 + }, + "4": { + "queries": 64, + "top1_hits": 42, + "top5_hits": 62, + "shortlist_hits": 63 + } + }, + "search": { + "0": { + "queries": 64, + "top1_hits": 2, + "top5_hits": 5, + "shortlist_hits": 7 + }, + "1": { + "queries": 64, + "top1_hits": 6, + "top5_hits": 10, + "shortlist_hits": 12 + }, + "2": { + "queries": 64, + "top1_hits": 10, + "top5_hits": 19, + "shortlist_hits": 22 + }, + "3": { + "queries": 64, + "top1_hits": 15, + "top5_hits": 36, + "shortlist_hits": 42 + }, + "4": { + "queries": 64, + "top1_hits": 24, + "top5_hits": 49, + "shortlist_hits": 53 + } + } + }, + "gguf_sha256": null, + "cache_hit": null, + "prediction_sha256": "359afe87eecb2eef3eef45adac2d2618f774834f06d65b9cae36eed38740df0e", + "peak_process_working_set_bytes": 389189632 + }, + "f32": { + "mode": "Neural", + "fixture_sha256": "5e7fe93fc126862da5895d6c7304414c2a49a48303688648349afb3db956fe76", + "baseline_sha256": "222253417d0a7a705823ffb7e599a3bcf5d5d3daf4a9d76161ac6b3e555aeaad", + "model_manifest_sha256": "5099892d4ea02608c0710c55358c4fd38e5890c9fd061fa3291b89b21cacf140", + "per_cell": 64, + "shortlist": 8, + "context_tokens": 64, + "cold_load_ms": 2803.5672, + "query_count": 1280, + "warm_median_ms": 165.99050000000003, + "warm_p95_ms": 226.72410000000002, + "warm_max_ms": 398.2863, + "failed_queries": 0, + "model_file_bytes": 269060552, + "baseline_payload_bytes": 16821218, + "cells": { + "documents": { + "0": { + "queries": 64, + "top1_hits": 6, + "top5_hits": 10, + "shortlist_hits": 10 + }, + "1": { + "queries": 64, + "top1_hits": 9, + "top5_hits": 10, + "shortlist_hits": 10 + }, + "2": { + "queries": 64, + "top1_hits": 21, + "top5_hits": 23, + "shortlist_hits": 23 + }, + "3": { + "queries": 64, + "top1_hits": 38, + "top5_hits": 41, + "shortlist_hits": 41 + }, + "4": { + "queries": 64, + "top1_hits": 47, + "top5_hits": 53, + "shortlist_hits": 53 + } + }, + "email": { + "0": { + "queries": 64, + "top1_hits": 11, + "top5_hits": 18, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 24, + "top5_hits": 30, + "shortlist_hits": 31 + }, + "2": { + "queries": 64, + "top1_hits": 32, + "top5_hits": 44, + "shortlist_hits": 44 + }, + "3": { + "queries": 64, + "top1_hits": 37, + "top5_hits": 47, + "shortlist_hits": 47 + }, + "4": { + "queries": 64, + "top1_hits": 43, + "top5_hits": 52, + "shortlist_hits": 52 + } + }, + "messages": { + "0": { + "queries": 64, + "top1_hits": 13, + "top5_hits": 18, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 25, + "top5_hits": 38, + "shortlist_hits": 41 + }, + "2": { + "queries": 64, + "top1_hits": 42, + "top5_hits": 56, + "shortlist_hits": 57 + }, + "3": { + "queries": 64, + "top1_hits": 50, + "top5_hits": 59, + "shortlist_hits": 59 + }, + "4": { + "queries": 64, + "top1_hits": 49, + "top5_hits": 63, + "shortlist_hits": 63 + } + }, + "search": { + "0": { + "queries": 64, + "top1_hits": 5, + "top5_hits": 7, + "shortlist_hits": 7 + }, + "1": { + "queries": 64, + "top1_hits": 11, + "top5_hits": 12, + "shortlist_hits": 12 + }, + "2": { + "queries": 64, + "top1_hits": 18, + "top5_hits": 22, + "shortlist_hits": 22 + }, + "3": { + "queries": 64, + "top1_hits": 32, + "top5_hits": 41, + "shortlist_hits": 42 + }, + "4": { + "queries": 64, + "top1_hits": 35, + "top5_hits": 53, + "shortlist_hits": 53 + } + } + }, + "gguf_sha256": null, + "cache_hit": { + "queries": 1280, + "median_ms": 128.81699999999998, + "p95_ms": 187.2048, + "max_ms": 381.0199 + }, + "prediction_sha256": "cf2bcb6d9847e9baeaa63c80cc89134d6c8db9f8dd5a540b6850edc04e8e25bc", + "peak_process_working_set_bytes": 1087184896 + }, + "q8": { + "mode": "Neural", + "fixture_sha256": "5e7fe93fc126862da5895d6c7304414c2a49a48303688648349afb3db956fe76", + "baseline_sha256": "222253417d0a7a705823ffb7e599a3bcf5d5d3daf4a9d76161ac6b3e555aeaad", + "model_manifest_sha256": "5099892d4ea02608c0710c55358c4fd38e5890c9fd061fa3291b89b21cacf140", + "per_cell": 64, + "shortlist": 8, + "context_tokens": 64, + "cold_load_ms": 2716.641, + "query_count": 1280, + "warm_median_ms": 223.93689999999998, + "warm_p95_ms": 319.8611, + "warm_max_ms": 539.563, + "failed_queries": 0, + "model_file_bytes": 143041952, + "baseline_payload_bytes": 16821218, + "cells": { + "documents": { + "0": { + "queries": 64, + "top1_hits": 7, + "top5_hits": 10, + "shortlist_hits": 10 + }, + "1": { + "queries": 64, + "top1_hits": 9, + "top5_hits": 10, + "shortlist_hits": 10 + }, + "2": { + "queries": 64, + "top1_hits": 22, + "top5_hits": 23, + "shortlist_hits": 23 + }, + "3": { + "queries": 64, + "top1_hits": 38, + "top5_hits": 41, + "shortlist_hits": 41 + }, + "4": { + "queries": 64, + "top1_hits": 47, + "top5_hits": 53, + "shortlist_hits": 53 + } + }, + "email": { + "0": { + "queries": 64, + "top1_hits": 11, + "top5_hits": 18, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 25, + "top5_hits": 30, + "shortlist_hits": 31 + }, + "2": { + "queries": 64, + "top1_hits": 34, + "top5_hits": 44, + "shortlist_hits": 44 + }, + "3": { + "queries": 64, + "top1_hits": 38, + "top5_hits": 47, + "shortlist_hits": 47 + }, + "4": { + "queries": 64, + "top1_hits": 43, + "top5_hits": 52, + "shortlist_hits": 52 + } + }, + "messages": { + "0": { + "queries": 64, + "top1_hits": 12, + "top5_hits": 17, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 26, + "top5_hits": 38, + "shortlist_hits": 41 + }, + "2": { + "queries": 64, + "top1_hits": 40, + "top5_hits": 56, + "shortlist_hits": 57 + }, + "3": { + "queries": 64, + "top1_hits": 50, + "top5_hits": 59, + "shortlist_hits": 59 + }, + "4": { + "queries": 64, + "top1_hits": 49, + "top5_hits": 63, + "shortlist_hits": 63 + } + }, + "search": { + "0": { + "queries": 64, + "top1_hits": 5, + "top5_hits": 7, + "shortlist_hits": 7 + }, + "1": { + "queries": 64, + "top1_hits": 11, + "top5_hits": 12, + "shortlist_hits": 12 + }, + "2": { + "queries": 64, + "top1_hits": 19, + "top5_hits": 22, + "shortlist_hits": 22 + }, + "3": { + "queries": 64, + "top1_hits": 32, + "top5_hits": 41, + "shortlist_hits": 42 + }, + "4": { + "queries": 64, + "top1_hits": 35, + "top5_hits": 53, + "shortlist_hits": 53 + } + } + }, + "gguf_sha256": "8d75e9b96c4b64e8a1180224cbaefbbc7744f21ca2e0be2319a429f2c589d342", + "cache_hit": { + "queries": 1280, + "median_ms": 136.65079999999998, + "p95_ms": 204.8758, + "max_ms": 443.2224 + }, + "prediction_sha256": "45631431c0a88eacc374195b0a99127f0aaa93f202f81a34ffe2b308d1db9056", + "peak_process_working_set_bytes": 538890240 + } + }, + "promote": false, + "cache_workload": "Each frozen query starts with no cached context and is then repeated once. Only context KV/logits are cached, never candidate scores. This is a best-case cache-hit probe, not a typing trace.", + "memory_method": "Windows peak process working set, including loading; unavailable on other platforms." + }, + "portable_batched": { + "source_commit": "381ec0945a52f2e712ac26a0a69eb91b3e93989d", + "f32": { + "mode": "Neural", + "fixture_sha256": "5e7fe93fc126862da5895d6c7304414c2a49a48303688648349afb3db956fe76", + "baseline_sha256": "222253417d0a7a705823ffb7e599a3bcf5d5d3daf4a9d76161ac6b3e555aeaad", + "model_manifest_sha256": "5099892d4ea02608c0710c55358c4fd38e5890c9fd061fa3291b89b21cacf140", + "per_cell": 64, + "shortlist": 8, + "context_tokens": 64, + "cold_load_ms": 2844.7086, + "query_count": 1280, + "warm_median_ms": 146.9601, + "warm_p95_ms": 225.8362, + "warm_max_ms": 304.25419999999997, + "failed_queries": 0, + "model_file_bytes": 269060552, + "baseline_payload_bytes": 16821218, + "cells": { + "documents": { + "0": { + "queries": 64, + "top1_hits": 6, + "top5_hits": 10, + "shortlist_hits": 10 + }, + "1": { + "queries": 64, + "top1_hits": 9, + "top5_hits": 10, + "shortlist_hits": 10 + }, + "2": { + "queries": 64, + "top1_hits": 21, + "top5_hits": 23, + "shortlist_hits": 23 + }, + "3": { + "queries": 64, + "top1_hits": 38, + "top5_hits": 41, + "shortlist_hits": 41 + }, + "4": { + "queries": 64, + "top1_hits": 47, + "top5_hits": 53, + "shortlist_hits": 53 + } + }, + "email": { + "0": { + "queries": 64, + "top1_hits": 11, + "top5_hits": 18, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 24, + "top5_hits": 30, + "shortlist_hits": 31 + }, + "2": { + "queries": 64, + "top1_hits": 32, + "top5_hits": 44, + "shortlist_hits": 44 + }, + "3": { + "queries": 64, + "top1_hits": 37, + "top5_hits": 47, + "shortlist_hits": 47 + }, + "4": { + "queries": 64, + "top1_hits": 43, + "top5_hits": 52, + "shortlist_hits": 52 + } + }, + "messages": { + "0": { + "queries": 64, + "top1_hits": 13, + "top5_hits": 18, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 25, + "top5_hits": 38, + "shortlist_hits": 41 + }, + "2": { + "queries": 64, + "top1_hits": 42, + "top5_hits": 56, + "shortlist_hits": 57 + }, + "3": { + "queries": 64, + "top1_hits": 50, + "top5_hits": 59, + "shortlist_hits": 59 + }, + "4": { + "queries": 64, + "top1_hits": 49, + "top5_hits": 63, + "shortlist_hits": 63 + } + }, + "search": { + "0": { + "queries": 64, + "top1_hits": 5, + "top5_hits": 7, + "shortlist_hits": 7 + }, + "1": { + "queries": 64, + "top1_hits": 11, + "top5_hits": 12, + "shortlist_hits": 12 + }, + "2": { + "queries": 64, + "top1_hits": 18, + "top5_hits": 22, + "shortlist_hits": 22 + }, + "3": { + "queries": 64, + "top1_hits": 32, + "top5_hits": 41, + "shortlist_hits": 42 + }, + "4": { + "queries": 64, + "top1_hits": 35, + "top5_hits": 53, + "shortlist_hits": 53 + } + } + }, + "gguf_sha256": null, + "cache_hit": { + "queries": 1280, + "median_ms": 59.4241, + "p95_ms": 132.7823, + "max_ms": 213.2565 + }, + "prediction_sha256": "cf2bcb6d9847e9baeaa63c80cc89134d6c8db9f8dd5a540b6850edc04e8e25bc", + "batch_candidates": true + }, + "peak_process_working_set_bytes": null, + "executable_sha256": null, + "limitation": "The later Q8 batch failed before the original runner saved its aggregate memory/executable metadata. The completed F32 per-mode timing and accuracy file survived.", + "failed_modes": { + "q8-batched": "Candle 0.11 non-contiguous output slice for batch>1 and prefill>1. Fixed through token-wise prefill for the later host-native run." + } + }, + "host_native": { + "protocol": "neural-latency-v1", + "hardware": "Windows-11-10.0.26200-SP0", + "logical_cpus": 24, + "rayon_threads": 4, + "executable_sha256": "06cb46125a0b73726fdf5df6d7c5b2e56cfb3c864f296d9283e392eedb4820e3", + "converter_sha256": "020c6be914c0d8be5086b4f8f1002863f5899ad298502f54a0319e3b68f2d365", + "build_label": "host-native", + "reports": { + "q8": { + "mode": "Neural", + "fixture_sha256": "5e7fe93fc126862da5895d6c7304414c2a49a48303688648349afb3db956fe76", + "baseline_sha256": "222253417d0a7a705823ffb7e599a3bcf5d5d3daf4a9d76161ac6b3e555aeaad", + "model_manifest_sha256": "5099892d4ea02608c0710c55358c4fd38e5890c9fd061fa3291b89b21cacf140", + "per_cell": 64, + "shortlist": 8, + "context_tokens": 64, + "cold_load_ms": 2710.7670000000003, + "query_count": 1280, + "warm_median_ms": 74.3587, + "warm_p95_ms": 101.78320000000001, + "warm_max_ms": 176.13690000000003, + "failed_queries": 0, + "model_file_bytes": 143041952, + "baseline_payload_bytes": 16821218, + "cells": { + "documents": { + "0": { + "queries": 64, + "top1_hits": 7, + "top5_hits": 10, + "shortlist_hits": 10 + }, + "1": { + "queries": 64, + "top1_hits": 9, + "top5_hits": 10, + "shortlist_hits": 10 + }, + "2": { + "queries": 64, + "top1_hits": 21, + "top5_hits": 23, + "shortlist_hits": 23 + }, + "3": { + "queries": 64, + "top1_hits": 38, + "top5_hits": 41, + "shortlist_hits": 41 + }, + "4": { + "queries": 64, + "top1_hits": 47, + "top5_hits": 53, + "shortlist_hits": 53 + } + }, + "email": { + "0": { + "queries": 64, + "top1_hits": 10, + "top5_hits": 18, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 24, + "top5_hits": 30, + "shortlist_hits": 31 + }, + "2": { + "queries": 64, + "top1_hits": 34, + "top5_hits": 44, + "shortlist_hits": 44 + }, + "3": { + "queries": 64, + "top1_hits": 38, + "top5_hits": 47, + "shortlist_hits": 47 + }, + "4": { + "queries": 64, + "top1_hits": 43, + "top5_hits": 52, + "shortlist_hits": 52 + } + }, + "messages": { + "0": { + "queries": 64, + "top1_hits": 12, + "top5_hits": 18, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 26, + "top5_hits": 38, + "shortlist_hits": 41 + }, + "2": { + "queries": 64, + "top1_hits": 41, + "top5_hits": 56, + "shortlist_hits": 57 + }, + "3": { + "queries": 64, + "top1_hits": 50, + "top5_hits": 59, + "shortlist_hits": 59 + }, + "4": { + "queries": 64, + "top1_hits": 50, + "top5_hits": 63, + "shortlist_hits": 63 + } + }, + "search": { + "0": { + "queries": 64, + "top1_hits": 5, + "top5_hits": 7, + "shortlist_hits": 7 + }, + "1": { + "queries": 64, + "top1_hits": 11, + "top5_hits": 12, + "shortlist_hits": 12 + }, + "2": { + "queries": 64, + "top1_hits": 19, + "top5_hits": 22, + "shortlist_hits": 22 + }, + "3": { + "queries": 64, + "top1_hits": 32, + "top5_hits": 41, + "shortlist_hits": 42 + }, + "4": { + "queries": 64, + "top1_hits": 35, + "top5_hits": 53, + "shortlist_hits": 53 + } + } + }, + "gguf_sha256": "8d75e9b96c4b64e8a1180224cbaefbbc7744f21ca2e0be2319a429f2c589d342", + "cache_hit": { + "queries": 1280, + "median_ms": 56.6761, + "p95_ms": 79.88149999999999, + "max_ms": 166.6237 + }, + "prediction_sha256": "c15a0a27ba6d863b5fe5f73cc2d0fc34208ca766ea038d058572a9cb18617c56", + "batch_candidates": false, + "peak_process_working_set_bytes": 539799552 + }, + "q8-batched": { + "mode": "Neural", + "fixture_sha256": "5e7fe93fc126862da5895d6c7304414c2a49a48303688648349afb3db956fe76", + "baseline_sha256": "222253417d0a7a705823ffb7e599a3bcf5d5d3daf4a9d76161ac6b3e555aeaad", + "model_manifest_sha256": "5099892d4ea02608c0710c55358c4fd38e5890c9fd061fa3291b89b21cacf140", + "per_cell": 64, + "shortlist": 8, + "context_tokens": 64, + "cold_load_ms": 2743.264, + "query_count": 1280, + "warm_median_ms": 159.8068, + "warm_p95_ms": 297.361, + "warm_max_ms": 436.3091, + "failed_queries": 0, + "model_file_bytes": 143041952, + "baseline_payload_bytes": 16821218, + "cells": { + "documents": { + "0": { + "queries": 64, + "top1_hits": 7, + "top5_hits": 10, + "shortlist_hits": 10 + }, + "1": { + "queries": 64, + "top1_hits": 9, + "top5_hits": 10, + "shortlist_hits": 10 + }, + "2": { + "queries": 64, + "top1_hits": 21, + "top5_hits": 23, + "shortlist_hits": 23 + }, + "3": { + "queries": 64, + "top1_hits": 38, + "top5_hits": 41, + "shortlist_hits": 41 + }, + "4": { + "queries": 64, + "top1_hits": 47, + "top5_hits": 53, + "shortlist_hits": 53 + } + }, + "email": { + "0": { + "queries": 64, + "top1_hits": 10, + "top5_hits": 18, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 24, + "top5_hits": 30, + "shortlist_hits": 31 + }, + "2": { + "queries": 64, + "top1_hits": 34, + "top5_hits": 44, + "shortlist_hits": 44 + }, + "3": { + "queries": 64, + "top1_hits": 38, + "top5_hits": 47, + "shortlist_hits": 47 + }, + "4": { + "queries": 64, + "top1_hits": 43, + "top5_hits": 52, + "shortlist_hits": 52 + } + }, + "messages": { + "0": { + "queries": 64, + "top1_hits": 12, + "top5_hits": 18, + "shortlist_hits": 19 + }, + "1": { + "queries": 64, + "top1_hits": 26, + "top5_hits": 38, + "shortlist_hits": 41 + }, + "2": { + "queries": 64, + "top1_hits": 41, + "top5_hits": 56, + "shortlist_hits": 57 + }, + "3": { + "queries": 64, + "top1_hits": 50, + "top5_hits": 59, + "shortlist_hits": 59 + }, + "4": { + "queries": 64, + "top1_hits": 50, + "top5_hits": 63, + "shortlist_hits": 63 + } + }, + "search": { + "0": { + "queries": 64, + "top1_hits": 5, + "top5_hits": 7, + "shortlist_hits": 7 + }, + "1": { + "queries": 64, + "top1_hits": 11, + "top5_hits": 12, + "shortlist_hits": 12 + }, + "2": { + "queries": 64, + "top1_hits": 19, + "top5_hits": 22, + "shortlist_hits": 22 + }, + "3": { + "queries": 64, + "top1_hits": 32, + "top5_hits": 41, + "shortlist_hits": 42 + }, + "4": { + "queries": 64, + "top1_hits": 35, + "top5_hits": 53, + "shortlist_hits": 53 + } + } + }, + "gguf_sha256": "8d75e9b96c4b64e8a1180224cbaefbbc7744f21ca2e0be2319a429f2c589d342", + "cache_hit": { + "queries": 1280, + "median_ms": 32.382099999999994, + "p95_ms": 69.1143, + "max_ms": 114.19269999999999 + }, + "prediction_sha256": "c15a0a27ba6d863b5fe5f73cc2d0fc34208ca766ea038d058572a9cb18617c56", + "batch_candidates": true, + "peak_process_working_set_bytes": 556535808 + } + }, + "failed_modes": {}, + "promote": false, + "cache_workload": "Each frozen query starts with no cached context and is then repeated once. Only context KV/logits are cached, never candidate scores. This is a best-case cache-hit probe, not a typing trace.", + "memory_method": "Windows peak process working set, including loading; unavailable on other platforms." + }, + "futo": { + "source_commit": "70a5d390c505a6bbcc4e14966e5628e43ca3f1fc", + "source_archive_sha256": "465677ea63dcd53ed7ba58da9353c0d1dadf744d55a9e4b7978ccf6da2c79b6d", + "one_thread": { + "query_count": 1280, + "successful_inference_queries": 1279, + "cold_load_ms": 86.5863, + "warm_median_ms": 13.8488, + "warm_p95_ms": 24.067, + "warm_max_ms": 30.3368, + "filtered_non_prefix_suggestions": 789, + "unsupported_prefix_queries": 1, + "failed_queries": 0, + "cells": { + "documents": { + "0": { + "queries": 64, + "native_top1_hits": 7, + "native_top3_hits": 11, + "exact_prefix_top1_hits": 7, + "exact_prefix_top3_hits": 11, + "empty_exact_results": 0 + }, + "1": { + "queries": 64, + "native_top1_hits": 1, + "native_top3_hits": 6, + "exact_prefix_top1_hits": 3, + "exact_prefix_top3_hits": 6, + "empty_exact_results": 10 + }, + "2": { + "queries": 64, + "native_top1_hits": 21, + "native_top3_hits": 36, + "exact_prefix_top1_hits": 23, + "exact_prefix_top3_hits": 36, + "empty_exact_results": 0 + }, + "3": { + "queries": 64, + "native_top1_hits": 29, + "native_top3_hits": 49, + "exact_prefix_top1_hits": 30, + "exact_prefix_top3_hits": 49, + "empty_exact_results": 1 + }, + "4": { + "queries": 64, + "native_top1_hits": 42, + "native_top3_hits": 57, + "exact_prefix_top1_hits": 42, + "exact_prefix_top3_hits": 57, + "empty_exact_results": 0 + } + }, + "email": { + "0": { + "queries": 64, + "native_top1_hits": 11, + "native_top3_hits": 20, + "exact_prefix_top1_hits": 11, + "exact_prefix_top3_hits": 20, + "empty_exact_results": 0 + }, + "1": { + "queries": 64, + "native_top1_hits": 1, + "native_top3_hits": 7, + "exact_prefix_top1_hits": 6, + "exact_prefix_top3_hits": 7, + "empty_exact_results": 11 + }, + "2": { + "queries": 64, + "native_top1_hits": 30, + "native_top3_hits": 41, + "exact_prefix_top1_hits": 33, + "exact_prefix_top3_hits": 41, + "empty_exact_results": 1 + }, + "3": { + "queries": 64, + "native_top1_hits": 27, + "native_top3_hits": 39, + "exact_prefix_top1_hits": 27, + "exact_prefix_top3_hits": 39, + "empty_exact_results": 3 + }, + "4": { + "queries": 64, + "native_top1_hits": 30, + "native_top3_hits": 53, + "exact_prefix_top1_hits": 32, + "exact_prefix_top3_hits": 53, + "empty_exact_results": 1 + } + }, + "messages": { + "0": { + "queries": 64, + "native_top1_hits": 11, + "native_top3_hits": 15, + "exact_prefix_top1_hits": 11, + "exact_prefix_top3_hits": 15, + "empty_exact_results": 0 + }, + "1": { + "queries": 64, + "native_top1_hits": 5, + "native_top3_hits": 12, + "exact_prefix_top1_hits": 6, + "exact_prefix_top3_hits": 12, + "empty_exact_results": 11 + }, + "2": { + "queries": 64, + "native_top1_hits": 29, + "native_top3_hits": 47, + "exact_prefix_top1_hits": 33, + "exact_prefix_top3_hits": 47, + "empty_exact_results": 1 + }, + "3": { + "queries": 64, + "native_top1_hits": 36, + "native_top3_hits": 53, + "exact_prefix_top1_hits": 38, + "exact_prefix_top3_hits": 53, + "empty_exact_results": 1 + }, + "4": { + "queries": 64, + "native_top1_hits": 38, + "native_top3_hits": 53, + "exact_prefix_top1_hits": 40, + "exact_prefix_top3_hits": 53, + "empty_exact_results": 2 + } + }, + "search": { + "0": { + "queries": 64, + "native_top1_hits": 8, + "native_top3_hits": 11, + "exact_prefix_top1_hits": 8, + "exact_prefix_top3_hits": 11, + "empty_exact_results": 0 + }, + "1": { + "queries": 64, + "native_top1_hits": 4, + "native_top3_hits": 7, + "exact_prefix_top1_hits": 6, + "exact_prefix_top3_hits": 7, + "empty_exact_results": 7 + }, + "2": { + "queries": 64, + "native_top1_hits": 14, + "native_top3_hits": 28, + "exact_prefix_top1_hits": 15, + "exact_prefix_top3_hits": 28, + "empty_exact_results": 0 + }, + "3": { + "queries": 64, + "native_top1_hits": 24, + "native_top3_hits": 41, + "exact_prefix_top1_hits": 26, + "exact_prefix_top3_hits": 41, + "empty_exact_results": 1 + }, + "4": { + "queries": 64, + "native_top1_hits": 27, + "native_top3_hits": 49, + "exact_prefix_top1_hits": 28, + "exact_prefix_top3_hits": 49, + "empty_exact_results": 2 + } + } + }, + "prediction_sha256": "12a96eefdb411389beda808f1293ffda4f9e472d4ed839ae6a93ceb9ddc5e373", + "protocol": "futo-native-v1", + "model_sha256": "6545c1c9ef2d76e9bfb87ad4fcf2061889513af84fcf30d907412be7fcdedb7b", + "model_file_bytes": 30662880, + "executable_sha256": "c2717fb98454fc6a44423a75f0b26b13c9a34cb962ab394d820bf743991d5a34", + "queries_sha256": "64c916f03f2741694c59cb0c74a3f513274eeb9a54fe6ac475a27e936df10b11", + "hardware": "Windows-11-10.0.26200-SP0", + "peak_process_working_set_bytes": 112271360, + "threads": 1, + "results_limit": 3, + "promote": false, + "note": "Native no-coordinate correction path, then exact-prefix filtering. No backfill. Not a five-slot reranker; not validated against an Android device." + }, + "four_threads": { + "query_count": 1280, + "successful_inference_queries": 1279, + "cold_load_ms": 82.8641, + "warm_median_ms": 7.5909, + "warm_p95_ms": 12.8327, + "warm_max_ms": 16.769, + "filtered_non_prefix_suggestions": 789, + "unsupported_prefix_queries": 1, + "failed_queries": 0, + "cells": { + "documents": { + "0": { + "queries": 64, + "native_top1_hits": 7, + "native_top3_hits": 11, + "exact_prefix_top1_hits": 7, + "exact_prefix_top3_hits": 11, + "empty_exact_results": 0 + }, + "1": { + "queries": 64, + "native_top1_hits": 1, + "native_top3_hits": 6, + "exact_prefix_top1_hits": 3, + "exact_prefix_top3_hits": 6, + "empty_exact_results": 10 + }, + "2": { + "queries": 64, + "native_top1_hits": 21, + "native_top3_hits": 36, + "exact_prefix_top1_hits": 23, + "exact_prefix_top3_hits": 36, + "empty_exact_results": 0 + }, + "3": { + "queries": 64, + "native_top1_hits": 29, + "native_top3_hits": 49, + "exact_prefix_top1_hits": 30, + "exact_prefix_top3_hits": 49, + "empty_exact_results": 1 + }, + "4": { + "queries": 64, + "native_top1_hits": 42, + "native_top3_hits": 57, + "exact_prefix_top1_hits": 42, + "exact_prefix_top3_hits": 57, + "empty_exact_results": 0 + } + }, + "email": { + "0": { + "queries": 64, + "native_top1_hits": 11, + "native_top3_hits": 20, + "exact_prefix_top1_hits": 11, + "exact_prefix_top3_hits": 20, + "empty_exact_results": 0 + }, + "1": { + "queries": 64, + "native_top1_hits": 1, + "native_top3_hits": 7, + "exact_prefix_top1_hits": 6, + "exact_prefix_top3_hits": 7, + "empty_exact_results": 11 + }, + "2": { + "queries": 64, + "native_top1_hits": 30, + "native_top3_hits": 41, + "exact_prefix_top1_hits": 33, + "exact_prefix_top3_hits": 41, + "empty_exact_results": 1 + }, + "3": { + "queries": 64, + "native_top1_hits": 27, + "native_top3_hits": 39, + "exact_prefix_top1_hits": 27, + "exact_prefix_top3_hits": 39, + "empty_exact_results": 3 + }, + "4": { + "queries": 64, + "native_top1_hits": 30, + "native_top3_hits": 53, + "exact_prefix_top1_hits": 32, + "exact_prefix_top3_hits": 53, + "empty_exact_results": 1 + } + }, + "messages": { + "0": { + "queries": 64, + "native_top1_hits": 11, + "native_top3_hits": 15, + "exact_prefix_top1_hits": 11, + "exact_prefix_top3_hits": 15, + "empty_exact_results": 0 + }, + "1": { + "queries": 64, + "native_top1_hits": 5, + "native_top3_hits": 12, + "exact_prefix_top1_hits": 6, + "exact_prefix_top3_hits": 12, + "empty_exact_results": 11 + }, + "2": { + "queries": 64, + "native_top1_hits": 29, + "native_top3_hits": 47, + "exact_prefix_top1_hits": 33, + "exact_prefix_top3_hits": 47, + "empty_exact_results": 1 + }, + "3": { + "queries": 64, + "native_top1_hits": 36, + "native_top3_hits": 53, + "exact_prefix_top1_hits": 38, + "exact_prefix_top3_hits": 53, + "empty_exact_results": 1 + }, + "4": { + "queries": 64, + "native_top1_hits": 38, + "native_top3_hits": 53, + "exact_prefix_top1_hits": 40, + "exact_prefix_top3_hits": 53, + "empty_exact_results": 2 + } + }, + "search": { + "0": { + "queries": 64, + "native_top1_hits": 8, + "native_top3_hits": 11, + "exact_prefix_top1_hits": 8, + "exact_prefix_top3_hits": 11, + "empty_exact_results": 0 + }, + "1": { + "queries": 64, + "native_top1_hits": 4, + "native_top3_hits": 7, + "exact_prefix_top1_hits": 6, + "exact_prefix_top3_hits": 7, + "empty_exact_results": 7 + }, + "2": { + "queries": 64, + "native_top1_hits": 14, + "native_top3_hits": 28, + "exact_prefix_top1_hits": 15, + "exact_prefix_top3_hits": 28, + "empty_exact_results": 0 + }, + "3": { + "queries": 64, + "native_top1_hits": 24, + "native_top3_hits": 41, + "exact_prefix_top1_hits": 26, + "exact_prefix_top3_hits": 41, + "empty_exact_results": 1 + }, + "4": { + "queries": 64, + "native_top1_hits": 27, + "native_top3_hits": 49, + "exact_prefix_top1_hits": 28, + "exact_prefix_top3_hits": 49, + "empty_exact_results": 2 + } + } + }, + "prediction_sha256": "12a96eefdb411389beda808f1293ffda4f9e472d4ed839ae6a93ceb9ddc5e373", + "protocol": "futo-native-v1", + "model_sha256": "6545c1c9ef2d76e9bfb87ad4fcf2061889513af84fcf30d907412be7fcdedb7b", + "model_file_bytes": 30662880, + "executable_sha256": "c2717fb98454fc6a44423a75f0b26b13c9a34cb962ab394d820bf743991d5a34", + "queries_sha256": "64c916f03f2741694c59cb0c74a3f513274eeb9a54fe6ac475a27e936df10b11", + "hardware": "Windows-11-10.0.26200-SP0", + "peak_process_working_set_bytes": 111886336, + "threads": 4, + "results_limit": 3, + "promote": false, + "note": "Native no-coordinate correction path, then exact-prefix filtering. No backfill. Not a five-slot reranker; not validated against an Android device." + } + }, + "baseline_database_bytes": 29802496, + "decision": { + "promote": false, + "neural_p95_target_passed": false, + "futo_four_thread_p95_target_passed": true, + "reason": "FUTO meets the speed target but uses a three-result correction decoder with weak one-character completion and unsupported prefixes. Neural variants still exceed 20 ms. Real-writing and Android equivalence tests remain outstanding." + } +} diff --git a/docs/neural-latency-results.md b/docs/neural-latency-results.md new file mode 100644 index 0000000..ca37521 --- /dev/null +++ b/docs/neural-latency-results.md @@ -0,0 +1,149 @@ +# Prediction speed follow-up + +FUTO's native decoder met the initial speed target on this Windows PC: **12.83 ms +p95 with four threads**, compared with 24.07 ms using its default one thread. +Both returned exactly the same ordered suggestions. It is a promising next +candidate, but its correction behavior needs adapting and evaluating before use +as an exact-prefix word-completion engine. + +Optimizing SmolLM2 helped, but did not meet 20 ms. The best measured cache-miss +p95 was **101.78 ms** for host-optimized Q8, down from 226.72 ms for F32. +Batching improved cache hits but made Q8 context preparation much slower. +No model is promoted and Switchify PC is unchanged. + +## Workload and limits + +Measured on 2026-10-03, Windows 11, AMD Ryzen AI 9 HX 370, 24 logical CPUs. +Each completed mode used the same 1,280 frozen synthetic queries across messages, +email, documents and search. Neural modes also measured 1,280 immediate repeated +requests with a retained context cache and checked exact ordered-result equality. +These repetitions measure best-case cache hits, not a typing trace or real hit +rate. Candidate scores were always recomputed. + +The fixtures are diagnostic comparisons, not representative user text or proof +of unseen-data accuracy. Training overlap beyond the checked exact sentence +matches is unknown. The statistical control retains its historical training +mixture. No personal learning, live field capture or input injection was used. +Benchmarks ran sequentially, without concurrent builds or other benchmarks. + +## Neural performance + +Times are milliseconds. Miss means the context cache was cleared before every +query. Hit means the immediately repeated request reused context KV/logits. +All completed neural modes had zero failed queries. + +| Mode | Miss median | Miss p95 | Miss max | Hit median | Hit p95 | Hit max | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Statistical baseline | 0.08 | 6.13 | 8.58 | n/a | n/a | n/a | +| Portable F32 | 165.99 | 226.72 | 398.29 | 128.82 | 187.20 | 381.02 | +| Portable F32, batched | 146.96 | 225.84 | 304.25 | 59.42 | 132.78 | 213.26 | +| Portable Q8 | 223.94 | 319.86 | 539.56 | 136.65 | 204.88 | 443.22 | +| Host-native Q8 | 74.36 | 101.78 | 176.14 | 56.68 | 79.88 | 166.62 | +| Host-native Q8, batched | 159.81 | 297.36 | 436.31 | 32.38 | 69.11 | 114.19 | + +Candle's Q8 kernels have compile-time AVX2 gates. The portable build did not +enable them; the separate `-C target-cpu=native` build did. Its binary is specific +to this PC's instruction set and is not a portable release artifact. Four Rayon +threads were used throughout. Q8 reduces weights on disk from 256.60 MiB to +136.42 MiB, but smaller weights alone did not make the portable build faster. + +The first portable Q8 batched attempt failed with a non-contiguous output slice +in Candle 0.11 when batch and prefill lengths both exceeded one. The subsequent +adapter uses token-wise prefill for batched Q8. This avoids modifying Candle, +but explains its expensive cache misses. That failed attempt has no accepted +timing/accuracy score and is retained in the machine-readable record. + +| Mode | Process cold load, ms | Peak process working set, MiB | +| --- | ---: | ---: | +| Statistical baseline | 2595.63 | 371.16 | +| Portable F32 | 2803.57 | 1036.82 | +| Portable F32, batched | 2844.71 | not retained | +| Portable Q8 | 2716.64 | 513.93 | +| Host-native Q8 | 2710.77 | 514.79 | +| Host-native Q8, batched | 2743.26 | 530.75 | + +Cold load excludes checksum verification and does not imply a cold OS file +cache. Working set is the Windows process high-water value and includes loading, +validation and scoring, not just model tensors. The baseline database is +28.42 MiB. The portable batched F32 per-mode report survived the later Q8 failure, +but its aggregate memory/executable metadata did not; neither is invented here. +The runner now saves each completed mode and failure record incrementally. + +## Neural accuracy + +Early accuracy averages zero, one and two typed Unicode graphemes equally across +the four domains, 768 queries. The same top-eight shortlist supplied every +reranker; each returned five words. Full domain/prefix cells are in the JSON. + +| Mode | Early top-one | Early top-five | +| --- | ---: | ---: | +| Statistical baseline | 19.27% | 32.94% | +| F32, sequential or batched | 28.26% | 37.50% | +| Portable Q8 | 28.78% | 37.37% | +| Host-native Q8, sequential or batched | 28.52% | 37.50% | + +F32 batching preserved every ordered result. Host-native Q8 batching also +preserved every result of its sequential counterpart. Quantized kernels changed +some rankings relative to F32 and between portable/native builds; matching +aggregate accuracy does not imply identical predictions. Every cache-hit pair +matched its cache-miss result. F32 GGUF conversion passed the fixed prefill and +continuation logit check before quantized results were accepted. + +## FUTO native decoder + +The probe uses the pinned upstream native decoder and matching 30,662,880-byte +model, verified locally against SHA-256 +`6545c1c9ef2d76e9bfb87ad4fcf2061889513af84fcf30d907412be7fcdedb7b`. +It retains upstream tokenization, sampling, three-result limit and the JNI +wrapper's exact-match preference. Typed prefixes use its no-coordinate +correction fallback. See [the probe protocol](futo-probe-protocol.md) and +[upstream sources](futo-investigation.md). + +| Threads | Median, ms | p95, ms | Max, ms | Cold load, ms | Peak working set, MiB | +| --- | ---: | ---: | ---: | ---: | ---: | +| 1 | 13.85 | 24.07 | 30.34 | 86.59 | 107.07 | +| 4 | 7.59 | 12.83 | 16.77 | 82.86 | 106.70 | + +These are internal native query timings; there is no per-query process startup. +The Windows C++ build enables AVX2. Both runs completed 1,279 supported inferences +and explicitly rejected one prefix containing a nonletter. No decode failures +occurred. All four native decode calls are checked, upstream error paths fail +closed, and the runner requires explicit query-success markers. The four-thread +run's ordered-output digest exactly matches the one-thread run. + +This is a three-result correction decoder, not a five-slot reranker. After +normalization, 789 suggestions across each full run did not match the typed +prefix and were filtered without backfill. Native and exact-prefix results are +therefore reported separately. Targets are longer than their tested prefixes, +so removing nonmatching corrections leaves top-three target hits unchanged. + +| Typed graphemes | Native top-one | Native top-three | Filtered top-one | Filtered top-three | Empty after filtering | +| --- | ---: | ---: | ---: | ---: | ---: | +| 0 | 14.45% | 22.27% | 14.45% | 22.27% | 0.00% | +| 1 | 4.30% | 12.50% | 8.20% | 12.50% | 15.23% | +| 2 | 36.72% | 59.38% | 40.62% | 59.38% | 0.78% | +| 3 | 45.31% | 71.09% | 47.27% | 71.09% | 2.34% | +| 4 | 53.52% | 82.81% | 55.47% | 82.81% | 1.95% | + +FUTO is fast enough in this experiment, but the one-letter results are a clear +weakness for Switchify's completion use. A constrained completion adapter and +real-writing evaluation should come before choosing it. Android output +equivalence has not been tested, macOS runtime performance is untested, and +model/code redistribution terms need a separate review. No upstream decoder +source or model weights are committed or published by this work. + +## Validation and reproduction + +Root formatting, Clippy with warnings denied and 22 Rust tests passed. The +separate neural crate passed formatting, Clippy and six fake/pure Rust tests; +14 Python tests passed. The native Windows build succeeded. Root dependency +audit passed; the isolated Candle experiment retains its documented +`RUSTSEC-2024-0436` unmaintained-`paste` exception. Production dependencies and +its strict audit remain unchanged. CI and independent latest-head review are +tracked on [PR #16](https://github.com/switchifyapp/switchify-prediction/pull/16). + +[Machine-readable results](neural-latency-results.json) retain all completed +domain/prefix cells, hashes, memory data, the failed portable Q8 batch attempt +and measurement limits. Reproduce with [the neural protocol](neural-latency-protocol.md) +and [the FUTO protocol](futo-probe-protocol.md). Use separate empty output +directories for each run. Keep the existing predictor as the production default. diff --git a/experiments/futo/probe.cpp b/experiments/futo/probe.cpp index 0e11c7f..1cf84f6 100644 --- a/experiments/futo/probe.cpp +++ b/experiments/futo/probe.cpp @@ -7,11 +7,14 @@ #include "decoder.inc" int probe_main(int argc, char **argv) { - if (argc != 3) return 2; + if (argc != 3 && argc != 4) return 2; + const int threads = argc == 4 ? std::stoi(argv[3]) : 1; + if (threads != 1 && threads != 4) return 2; llama_backend_init(false); auto start = std::chrono::steady_clock::now(); LanguageModelState state; if (!state.Initialize(argv[1])) return 3; + llama_set_n_threads(state.model->context(), threads, threads); auto elapsed = [](auto start) { return std::chrono::duration(std::chrono::steady_clock::now() - start).count(); }; diff --git a/scripts/futo_experiment.py b/scripts/futo_experiment.py index 70a5177..1560667 100644 --- a/scripts/futo_experiment.py +++ b/scripts/futo_experiment.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Score the pinned native FUTO three-suggestion decoder on exported synthetic queries.""" import argparse +import hashlib import json import math import os @@ -36,7 +37,7 @@ def summarize(queries, lines): prime = lines[1].split('\t') if not math.isfinite(cold_load) or cold_load < 0 or len(prime) < 3 or prime[:2] != ['0', 'ok']: raise ValueError('Invalid load or warm-up result') - cells, latencies = {}, [] + cells, latencies, predictions = {}, [], [] filtered, unsupported = 0, 0 # First query is an untimed prime, then each frozen query is measured once. for index, (q, line) in enumerate(zip(queries, lines[2:])): @@ -60,6 +61,7 @@ def summarize(queries, lines): filtered += 1 if any(c < 'a' or c > 'z' for c in q['prefix']): unsupported += 1 + predictions.append((q['key'] if 'key' in q else index, native, exact)) cell = cells.setdefault(q['domain'], {}).setdefault(str(q['prefix_chars']), dict(queries=0, native_top1_hits=0, native_top3_hits=0, exact_prefix_top1_hits=0, exact_prefix_top3_hits=0, empty_exact_results=0)) @@ -74,13 +76,15 @@ def summarize(queries, lines): return dict(query_count=len(queries), successful_inference_queries=len(queries) - unsupported, cold_load_ms=cold_load, warm_median_ms=percentile(.5), warm_p95_ms=percentile(.95), warm_max_ms=latencies[-1], filtered_non_prefix_suggestions=filtered, unsupported_prefix_queries=unsupported, - failed_queries=0, cells=cells) + failed_queries=0, cells=cells, + prediction_sha256=hashlib.sha256(json.dumps(predictions, ensure_ascii=False).encode()).hexdigest()) def main(): parser = argparse.ArgumentParser(description=__doc__) for name in ('executable', 'model', 'queries', 'output'): parser.add_argument('--' + name, type=Path, required=True) + parser.add_argument('--threads', type=int, choices=(1, 4), default=1) args = parser.parse_args() if sha(args.model) != MODEL_SHA: raise ValueError('FUTO model checksum mismatch') @@ -99,7 +103,7 @@ def main(): for q in [queries[0]] + queries).encode()) peak = None with (out / 'native.tsv').open('wb') as stdout, (out / 'native.log').open('wb') as stderr: - process = subprocess.Popen([str(args.executable.resolve()), str(args.model.resolve()), str(fixture)], + process = subprocess.Popen([str(args.executable.resolve()), str(args.model.resolve()), str(fixture), str(args.threads)], stdout=stdout, stderr=stderr) deadline = time.monotonic() + 1800 while process.poll() is None: @@ -118,7 +122,7 @@ def main(): report.update(protocol='futo-native-v1', model_sha256=MODEL_SHA, model_file_bytes=args.model.stat().st_size, executable_sha256=sha(args.executable), queries_sha256=sha(args.queries), hardware=platform.platform(), - peak_process_working_set_bytes=peak, threads=1, results_limit=3, + peak_process_working_set_bytes=peak, threads=args.threads, results_limit=3, promote=False, note='Native no-coordinate correction path, then exact-prefix filtering. No backfill. Not a five-slot reranker; not validated against an Android device.') (out / 'results.json').write_bytes((json.dumps(report, indent=2) + '\n').encode()) diff --git a/scripts/neural_latency_experiment.py b/scripts/neural_latency_experiment.py index 56de36f..b8d117b 100644 --- a/scripts/neural_latency_experiment.py +++ b/scripts/neural_latency_experiment.py @@ -45,7 +45,7 @@ def main(): 'logical_cpus': os.cpu_count(), 'rayon_threads': 4, 'executable_sha256': sha(binary), 'converter_sha256': sha(converter), 'build_label': args.build_label, - 'reports': {}, 'promote': False, + 'reports': {}, 'failed_modes': {}, 'promote': False, 'cache_workload': 'Each frozen query starts with no cached context and is then repeated once. Only context KV/logits are cached, never candidate scores. This is a best-case cache-hit probe, not a typing trace.', 'memory_method': 'Windows peak process working set, including loading; unavailable on other platforms.'} modes = args.modes or (('f32-batched', 'q8-batched') if args.only_batched else ('baseline', 'f32', 'q8', 'f32-batched', 'q8-batched')) @@ -59,11 +59,17 @@ def main(): if mode.startswith('q8'): quantized = out / 'q8.gguf' command += ['--gguf', str(quantized), '--gguf-sha256', sha(quantized)] - peak = run(command, env) + try: + peak = run(command, env) + except Exception as error: + result['failed_modes'][mode] = type(error).__name__ + (out / 'results.json').write_bytes((json.dumps(result, indent=2) + '\n').encode()) + raise report = json.loads(dest.read_text(encoding='utf-8')) report['peak_process_working_set_bytes'] = peak + dest.write_bytes((json.dumps(report, indent=2) + '\n').encode()) result['reports'][mode] = report - (out / 'results.json').write_bytes((json.dumps(result, indent=2) + '\n').encode()) + (out / 'results.json').write_bytes((json.dumps(result, indent=2) + '\n').encode()) if __name__ == '__main__': diff --git a/scripts/prepare_futo_probe.py b/scripts/prepare_futo_probe.py index 87c0a68..e642666 100644 --- a/scripts/prepare_futo_probe.py +++ b/scripts/prepare_futo_probe.py @@ -45,6 +45,8 @@ def main(): # Mechanical portability/safety fixes only, no ranking or sampling changes. text = text.replace('int seq_id_use_count[n_results];', 'std::vector seq_id_use_count(n_results);') text = text.replace('auto start = s.begin();', 'if (s.empty()) return {};\n auto start = s.begin();') + text = text.replace('auto end = s.end();', 'if (start == s.end()) return {};\n auto end = s.end();') + text = text.replace('std::isspace(*start)', 'std::isspace(static_cast(*start))').replace('std::isspace(*end)', 'std::isspace(static_cast(*end))') # Upstream sometimes returns default/empty state on errors, and one decode # call is unchecked. A benchmark must not count those as successful queries. head, state = text.split('struct LanguageModelState {', 1)