diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml
new file mode 100644
index 0000000..f9e2f9f
--- /dev/null
+++ b/.github/workflows/ci-python.yml
@@ -0,0 +1,256 @@
+# This file is autogenerated by maturin v1.13.3
+# To update, run
+#
+#    maturin generate-ci github
+#
+name: CI - Python Bindings
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "crates/**"
+      - ".github/workflows/ci-python.yml"
+    tags:
+      - "python@**"
+  pull_request:
+    paths:
+      - "crates/**"
+      - ".github/workflows/ci-python.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  stubs:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.x"
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+      - name: Cache cargo registry and build
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: crates/statembed-py
+      - name: Generate Python stubs
+        working-directory: crates/statembed-py
+        run: |
+          python3 -m venv .venv
+          source .venv/bin/activate
+          export PYO3_PYTHON=$(which python3)
+          cargo run --bin stub_gen
+      - name: Upload stubs
+        uses: actions/upload-artifact@v6
+        with:
+          name: python-stubs
+          path: crates/statembed-py/*.pyi
+
+  linux:
+    runs-on: ${{ matrix.platform.runner }}
+    needs: [stubs]
+    strategy:
+      matrix:
+        platform:
+          - runner: ubuntu-22.04
+            target: x86_64
+          - runner: ubuntu-22.04
+            target: x86
+          - runner: ubuntu-22.04
+            target: aarch64
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.x"
+      - name: Download stubs
+        uses: actions/download-artifact@v7
+        with:
+            name: python-stubs
+            path: crates/statembed-py/
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --interpreter python3.10 python3.11 python3.12 python3.13
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+          manylinux: auto
+          working-directory: crates/statembed-py
+      - name: Upload wheels
+        uses: actions/upload-artifact@v6
+        with:
+          name: wheels-linux-${{ matrix.platform.target }}
+          path: crates/statembed-py/dist
+
+  musllinux:
+    needs: [stubs]
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: ubuntu-22.04
+            target: x86_64
+          - runner: ubuntu-22.04
+            target: x86
+          - runner: ubuntu-22.04
+            target: aarch64
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.x"
+      - name: Download stubs
+        uses: actions/download-artifact@v7
+        with:
+            name: python-stubs
+            path: crates/statembed-py/
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --interpreter python3.10 python3.11 python3.12 python3.13
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+          manylinux: musllinux_1_2
+          working-directory: crates/statembed-py
+      - name: Upload wheels
+        uses: actions/upload-artifact@v6
+        with:
+          name: wheels-musllinux-${{ matrix.platform.target }}
+          path: crates/statembed-py/dist
+
+  windows:
+    runs-on: ${{ matrix.platform.runner }}
+    needs: [stubs]
+    strategy:
+      matrix:
+        platform:
+          - runner: windows-latest
+            target: x64
+            python_arch: x64
+          - runner: windows-latest
+            target: x86
+            python_arch: x86
+          - runner: windows-11-arm
+            target: aarch64
+            python_arch: arm64
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+          architecture: ${{ matrix.platform.python_arch }}
+      - name: Download stubs
+        uses: actions/download-artifact@v7
+        with:
+            name: python-stubs
+            path: crates/statembed-py/
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --find-interpreter
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+          working-directory: crates/statembed-py
+      - name: Upload wheels
+        uses: actions/upload-artifact@v6
+        with:
+          name: wheels-windows-${{ matrix.platform.target }}
+          path: crates/statembed-py/dist
+
+  macos:
+    needs: [stubs]
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: macos-15-intel
+            target: x86_64
+          - runner: macos-latest
+            target: aarch64
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.x"
+      - name: Download stubs
+        uses: actions/download-artifact@v7
+        with:
+            name: python-stubs
+            path: crates/statembed-py/
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --find-interpreter
+          sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
+          working-directory: crates/statembed-py
+      - name: Upload wheels
+        uses: actions/upload-artifact@v6
+        with:
+          name: wheels-macos-${{ matrix.platform.target }}
+          path: crates/statembed-py/dist
+
+  sdist:
+    needs: [stubs]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - name: Download stubs
+        uses: actions/download-artifact@v7
+        with:
+            name: python-stubs
+            path: crates/statembed-py/
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+          working-directory: crates/statembed-py
+      - name: Upload sdist
+        uses: actions/upload-artifact@v6
+        with:
+          name: wheels-sdist
+          path: crates/statembed-py/dist
+
+  publish:
+    name: Publish - Python Bindings
+    runs-on: ubuntu-latest
+    if: ${{ startsWith(github.ref, 'refs/tags/') }}
+    needs: [linux, musllinux, windows, macos, sdist]
+    permissions:
+      # Use to sign the release artifacts
+      id-token: write
+      # Used to upload release artifacts
+      contents: write
+      # Used to generate artifact attestation
+      attestations: write
+    steps:
+      - uses: actions/download-artifact@v7
+      - name: Generate artifact attestation
+        uses: actions/attest@v4
+        with:
+          subject-path: "wheels-*/*"
+      - name: Install uv
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        uses: astral-sh/setup-uv@v7
+      - name: Publish to PyPI
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        run: uv publish 'wheels-*/*'
+        env:
+          UV_PUBLISH_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c99ce06..6dd97a4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -164,7 +164,7 @@ jobs:
           CARGO_API_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
 
       - name: Publish to crates
-        run: cargo publish
+        run: cargo publish -p statembed
         env:
           CARGO_API_TOKEN: ${{ secrets.CARGO_API_TOKEN }}
 
diff --git a/Cargo.lock b/Cargo.lock
index 33a1a21..7f30766 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -348,6 +348,16 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "chunk"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc04d623ff6273fccafb4dae2e022d973b81a7fac748ad54beddfbffa6092ff1"
+dependencies = [
+ "daggrs",
+ "memchr",
+]
+
 [[package]]
 name = "ciborium"
 version = "0.2.2"
@@ -652,10 +662,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "daachorse"
-version = "1.0.1"
+name = "daggrs"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f55d7153ba3b507595872a3874803f07a8a81d1e888abed8e5db7da0597d6e2"
+checksum = "0d1092288af9ecfd5378565a17df11a13dbe172d4a71abe9b55a9689f1336b8e"
 
 [[package]]
 name = "darling"
@@ -861,6 +871,12 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foldhash"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.2"
@@ -973,6 +989,15 @@ dependencies = [
  "cfg-if 0.1.10",
 ]
 
+[[package]]
+name = "getopts"
+version = "0.2.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df"
+dependencies = [
+ "unicode-width",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.17"
@@ -1305,6 +1330,18 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "icu_collections"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
+dependencies = [
+ "displaydoc",
+ "yoke 0.7.5",
+ "zerofrom",
+ "zerovec 0.10.4",
+]
+
 [[package]]
 name = "icu_collections"
 version = "2.2.0"
@@ -1314,9 +1351,9 @@ dependencies = [
  "displaydoc",
  "potential_utf",
  "utf8_iter",
- "yoke",
+ "yoke 0.8.3",
  "zerofrom",
- "zerovec",
+ "zerovec 0.11.6",
 ]
 
 [[package]]
@@ -1326,10 +1363,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"
 dependencies = [
  "displaydoc",
- "litemap",
- "tinystr",
- "writeable",
- "zerovec",
+ "litemap 0.8.2",
+ "tinystr 0.8.3",
+ "writeable 0.6.3",
+ "zerovec 0.11.6",
+]
+
+[[package]]
+name = "icu_locid"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
+dependencies = [
+ "displaydoc",
+ "litemap 0.7.5",
+ "tinystr 0.7.6",
+ "writeable 0.5.5",
+ "zerovec 0.10.4",
+]
+
+[[package]]
+name = "icu_locid_transform"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
+dependencies = [
+ "displaydoc",
+ "icu_locid",
+ "icu_locid_transform_data",
+ "icu_provider 1.5.0",
+ "tinystr 0.7.6",
+ "zerovec 0.10.4",
+]
+
+[[package]]
+name = "icu_locid_transform_data"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d"
+
+[[package]]
+name = "icu_normalizer"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
+dependencies = [
+ "displaydoc",
+ "icu_collections 1.5.0",
+ "icu_normalizer_data 1.5.1",
+ "icu_properties 1.5.1",
+ "icu_provider 1.5.0",
+ "smallvec",
+ "utf16_iter",
+ "utf8_iter",
+ "write16",
+ "zerovec 0.10.4",
 ]
 
 [[package]]
@@ -1338,40 +1426,84 @@ version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"
 dependencies = [
- "icu_collections",
- "icu_normalizer_data",
- "icu_properties",
- "icu_provider",
+ "icu_collections 2.2.0",
+ "icu_normalizer_data 2.2.0",
+ "icu_properties 2.2.0",
+ "icu_provider 2.2.0",
  "smallvec",
- "zerovec",
+ "zerovec 0.11.6",
 ]
 
+[[package]]
+name = "icu_normalizer_data"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7"
+
 [[package]]
 name = "icu_normalizer_data"
 version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"
 
+[[package]]
+name = "icu_properties"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5"
+dependencies = [
+ "displaydoc",
+ "icu_collections 1.5.0",
+ "icu_locid_transform",
+ "icu_properties_data 1.5.1",
+ "icu_provider 1.5.0",
+ "tinystr 0.7.6",
+ "zerovec 0.10.4",
+]
+
 [[package]]
 name = "icu_properties"
 version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"
 dependencies = [
- "icu_collections",
+ "icu_collections 2.2.0",
  "icu_locale_core",
- "icu_properties_data",
- "icu_provider",
+ "icu_properties_data 2.2.0",
+ "icu_provider 2.2.0",
  "zerotrie",
- "zerovec",
+ "zerovec 0.11.6",
 ]
 
+[[package]]
+name = "icu_properties_data"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2"
+
 [[package]]
 name = "icu_properties_data"
 version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"
 
+[[package]]
+name = "icu_provider"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
+dependencies = [
+ "displaydoc",
+ "icu_locid",
+ "icu_provider_macros",
+ "stable_deref_trait",
+ "tinystr 0.7.6",
+ "writeable 0.5.5",
+ "yoke 0.7.5",
+ "zerofrom",
+ "zerovec 0.10.4",
+]
+
 [[package]]
 name = "icu_provider"
 version = "2.2.0"
@@ -1380,11 +1512,22 @@ checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"
 dependencies = [
  "displaydoc",
  "icu_locale_core",
- "writeable",
- "yoke",
+ "writeable 0.6.3",
+ "yoke 0.8.3",
  "zerofrom",
  "zerotrie",
- "zerovec",
+ "zerovec 0.11.6",
+]
+
+[[package]]
+name = "icu_provider_macros"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -1410,8 +1553,8 @@ version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714"
 dependencies = [
- "icu_normalizer",
- "icu_properties",
+ "icu_normalizer 2.2.0",
+ "icu_properties 2.2.0",
 ]
 
 [[package]]
@@ -1437,18 +1580,48 @@ dependencies = [
  "web-time",
 ]
 
+[[package]]
+name = "inventory"
+version = "0.3.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "ipnet"
 version = "2.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78"
 
+[[package]]
+name = "is-macro"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d57a3e447e24c22647738e4607f1df1e0ec6f72e16182c4cd199f647cdfb0e4"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "is_terminal_polyfill"
 version = "1.70.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
 
+[[package]]
+name = "itertools"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itertools"
 version = "0.13.0"
@@ -1560,6 +1733,12 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
+[[package]]
+name = "lalrpop-util"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553"
+
 [[package]]
 name = "lazy_static"
 version = "1.5.0"
@@ -1599,6 +1778,12 @@ version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
 
+[[package]]
+name = "litemap"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
+
 [[package]]
 name = "litemap"
 version = "0.8.2"
@@ -1642,6 +1827,12 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c"
 
+[[package]]
+name = "maplit"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
+
 [[package]]
 name = "matchers"
 version = "0.2.0"
@@ -1733,7 +1924,7 @@ dependencies = [
  "safetensors",
  "serde",
  "serde_json",
- "tokenizers 0.21.4",
+ "tokenizers",
  "ureq",
 ]
 
@@ -1806,6 +1997,16 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "num-bigint"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367"
+dependencies = [
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-complex"
 version = "0.4.6"
@@ -1845,6 +2046,22 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
 
+[[package]]
+name = "numpy"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a5b15d63a5ff39e378daed0e1340d3a5964703ea9712eb09a0dc66fade996f4"
+dependencies = [
+ "libc",
+ "ndarray",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "pyo3",
+ "pyo3-build-config",
+ "rustc-hash 2.1.3",
+]
+
 [[package]]
 name = "objc2-core-foundation"
 version = "0.3.2"
@@ -1931,6 +2148,15 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
 
+[[package]]
+name = "ordered-float"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "os_str_bytes"
 version = "6.6.1"
@@ -1974,6 +2200,44 @@ version = "2.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
 
+[[package]]
+name = "phf"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
+dependencies = [
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_codegen"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
+dependencies = [
+ "phf_shared",
+ "rand 0.8.7",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "pin-project"
 version = "1.1.13"
@@ -2046,7 +2310,7 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564"
 dependencies = [
- "zerovec",
+ "zerovec 0.11.6",
 ]
 
 [[package]]
@@ -2064,6 +2328,16 @@ dependencies = [
  "zerocopy",
 ]
 
+[[package]]
+name = "pretokie"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a57fe333385b85c2ddf83ee5c921845aa6c24a3783c70bf8182a26cb8f9af55"
+dependencies = [
+ "regex-automata",
+ "unicode-general-category",
+]
+
 [[package]]
 name = "prettyplease"
 version = "0.2.37"
@@ -2083,6 +2357,103 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "pyo3"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b"
+dependencies = [
+ "libc",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9"
+dependencies = [
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "pyo3-stub-gen"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a267ea4da9a831f534def7f1d8e14f581d5640d3e5af6527072724f216f4dbbf"
+dependencies = [
+ "anyhow",
+ "chrono",
+ "either",
+ "indexmap",
+ "inventory",
+ "itertools 0.14.0",
+ "log",
+ "maplit",
+ "num-complex",
+ "numpy",
+ "ordered-float",
+ "pyo3",
+ "pyo3-stub-gen-derive",
+ "rustpython-parser",
+ "serde",
+ "serde_json",
+ "time",
+ "toml",
+]
+
+[[package]]
+name = "pyo3-stub-gen-derive"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6573423a5e8cc43ec7565eccccbc2dd35e1fc9032d9ca404afab875429eb8cf0"
+dependencies = [
+ "heck",
+ "indexmap",
+ "proc-macro2",
+ "quote",
+ "rustpython-parser",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "quinn"
 version = "0.11.11"
@@ -2094,7 +2465,7 @@ dependencies = [
  "pin-project-lite",
  "quinn-proto",
  "quinn-udp",
- "rustc-hash",
+ "rustc-hash 2.1.3",
  "rustls",
  "socket2",
  "thiserror",
@@ -2116,7 +2487,7 @@ dependencies = [
  "rand 0.10.2",
  "rand_pcg",
  "ring",
- "rustc-hash",
+ "rustc-hash 2.1.3",
  "rustls",
  "rustls-pki-types",
  "slab",
@@ -2161,13 +2532,24 @@ version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
 
+[[package]]
+name = "rand"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a"
+dependencies = [
+ "libc",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
+]
+
 [[package]]
 name = "rand"
 version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
 dependencies = [
- "rand_chacha",
+ "rand_chacha 0.9.0",
  "rand_core 0.9.5",
 ]
 
@@ -2182,6 +2564,16 @@ dependencies = [
  "rand_core 0.10.1",
 ]
 
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.6.4",
+]
+
 [[package]]
 name = "rand_chacha"
 version = "0.9.0"
@@ -2192,6 +2584,15 @@ dependencies = [
  "rand_core 0.9.5",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom 0.2.17",
+]
+
 [[package]]
 name = "rand_core"
 version = "0.9.5"
@@ -2375,6 +2776,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
 [[package]]
 name = "rustc-hash"
 version = "2.1.3"
@@ -2480,6 +2887,63 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "rustpython-ast"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4cdaf8ee5c1473b993b398c174641d3aa9da847af36e8d5eb8291930b72f31a5"
+dependencies = [
+ "is-macro",
+ "num-bigint",
+ "rustpython-parser-core",
+ "static_assertions",
+]
+
+[[package]]
+name = "rustpython-parser"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "868f724daac0caf9bd36d38caf45819905193a901e8f1c983345a68e18fb2abb"
+dependencies = [
+ "anyhow",
+ "is-macro",
+ "itertools 0.11.0",
+ "lalrpop-util",
+ "log",
+ "num-bigint",
+ "num-traits",
+ "phf",
+ "phf_codegen",
+ "rustc-hash 1.1.0",
+ "rustpython-ast",
+ "rustpython-parser-core",
+ "tiny-keccak",
+ "unic-emoji-char",
+ "unic-ucd-ident",
+ "unicode_names2",
+]
+
+[[package]]
+name = "rustpython-parser-core"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4b6c12fa273825edc7bccd9a734f0ad5ba4b8a2f4da5ff7efe946f066d0f4ad"
+dependencies = [
+ "is-macro",
+ "memchr",
+ "rustpython-parser-vendored",
+]
+
+[[package]]
+name = "rustpython-parser-vendored"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04fcea49a4630a3a5d940f4d514dc4f575ed63c14c3e3ed07146634aed7f67a6"
+dependencies = [
+ "memchr",
+ "once_cell",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.23"
@@ -2619,6 +3083,15 @@ dependencies = [
  "syn 3.0.3",
 ]
 
+[[package]]
+name = "serde_spanned"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26"
+dependencies = [
+ "serde_core",
+]
+
 [[package]]
 name = "serde_urlencoded"
 version = "0.7.1"
@@ -2690,6 +3163,12 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
 
+[[package]]
+name = "siphasher"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
+
 [[package]]
 name = "slab"
 version = "0.4.12"
@@ -2743,7 +3222,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
 
 [[package]]
 name = "statembed"
-version = "0.1.0"
+version = "0.1.1"
 dependencies = [
  "criterion",
  "dirs",
@@ -2753,11 +3232,20 @@ dependencies = [
  "model2vec-rs",
  "serde",
  "serde_json",
- "tokenizers 0.23.1",
+ "tokie",
  "tokio",
  "wide",
 ]
 
+[[package]]
+name = "statembed-py"
+version = "0.1.0"
+dependencies = [
+ "pyo3",
+ "pyo3-stub-gen",
+ "statembed",
+]
+
 [[package]]
 name = "static_assertions"
 version = "1.1.0"
@@ -2869,6 +3357,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
 [[package]]
 name = "tempfile"
 version = "3.27.0"
@@ -2941,6 +3435,25 @@ dependencies = [
  "time-core",
 ]
 
+[[package]]
+name = "tiny-keccak"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
+dependencies = [
+ "crunchy",
+]
+
+[[package]]
+name = "tinystr"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
+dependencies = [
+ "displaydoc",
+ "zerovec 0.10.4",
+]
+
 [[package]]
 name = "tinystr"
 version = "0.8.3"
@@ -2948,7 +3461,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"
 dependencies = [
  "displaydoc",
- "zerovec",
+ "zerovec 0.11.6",
 ]
 
 [[package]]
@@ -3011,36 +3524,25 @@ dependencies = [
 ]
 
 [[package]]
-name = "tokenizers"
-version = "0.23.1"
+name = "tokie"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44e5bea67576e04b6ff8564c5d9e09c2ef0cf476502245f2f120e497769d3112"
+checksum = "dfb5ea1612e3fc71e8bfd1a61a8101d89310d80309fa617273bcaf66624bbcb0"
 dependencies = [
- "ahash",
- "compact_str",
- "daachorse",
- "dary_heap",
- "derive_builder",
- "esaxx-rs",
- "getrandom 0.3.4",
- "itertools 0.14.0",
- "log",
- "macro_rules_attribute",
- "monostate",
- "onig",
- "paste",
- "rand 0.9.5",
- "rayon",
- "rayon-cond",
- "regex",
- "regex-syntax",
+ "base64 0.22.1",
+ "chunk",
+ "crc32fast",
+ "daggrs",
+ "foldhash",
+ "icu_normalizer 1.5.0",
+ "libc",
+ "memchr",
+ "pretokie",
  "serde",
  "serde_json",
- "spm_precompiled",
- "thiserror",
- "unicode-normalization-alignments",
+ "smallvec",
+ "unicode-general-category",
  "unicode-segmentation",
- "unicode_categories",
 ]
 
 [[package]]
@@ -3128,6 +3630,45 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "toml"
+version = "1.1.4+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5"
+dependencies = [
+ "indexmap",
+ "serde_core",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_parser",
+ "toml_writer",
+ "winnow",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "1.1.1+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "toml_parser"
+version = "1.1.3+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56"
+dependencies = [
+ "winnow",
+]
+
+[[package]]
+name = "toml_writer"
+version = "1.1.2+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2"
+
 [[package]]
 name = "tower"
 version = "0.5.3"
@@ -3284,12 +3825,70 @@ version = "1.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6"
 
+[[package]]
+name = "unic-char-property"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221"
+dependencies = [
+ "unic-char-range",
+]
+
+[[package]]
+name = "unic-char-range"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc"
+
+[[package]]
+name = "unic-common"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc"
+
+[[package]]
+name = "unic-emoji-char"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d"
+dependencies = [
+ "unic-char-property",
+ "unic-char-range",
+ "unic-ucd-version",
+]
+
+[[package]]
+name = "unic-ucd-ident"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987"
+dependencies = [
+ "unic-char-property",
+ "unic-char-range",
+ "unic-ucd-version",
+]
+
+[[package]]
+name = "unic-ucd-version"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4"
+dependencies = [
+ "unic-common",
+]
+
 [[package]]
 name = "unicase"
 version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
 
+[[package]]
+name = "unicode-general-category"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"
@@ -3323,6 +3922,28 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
 
+[[package]]
+name = "unicode_names2"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd"
+dependencies = [
+ "phf",
+ "unicode_names2_generator",
+]
+
+[[package]]
+name = "unicode_names2_generator"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e"
+dependencies = [
+ "getopts",
+ "log",
+ "phf_codegen",
+ "rand 0.8.7",
+]
+
 [[package]]
 name = "untrusted"
 version = "0.9.0"
@@ -3366,6 +3987,12 @@ version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
 
+[[package]]
+name = "utf16_iter"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
+
 [[package]]
 name = "utf8_iter"
 version = "1.0.4"
@@ -3908,12 +4535,30 @@ version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
 
+[[package]]
+name = "winnow"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81"
+
 [[package]]
 name = "wit-bindgen"
 version = "0.57.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
 
+[[package]]
+name = "write16"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
+
+[[package]]
+name = "writeable"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
+
 [[package]]
 name = "writeable"
 version = "0.6.3"
@@ -4061,6 +4706,18 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "yoke"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40"
+dependencies = [
+ "serde",
+ "stable_deref_trait",
+ "yoke-derive 0.7.5",
+ "zerofrom",
+]
+
 [[package]]
 name = "yoke"
 version = "0.8.3"
@@ -4068,10 +4725,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
 dependencies = [
  "stable_deref_trait",
- "yoke-derive",
+ "yoke-derive 0.8.2",
  "zerofrom",
 ]
 
+[[package]]
+name = "yoke-derive"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+ "synstructure",
+]
+
 [[package]]
 name = "yoke-derive"
 version = "0.8.2"
@@ -4138,8 +4807,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"
 dependencies = [
  "displaydoc",
- "yoke",
+ "yoke 0.8.3",
+ "zerofrom",
+]
+
+[[package]]
+name = "zerovec"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"
+dependencies = [
+ "yoke 0.7.5",
  "zerofrom",
+ "zerovec-derive 0.10.3",
 ]
 
 [[package]]
@@ -4148,9 +4828,20 @@ version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"
 dependencies = [
- "yoke",
+ "yoke 0.8.3",
  "zerofrom",
- "zerovec-derive",
+ "zerovec-derive 0.11.3",
+]
+
+[[package]]
+name = "zerovec-derive"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 05d1898..24ddbf9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,45 +1,5 @@
-[package]
-name = "statembed"
-version = "0.1.0"
-edition = "2024"
-license = "MIT"
-readme = "README.md"
-repository = "https://github.com/AstraBert/statembed"
-description = "Fast, lightweight static text embeddings for Rust"
-keywords = ["static-embeddings", "retrieval", "rust", "huggingface", "tokenizers"]
-include = [
-    "src/**/*",
-    "Cargo.toml",
-    "README.md",
-    "LICENSE",
+[workspace]
+resolver = "3"
+members = [
+    "crates/*"
 ]
-
-[dependencies]
-memmap2 = { version = "0.9", optional = true }
-half = "2.7"
-serde = { version = "1", features = ["derive"] }
-serde_json = { version = "1", features = ["preserve_order"] }
-tokenizers = { version = "0.23", default-features = false, features = ["onig", "esaxx_fast"], optional = true }
-hf-hub = { version = "1.0.0", optional = true }
-dirs = { version = "6", optional = true }
-wide = { version = "1.6.1", optional = true }
-
-[features]
-default = ["tokenizers", "simd"]
-tokenizers = ["dep:tokenizers"]
-hf-hub = ["dep:hf-hub", "dep:dirs"]
-mmap = ["dep:memmap2"]
-simd = ["dep:wide"]
-
-[dev-dependencies]
-model2vec-rs = "0.2"
-criterion = { version = "0.8", features = ["html_reports"] }
-tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
-
-[[bench]]
-name = "embed_benchmark"
-harness = false
-
-[[bench]]
-name = "m2vec_benchmark"
-harness = false
diff --git a/crates/statembed-py/.gitignore b/crates/statembed-py/.gitignore
new file mode 100644
index 0000000..f8e6ee5
--- /dev/null
+++ b/crates/statembed-py/.gitignore
@@ -0,0 +1,75 @@
+/target
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+.pytest_cache/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+.venv/
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+include/
+man/
+venv/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+pip-selfcheck.json
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+.DS_Store
+
+# Sphinx documentation
+docs/_build/
+
+# PyCharm
+.idea/
+
+# VSCode
+.vscode/
+
+# Pyenv
+.python-version
+
+!src/bin/
+*.pyi
diff --git a/crates/statembed-py/Cargo.toml b/crates/statembed-py/Cargo.toml
new file mode 100644
index 0000000..eb234ed
--- /dev/null
+++ b/crates/statembed-py/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "statembed-py"
+version = "0.1.0"
+edition = "2024"
+readme = "README.md"
+license = "MIT"
+description = "Fast, lightweight static text embeddings for Python"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+name = "statembed_py"
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+pyo3 = "0.29.0"
+pyo3-stub-gen = "0.23.0"
+statembed = { version = "0.1", path = "../statembed", default-features=false, features=["simd"] }
diff --git a/crates/statembed-py/README.md b/crates/statembed-py/README.md
new file mode 100644
index 0000000..fe75f96
--- /dev/null
+++ b/crates/statembed-py/README.md
@@ -0,0 +1,70 @@
+# statembed-py
+
+Fast, lightweight static text embeddings.
+
+> _Python bindings for the [`statembed`](../statembed/README.md) Rust crate, built with [PyO3](https://pyo3.rs)._
+
+`statembed-py` loads a static embedding model (a `model.safetensors` file) and produces mean-pooled, optionally L2-normalized embeddings from token IDs. It does not tokenize text itself — pair it with a tokenizer library such as [`tokenizers`](https://pypi.org/project/tokenizers/).
+
+## Installation
+
+```bash
+# with uv
+uv add statembed-py
+# with pip
+pip install statembed-py
+```
+
+### Building from source
+
+Requires [Rust](https://rustup.rs) and [maturin](https://www.maturin.rs):
+
+```bash
+pip install maturin
+maturin develop --release
+```
+
+## Usage
+
+```python
+from functools import lru_cache
+from tokenizers import Tokenizer
+from statembed_py import StaticEmbedding
+
+@lru_cache(maxsize=1)
+def get_embedding_model() -> StaticEmbedding:
+    return StaticEmbedding(model_dir="./my-model")  # must contain model.safetensors
+
+@lru_cache(maxsize=1)
+def get_tokenizer() -> Tokenizer:
+    return Tokenizer.from_file("./my-model/tokenizer.json")
+
+def embed(text: str) -> list[float]:
+    tokens = get_tokenizer().encode(text).ids
+    embedding = get_embedding_model().embed_tokens(tokens)
+    return embedding
+```
+
+## API
+
+### `StaticEmbedding(model_dir, normalize=True)`
+
+Loads a model from a local directory containing `model.safetensors`. The tensor is loaded lazily on the first `embed_tokens` call.
+
+- `model_dir: str` — path to the model directory.
+- `normalize: bool` — if `True` (default), output embeddings are L2-normalized.
+
+### `embed_tokens(tokens: Sequence[int]) -> list[float]`
+
+Mean-pools the embedding rows for the given token IDs into a single fixed-length vector, applying normalization if enabled.
+
+Type stubs (`statembed_py.pyi`) are bundled for editor and type-checker support.
+
+## Development
+
+- `maturin develop` — build and install the extension into the active virtualenv.
+- `cargo run --bin stub_gen` — regenerate `statembed_py.pyi` after changing the PyO3 bindings.
+
+## License
+
+MIT
diff --git a/crates/statembed-py/jakefile.toml b/crates/statembed-py/jakefile.toml
new file mode 100644
index 0000000..42b0b92
--- /dev/null
+++ b/crates/statembed-py/jakefile.toml
@@ -0,0 +1,3 @@
+dev = "maturin develop"
+stubs = "cargo run --bin stub_gen"
+build = "maturin build --release"
diff --git a/crates/statembed-py/pyproject.toml b/crates/statembed-py/pyproject.toml
new file mode 100644
index 0000000..7c2e3db
--- /dev/null
+++ b/crates/statembed-py/pyproject.toml
@@ -0,0 +1,13 @@
+[build-system]
+requires = ["maturin>=1.14,<2.0"]
+build-backend = "maturin"
+
+[project]
+name = "statembed-py"
+requires-python = ">=3.10"
+classifiers = [
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+]
+dynamic = ["version", "readme", "description", "license"]
diff --git a/crates/statembed-py/src/bin/stub_gen.rs b/crates/statembed-py/src/bin/stub_gen.rs
new file mode 100644
index 0000000..ea668ed
--- /dev/null
+++ b/crates/statembed-py/src/bin/stub_gen.rs
@@ -0,0 +1,8 @@
+use pyo3_stub_gen::Result;
+
+fn main() -> Result<()> {
+    // `stub_info` is a function defined by `define_stub_info_gatherer!` macro.
+    let stub = statembed_py::stub_info()?;
+    stub.generate()?;
+    Ok(())
+}
diff --git a/crates/statembed-py/src/lib.rs b/crates/statembed-py/src/lib.rs
new file mode 100644
index 0000000..ff8aab3
--- /dev/null
+++ b/crates/statembed-py/src/lib.rs
@@ -0,0 +1,54 @@
+use pyo3::prelude::*;
+use pyo3_stub_gen::define_stub_info_gatherer;
+
+/// A Python module implemented in Rust.
+#[pymodule]
+mod statembed_py {
+    use pyo3::{
+        exceptions::{PyRuntimeError, PyValueError},
+        prelude::*,
+    };
+    use pyo3_stub_gen::derive::*;
+    use statembed::StaticEmbedding as CoreEmbedding;
+
+    #[gen_stub_pyclass]
+    #[pyclass(from_py_object)]
+    #[derive(Clone)]
+    struct StaticEmbedding {
+        core: CoreEmbedding,
+    }
+
+    #[gen_stub_pymethods]
+    #[pymethods]
+    impl StaticEmbedding {
+        #[new]
+        #[pyo3(signature = (model_dir, normalize = true))]
+        /// Creates a `StaticEmbedding` from a local directory.
+        ///
+        /// The directory must contain at least `model.safetensors`. When the
+        /// `tokenizers` feature is enabled, `tokenizer.json` is also required.
+        ///
+        /// # Arguments
+        /// * `model_dir` - Path to the model directory.
+        /// * `normalize` - If `True`, output embeddings will be L2-normalized.
+        fn new(model_dir: String, normalize: bool) -> PyResult<Self> {
+            Ok(Self {
+                core: CoreEmbedding::from_dir(&model_dir, Some(normalize))
+                    .map_err(|e| PyValueError::new_err(e.to_string()))?,
+            })
+        }
+
+        /// Generates an embedding for a pre-tokenized sequence of token IDs (use whatever tokenization
+        /// libary you prefer to generate tokens).
+        ///
+        /// The tensor is loaded lazily on first call. Embeddings are mean-pooled
+        /// and optionally normalized.
+        fn embed_tokens(&mut self, tokens: Vec<u32>) -> PyResult<Vec<f32>> {
+            self.core
+                .embed_tokens(tokens)
+                .map_err(|e| PyRuntimeError::new_err(e.to_string()))
+        }
+    }
+}
+
+define_stub_info_gatherer!(stub_info);
diff --git a/crates/statembed/Cargo.toml b/crates/statembed/Cargo.toml
new file mode 100644
index 0000000..5d2ff0c
--- /dev/null
+++ b/crates/statembed/Cargo.toml
@@ -0,0 +1,45 @@
+[package]
+name = "statembed"
+version = "0.1.1"
+edition = "2024"
+license = "MIT"
+readme = "README.md"
+repository = "https://github.com/AstraBert/statembed"
+description = "Fast, lightweight static text embeddings for Rust"
+keywords = ["static-embeddings", "retrieval", "rust", "huggingface", "tokenizers"]
+include = [
+    "src/**/*",
+    "Cargo.toml",
+    "README.md",
+    "LICENSE",
+]
+
+[dependencies]
+memmap2 = { version = "0.9", optional = true }
+half = "2.7"
+serde = { version = "1", features = ["derive"] }
+serde_json = { version = "1", features = ["preserve_order"] }
+tokie = { version = "0.1", optional = true }
+hf-hub = { version = "1.0.0", optional = true }
+dirs = { version = "6", optional = true }
+wide = { version = "1.6.1", optional = true }
+
+[features]
+default = ["tokenizers", "simd"]
+tokenizers = ["dep:tokie"]
+hf-hub = ["dep:hf-hub", "dep:dirs"]
+mmap = ["dep:memmap2"]
+simd = ["dep:wide"]
+
+[dev-dependencies]
+model2vec-rs = "0.2"
+criterion = { version = "0.8", features = ["html_reports"] }
+tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
+
+[[bench]]
+name = "embed_benchmark"
+harness = false
+
+[[bench]]
+name = "m2vec_benchmark"
+harness = false
diff --git a/README.md b/crates/statembed/README.md
similarity index 100%
rename from README.md
rename to crates/statembed/README.md
diff --git a/benches/embed_benchmark.rs b/crates/statembed/benches/embed_benchmark.rs
similarity index 100%
rename from benches/embed_benchmark.rs
rename to crates/statembed/benches/embed_benchmark.rs
diff --git a/benches/m2vec_benchmark.rs b/crates/statembed/benches/m2vec_benchmark.rs
similarity index 100%
rename from benches/m2vec_benchmark.rs
rename to crates/statembed/benches/m2vec_benchmark.rs
diff --git a/src/errors.rs b/crates/statembed/src/errors.rs
similarity index 92%
rename from src/errors.rs
rename to crates/statembed/src/errors.rs
index c7b0d52..355d5a0 100644
--- a/src/errors.rs
+++ b/crates/statembed/src/errors.rs
@@ -44,6 +44,15 @@ pub struct TokenizationError {
     pub cause: String,
 }
 
+#[cfg(feature = "tokenizers")]
+impl From<io::Error> for TokenizationError {
+    fn from(value: io::Error) -> Self {
+        Self {
+            cause: format!("IO Error: {}. Details: {}", value.kind(), value),
+        }
+    }
+}
+
 #[cfg(feature = "tokenizers")]
 impl From<serde_json::Error> for TokenizationError {
     fn from(value: serde_json::Error) -> Self {
diff --git a/src/lib.rs b/crates/statembed/src/lib.rs
similarity index 98%
rename from src/lib.rs
rename to crates/statembed/src/lib.rs
index 327c9b6..8bbe799 100644
--- a/src/lib.rs
+++ b/crates/statembed/src/lib.rs
@@ -21,10 +21,12 @@ use crate::{errors::TokenizationError, tokenize::load_tokenizer};
 use hf_hub::{HFClient, RepoTypeModel};
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
+#[cfg(feature = "tokenizers")]
+use std::sync::Arc;
 #[cfg(feature = "hf-hub")]
 use std::sync::OnceLock;
 #[cfg(feature = "tokenizers")]
-use tokenizers::Tokenizer;
+use tokie::Tokenizer;
 #[cfg(feature = "simd")]
 use wide::f32x8;
 
@@ -217,6 +219,7 @@ fn sequential_mean_pooling(
 /// `StaticEmbedding` lazily loads the underlying tensor and tokenizer on first
 /// use, then caches them for subsequent calls. It supports mean-pooled
 /// embeddings with optional L2 normalization.
+#[derive(Clone)]
 pub struct StaticEmbedding {
     /// Filesystem path to the model directory.
     pub base_path: PathBuf,
@@ -226,7 +229,7 @@ pub struct StaticEmbedding {
     pub normalize: bool,
     tensor: Option<Vec<u8>>,
     #[cfg(feature = "tokenizers")]
-    tokenizer: Option<Tokenizer>,
+    tokenizer: Option<Arc<Tokenizer>>,
     #[cfg(feature = "tokenizers")]
     median_token_length: Option<usize>,
     #[cfg(feature = "tokenizers")]
@@ -348,9 +351,9 @@ impl StaticEmbedding {
     fn load_tokenizer(&mut self) -> Result<(), TokenizationError> {
         use crate::tokenize::extract_tokenizer_details;
 
-        let tokenizer = load_tokenizer(self.base_path.join("tokenizer.json"))?;
-        let (median_length, unk_tok) = extract_tokenizer_details(&tokenizer)?;
-        self.tokenizer = Some(tokenizer);
+        let (tokenizer, unk_tok) = load_tokenizer(self.base_path.join("tokenizer.json"))?;
+        let median_length = extract_tokenizer_details(&tokenizer);
+        self.tokenizer = Some(Arc::new(tokenizer));
         self.median_token_length = Some(median_length);
         self.unknown_token = unk_tok;
         Ok(())
diff --git a/src/load.rs b/crates/statembed/src/load.rs
similarity index 100%
rename from src/load.rs
rename to crates/statembed/src/load.rs
diff --git a/crates/statembed/src/tokenize.rs b/crates/statembed/src/tokenize.rs
new file mode 100644
index 0000000..43de141
--- /dev/null
+++ b/crates/statembed/src/tokenize.rs
@@ -0,0 +1,117 @@
+//! Tokenization utilities for the `statembed` library.
+//!
+//! This module provides helpers for loading `tokie` from JSON files,
+//! encoding text, and extracting vocabulary statistics such as median token length.
+
+use std::path::PathBuf;
+
+use tokie::Tokenizer;
+
+use crate::errors::TokenizationError;
+use serde::Deserialize;
+use std::fs::File;
+use std::io::BufReader;
+
+#[derive(Deserialize)]
+struct TokenizerSpec {
+    model: Option<ModelSpec>,
+}
+
+#[derive(Deserialize)]
+struct ModelSpec {
+    unk_token: Option<String>,
+    // any other fields (vocab, merges, etc.) are simply skipped by serde,
+    // not allocated, since we don't declare them here.
+}
+
+/// Loads a `Tokenizer` from a JSON file on disk.
+///
+/// Returns also the ID of the `unk_token` if one is defined in the model config.
+///
+/// # Arguments
+/// * `path` - Path to the `tokenizer.json` file.
+pub fn load_tokenizer(
+    path: impl Into<PathBuf>,
+) -> Result<(Tokenizer, Option<u32>), TokenizationError> {
+    let p = path.into();
+
+    let file = File::open(&p)?;
+    let reader = BufReader::new(file);
+    let spec: TokenizerSpec = serde_json::from_reader(reader)?;
+
+    let tokenizer = Tokenizer::from_json(&p).map_err(|e| TokenizationError {
+        cause: e.to_string(),
+    })?;
+
+    let unk_token_id = if let Some(tok) = spec.model.and_then(|m| m.unk_token) {
+        let id = tokenizer
+            .token_to_id(&tok)
+            .ok_or_else(|| TokenizationError {
+                cause: format!("unk_token '{tok}' not found in vocabulary"),
+            })?;
+        Some(id)
+    } else {
+        None
+    };
+
+    Ok((tokenizer, unk_token_id))
+}
+
+/// Encodes a string into a vector of token IDs.
+///
+/// # Arguments
+/// * `tk` - The tokenizer to use.
+/// * `text` - The input text to encode.
+/// * `unknown_token` - If provided, token IDs matching this value are filtered out.
+pub fn tokenize(
+    tk: &Tokenizer,
+    text: &str,
+    unknown_token: Option<u32>,
+) -> Result<Vec<u32>, TokenizationError> {
+    let encoding = tk.encode(text, false);
+    let mut ids = encoding.ids;
+    if let Some(unk_id) = unknown_token {
+        ids.retain(|&id| id != unk_id);
+    }
+    Ok(ids)
+}
+
+/// Extracts useful statistics from a tokenizer's vocabulary.
+///
+/// Returns `median_token_length`, i.e. the median length (in bytes) of all vocabulary tokens.
+pub fn extract_tokenizer_details(tk: &Tokenizer) -> usize {
+    let mut lens: Vec<usize> = tk.get_vocab().keys().map(|tk| tk.len()).collect();
+    lens.sort_unstable();
+
+    lens.get(lens.len() / 2).copied().unwrap_or(1)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[cfg(feature = "tokenizers")]
+    fn test_load_tokenizer() {
+        let _ = load_tokenizer("testfiles/tokenizer.json").expect("Should not fail");
+    }
+
+    #[test]
+    #[cfg(feature = "tokenizers")]
+    fn test_extract_tokenizer_details() {
+        let (expected_median_len, expected_unk_token) = (6, Some(1));
+        let (tok, unk_token) =
+            load_tokenizer("testfiles/tokenizer.json").expect("Should load tokenizer");
+        let median_len = extract_tokenizer_details(&tok);
+        assert_eq!(expected_median_len, median_len);
+        assert_eq!(expected_unk_token, unk_token);
+    }
+
+    #[test]
+    #[cfg(feature = "tokenizers")]
+    fn test_tokenize() {
+        let (tok, _) = load_tokenizer("testfiles/tokenizer.json").expect("Should load tokenizer");
+        let tokens = tokenize(&tok, "hello", Some(1)).expect("Should tokenize successfully");
+        assert_eq!(tokens[0], 6598);
+    }
+}
diff --git a/testfiles/model.safetensors b/crates/statembed/testfiles/model.safetensors
similarity index 100%
rename from testfiles/model.safetensors
rename to crates/statembed/testfiles/model.safetensors
diff --git a/testfiles/tokenizer.json b/crates/statembed/testfiles/tokenizer.json
similarity index 100%
rename from testfiles/tokenizer.json
rename to crates/statembed/testfiles/tokenizer.json
diff --git a/tests/integration_test.rs b/crates/statembed/tests/integration_test.rs
similarity index 100%
rename from tests/integration_test.rs
rename to crates/statembed/tests/integration_test.rs
diff --git a/jakefile.toml b/jakefile.toml
index 0aae87c..992cf6f 100644
--- a/jakefile.toml
+++ b/jakefile.toml
@@ -1,8 +1,8 @@
-build="cargo build --release"
-test="cargo test --release"
-test-mmap="cargo test --features mmap --release"
-test-hf="cargo test --features hf-hub --release"
-test-nodef="cargo test --no-default-features --release"
+build="cargo build --release -p statembed"
+test="cargo test --release -p statembed"
+test-mmap="cargo test --features mmap --release -p statembed"
+test-hf="cargo test --features hf-hub --release -p statembed"
+test-nodef="cargo test --no-default-features --release -p statembed"
 clippy = "cargo clippy --all-features"
 clippy-fix = "cargo clippy --all-features --fix --allow-dirty"
 format = "cargo fmt --all -- --check"
diff --git a/src/tokenize.rs b/src/tokenize.rs
deleted file mode 100644
index 1f45f3b..0000000
--- a/src/tokenize.rs
+++ /dev/null
@@ -1,101 +0,0 @@
-//! Tokenization utilities for the `statembed` library.
-//!
-//! This module provides helpers for loading Hugging Face `tokenizers` JSON files,
-//! encoding text, and extracting vocabulary statistics such as median token length.
-
-use std::path::PathBuf;
-
-use tokenizers::Tokenizer;
-
-use crate::errors::TokenizationError;
-
-/// Loads a `Tokenizer` from a JSON file on disk.
-///
-/// # Arguments
-/// * `path` - Path to the `tokenizer.json` file.
-pub fn load_tokenizer(path: impl Into<PathBuf>) -> Result<Tokenizer, TokenizationError> {
-    let tokenizer = Tokenizer::from_file(path.into()).map_err(|e| TokenizationError {
-        cause: e.to_string(),
-    })?;
-    Ok(tokenizer)
-}
-
-/// Encodes a string into a vector of token IDs.
-///
-/// # Arguments
-/// * `tk` - The tokenizer to use.
-/// * `text` - The input text to encode.
-/// * `unknown_token` - If provided, token IDs matching this value are filtered out.
-pub fn tokenize(
-    tk: &Tokenizer,
-    text: &str,
-    unknown_token: Option<u32>,
-) -> Result<Vec<u32>, TokenizationError> {
-    let encoding = tk.encode(text, false).map_err(|e| TokenizationError {
-        cause: format!("Error while encoding text: {}", e),
-    })?;
-    let mut ids = encoding.get_ids().to_vec();
-    if let Some(unk_id) = unknown_token {
-        ids.retain(|&id| id != unk_id);
-    }
-    Ok(ids.to_vec())
-}
-
-/// Extracts useful statistics from a tokenizer's vocabulary.
-///
-/// Returns a tuple of `(median_token_length, unknown_token_id)` where:
-/// * `median_token_length` - The median length (in bytes) of all vocabulary tokens.
-/// * `unknown_token_id` - The ID of the `unk_token` if one is defined in the model config.
-pub fn extract_tokenizer_details(
-    tk: &Tokenizer,
-) -> Result<(usize, Option<u32>), TokenizationError> {
-    let mut lens: Vec<usize> = tk.get_vocab(false).keys().map(|tk| tk.len()).collect();
-    lens.sort_unstable();
-    let median_token_length = lens.get(lens.len() / 2).copied().unwrap_or(1);
-
-    let spec: serde_json::Value = serde_json::to_value(tk)?;
-    let unk_token = spec
-        .get("model")
-        .and_then(|m| m.get("unk_token"))
-        .and_then(serde_json::Value::as_str);
-    let unk_token_id = if let Some(tok) = unk_token {
-        let id = tk.token_to_id(tok).ok_or_else(|| TokenizationError {
-            cause: "unk_token '{tok}' not found in vocabulary".to_string(),
-        })?;
-        Some(id)
-    } else {
-        None
-    };
-
-    Ok((median_token_length, unk_token_id))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    #[cfg(feature = "tokenizers")]
-    fn test_load_tokenizer() {
-        let _ = load_tokenizer("testfiles/tokenizer.json").expect("Should not fail");
-    }
-
-    #[test]
-    #[cfg(feature = "tokenizers")]
-    fn test_extract_tokenizer_details() {
-        let (expected_median_len, expected_unk_token) = (6, Some(1));
-        let tok = load_tokenizer("testfiles/tokenizer.json").expect("Should load tokenizer");
-        let (median_len, unk_token) =
-            extract_tokenizer_details(&tok).expect("Should be able to extract tokenizer details");
-        assert_eq!(expected_median_len, median_len);
-        assert_eq!(expected_unk_token, unk_token);
-    }
-
-    #[test]
-    #[cfg(feature = "tokenizers")]
-    fn test_tokenize() {
-        let tok = load_tokenizer("testfiles/tokenizer.json").expect("Should load tokenizer");
-        let tokens = tokenize(&tok, "hello", Some(1)).expect("Should tokenize successfully");
-        assert_eq!(tokens[0], 6598);
-    }
-}