diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 965c3a3..01bd080 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -19,6 +19,7 @@ jobs:
with:
components: rustfmt, clippy
- run: cargo fmt --all --check
+ - run: python3 scripts/check-release-versions.py
- run: cargo clippy --workspace --all-targets -- -D warnings
- run: cargo doc --workspace --no-deps --locked
env:
@@ -84,8 +85,8 @@ jobs:
- uses: actions/setup-python@v6
with:
python-version: "3.12"
- # CPU-only development package: no ML dependencies or model files.
- - run: python -m pip install setuptools==78.1.0
+ # Installed CPU/mock contracts only; no weights, PyTorch or inference.
+ - run: python -m pip install setuptools==78.1.0 numpy==2.5.3
- run: python -m pip install --no-deps --no-build-isolation ./packages/tokenfold-select
- run: python -I -m unittest discover -s "${{ github.workspace }}/packages/tokenfold-select/tests" -v
working-directory: ${{ runner.temp }}
diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml
index d3964da..61ec52c 100644
--- a/.github/workflows/publish-packages.yml
+++ b/.github/workflows/publish-packages.yml
@@ -6,7 +6,7 @@ on:
version:
description: Existing GitHub release version to publish
required: true
- default: 0.5.1
+ default: 0.5.2
permissions:
contents: read
@@ -35,6 +35,7 @@ jobs:
npm_version=$(node -p "require('./packages/tokenfold/package.json').version")
test "$VERSION" = "$cargo_version"
test "$VERSION" = "$npm_version"
+ python3 scripts/check-release-versions.py
test "v$VERSION" = "$(git describe --tags --exact-match)"
- run: cargo publish -p tokenfold-core --dry-run --locked
- name: Validate registry packages from the locked source tree
@@ -143,8 +144,29 @@ jobs:
name: python-sdist
path: dist/*
+ select-python:
+ needs: validate
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+ with:
+ ref: v${{ inputs.version }}
+ - uses: actions/setup-python@v6
+ with:
+ python-version: "3.12"
+ - run: python -m pip install build twine numpy==2.5.3
+ - run: python -m build --outdir dist-select packages/tokenfold-select
+ - run: python -m twine check dist-select/*
+ - run: python -m pip install --no-deps dist-select/*.whl
+ - run: python -I -m unittest discover -s "${{ github.workspace }}/packages/tokenfold-select/tests" -v
+ working-directory: ${{ runner.temp }}
+ - uses: actions/upload-artifact@v7
+ with:
+ name: python-select
+ path: dist-select/*
+
publish-python:
- needs: [publish-crates, python-wheels, python-sdist]
+ needs: [publish-crates, python-wheels, python-sdist, select-python]
runs-on: ubuntu-latest
permissions:
id-token: write
@@ -245,7 +267,7 @@ jobs:
! readelf -l "$binary" | grep -q INTERP
release-ready:
- needs: [validate, python-wheels, python-sdist, npm-smoke]
+ needs: [validate, python-wheels, python-sdist, select-python, npm-smoke]
runs-on: ubuntu-latest
steps:
- run: echo "All registry artifacts passed their pre-publication checks."
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1433919..267df01 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,23 @@
## [Unreleased]
+## [0.5.2] - 2026-10-17
+
+- Align Cargo, Python binding, npm and five platform packages at 0.5.2; add the
+ optional `tokenfold-select` wheel/sdist to the existing publication gate.
+- Default the optional Select runtime worker to the pinned Select v2 FP32 ONNX
+ CPU backend. Add integrity-only inspection and explicit approval-config setup;
+ preserve the legacy PyTorch development backend as an explicit option.
+ No weights or automatic downloads are bundled. Core defaults and experimental
+ Select/approval/fallback boundaries remain unchanged. ONNX task quality and
+ cold/warm performance are not yet qualified.
+- Prepare the separate Select v2 model distribution in both ONNX and Safetensors
+ formats, with custom-loader instructions; retain ONNX as the worker default.
+- Update the offline quickstart notebook for 0.5.2, with a disabled-by-default
+ Select v2 ONNX example pinned to the published model commit and Safetensors
+ guidance; keep legacy v1 results explicitly separate from v2 evidence.
+- Check registry-version consistency in CI and before publication.
+
- Explicitly release the persistent observation ledger's exclusive lock when its
owner drops, so a briefly inherited Unix descriptor cannot delay restart.
Keep live-owner refusal and snapshot safety unchanged; add a Unix regression.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7836a17..9282e5b 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -51,6 +51,11 @@ same ones.
Release commits go through a PR like everything else.
+Also align `packages/tokenfold-select/pyproject.toml` and its package version;
+`python scripts/check-release-versions.py` checks the registry version set. The
+optional Select wheel/sdist is published alongside the `tokenfold` binding.
+Model weights remain a separate, explicitly approved Hugging Face release.
+
```sh
git switch main && git pull
git switch -c release/v0.4.1
diff --git a/Cargo.lock b/Cargo.lock
index 37a0baf..188f318 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1109,7 +1109,7 @@ dependencies = [
[[package]]
name = "tokenfold-adapters"
-version = "0.5.1"
+version = "0.5.2"
dependencies = [
"serde",
"serde_json",
@@ -1118,7 +1118,7 @@ dependencies = [
[[package]]
name = "tokenfold-admin"
-version = "0.5.1"
+version = "0.5.2"
dependencies = [
"ed25519-dalek",
"serde",
@@ -1128,7 +1128,7 @@ dependencies = [
[[package]]
name = "tokenfold-cli"
-version = "0.5.1"
+version = "0.5.2"
dependencies = [
"clap",
"serde",
@@ -1143,7 +1143,7 @@ dependencies = [
[[package]]
name = "tokenfold-core"
-version = "0.5.1"
+version = "0.5.2"
dependencies = [
"proptest",
"regex",
@@ -1158,25 +1158,25 @@ dependencies = [
[[package]]
name = "tokenfold-image"
-version = "0.5.1"
+version = "0.5.2"
[[package]]
name = "tokenfold-learn"
-version = "0.5.1"
+version = "0.5.2"
dependencies = [
"tokenfold-core",
]
[[package]]
name = "tokenfold-output"
-version = "0.5.1"
+version = "0.5.2"
dependencies = [
"tokenfold-core",
]
[[package]]
name = "tokenfold-proxy"
-version = "0.5.1"
+version = "0.5.2"
dependencies = [
"clap",
"ctrlc",
@@ -1190,7 +1190,7 @@ dependencies = [
[[package]]
name = "tokenfold-py"
-version = "0.5.1"
+version = "0.5.2"
dependencies = [
"pyo3",
"serde_json",
@@ -1199,7 +1199,7 @@ dependencies = [
[[package]]
name = "tokenfold-rag"
-version = "0.5.1"
+version = "0.5.2"
dependencies = [
"tokenfold-core",
]
diff --git a/Cargo.toml b/Cargo.toml
index 1e59ab0..e10b1de 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -14,7 +14,7 @@ members = [
]
[workspace.package]
-version = "0.5.1"
+version = "0.5.2"
edition = "2024"
rust-version = "1.98"
license = "Apache-2.0"
diff --git a/README.md b/README.md
index 2127519..d6dd9a2 100644
--- a/README.md
+++ b/README.md
@@ -339,7 +339,53 @@ Tokenfold solves a different problem first: repeated keys, values, schemas, rows
**When structure ends, rank what matters.**
-Tokenfold Select is an Apache-2.0 LoRA adapter on
+**Select v2 is the default scorer backend in the optional `tokenfold-select`
+0.5.2 package: local FP32 ONNX inference, original text, whole-group ranking.**
+Provision the separate pinned model artifact, generate explicit runtime approval,
+and invoke `tokenfold --experimental select`. Core remains model-free and
+deterministic; installing or upgrading Tokenfold does not silently load a model
+or enable lossy selection. See the [Select v2 setup and limits](packages/tokenfold-select/README.md).
+The separate model bundle also carries Safetensors weights and a custom reference
+loader; ONNX remains the CLI worker default. V2 is an experimental preview:
+[Both formats are published under Apache-2.0](https://huggingface.co/snchimata/tokenfold-select-v2).
+Synthetic CPU parity is verified, but task-quality and cold/warm performance
+qualification are still open. The [example notebook](examples/quickstart.ipynb#10-tokenfold-select-v2-optional-local-ranking)
+keeps model execution off until you explicitly enable it.
+
+
+Python: try Tokenfold Select v2 (ONNX)
+
+Install `tokenfold-select[onnx]==0.5.2` and `huggingface_hub==1.33.0`, then download
+only the ONNX profile at the pinned release commit:
+
+```python
+from pathlib import Path
+from huggingface_hub import snapshot_download
+from tokenfold_select import OnnxScorer
+
+root = Path(snapshot_download(
+ "snchimata/tokenfold-select-v2",
+ revision="78736d9afea819322f212d6d53eedcf68ac7a39b",
+ allow_patterns=["model.onnx", "model.onnx.data", "onnx-manifest.json",
+ "tokenizer/*", "runtime/*", "LICENSE", "NOTICE.md"],
+)).resolve()
+scorer = OnnxScorer(root, development=True)
+response = scorer.score({
+ "schema_version": 1, "model_revision": scorer.revision,
+ "query": "Which delivery city?",
+ "groups": [{"id": "delivery", "text": "Delivery city: Paris"}],
+})
+print(response) # scores only; your controller owns protection and allocation
+```
+
+For Safetensors, use the model card's custom loader rather than a stock
+Transformers classification class. Neither format silently changes Core behavior.
+
+
+
+### Tokenfold Select v1 (legacy model)
+
+The separately published Select v1 is an Apache-2.0 LoRA adapter on
`ibm-granite/granite-embedding-reranker-english-r2`. It scores candidate spans
against a query; your allocator applies the token budget and force-keeps
required content. It is a separately distributed offering. Core remains
@@ -352,7 +398,7 @@ model-free and deterministic and neither loads nor invokes Select.
| Output | Compressed payload + exact receipt | Ranking logits |
-Python: load the model and score spans
+Legacy v1 Python: load the LoRA adapter and score spans
```python
from pathlib import Path
@@ -393,7 +439,7 @@ def score(query: str, spans: list[str]) -> list[float]:
-### Tokenfold Select benchmarks
+### Tokenfold Select v1 benchmarks (not v2 evidence)
[Tokenfold Select][tokenfold-select] is Tokenfold's query-aware model for choosing
what fills a tight context window after structural compression ends.
diff --git a/crates/tokenfold-adapters/Cargo.toml b/crates/tokenfold-adapters/Cargo.toml
index 8a189a5..1fb0ce7 100644
--- a/crates/tokenfold-adapters/Cargo.toml
+++ b/crates/tokenfold-adapters/Cargo.toml
@@ -9,6 +9,6 @@ description = "Explicit, model-free provider observation adapters for Tokenfold.
readme = "README.md"
[dependencies]
-tokenfold-core = { version = "0.5.1", path = "../tokenfold-core" }
+tokenfold-core = { version = "0.5.2", path = "../tokenfold-core" }
serde = { version = "1", features = ["derive"] }
serde_json = { version = "1", features = ["preserve_order"] }
diff --git a/crates/tokenfold-cli/Cargo.toml b/crates/tokenfold-cli/Cargo.toml
index 48bff0a..f75885e 100644
--- a/crates/tokenfold-cli/Cargo.toml
+++ b/crates/tokenfold-cli/Cargo.toml
@@ -14,11 +14,11 @@ name = "tokenfold"
path = "src/main.rs"
[dependencies]
-tokenfold-adapters = { version = "0.5.1", path = "../tokenfold-adapters" }
-tokenfold-core = { version = "0.5.1", path = "../tokenfold-core" }
-tokenfold-output = { version = "0.5.1", path = "../tokenfold-output" }
-tokenfold-learn = { version = "0.5.1", path = "../tokenfold-learn" }
-tokenfold-rag = { version = "0.5.1", path = "../tokenfold-rag" }
+tokenfold-adapters = { version = "0.5.2", path = "../tokenfold-adapters" }
+tokenfold-core = { version = "0.5.2", path = "../tokenfold-core" }
+tokenfold-output = { version = "0.5.2", path = "../tokenfold-output" }
+tokenfold-learn = { version = "0.5.2", path = "../tokenfold-learn" }
+tokenfold-rag = { version = "0.5.2", path = "../tokenfold-rag" }
clap = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
diff --git a/crates/tokenfold-learn/Cargo.toml b/crates/tokenfold-learn/Cargo.toml
index 8e332e8..f1ec905 100644
--- a/crates/tokenfold-learn/Cargo.toml
+++ b/crates/tokenfold-learn/Cargo.toml
@@ -10,4 +10,4 @@ keywords = ["llm", "tokens", "compression", "policy", "token-optimization"]
categories = ["compression"]
[dependencies]
-tokenfold-core = { version = "0.5.1", path = "../tokenfold-core" }
+tokenfold-core = { version = "0.5.2", path = "../tokenfold-core" }
diff --git a/crates/tokenfold-output/Cargo.toml b/crates/tokenfold-output/Cargo.toml
index 83c8f31..4544ddd 100644
--- a/crates/tokenfold-output/Cargo.toml
+++ b/crates/tokenfold-output/Cargo.toml
@@ -10,4 +10,4 @@ keywords = ["llm", "tokens", "compression", "reports", "token-optimization"]
categories = ["compression"]
[dependencies]
-tokenfold-core = { version = "0.5.1", path = "../tokenfold-core" }
+tokenfold-core = { version = "0.5.2", path = "../tokenfold-core" }
diff --git a/crates/tokenfold-py/Cargo.toml b/crates/tokenfold-py/Cargo.toml
index 1a1a163..0e867f0 100644
--- a/crates/tokenfold-py/Cargo.toml
+++ b/crates/tokenfold-py/Cargo.toml
@@ -16,7 +16,7 @@ name = "tokenfold_py"
crate-type = ["cdylib", "rlib"]
[dependencies]
-tokenfold-core = { version = "0.5.1", path = "../tokenfold-core" }
+tokenfold-core = { version = "0.5.2", path = "../tokenfold-core" }
serde_json = { workspace = true }
pyo3 = { version = "0.29.0", features = ["abi3-py39"] }
diff --git a/crates/tokenfold-rag/Cargo.toml b/crates/tokenfold-rag/Cargo.toml
index 1d7cfe9..a857fbb 100644
--- a/crates/tokenfold-rag/Cargo.toml
+++ b/crates/tokenfold-rag/Cargo.toml
@@ -9,4 +9,4 @@ description = "Bounded BM25 search over authorized Tokenfold evidence."
readme = "README.md"
[dependencies]
-tokenfold-core = { version = "0.5.1", path = "../tokenfold-core" }
+tokenfold-core = { version = "0.5.2", path = "../tokenfold-core" }
diff --git a/examples/quickstart.ipynb b/examples/quickstart.ipynb
index 7c9dcd7..bec7452 100644
--- a/examples/quickstart.ipynb
+++ b/examples/quickstart.ipynb
@@ -11,13 +11,13 @@
"changed. **Lossless compression is the default**; recoverable lossy pruning happens only\n",
"when you explicitly request it.\n",
"\n",
- "This tour runs entirely offline against the fixtures next to this notebook: no network\n",
+ "The default tour runs entirely offline against the fixtures next to this notebook: no network\n",
"call, no API key, and no provider request. It runs in seconds once the kernel is up.\n",
"\n",
"## Setup\n",
"\n",
"```bash\n",
- "pip install \"tokenfold>=0.5.0\"\n",
+ "pip install \"tokenfold==0.5.2\"\n",
"pip install jupyter # or open this notebook in a Jupyter environment you already have\n",
"```\n",
"\n",
@@ -38,7 +38,8 @@
"| Emit verified TOON output | `compress(..., encoding=Encoding.TOON)` |\n",
"| Enforce a token target | `require_target=True` + `BudgetUnmetError` |\n",
"| Project recoverable pruning | `PruningPolicy` + `inspect` |\n",
- "| Read typed receipts | `CompressionResult` / `CompressionReport` |\n"
+ "| Read typed receipts | `CompressionResult` / `CompressionReport` |\n",
+ "| Optional local query-aware ranking | Select v2 ONNX (section 10; opt-in download) |\n"
]
},
{
@@ -603,54 +604,56 @@
"id": "select",
"metadata": {},
"source": [
- "## 10. Tokenfold Select: optional fine-tuned ranking\n",
+ "## 10. Tokenfold Select v2: optional local ranking\n",
"\n",
- "**When structural compression ends and the remaining context still exceeds the budget, the\n",
- "question stops being \"what folds?\" and becomes \"what matters for this query?\"**\n",
+ "**When structural compression ends, rank what matters for the query.**\n",
"\n",
- "[Tokenfold Select](https://huggingface.co/snchimata/tokenfold-select) is an Apache-2.0\n",
- "LoRA adapter on\n",
- "[`ibm-granite/granite-embedding-reranker-english-r2`](https://huggingface.co/ibm-granite/granite-embedding-reranker-english-r2).\n",
- "It jointly scores each `(query, span)` pair and returns one ranking logit per span. It is a\n",
- "**ranker** — not a compressor, not an allocator, and not a safety filter — and its logits\n",
- "are not probabilities.\n",
+ "[Tokenfold Select v2](https://huggingface.co/snchimata/tokenfold-select-v2) is a\n",
+ "trained 98.9M-parameter joint-group encoder: one bounded forward scores the admitted\n",
+ "original text groups together. It returns ranking scores, not generated text,\n",
+ "probabilities or a budget-enforced compressed document.\n",
"\n",
- "| | Tokenfold Core | Tokenfold Select |\n",
+ "| | Tokenfold Core | Tokenfold Select v2 |\n",
"| --- | --- | --- |\n",
- "| Best at | Deterministic structural compression | Query-conditioned span ranking |\n",
- "| Runtime | Rust core / Python binding | Granite reranker plus LoRA adapter |\n",
- "| Output | Compressed payload plus exact receipt | One ranking logit per span |\n",
- "\n",
- "The intended composition is a pipeline: run deterministic Core compression first, segment\n",
- "what remains into candidate spans, use Select to order the eligible spans, then let a\n",
- "separate allocator force-keep required content and enforce the exact token budget.\n",
- "\n",
- "### Published held-out results\n",
- "\n",
- "Task success is the fraction of fixtures whose literal gold-answer span survives the\n",
- "matched allocator budget. Values are means over three stratified repeated-subsampling\n",
- "runs, with roughly 24,000 held-out fixtures per run.\n",
- "\n",
- "| Token budget kept | Tokenfold Select | BM25 | Relative lift over BM25 |\n",
- "| ---: | ---: | ---: | ---: |\n",
- "| 50% | 86.3% | 79.4% | 8.7% |\n",
- "| 25% | 70.3% | 60.7% | 15.8% |\n",
- "| 10% | 39.9% | 37.7% | 5.8% |\n",
- "\n",
- "### Limitations\n",
- "\n",
- "- Select is a separate PyTorch/Transformers runtime. It is **not** a function exported by\n",
- " the `tokenfold` Python package, and the adapter repository does not contain the Granite\n",
- " base weights — that is a second download.\n",
- "- Evaluation is English-centric, and each `(query, span)` pair is truncated at 8,192 tokens.\n",
- "- Logits are uncalibrated: compare them within one query, never across queries.\n",
- "- Ranking alone guarantees nothing about required or safety-critical text surviving. Your\n",
- " allocator must force-keep it and enforce the provider budget.\n",
- "\n",
- "The cell below is **disabled by default** and imports and downloads nothing until you\n",
- "change the flag yourself. See the\n",
- "[model card](https://huggingface.co/snchimata/tokenfold-select) for the full training,\n",
- "evaluation, and licensing details."
+ "| Best at | Deterministic structural compression | Query-conditioned whole-group ranking |\n",
+ "| Runtime | Rust core / Python binding | FP32 ONNX CPU by default |\n",
+ "| Output | Compressed payload plus receipt | Ordered group IDs and scores |\n",
+ "\n",
+ "The optional `tokenfold-select` 0.5.2 package uses ONNX v2 for its approved worker.\n",
+ "Core does **not** enable a model when installed or upgraded. The CLI still requires\n",
+ "`--experimental`, explicit runtime approval and a caller-owned model directory;\n",
+ "see the [Select setup](../packages/tokenfold-select/README.md).\n",
+ "\n",
+ "### Enable only when you want to download and run the model\n",
+ "\n",
+ "Use Python 3.12 and separately install the optional dependencies:\n",
+ "\n",
+ "```bash\n",
+ "pip install \"tokenfold-select[onnx]==0.5.2\" \"huggingface_hub==1.33.0\"\n",
+ "```\n",
+ "\n",
+ "The cell below is **off by default**: no model imports, downloads or inference.\n",
+ "When enabled, it downloads only the ONNX profile at a pinned full Hub commit and\n",
+ "scores three short groups from the bundled example fixture. No hosted API is used.\n",
+ "The model reference admits at most 8,192 tokens and 512 complete groups; overflow\n",
+ "is refused, never silently truncated or windowed. Your process owns its deadline.\n",
+ "\n",
+ "### Safetensors alternative and evidence\n",
+ "\n",
+ "The same model repository includes `model.safetensors`, a custom joint-group\n",
+ "architecture/CPU reference loader and its pinned manifest. Follow the\n",
+ "[model card's Safetensors instructions](https://huggingface.co/snchimata/tokenfold-select-v2#run-tokenfold-select-v2-with-safetensors):\n",
+ "this is **not** a stock `AutoModelForSequenceClassification` or v1 LoRA checkpoint.\n",
+ "ONNX remains the CLI worker default.\n",
+ "\n",
+ "Both formats passed synthetic CPU conversion/loader checks. Safetensors matches\n",
+ "all 76 source tensors bit-for-bit; this is **not** representative quality,\n",
+ "cold/warm speed or full-8k execution qualification. Published Select v1 held-out\n",
+ "results remain [legacy v1 evidence](https://huggingface.co/snchimata/tokenfold-select),\n",
+ "not v2 results. V2 is an opt-in experimental preview.\n",
+ "\n",
+ "Your controller must enforce required content, redaction, source order and the\n",
+ "final token budget. Sorting scores alone supplies none of those guarantees.\n"
]
},
{
@@ -667,67 +670,47 @@
},
"outputs": [],
"source": [
- "RUN_TOKENFOLD_SELECT = False # set to True to download the model weights and score locally\n",
+ "RUN_TOKENFOLD_SELECT = False # explicit opt-in: downloads ~400 MB, then local CPU inference\n",
+ "SELECT_MODEL_REVISION = \"78736d9afea819322f212d6d53eedcf68ac7a39b\"\n",
"\n",
"if not RUN_TOKENFOLD_SELECT:\n",
- " print(\"Tokenfold Select is off by default: nothing imported, nothing downloaded.\")\n",
- " print(\"To enable: pip install torch transformers peft huggingface_hub\")\n",
- " print(\"Model card: https://huggingface.co/snchimata/tokenfold-select\")\n",
+ " print(\"Tokenfold Select v2 is off: no model imports, downloads or inference.\")\n",
+ " print('To enable: pip install \"tokenfold-select[onnx]==0.5.2\" \"huggingface_hub==1.33.0\"')\n",
+ " print(\"Model card: https://huggingface.co/snchimata/tokenfold-select-v2\")\n",
"else:\n",
" import math\n",
- "\n",
- " import torch\n",
+ " import os\n",
+ " os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"-1\"\n",
" from huggingface_hub import snapshot_download\n",
- " from peft import PeftModel\n",
- " from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
+ " from tokenfold_select import OnnxScorer\n",
"\n",
- " # Three short spans straight from the section-8 fixture, loaded here so this cell\n",
- " # does not depend on earlier notebook state.\n",
" feed = json.loads((EXAMPLES / \"incident_feed.json\").read_text(encoding=\"utf-8\"))\n",
" events = feed[\"events\"]\n",
- " query = \"why did the object-store writer fail?\"\n",
- " spans = [\n",
- " event[\"detail\"]\n",
- " for event in (\n",
- " next(e for e in events if e[\"event\"] == \"incident\"),\n",
- " events[0],\n",
- " events[1],\n",
- " )\n",
+ " groups = [\n",
+ " {\"id\": str(i), \"text\": event[\"detail\"]}\n",
+ " for i, event in enumerate((\n",
+ " next(e for e in events if e[\"event\"] == \"incident\"), events[0], events[1],\n",
+ " ))\n",
" ]\n",
- "\n",
- " repo_dir = Path(snapshot_download(\"snchimata/tokenfold-select\"))\n",
- " adapter_dir = repo_dir / \"adapter\"\n",
- " tokenizer = AutoTokenizer.from_pretrained(adapter_dir)\n",
- " base = AutoModelForSequenceClassification.from_pretrained(\n",
- " \"ibm-granite/granite-embedding-reranker-english-r2\",\n",
- " dtype=torch.float32,\n",
- " )\n",
- " model = PeftModel.from_pretrained(base, adapter_dir).eval()\n",
- "\n",
- " encoded = tokenizer(\n",
- " [query] * len(spans),\n",
- " spans,\n",
- " padding=True,\n",
- " truncation=True,\n",
- " max_length=8192,\n",
- " return_tensors=\"pt\",\n",
- " )\n",
- " with torch.no_grad():\n",
- " logits = model(\n",
- " input_ids=encoded[\"input_ids\"],\n",
- " attention_mask=encoded[\"attention_mask\"],\n",
- " ).logits\n",
- " scores = logits.view(-1).float().tolist()\n",
- "\n",
- " # One finite score per span is the contract; exact values and order are not.\n",
- " assert len(scores) == len(spans)\n",
- " assert all(math.isfinite(score) for score in scores)\n",
- "\n",
- " print(f\"query: {query}\\n\")\n",
- " for score, span in sorted(zip(scores, spans), key=lambda pair: -pair[0]):\n",
- " print(f\"{score:8.3f} {span[:88]}...\")\n",
- " print(\"\\nSorting is not allocation: a real consumer must force-keep required spans\")\n",
- " print(\"and enforce the provider token budget separately.\")"
+ " repo_dir = Path(snapshot_download(\n",
+ " \"snchimata/tokenfold-select-v2\",\n",
+ " revision=SELECT_MODEL_REVISION,\n",
+ " allow_patterns=[\n",
+ " \"model.onnx\", \"model.onnx.data\", \"onnx-manifest.json\",\n",
+ " \"tokenizer/*\", \"runtime/*\", \"LICENSE\", \"NOTICE.md\",\n",
+ " ],\n",
+ " )).resolve()\n",
+ " scorer = OnnxScorer(repo_dir, development=True)\n",
+ " response = scorer.score({\n",
+ " \"schema_version\": 1,\n",
+ " \"model_revision\": scorer.revision,\n",
+ " \"query\": \"why did the object-store writer fail?\",\n",
+ " \"groups\": groups,\n",
+ " })\n",
+ " assert [item[0] for item in response[\"scores\"]] == [g[\"id\"] for g in groups]\n",
+ " assert all(math.isfinite(score) for _, score in response[\"scores\"])\n",
+ " print(\"Ordered group scores:\", response[\"scores\"])\n",
+ " print(\"Ranking is not allocation: force-keep required content and enforce the budget separately.\")\n"
]
},
{
@@ -748,7 +731,7 @@
"| Node application | [`quickstart.mjs`](quickstart.mjs) |\n",
"| Native embedding | `tokenfold-core` and [`quickstart.rs`](quickstart.rs) |\n",
"| Deeper pruning behavior | [`lossy_pruning.py`](lossy_pruning.py) |\n",
- "| Query-aware learned ranking | Section 9 and the [Tokenfold Select model card](https://huggingface.co/snchimata/tokenfold-select) |\n",
+ "| Query-aware learned ranking | Section 10 and the [Tokenfold Select v2 model card](https://huggingface.co/snchimata/tokenfold-select-v2) |\n",
"\n",
"### Which path is yours?\n",
"\n",
diff --git a/packages/tokenfold-select/README.md b/packages/tokenfold-select/README.md
index e89544e..93ab208 100644
--- a/packages/tokenfold-select/README.md
+++ b/packages/tokenfold-select/README.md
@@ -1,79 +1,117 @@
-# Tokenfold Select development scorer
+# Tokenfold Select v2
-**Private development prototype, not a released or qualified Select v2 preview.**
-No weights, datasets, hosted inference or model downloads are included. Nothing
-changes Tokenfold's production defaults. Artifact inspection and protocol helpers
-use only Python's standard library; actual scoring requires the separately
-provisioned, pinned local ML environment recorded in the clean model recipe.
+**Smaller contexts. Original words. Local ONNX scoring.**
-The package contains exact copies of the trained model's canonical joint encoder
-and source-context packer. Their content hashes must match the caller-provisioned
-clean compact12 artifact. No inference code imports a research checkout, ignored
-training directory or developer-local path.
+The optional `tokenfold-select` package defaults to the **Select v2 FP32 ONNX CPU
+scorer**. It ranks whole caller-declared groups jointly in one bounded encoder
+window; Tokenfold's existing allocator keeps required content, original order
+and the final token-budget check. It never generates replacement text.
-## Inspect without loading a model
+This is an **experimental preview**, not a quality- or performance-qualified
+production model. Core compression stays model-free and deterministic. Installing
+this package does not enable selection, download weights, send data to a service,
+or change the `tokenfold` Python/TypeScript compression APIs.
+
+## Install and inspect
+
+```sh
+python -m pip install 'tokenfold-select[onnx]==0.5.2'
+python -m tokenfold_select inspect-onnx --artifact /absolute/select-v2-model
+```
+
+Provision the separate ONNX model directory before use. It must contain
+`model.onnx`, `model.onnx.data`, the pinned tokenizer, runtime reference files and
+`onnx-manifest.json`. The default manifest hash is
+`267313718e80aa91a56923dea30363fd5c24fde3782c716d6d048891860a2054`;
+model revision is the source checkpoint hash
+`4dbdb5071e7d6316d8eb70dc161b1f2f77f555012f6616d7fa34311188e24cec`.
+The package verifies every computational file before inference. Do not substitute
+an untrusted manifest pin. Artifact inspection requires no inference dependencies.
+
+## Use with the CLI
+
+Create an existing, private scratch directory. Explicitly approve your native
+Python executable, installed package/dependency directory and model files:
```sh
-python -m tokenfold_select inspect --artifact /absolute/model-directory --revision CHECKPOINT_SHA256
+python -m tokenfold_select configure \
+ --artifact /absolute/select-v2-model \
+ --site-packages /absolute/venv/lib/python3.12/site-packages \
+ --scratch-root /absolute/private-scratch \
+ --approved-by your-name > select-scorer.json
+
+tokenfold --experimental select context.json --query 'find the relevant item' \
+ --target-tokens 256 --scorer-config select-scorer.json \
+ --inference-timeout-ms 30000
```
-This verifies bounded metadata/file integrity and a declared completed, support-
-passing development decision. It does **not** validate checkpoint tensors,
-redistribution rights, task quality, latency, release approval or the local GPU.
-Failed/unfinished student artifacts are refused.
+Use actual absolute paths for your platform (Windows uses `Lib/site-packages`).
+`configure` checks model-file integrity and emits the existing approved-scorer
+JSON; it performs no inference. Inspect the file before use. The command pins the
+**native base interpreter**, not a Windows venv shim; the isolated bootstrap loads
+this installed package and the explicitly provisioned site-packages, without
+`.pth` processing. Protect auxiliary code, libraries, model files and scratch
+storage. This is not an OS sandbox.
-## Existing direct-child protocol
+A minimal context document is:
-Use the existing experimental Select approved-scorer configuration. The executable
-must be the **native** Python interpreter, not a Windows venv shim or shell. Its
-arguments are:
+```json
+{"prefix":"","suffix":"","groups":[{"id":"required","text":"Keep this.\n","required":true},{"id":"optional","text":"Other context.\n","fallback_score":0.1}]}
+```
-```text
--I -S /installed/tokenfold_select/bootstrap.py
---site-packages /absolute/provisioned/site-packages
-worker --artifact /absolute/model-directory --revision CHECKPOINT_SHA256 --development
+The worker defaults to `--backend onnx`, explicitly selects
+`CPUExecutionProvider`, disables provider fallback and hides GPUs before loading
+inference libraries. It uses bounded file IPC, no network or descendants, and an
+owned direct-child deadline. A receipt with `used_scorer: false` means fallback,
+not successful model-backed selection. Missing approval retains the existing
+caller-declared deterministic fallback; there is no implicit model execution.
+
+## Resident API
+
+```python
+from tokenfold_select import OnnxScorer
+
+scorer = OnnxScorer('/absolute/select-v2-model', development=True)
+request = {
+ 'schema_version': 1,
+ 'model_revision': scorer.revision,
+ 'query': 'find the relevant item',
+ 'groups': [{'id': 'item', 'text': 'Original context.'}],
+}
+response = scorer.score(request) # finite, id-aligned scores only
```
-The caller supplies the existing `TOKENFOLD_SELECT_REQUEST_PATH` and
-`TOKENFOLD_SELECT_RESPONSE_PATH`, an executable SHA256, model revision, scratch
-root, explicit approval and a maximum 30-second child deadline. Protect auxiliary
-code/model files and scratch storage as required by the existing Select adapter.
-This is not an OS sandbox. The worker forbids network and descendants, emits only
-finite id-aligned scores, refuses rather than truncates, and leaves renderer,
-redaction, protected content, budgets and declared fallback to the existing
-Tokenfold CLI. Never interpret fallback as a successful model-backed result.
-
-The direct-child prototype is **FP32 only**. It does not enable an unqualified
-FP16 cold profile. Actual native CUDA integration/parity and timing remain to be
-verified before distribution.
-
-## Dedicated-process resident API
-
-`DevelopmentScorer(artifact, revision, cache_directory=absolute_path,
-development=True, precision="fp32")` lazily admits a request and then loads the
-model. `score(request)` uses the same schema-version-one query/id/text packet and
-returns its standard score response. Model/tokenizer objects, not model results,
-are retained. The cache must be outside the immutable artifact directory.
-
-An explicit `precision="fp16"` casts the verified FP32 source model on the tested
-RTX 5080/capability-12.0 environment only. No CPU/dtype fallback is attempted.
-This experimental choice has not passed a complete launch comparison; it is not
-a quality or speed claim. Use a dedicated process because execution settings
-(threads, deterministic algorithms, TF32) are intentionally pinned there.
-
-## Bounds and remaining gates
-
-The component admits one complete 8,192-token window, 1–512 original groups,
-a 1 MiB request and 64 KiB response. It refuses multi-window input before loading
-a model. No source text is generated, cropped or split.
-
-The unchanged score protocol contains no workload or retention-ratio field: it
-cannot certify generic API-response grouping or 50% retention. Those belong to
-the controller/renderer. This component is not the complete proposed bounded
-JSON/API-response preview and does not replace full-v2 workload/long-source scope.
-
-Remaining: real CUDA/native parity, complete matched-budget quality and warm/cold
-latency evidence, separate contiguous pipeline reporting, grouping/controller
-integration, source isolation and rights/accurate trained-model notices, clean-user
-GPU provisioning, CI and publication approval. Do not ship model weights or claim
-release readiness from CPU/mock/install checks.
+Use a dedicated process with caller-owned deadlines for the resident API. The
+CLI owns and waits for its one-shot worker; resident callers own their process.
+Neither API owns grouping, protected-content policy or final allocation.
+
+## Bounds and evidence
+
+One complete window: at most 8,192 tokens, 512 groups, a 1 MiB request and a
+64 KiB response. Overflow is refused, not truncated or silently windowed. ONNX
+Runtime 1.30.0, NumPy 2.5.3 and tokenizers 0.23.2 are pinned; no PyTorch inference
+dependency is required for this backend. No runtime binaries or weights are bundled.
+
+The exported graph passed six synthetic CPU parity cases against the source
+model, including dynamic lengths through 2,048 tokens and 512 pooled ranges.
+This is mechanics evidence, **not** full-8k execution, representative downstream
+quality, clean-user capacity or cold/warm compression latency qualification.
+Historical PyTorch FP16 timings do not measure this ONNX CPU backend. Keep
+`--experimental` and the explicit development acknowledgement until qualified.
+
+## Legacy development backend
+
+`DevelopmentScorer` and `inspect` retain the private PyTorch artifact contract.
+Use `worker --backend pytorch` explicitly for that FP32 direct-child prototype;
+it still requires its original pinned environment and artifact. The resident API
+also retains its explicit development-only FP16 option. Select v1's separately
+published LoRA model and historical benchmark results are not v2 evidence.
+
+## Safetensors model distribution
+
+The separate model release bundle also includes `model.safetensors` plus its
+custom joint-group architecture and pinned CPU reference loader. Safetensors is
+an alternative weight format, not a stock Transformers classification checkpoint
+or an automatic backend in this package. ONNX remains this worker's default.
+Use the model card's `SafetensorsScorer` instructions and manifest; representative
+quality/performance qualification is separate from format/parity validation.
diff --git a/packages/tokenfold-select/pyproject.toml b/packages/tokenfold-select/pyproject.toml
index 83e06e1..c2f4c69 100644
--- a/packages/tokenfold-select/pyproject.toml
+++ b/packages/tokenfold-select/pyproject.toml
@@ -4,12 +4,15 @@ build-backend = "setuptools.build_meta"
[project]
name = "tokenfold-select"
-version = "0.0.0.dev0"
-description = "Unqualified local development scorer for Tokenfold Select"
+version = "0.5.2"
+description = "Opt-in ONNX Select v2 scorer for Tokenfold"
readme = "README.md"
requires-python = ">=3.12"
license = { file = "LICENSE" }
dependencies = []
+[project.optional-dependencies]
+onnx = ["onnxruntime==1.30.0", "numpy==2.5.3", "tokenizers==0.23.2"]
+
[tool.setuptools.packages.find]
where = ["src"]
diff --git a/packages/tokenfold-select/src/tokenfold_select/__init__.py b/packages/tokenfold-select/src/tokenfold_select/__init__.py
index 6f148ff..5402f42 100644
--- a/packages/tokenfold-select/src/tokenfold_select/__init__.py
+++ b/packages/tokenfold-select/src/tokenfold_select/__init__.py
@@ -1,6 +1,7 @@
"""Optional development-only scorer; importing never loads a model/framework."""
from .artifact import inspect_artifact
from .runtime import DevelopmentScorer
+from .onnx import OnnxScorer, inspect_onnx
-__all__ = ["DevelopmentScorer", "inspect_artifact"]
-__version__ = "0.0.0.dev0"
+__all__ = ["OnnxScorer", "inspect_onnx", "DevelopmentScorer", "inspect_artifact"]
+__version__ = "0.5.2"
diff --git a/packages/tokenfold-select/src/tokenfold_select/__main__.py b/packages/tokenfold-select/src/tokenfold_select/__main__.py
index 215a4d9..08959e4 100644
--- a/packages/tokenfold-select/src/tokenfold_select/__main__.py
+++ b/packages/tokenfold-select/src/tokenfold_select/__main__.py
@@ -1,5 +1,6 @@
"""Inspection or the existing one-shot Select file protocol; no downloads."""
import argparse
+import hashlib
import json
import os
from pathlib import Path
@@ -9,6 +10,30 @@
from .artifact import inspect_artifact
from .protocol import REQUEST_BYTES, encode_response, load_request
from .runtime import DevelopmentScorer
+from .onnx import DEFAULT_MANIFEST_SHA256, DEFAULT_REVISION, OnnxScorer, inspect_onnx
+
+
+def configure(args):
+ """Generate approval only on explicit request; never download or invoke a model."""
+ model = inspect_onnx(args.artifact, args.manifest_sha256)
+ if model['source_checkpoint_sha256'] != args.revision:
+ raise ValueError('model_revision_mismatch')
+ if (not args.approved_by.strip() or not args.scratch_root.is_absolute()
+ or not args.scratch_root.is_dir() or not args.site_packages.is_absolute()
+ or not args.site_packages.is_dir()):
+ raise ValueError('explicit_approval_and_absolute_existing_directories_required')
+ executable = Path(sys._base_executable).resolve(strict=True)
+ if executable.stat().st_size > 64 * 1024 * 1024:
+ raise ValueError('executable_byte_limit')
+ bootstrap = Path(__file__).with_name('bootstrap.py').resolve()
+ with executable.open('rb') as stream:
+ pin = hashlib.file_digest(stream, 'sha256').hexdigest()
+ return dict(executable=str(executable), executable_sha256=pin,
+ arguments=['-I', '-S', str(bootstrap), '--site-packages', str(args.site_packages.resolve()),
+ 'worker', '--artifact', str(args.artifact.resolve()), '--revision', args.revision,
+ '--manifest-sha256', args.manifest_sha256, '--development'],
+ model_revision=args.revision, approved_by=args.approved_by,
+ scratch_root=str(args.scratch_root.resolve()))
def _worker(args):
@@ -18,6 +43,19 @@ def _worker(args):
timer.start()
code = 1
try:
+ # Set before importing any inference library. ONNX is explicitly CPU-only.
+ if args.backend == 'onnx':
+ os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
+ os.environ['TOKENIZERS_PARALLELISM'] = 'false'
+ if os.name == 'nt':
+ # The Rust parent clears the environment. Obtain the system path
+ # from Windows, not an inherited/user-controlled environment value.
+ import ctypes
+ system = ctypes.create_unicode_buffer(32768)
+ length = ctypes.windll.kernel32.GetWindowsDirectoryW(system, len(system))
+ if not 0 < length < len(system):
+ raise ValueError('windows_system_directory_unavailable')
+ os.environ['SystemRoot'] = system.value
request_path = Path(os.environ['TOKENFOLD_SELECT_REQUEST_PATH'])
response_path = Path(os.environ['TOKENFOLD_SELECT_RESPONSE_PATH'])
if (not request_path.is_absolute() or not response_path.is_absolute()
@@ -30,13 +68,19 @@ def offline(event, _):
sys.addaudithook(offline)
with request_path.open('rb') as stream:
request = load_request(stream.read(REQUEST_BYTES + 1), args.revision)
- scorer = DevelopmentScorer(args.artifact, args.revision, cache_directory=request_path.parent / 'model-cache',
- development=args.development, precision='fp32')
+ if args.backend == 'onnx':
+ scorer = OnnxScorer(args.artifact, args.manifest_sha256, development=args.development)
+ if scorer.revision != args.revision:
+ raise ValueError('model_revision_mismatch')
+ else:
+ scorer = DevelopmentScorer(args.artifact, args.revision, cache_directory=request_path.parent / 'model-cache',
+ development=args.development, precision='fp32')
response = scorer.score(request)
raw = encode_response(request, [value for _, value in response['scores']])
with response_path.open('xb') as stream:
stream.write(raw)
- scorer._torch.cuda.synchronize()
+ if args.backend == 'pytorch':
+ scorer._torch.cuda.synchronize()
code = 0
except BaseException:
print('tokenfold-select: refused', file=sys.stderr, flush=True)
@@ -50,17 +94,32 @@ def offline(event, _):
def main(arguments=None):
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest='command', required=True)
- for command in ('inspect', 'worker'):
+ for command in ('inspect', 'inspect-onnx', 'configure', 'worker'):
mode = sub.add_parser(command)
mode.add_argument('--artifact', type=Path, required=True)
- mode.add_argument('--revision', required=True)
+ mode.add_argument('--revision', default=DEFAULT_REVISION)
+ mode.add_argument('--manifest-sha256', default=DEFAULT_MANIFEST_SHA256)
if command == 'worker':
+ mode.add_argument('--backend', choices=('onnx', 'pytorch'), default='onnx')
mode.add_argument('--development', action='store_true', required=True)
+ if command == 'configure':
+ mode.add_argument('--approved-by', required=True)
+ mode.add_argument('--site-packages', type=Path, required=True)
+ mode.add_argument('--scratch-root', type=Path, required=True)
args = parser.parse_args(arguments)
if args.command == 'worker':
_worker(args)
try:
- report = inspect_artifact(args.artifact, args.revision)
+ if args.command == 'configure':
+ print(json.dumps(configure(args)))
+ return 0
+ if args.command == 'inspect-onnx':
+ report = inspect_onnx(args.artifact, args.manifest_sha256)
+ if report['source_checkpoint_sha256'] != args.revision:
+ raise ValueError('model_revision_mismatch')
+ report = dict(checkpoint_sha256=report['source_checkpoint_sha256'])
+ else:
+ report = inspect_artifact(args.artifact, args.revision)
except (OSError, ValueError, KeyError, TypeError, AttributeError):
print('tokenfold-select: refused', file=sys.stderr)
return 1
diff --git a/packages/tokenfold-select/src/tokenfold_select/onnx.py b/packages/tokenfold-select/src/tokenfold_select/onnx.py
new file mode 100644
index 0000000..dd412a1
--- /dev/null
+++ b/packages/tokenfold-select/src/tokenfold_select/onnx.py
@@ -0,0 +1,85 @@
+"""Bounded ONNX CPU reference scorer; no PyTorch, downloads or generated text."""
+import hashlib
+import importlib.metadata
+import json
+from pathlib import Path
+import re
+
+from . import _windows as packing
+from . import protocol
+
+CAPS={'model.onnx':64*1024**2,'model.onnx.data':1024**3,
+ 'tokenizer/tokenizer.json':32*1024**2,'tokenizer/tokenizer_config.json':65536,
+ 'tokenizer/special_tokens_map.json':65536,
+ 'runtime/onnx_score.py':65536,'runtime/packing.py':65536,'runtime/protocol.py':65536}
+DEFAULT_REVISION='4dbdb5071e7d6316d8eb70dc161b1f2f77f555012f6616d7fa34311188e24cec'
+DEFAULT_MANIFEST_SHA256='267313718e80aa91a56923dea30363fd5c24fde3782c716d6d048891860a2054'
+ENVIRONMENT={'onnxruntime':'1.30.0','numpy':'2.5.3','tokenizers':'0.23.2'}
+
+def digest(path):
+ with Path(path).open('rb') as stream:return hashlib.file_digest(stream,'sha256').hexdigest()
+
+def inspect_onnx(directory,manifest_sha256=DEFAULT_MANIFEST_SHA256):
+ root=Path(directory)
+ if not root.is_absolute() or not re.fullmatch(r'[0-9a-f]{64}',manifest_sha256):
+ raise ValueError('absolute_root_and_manifest_pin_required')
+ root=root.resolve(strict=True);path=root/'onnx-manifest.json'
+ if path.is_symlink() or path.stat().st_size>65536 or digest(path)!=manifest_sha256:
+ raise ValueError('manifest_pin_mismatch')
+ m=protocol.decode(path.read_bytes())
+ if (m.get('status')!='development_ONNX_export' or m.get('precision')!='fp32'
+ or m.get('provider')!='CPUExecutionProvider' or m.get('opset')!=18
+ or m.get('limits')!={'tokens':8192,'groups':512,'request_bytes':1048576,'response_bytes':65536}
+ or not re.fullmatch(r'[0-9a-f]{64}',m.get('source_checkpoint_sha256',''))
+ or m.get('qualification') is not False or set(m.get('files',{}))!=set(CAPS)):
+ raise ValueError('bounded_development_ONNX_profile_required')
+ for name,cap in CAPS.items():
+ path=root/name;pin=m['files'][name]
+ if (path.is_symlink() or not path.resolve(strict=True).is_relative_to(root)
+ or type(pin.get('bytes')) is not int or not 0=22"
},
"optionalDependencies": {
- "@tokenfold/cli-darwin-arm64": "0.5.1",
- "@tokenfold/cli-darwin-x64": "0.5.1",
- "@tokenfold/cli-linux-arm64": "0.5.1",
- "@tokenfold/cli-linux-x64": "0.5.1",
- "@tokenfold/cli-win32-x64": "0.5.1"
+ "@tokenfold/cli-darwin-arm64": "0.5.2",
+ "@tokenfold/cli-darwin-x64": "0.5.2",
+ "@tokenfold/cli-linux-arm64": "0.5.2",
+ "@tokenfold/cli-linux-x64": "0.5.2",
+ "@tokenfold/cli-win32-x64": "0.5.2"
}
},
"node_modules/@tokenfold/cli-darwin-arm64": {
- "version": "0.5.1",
- "resolved": "https://registry.npmjs.org/@tokenfold/cli-darwin-arm64/-/cli-darwin-arm64-0.5.1.tgz",
- "integrity": "sha512-JJ+sEDXtxN9vybvfzBHhe6yZ/jV2lS8uMesVC0MjAC1B5zp88GSMLNvXwInieEDcLJQ83kgvvgwZkeq59hLj2Q==",
+ "version": "0.5.2",
+ "resolved": "https://registry.npmjs.org/@tokenfold/cli-darwin-arm64/-/cli-darwin-arm64-0.5.2.tgz",
"cpu": [
"arm64"
],
@@ -43,9 +42,8 @@
}
},
"node_modules/@tokenfold/cli-darwin-x64": {
- "version": "0.5.1",
- "resolved": "https://registry.npmjs.org/@tokenfold/cli-darwin-x64/-/cli-darwin-x64-0.5.1.tgz",
- "integrity": "sha512-dt+g44lk2c1zS1Jrg2B/3vVmKAtcx5JtfBbLCdPq1dkP61Wg5a1OXOxVu9YfkXY7IHwPGVcSGcCvqptq7gGGxA==",
+ "version": "0.5.2",
+ "resolved": "https://registry.npmjs.org/@tokenfold/cli-darwin-x64/-/cli-darwin-x64-0.5.2.tgz",
"cpu": [
"x64"
],
@@ -59,9 +57,8 @@
}
},
"node_modules/@tokenfold/cli-linux-arm64": {
- "version": "0.5.1",
- "resolved": "https://registry.npmjs.org/@tokenfold/cli-linux-arm64/-/cli-linux-arm64-0.5.1.tgz",
- "integrity": "sha512-cSewwgR63Lgqlfg5y6d+xMk8JNNHgekxgXHT23oCBoNJ03GtJfOTmST/SqMSaHBjV47r0SjNobtsxGwjrVi8Yw==",
+ "version": "0.5.2",
+ "resolved": "https://registry.npmjs.org/@tokenfold/cli-linux-arm64/-/cli-linux-arm64-0.5.2.tgz",
"cpu": [
"arm64"
],
@@ -75,9 +72,8 @@
}
},
"node_modules/@tokenfold/cli-linux-x64": {
- "version": "0.5.1",
- "resolved": "https://registry.npmjs.org/@tokenfold/cli-linux-x64/-/cli-linux-x64-0.5.1.tgz",
- "integrity": "sha512-CTMH5h4BSb4FZIUBk+G3VQiuZDSxsNTC7knwzgpA8OrzX40l5fTLZMtVGXVaZ5FwznUu5XjJOGaKKkxCobk1eQ==",
+ "version": "0.5.2",
+ "resolved": "https://registry.npmjs.org/@tokenfold/cli-linux-x64/-/cli-linux-x64-0.5.2.tgz",
"cpu": [
"x64"
],
@@ -91,9 +87,8 @@
}
},
"node_modules/@tokenfold/cli-win32-x64": {
- "version": "0.5.1",
- "resolved": "https://registry.npmjs.org/@tokenfold/cli-win32-x64/-/cli-win32-x64-0.5.1.tgz",
- "integrity": "sha512-oMX+hDr9dmoGWVI6S0JwYahLSruF70HI6byb6XgQNHE4BudIjVfIDYyK5Ocm8Ga5Xlf6qGIIM3mafd6PkGv95g==",
+ "version": "0.5.2",
+ "resolved": "https://registry.npmjs.org/@tokenfold/cli-win32-x64/-/cli-win32-x64-0.5.2.tgz",
"cpu": [
"x64"
],
diff --git a/packages/tokenfold/package.json b/packages/tokenfold/package.json
index 70c8d82..d94605f 100644
--- a/packages/tokenfold/package.json
+++ b/packages/tokenfold/package.json
@@ -1,6 +1,6 @@
{
"name": "tokenfold",
- "version": "0.5.1",
+ "version": "0.5.2",
"description": "Token-aware compression for LLM payloads, backed by the tokenfold Rust CLI.",
"license": "Apache-2.0",
"repository": {
@@ -49,11 +49,11 @@
"prepack": "npm run build"
},
"optionalDependencies": {
- "@tokenfold/cli-darwin-x64": "0.5.1",
- "@tokenfold/cli-darwin-arm64": "0.5.1",
- "@tokenfold/cli-linux-x64": "0.5.1",
- "@tokenfold/cli-linux-arm64": "0.5.1",
- "@tokenfold/cli-win32-x64": "0.5.1"
+ "@tokenfold/cli-darwin-x64": "0.5.2",
+ "@tokenfold/cli-darwin-arm64": "0.5.2",
+ "@tokenfold/cli-linux-x64": "0.5.2",
+ "@tokenfold/cli-linux-arm64": "0.5.2",
+ "@tokenfold/cli-win32-x64": "0.5.2"
},
"devDependencies": {
"@types/node": "^24.0.0",
diff --git a/python-tests/test_notebook.py b/python-tests/test_notebook.py
new file mode 100644
index 0000000..772d4d0
--- /dev/null
+++ b/python-tests/test_notebook.py
@@ -0,0 +1,85 @@
+"""Keep the quickstart executable offline and its optional model path pinned."""
+import ast
+import builtins
+from contextlib import redirect_stdout
+import io
+import json
+from pathlib import Path
+import re
+import sys
+from types import SimpleNamespace
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+NOTEBOOK = ROOT / "examples/quickstart.ipynb"
+
+
+def cells():
+ return json.loads(NOTEBOOK.read_bytes())["cells"]
+
+
+def select_cell():
+ return next("".join(c["source"]) for c in cells()
+ if c["cell_type"] == "code" and "RUN_TOKENFOLD_SELECT =" in "".join(c["source"]))
+
+
+def test_notebook_sources_compile_and_have_no_saved_outputs():
+ for i, cell in enumerate(cells()):
+ if cell["cell_type"] == "code":
+ ast.parse("".join(cell["source"]), filename=f"notebook-cell-{i}")
+ assert cell["outputs"] == [] and cell["execution_count"] is None
+ source = select_cell()
+ assert "RUN_TOKENFOLD_SELECT = False" in source
+ assert re.search(r'SELECT_MODEL_REVISION = "[0-9a-f]{40}"', source)
+ assert "truncation=True" not in source and "peft" not in source
+
+
+def test_default_notebook_executes_without_model_imports_or_downloads(monkeypatch):
+ monkeypatch.chdir(ROOT)
+ original = builtins.__import__
+ forbidden = {"torch", "transformers", "peft", "huggingface_hub",
+ "tokenfold_select", "onnxruntime", "tokenizers"}
+
+ def offline_import(name, *args, **kwargs):
+ assert name.split(".")[0] not in forbidden, f"Unexpected model import: {name}"
+ return original(name, *args, **kwargs)
+
+ namespace = {"__name__": "__main__"}
+ with patch("builtins.__import__", offline_import), redirect_stdout(io.StringIO()):
+ for i, cell in enumerate(cells()):
+ if cell["cell_type"] == "code":
+ exec(compile("".join(cell["source"]), f"notebook-cell-{i}", "exec"), namespace)
+ assert namespace["RUN_TOKENFOLD_SELECT"] is False
+
+
+def test_optional_cell_uses_pinned_onnx_profile_and_original_groups(tmp_path):
+ calls = {}
+
+ def download(repo, **kwargs):
+ calls.update(repo=repo, **kwargs)
+ return str(tmp_path)
+
+ class Scorer:
+ revision = "4dbdb5071e7d6316d8eb70dc161b1f2f77f555012f6616d7fa34311188e24cec"
+
+ def __init__(self, directory, *, development):
+ assert directory == tmp_path and development is True
+
+ def score(self, request):
+ calls["request"] = request
+ return {"scores": [[g["id"], 0.0] for g in request["groups"]]}
+
+ source = select_cell().replace("RUN_TOKENFOLD_SELECT = False", "RUN_TOKENFOLD_SELECT = True", 1)
+ namespace = dict(json=json, Path=Path, EXAMPLES=ROOT / "examples")
+ with patch.dict(sys.modules, huggingface_hub=SimpleNamespace(snapshot_download=download),
+ tokenfold_select=SimpleNamespace(OnnxScorer=Scorer)), patch.dict("os.environ"), redirect_stdout(io.StringIO()):
+ exec(compile(source, "optional-notebook-cell", "exec"), namespace)
+ assert calls["repo"] == "snchimata/tokenfold-select-v2"
+ assert re.fullmatch("[0-9a-f]{40}", calls["revision"])
+ assert "model.onnx" in calls["allow_patterns"] and "model.onnx.data" in calls["allow_patterns"]
+ assert "model.safetensors" not in calls["allow_patterns"]
+ request = calls["request"]
+ assert request["model_revision"] == Scorer.revision
+ assert [g["id"] for g in request["groups"]] == ["0", "1", "2"]
+ events = json.loads((ROOT / "examples/incident_feed.json").read_bytes())["events"]
+ assert request["groups"][0]["text"] == next(e["detail"] for e in events if e["event"] == "incident")
diff --git a/scripts/check-release-versions.py b/scripts/check-release-versions.py
new file mode 100644
index 0000000..94345ef
--- /dev/null
+++ b/scripts/check-release-versions.py
@@ -0,0 +1,38 @@
+"""Fail before publication if a supported registry package has version drift."""
+import ast
+import json
+from pathlib import Path
+import tomllib
+
+root = Path(__file__).resolve().parents[1]
+
+
+def toml(path):
+ return tomllib.loads((root / path).read_text(encoding="utf8"))
+
+
+version = toml("Cargo.toml")["workspace"]["package"]["version"]
+for path in (root / "crates").glob("*/Cargo.toml"):
+ manifest = toml(path)
+ assert manifest["package"]["version"] == {"workspace": True}, path
+ for dep in manifest.get("dependencies", {}).values():
+ if isinstance(dep, dict) and "path" in dep and "version" in dep:
+ assert dep["version"] == version, path
+for package in toml("Cargo.lock")["package"]:
+ if package["name"].startswith("tokenfold-") and "source" not in package:
+ assert package["version"] == version, package["name"]
+npm = json.loads((root / "packages/tokenfold/package.json").read_bytes())
+lock = json.loads((root / "packages/tokenfold/package-lock.json").read_bytes())
+assert npm["version"] == lock["version"] == lock["packages"][""]["version"] == version
+assert len(npm["optionalDependencies"]) == 5
+assert set(npm["optionalDependencies"].values()) == {version}
+assert lock["packages"][""]["optionalDependencies"] == npm["optionalDependencies"]
+for name in npm["optionalDependencies"]:
+ entry = lock["packages"]["node_modules/" + name]
+ assert entry["version"] == version, name
+assert toml("packages/tokenfold-select/pyproject.toml")["project"]["version"] == version
+module = ast.parse((root / "packages/tokenfold-select/src/tokenfold_select/__init__.py").read_text(encoding="utf8"))
+assert next(ast.literal_eval(node.value) for node in module.body if isinstance(node, ast.Assign)
+ and any(isinstance(t, ast.Name) and t.id == "__version__" for t in node.targets)) == version
+assert toml("crates/tokenfold-py/pyproject.toml")["project"]["dynamic"] == ["version"]
+print(f"All registry package versions agree: {version}")