From fb3ac05728691bd801869ac74992a6c9d88074ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:19:31 +0000 Subject: [PATCH] chore(deps): update tokenizers requirement from 0.22 to 0.23 Updates the requirements on [tokenizers](https://github.com/huggingface/tokenizers) to permit the latest version. - [Release notes](https://github.com/huggingface/tokenizers/releases) - [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md) - [Commits](https://github.com/huggingface/tokenizers/compare/v0.22.0...v0.23.1) --- updated-dependencies: - dependency-name: tokenizers dependency-version: 0.23.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c401f49..16a8c15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ safetensors = "0.7" # cut in Phase-0 dossier 02 §tokenize. `default-features = false` strips # the HTTP hub client, progressbar, and C++ esaxx dep; we keep only the # onig regex engine (required by BPE pre-tokenizers). -tokenizers = { version = "0.22", default-features = false, features = ["onig"] } +tokenizers = { version = "0.23", default-features = false, features = ["onig"] } # memmap2: read-only mmap for bulk-tensor files. No alternatives in the # Rust ecosystem; used only inside loader's safetensors path. memmap2 = "0.9"