From f77cf81078f473bc246857171197286166b2e7c8 Mon Sep 17 00:00:00 2001 From: Peter Staar Date: Tue, 6 Oct 2026 16:14:21 +0200 Subject: [PATCH] feat: implement step 3 Signed-off-by: Peter Staar --- .gitignore | 1 + .plans/implemented/implementation_step_3.md | 46 ++ README.md | 11 +- docs/cli.md | 51 ++ docs/index.md | 8 +- docs/retrieval.md | 169 ++++ docs/storage.md | 18 + pyproject.toml | 9 +- src/docling_context/__init__.py | 29 + src/docling_context/cli.py | 323 +++++++- src/docling_context/embeddings.py | 184 +++++ src/docling_context/local.py | 188 ++++- src/docling_context/package.py | 4 +- src/docling_context/retrieval.py | 585 ++++++++++++++ src/docling_context/vectors.py | 452 +++++++++++ tests/test_cli.py | 8 +- tests/test_embeddings.py | 147 ++++ .../{test_step2.py => test_ingestion_jobs.py} | 8 +- tests/test_retrieval.py | 723 ++++++++++++++++++ tests/{test_step1.py => test_storage.py} | 2 +- uv.lock | 518 ++++++++++++- 21 files changed, 3426 insertions(+), 58 deletions(-) create mode 100644 .plans/implemented/implementation_step_3.md create mode 100644 docs/retrieval.md create mode 100644 src/docling_context/embeddings.py create mode 100644 src/docling_context/retrieval.py create mode 100644 src/docling_context/vectors.py create mode 100644 tests/test_embeddings.py rename tests/{test_step2.py => test_ingestion_jobs.py} (97%) create mode 100644 tests/test_retrieval.py rename tests/{test_step1.py => test_storage.py} (99%) diff --git a/.gitignore b/.gitignore index b88434e..ec862d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .references/** **~ +*.DS_Store # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/.plans/implemented/implementation_step_3.md b/.plans/implemented/implementation_step_3.md new file mode 100644 index 0000000..784b88c --- /dev/null +++ b/.plans/implemented/implementation_step_3.md @@ -0,0 +1,46 @@ +# Step 3: Scoped retrieval and context assembly + +## Goal and dependency + +Find relevant DocLang nodes within authorized URI and XPath scopes, then assemble bounded context with exact citations. Depends on Steps 1–2. The system must answer lexical queries with no embedding model installed. + +## Index records and interfaces + +- A retrieval unit is a source document or semantic node with `tenant_id`, `uri`, `document_id`, `revision_id`, `xpath`, `parent_xpath`, `tier`, page, bounding box, text for indexing, and an internal stable `uint64` vector ID. SQLite maps the vector ID to the full address and stores lifecycle state. +- Use SQLite tables for tree/path filters and FTS5 for lexical search. Keep a `VectorIndex` interface (`upsert`, `remove`, `search(query, allowed_ids, k)`, `checkpoint`, `rebuild`) so the retrieval planner is independent of index implementation. +- Store the embedding model ID, dimensions, normalization rule, input hash, and index generation with each record. Never mix vectors from different model/dimension generations; rebuild a new generation and switch it atomically. +- Index L0 collection/document summaries, L1 outlines/section descriptions, and selected L2 semantic units. Preserve DocLang structure: split at headings and safe semantic boundaries; tables, formulas, and captions remain addressable units. Keep source XML untouched. + +## Local vector store choice + +- Evaluate [turbovec `IdMapIndex`](https://github.com/ryancodrai/turbovec) as the initial local implementation because it exposes stable external IDs, `remove`, `search(..., allowlist=...)`, and `write`/`load`/`sync` persistence. Use `IdMapIndex`, not positional IDs that can move after deletion. Its Python API expects contiguous `float32` arrays; document and validate dimensional requirements. +- Resolve tenant, URI-prefix, tier, and XPath-prefix constraints in SQLite first. Supply the resulting vector IDs as turbovec's allowlist, then combine vector results with FTS results. Recheck every returned ID against current revision and principal before opening a package. +- Treat turbovec's index file as a rebuildable projection, not the canonical document store. Record an index generation and committed outbox offset in SQLite. On restart, load a compatible snapshot and replay later events; if missing or inconsistent, rebuild from canonical DCLX packages and metadata. +- Benchmark filtered recall, search latency, index size, restart time, update/delete behavior, and embedding dimensions on representative local corpora. If a platform or dimension is unsupported, retain the FTS path and switch the vector adapter rather than changing public APIs. Repository benchmark claims are hypotheses for this workload, not acceptance results. + +## Retrieval algorithm + +1. Authenticate and normalize tenant, URI subtree, optional document, and optional XPath scope. +2. Read permitted candidates from SQLite, then run FTS and optional allowlist vector search only within those IDs. +3. Merge and rerank by relevance, tier, freshness, and structural proximity; collapse overlapping parent/child hits. +4. Fetch cited nodes from the exact DCLX revision. Return L0/L1 first when they satisfy the requested budget; load L2 only for selected nodes. Respect per-result and total token limits. +5. Return score components, URI, revision, XPath, page/bbox when present, tier, and truncation status. No citation may point to a stale or missing revision. + +## Work sequence + +1. Build deterministic node extraction and SQLite FTS records. +2. Implement scoped lexical search and cited XPath reads. +3. Add embedding provider interface and turbovec adapter with checkpoint/rebuild logic. +4. Add hybrid ranking and context assembly. +5. Measure quality and latency against a flat-search baseline with a fixed corpus and query set. + +## Acceptance checks + +- Scoped results never include another tenant, sibling subtree, or excluded XPath branch, including when an index contains stale IDs. +- Delete and replace a document, restart, and verify no old vector result is returned. Corrupt or remove the vector snapshot and verify a correct rebuild. +- Return the same cited DocLang content before and after index rebuild; enforce requested budgets and deterministic truncation. +- Report measured recall and latency for lexical-only, vector-only, and hybrid modes; set performance targets from the baseline rather than promising a fixed subsecond result in advance. + +## Exit artifact + +A local retrieval engine with exact DocLang citations, optional embeddings, scoped hybrid search, and a replaceable vector backend. diff --git a/README.md b/README.md index 75c9f50..83f8502 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ A local context tree that stores documents as immutable DocLang `.dclx` packages. -See the [capability guide](docs/index.md), [CLI reference](docs/cli.md), and +See the [capability guide](docs/index.md), [retrieval guide](docs/retrieval.md), +[CLI reference](docs/cli.md), and [single PDF and folder examples](examples/adding_resources/README.md). Ingest native `.dclg`, `.dclg.xml`, and `.dclx` sources with `Ingestor.add_resource`. @@ -40,11 +41,19 @@ uv run dc --store ./context-data add-resource ./paper.pdf --collection papers -- uv run dc --store ./context-data add-resource ./documents -r --from docx --from image uv run dc --store ./context-data worker --once uv run dc --store ./context-data tree docling://resources/papers -L 2 +uv run dc --store ./context-data index status +uv run dc --store ./context-data search "invoice" --uri docling://resources/papers +uv run --extra vectors dc --store ./context-data index vector rebuild +uv run --extra vectors dc --store ./context-data search "invoice" --mode hybrid ``` Use `uv run dc` or activate the project environment before running `dc` by itself; some systems also provide a calculator named `dc`. See the [full CLI guide](docs/cli.md) and [resource examples](examples/adding_resources/README.md). +Ingestion builds the lexical index automatically. Vector indexing is optional; +the `vectors` extra provides local embeddings and `dc index vector rebuild`. +`dc index lex rebuild` reconstructs search records from stored DCLX packages. +Search results cite exact source nodes or `@summary:` and `@toc:` sidecars. ## Local usage diff --git a/docs/cli.md b/docs/cli.md index 4f3f112..5ab650b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -13,10 +13,17 @@ uv run --extra conversion dc --store ./context-data add-resource ./paper.pdf \ uv run dc --store ./context-data task status JOB_ID uv run dc --store ./context-data worker --once uv run dc --store ./context-data status +uv run dc --store ./context-data index status +uv run dc --store ./context-data index lex status +uv run dc --store ./context-data index lex rebuild +uv run --extra vectors dc --store ./context-data index vector status +uv run --extra vectors dc --store ./context-data index vector rebuild +uv run --extra vectors dc --store ./context-data search "invoice" --mode hybrid uv run dc --store ./context-data ls docling://resources/ uv run dc --store ./context-data tree docling://resources/papers -L 2 uv run dc --store ./context-data find "invoice" --uri docling://resources/papers uv run dc --store ./context-data grep "total" --uri docling://resources/papers +uv run dc --store ./context-data search "invoice" --uri docling://resources/papers ``` `add-resource` accepts one document or a folder of documents. With the @@ -41,8 +48,52 @@ job; `add-resource` prints the revision and its `task_id`. `task status` reports that job. Run `worker --once` for one due job, or `worker` continuously. A collection summary may remain stale until its latest job completes. +Every `status` command prints a table by default. Pass `--json` for structured +output, for example `dc status --json`, `dc task status JOB_ID --json`, or +`dc index vector status --json`. + `find` and `grep` are case-insensitive substring scans of current documents. They scan at most 1,000 documents, 2,000 nodes per document, and 50,000 nodes total, returning at most 100 hits. They accept an optional URI subtree scope and enforce tenant and user access. They do not yet provide embedding search or relevance ranking. + +`search` uses the SQLite retrieval index and returns exact DCLX citations. +New or changed documents are indexed during ingestion. `index status` reports +both lexical and vector coverage. `index lex status` and `index vector status` +show each index separately. `index lex rebuild` regenerates the SQLite index +from stored DCLX packages without rerunning conversion or OCR. It invalidates +an existing vector snapshot; a configured embedding provider rebuilds it on its +next use. Status and lexical rebuild honor the selected tenant and user scope. + +The CLI defaults to lexical search until a vector model has been built, then +uses hybrid search. `--mode lexical|vector|hybrid` selects a mode explicitly. +Install the `vectors` extra and run `dc index vector rebuild` to generate +embeddings with the local default `BAAI/bge-small-en-v1.5` model. +`--model MODEL` selects another supported FastEmbed text model; +`Qdrant/clip-ViT-B-32` selects paired +text and image encoders. The first build downloads model weights unless +`--offline` is passed. On Apple Silicon, `--backend auto` selects MLX for +supported BGE models and uses ONNX for other models. Use `--backend mlx` or +`--backend onnx` to choose explicitly. MLX uses separate model weights and +cannot be combined with `--offline`; `--backend auto --offline` uses ONNX. +The selected backend is saved with the vector generation, so subsequent +searches use the same encoder. Both lexical and vector rebuilds display a +progress bar on interactive terminals; progress goes to stderr. +After a vector build, document writes and deletes replay from the outbox on +the next vector search. Missing or corrupt snapshots trigger a full rebuild. +`dc index vector status` reports embedding coverage and the +stored model. `search --image PATH --mode vector` accepts an image query with +CLIP; text and image may be supplied together. A text-only model skips the +image input. Vector builds currently require all documents in the store to be +accessible to the selected principal because the snapshot is store-wide. +With an image-capable model, referenced picture assets are indexed alongside +text nodes. Search results include the picture XPath and asset path; image-only +results show an image marker in the table. After upgrading an existing store, +run `dc index lex rebuild` and `dc index vector rebuild` to add picture assets. + +Use `--uri`, `--document`, and `--xpath` to restrict scope; repeat `--tier` +to select context tiers. `-k`, `--max-tokens`, and `--per-result-tokens` +control output size. It prints a compact table by default; pass `--json` for +full citation records. See [scoped retrieval](retrieval.md) for the Python +vector and hybrid API. diff --git a/docs/index.md b/docs/index.md index d32de44..2d0b56f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,12 +13,14 @@ them through local Python and `dc` command-line APIs. | Conversion | Typed PDF and image settings for OCR, tables, charts, enrichment, images, and limits | | Remote conversion | Opt-in Docling Serve adapter; explicit endpoint and bounded response | | Inspection | `dc status`, `ls`, `tree`, and task status | -| Search | `dc find` and `grep` provide bounded lexical scans of current documents | +| Search | `dc search` provides scoped lexical, vector, and hybrid retrieval, including picture assets with an image-capable model | -Search currently scans stored documents and has no semantic ranking or vector -index. The indexed retrieval work is planned separately. +`find` and `grep` retain their bounded substring scans. Indexed retrieval +verifies citations against the current DCLX revision before returning text. - [Conversion settings](conversion.md) +- [Scoped retrieval](retrieval.md) +- [Retrieval benchmark](../examples/retrieval/benchmark_corpus.py) - [CLI commands](cli.md) - [Storage and jobs](storage.md) - [Single PDF and folder examples](../examples/adding_resources/README.md) diff --git a/docs/retrieval.md b/docs/retrieval.md new file mode 100644 index 0000000..427cf25 --- /dev/null +++ b/docs/retrieval.md @@ -0,0 +1,169 @@ +# Scoped retrieval + +`dc search` uses the local SQLite index and works without an embedding model: + +```bash +uv run dc search "glacier" --uri docling://resources/articles \ + --xpath '/doclang[1]/text[1]' -k 5 --max-tokens 1000 +``` + +Ingestion indexes each document automatically. Run `dc index status` to inspect +both indexes, or `dc index lex rebuild` to regenerate lexical records from stored DCLX +packages. Rebuilding the SQLite index does not generate embeddings. Run +`uv run --extra vectors dc index vector rebuild` to generate them with the default +local model, then `dc search` uses hybrid mode automatically. Use +`dc search --mode lexical` to request lexical search explicitly. +On Apple Silicon, vector rebuilding selects MLX for supported BGE models when +installed; other models use FastEmbed's ONNX runtime. The backend is part of the +stored model ID, so changing backends regenerates vectors. Both rebuild +commands report progress on interactive terminals. + +By default, the command prints a table with total, lexical, vector, tier, +freshness, and structural proximity scores, document ID, XPath, and a compact +text excerpt. The excerpt shows the entire text through 128 characters, or +the first and last 64 characters separated by ` ... `. Use `--json` for the +complete citation records, including URI, revision, page, bounding box, full +bounded text, and truncation status. A footer reports how many nodes are shown +out of all indexed lexical matches in scope, before result and token limits. +`--document` restricts a query to one URI; +`--tier` can be repeated. `--max-tokens` and +`--per-result-tokens` limit the returned text using whitespace-delimited tokens. +Multiword lexical searches first look for nodes containing every query word; +if none match, they use any-word matching. The footer counts matches for the +selected lexical query. +The `@summary:` and `@toc:` XPath prefixes denote sidecars within the cited +DCLX revision. The default summary is the first 600 characters of extracted +heading and text content, so it can include publisher boilerplate. The TOC is +made from headings and may duplicate a heading hit. Source XML is never +rewritten for indexing. Matching summary and outline hits are assembled before +body nodes, with relevance ordering within each tier. + +The Python API supports lexical, vector, and hybrid modes: + +```python +from docling_context import LocalContextStore, Principal, Retriever, SearchScope + +with LocalContextStore("./context-data") as store: + result = Retriever(store).search( + Principal("default", "local"), + "glacier", + scope=SearchScope(uri="docling://resources/articles"), + mode="lexical", + ) +``` + +Install `docling-context[vectors]` and supply an embedding provider for vector +and hybrid queries. The provider implements `model_id`, `dimensions`, and +`embed(list[str]) -> list[list[float]]`. The `TurbovecIndex` adapter uses +stable `uint64` IDs from SQLite, contiguous `float32` arrays, L2 normalization, +and `IdMapIndex` with a SQL-derived allowlist. Dimensions must be a multiple of +eight, from 8 to 16,384. One model and dimension generation is active per +store. Replacing the provider rebuilds the vector generation. + +```python +retriever = Retriever(store, embedder=my_embedding_provider) +result = retriever.search(principal, "glacier", mode="hybrid") +``` + +The built-in local adapter supports direct text, image, and combined inputs: + +```python +from docling_context import FastEmbedProvider, embed_text, embed_image, embed_text_image + +provider = FastEmbedProvider("Qdrant/clip-ViT-B-32") +text_vector = embed_text(provider, "a geological cross section") +image_vector = embed_image(provider, image_bytes) +combined_vector = embed_text_image(provider, "a geological cross section", image_bytes) +``` + +The default `BAAI/bge-small-en-v1.5` model is text-only. With a text-only +provider, image input returns `None`, and a combined input uses the text only. +The API can search with an image via +`retriever.search(principal, "", image=image_bytes, mode="vector")` when the +selected provider supports images. A CLIP model can compare text queries with +image vectors in the same space. Picture assets referenced by source `` +nodes are separate vector units with the picture XPath and an `asset_path` in +the search result. Image-only picture results have empty `text`; the CLI table +shows their asset path. Picture nodes with text use combined text and image +embeddings when the model supports both. Text-only models skip image-only +pictures and use the text from pictures that contain text. Existing stores +must run `dc index lex rebuild` before picture units are available. +For a picture hit, `store.get_document(principal, hit.uri).get_part_bytes(hit.asset_path)` +reads the cited image from its DCLX package. + +SQLite stores vector metadata and embedding bytes. The turbovec snapshot is a +rebuildable projection. A missing or corrupt snapshot triggers a full rebuild; +compatible snapshots replay later outbox changes for only the affected URIs. +Changing the embedding model also requires a full rebuild. Search checks each +candidate against the current document revision and reads cited text from DCLX, +so stale index records are excluded. +The retriever enforces tenant, URI subtree, optional document, XPath subtree, +and tier constraints before searching. Parent and child hits are collapsed. +Source nodes retain their XML `parent_xpath`; `section_xpath` records the +preceding heading and contributes to structural proximity scoring. + +SQLite builds with FTS5 use FTS5; builds without it use FTS3. The latter is +common in some Python distributions on macOS. Both remain indexed and support +the same lexical API. `find` and `grep` retain their existing substring-scan +behavior; `search` uses the retrieval index. + +Run the fixed 60-document benchmark with `uv run --extra vectors +python examples/retrieval/benchmark.py`. It reports recall at five and median +and p95 query latency for a flat package scan, lexical, vector, and hybrid +search. The bundled hash embedder is a deterministic fixture for comparison, +not a semantic embedding model. Measure with a representative corpus and model +before setting production latency or recall targets. For a labeled corpus in +an existing store, create a JSON list such as: + +```json +[ + { + "query": "Koonap formation in the Karoo Basin", + "relevant_uris": ["docling://resources/documents/example-paper"] + } +] +``` + +Run `uv run --extra vectors python examples/retrieval/benchmark_corpus.py +--store ~/.local/share/docling-context --queries queries.json`. The command +reports recall at five and median and p95 latency for a flat package scan, +lexical retrieval, and, when a vector generation exists, vector and hybrid +retrieval. Supply several labeled queries and relevant URIs from your corpus +before using the results to set performance targets. Opening the store can +reconcile its index or vector snapshot. + +On a local macOS Python 3.12 environment using SQLite FTS3, the fixture +benchmark measured the following warm-query results. Each mode searched 60 +documents with 12 queries, repeated five times; recall is against the known +topic document at rank five. These figures are for the fixture only. + +| Mode | Recall@5 | Median query (ms) | p95 query (ms) | +| --- | ---: | ---: | ---: | +| Flat package scan | 1.00 | 8.060 | 9.437 | +| Indexed lexical | 1.00 | 1.337 | 1.403 | +| Vector | 1.00 | 1.717 | 2.045 | +| Hybrid | 1.00 | 1.803 | 1.895 | + +The filtered vector query returned the target at rank one for all 12 queries, +with a 0.349 ms median. The turbovec snapshot occupied 114,964 bytes. +Replacing one document and replaying its change took 19.323 ms; deleting +one and replaying its change took 15.121 ms. Restart and first vector query took +7.706 ms. These results only characterize this small fixture and host. + +On a separate 71-document PDF corpus, five title-derived queries with one +known relevant document each produced these one-pass measurements on the same +macOS host. The corpus used the default BGE text model and SQLite FTS3; each +mode returned a relevant document in the top five for all five queries. + +| Mode | Recall@5 | Median query (ms) | p95 query (ms) | +| --- | ---: | ---: | ---: | +| Flat package scan | 1.00 | 4808.376 | 4811.038 | +| Indexed lexical | 1.00 | 100.376 | 271.041 | +| Vector | 1.00 | 382.161 | 407.565 | +| Hybrid | 1.00 | 438.324 | 845.513 | + +For this corpus, an initial regression target is p95 under one second for each +indexed mode. These five queries are drawn from document titles, so their +recall does not establish quality on paraphrased or cross-document questions. +Use a larger labeled query set and repeated runs before setting a production +target. diff --git a/docs/storage.md b/docs/storage.md index 96cb6d2..d4907ee 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -24,5 +24,23 @@ with leases, retries failures, and can resume after a crash without adding a duplicate summary revision. Its `status` method exposes staleness and the summary revision ID. +Schema version 3 adds `retrieval_units`, an FTS table, and `vector_state`. +Schema version 4 adds `vector_members`, which records the stable IDs in each +snapshot so later outbox events can remove replaced or deleted nodes. +Schema version 5 records picture asset paths on retrieval units. Existing +stores can run `dc index lex rebuild` to add picture units from saved DCLX +packages, then `dc index vector rebuild` to embed them with an image-capable +model. +Schema version 6 records each source node's active heading in `section_xpath`. +Rebuild the lexical index after upgrading an existing store to populate that +field for previously indexed nodes. +Document writes replace their retrieval records in the same SQLite transaction +as the revision and outbox event; deletes remove those records. Existing stores +are backfilled from their DCLX packages on upgrade. The FTS table is a +rebuildable lexical projection. Optional vector snapshots are held in +`vectors-{generation}.tvim` files and can be reconstructed from SQLite +embedding bytes. The package and current revision remain authoritative for +all citations. + The local and in-memory stores implement the same basic `ContextStore` contract, leaving room for another storage backend later. diff --git a/pyproject.toml b/pyproject.toml index a33ff0f..caeeee5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,10 +19,17 @@ classifiers = [ "Topic :: Database", "Topic :: Text Processing :: Markup :: XML", ] -dependencies = ["doclang"] +dependencies = ["doclang", "tabulate>=0.10,<1", "tqdm>=4.66,<5"] [project.optional-dependencies] conversion = ["docling[rapidocr]>=2.134.0,<3"] +vectors = [ + "numpy>=1.26", + "turbovec>=1,<2", + "fastembed>=0.8.1,<1", + "mlx-embedding-models==0.0.11; sys_platform == 'darwin' and platform_machine == 'arm64'", + "transformers>=4.47,<5; sys_platform == 'darwin' and platform_machine == 'arm64'", +] [project.scripts] dc = "docling_context.cli:main" diff --git a/src/docling_context/__init__.py b/src/docling_context/__init__.py index 63ddd68..fd88448 100644 --- a/src/docling_context/__init__.py +++ b/src/docling_context/__init__.py @@ -8,6 +8,17 @@ DoclingServeConverter, LocalDoclingConverter, ) +from .embeddings import ( + CLIP_MODEL, + DEFAULT_MODEL, + FastEmbedProvider, + MlxEmbeddingProvider, + embed_image, + embed_text, + embed_text_image, + local_embedding_provider, + stored_embedding_provider, +) from .ingestion import Ingestor, SummaryProvider from .jobs import CollectionStatus, CollectionWorker, Job from .local import LocalContextStore, RecordMissing, RevisionConflict @@ -23,19 +34,26 @@ TreeEntry, ) from .package import FilePackageStore, PackageError, PackageMissing +from .retrieval import ContextResult, RetrievalHit, Retriever, SearchScope from .uri import AccessDenied, ContextURI, InvalidURI, authorize_uri, parse_uri +from .vectors import EmbeddingProvider, TurbovecIndex, VectorIndex __all__ = [ + "CLIP_MODEL", + "DEFAULT_MODEL", "AccessDenied", "ChangeEvent", "CollectionStatus", "CollectionWorker", + "ContextResult", "ContextStore", "ContextURI", "ConversionResult", "Converter", "DoclingServeConverter", "DocumentRecord", + "EmbeddingProvider", + "FastEmbedProvider", "FilePackageStore", "Ingestor", "InvalidURI", @@ -43,6 +61,7 @@ "LocalContextStore", "LocalDoclingConverter", "MemoryContextStore", + "MlxEmbeddingProvider", "NodeAddress", "NodeContent", "PackageError", @@ -51,10 +70,20 @@ "PdfConversionConfig", "Principal", "RecordMissing", + "RetrievalHit", + "Retriever", "RevisionConflict", + "SearchScope", "SummaryProvider", "TenantScope", "TreeEntry", + "TurbovecIndex", + "VectorIndex", "authorize_uri", + "embed_image", + "embed_text", + "embed_text_image", + "local_embedding_provider", "parse_uri", + "stored_embedding_provider", ] diff --git a/src/docling_context/cli.py b/src/docling_context/cli.py index 6e29a1f..6a0f284 100644 --- a/src/docling_context/cli.py +++ b/src/docling_context/cli.py @@ -9,23 +9,115 @@ import re import sys from collections import Counter +from collections.abc import Mapping from dataclasses import asdict, replace from pathlib import Path +from tabulate import tabulate # type: ignore[import-untyped] +from tqdm import tqdm # type: ignore[import-untyped] + from .conversion_config import PdfConversionConfig from .converters import LocalDoclingConverter +from .embeddings import ( + CLIP_MODEL, + DEFAULT_MODEL, + local_embedding_provider, + stored_embedding_provider, +) from .ingestion import Ingestor from .jobs import CollectionWorker from .local import LocalContextStore from .models import Principal from .package import bounded_nodes +from .retrieval import RetrievalHit, Retriever, SearchScope from .uri import authorize_uri, parse_uri +from .vectors import TurbovecIndex def _json(value: object) -> None: print(json.dumps(value, ensure_ascii=False, sort_keys=True)) +def _status( + values: Mapping[str, object], + *, + json_output: bool, + fields: tuple[str, ...] | None = None, +) -> None: + selected = fields or tuple(values) + if json_output: + _json({field: values[field] for field in selected}) + return + rows = [] + for field in selected: + value = values[field] + if isinstance(value, bool): + display = "yes" if value else "no" + elif value is None: + display = "—" + else: + display = str(value) + rows.append((field.replace("_", " ").capitalize(), display)) + print( + tabulate( + rows, + headers=("Field", "Value"), + tablefmt="rounded_grid", + disable_numparse=True, + ) + ) + + +def _advance_progress(bar: tqdm, completed: int, total: int) -> None: + if bar.total != total: + bar.total = total + bar.refresh() + bar.update(completed - bar.n) + + +def _search_excerpt(text: str) -> str: + compact = " ".join(text.split()) + if len(compact) <= 128: + return compact + return f"{compact[:64]} ... {compact[-64:]}" + + +def _search_table(hits: tuple[RetrievalHit, ...]) -> str: + if not hits: + return "No matches." + rows = [ + ( + f"{hit.score:.3f}", + f"{hit.lexical_score:.3f}", + f"{hit.vector_score:.3f}", + f"{hit.tier_score:.3f}", + f"{hit.freshness_score:.3f}", + f"{hit.proximity_score:.3f}", + hit.document_id, + hit.xpath, + _search_excerpt(hit.text) if hit.text else f"[image: {hit.asset_path}]", + ) + for hit in hits + ] + return tabulate( + rows, + headers=( + "Score", + "Lex", + "Vec", + "Tier", + "Fresh", + "Near", + "Document ID", + "XPath", + "Excerpt", + ), + tablefmt="rounded_grid", + maxcolwidths=(None, None, None, None, None, None, None, 48, 72), + disable_numparse=True, + ) + + def _prefix(value: str) -> str: return parse_uri(value.rstrip("/"), prefix=True).value @@ -248,7 +340,62 @@ def _parser() -> argparse.ArgumentParser: "--user", default=os.environ.get("DOCLING_CONTEXT_USER", "local") ) commands = parser.add_subparsers(dest="command", required=True) - commands.add_parser("status", help="Show local store and job counts") + status = commands.add_parser("status", help="Show store and job counts") + status.add_argument( + "--json", action="store_true", help="Print JSON instead of a table." + ) + index = commands.add_parser( + "index", help="Inspect or rebuild lexical and vector indexes" + ) + index_commands = index.add_subparsers(dest="index_command", required=True) + index_status = index_commands.add_parser( + "status", help="Show lexical and vector index status" + ) + index_status.add_argument( + "--json", action="store_true", help="Print JSON instead of a table." + ) + lex = index_commands.add_parser( + "lex", help="Inspect or rebuild indexed document nodes" + ) + lex_commands = lex.add_subparsers(dest="index_operation", required=True) + lex_status = lex_commands.add_parser( + "status", help="Show indexed document and node counts" + ) + lex_status.add_argument( + "--json", action="store_true", help="Print JSON instead of a table." + ) + lex_commands.add_parser( + "rebuild", help="Rebuild search records from stored DCLX documents" + ) + vector = index_commands.add_parser( + "vector", help="Inspect or rebuild vector embeddings" + ) + vector_commands = vector.add_subparsers(dest="index_operation", required=True) + vector_status = vector_commands.add_parser( + "status", help="Show vector model and coverage" + ) + vector_status.add_argument( + "--json", action="store_true", help="Print JSON instead of a table." + ) + vector_rebuild = vector_commands.add_parser( + "rebuild", help="Embed indexed nodes and rebuild the vector snapshot" + ) + vector_rebuild.add_argument( + "--model", + default=DEFAULT_MODEL, + help=f"FastEmbed text model or {CLIP_MODEL} (default: {DEFAULT_MODEL}).", + ) + vector_rebuild.add_argument( + "--offline", + action="store_true", + help="Use cached model files only (default: download missing files).", + ) + vector_rebuild.add_argument( + "--backend", + choices=("auto", "mlx", "onnx"), + default="auto", + help="Embedding runtime (default: MLX for supported Apple Silicon models, else ONNX).", + ) add = commands.add_parser( "add-resource", help="Import one supported document or a folder of documents", @@ -336,6 +483,9 @@ def _parser() -> argparse.ArgumentParser: task_commands = task.add_subparsers(dest="task_command", required=True) task_status = task_commands.add_parser("status") task_status.add_argument("job_id") + task_status.add_argument( + "--json", action="store_true", help="Print JSON instead of a table." + ) worker = commands.add_parser("worker", help="Process queued collection summaries") worker.add_argument("--once", action="store_true") listing = commands.add_parser("ls", help="List direct children") @@ -347,6 +497,59 @@ def _parser() -> argparse.ArgumentParser: search = commands.add_parser(name, help="Bounded lexical lookup") search.add_argument("term") search.add_argument("--uri", default="docling://resources/") + retrieval = commands.add_parser( + "search", help="Search indexed DocLang nodes with exact citations" + ) + retrieval.add_argument( + "query", nargs="?", default="", help="Query text (optional with --image)." + ) + retrieval.add_argument( + "--image", type=Path, help="Image query file (vector mode only)." + ) + retrieval.add_argument( + "--mode", + choices=("auto", "lexical", "vector", "hybrid"), + default="auto", + help="Search mode (default: hybrid if vectors exist, else lexical).", + ) + retrieval.add_argument( + "--uri", + default="docling://resources/", + help="Authorized URI subtree (default: all resources).", + ) + retrieval.add_argument( + "--document", help="Restrict to one document URI (default: all in scope)." + ) + retrieval.add_argument( + "--xpath", help="Restrict to one source XPath subtree (default: all nodes)." + ) + retrieval.add_argument( + "--tier", + type=int, + choices=(0, 1, 2), + action="append", + help="Include a context tier: 0 summary, 1 outline, 2 source (repeatable; default: all).", + ) + retrieval.add_argument( + "-k", type=int, default=10, help="Maximum cited results (default: 10)." + ) + retrieval.add_argument( + "--max-tokens", + type=int, + default=2_000, + help="Maximum tokens across returned citations (default: 2000).", + ) + retrieval.add_argument( + "--per-result-tokens", + type=int, + default=500, + help="Maximum tokens per citation (default: 500).", + ) + retrieval.add_argument( + "--json", + action="store_true", + help="Print complete JSON records instead of the compact table (default: table).", + ) return parser @@ -364,11 +567,79 @@ def main(argv: list[str] | None = None) -> int: ).fetchone()[0] for table in ("documents", "revisions", "jobs") } - _json({"store": str(store.root.resolve()), **counts}) + _status( + {"store": str(store.root.resolve()), **counts}, + json_output=args.json, + ) + elif args.command == "index": + if args.index_command == "status": + _status(store.index_status(principal), json_output=args.json) + elif args.index_command == "lex": + if args.index_operation == "status": + _status( + store.index_status(principal), + json_output=args.json, + fields=("documents", "indexed_documents", "indexed_nodes"), + ) + else: + total = store.index_status(principal)["documents"] + assert isinstance(total, int) + with tqdm( + total=total, + desc="Indexing documents", + unit="doc", + file=sys.stderr, + disable=not sys.stderr.isatty(), + ) as bar: + rebuild_result = store.rebuild_index( + principal, + progress=lambda done, size: _advance_progress( + bar, done, size + ), + ) + _json(rebuild_result) + else: + if args.index_operation == "status": + _status( + store.index_status(principal), + json_output=args.json, + fields=( + "vector_model", + "indexed_nodes", + "embedded_nodes", + "vector_snapshot_current", + ), + ) + else: + store.require_exclusive_vector_scope(principal) + total = store.index_status(principal)["indexed_nodes"] + assert isinstance(total, int) + with tqdm( + total=total, + desc="Indexing vectors", + unit="node", + file=sys.stderr, + disable=not sys.stderr.isatty(), + ) as bar: + provider = local_embedding_provider( + args.model, backend=args.backend, offline=args.offline + ) + TurbovecIndex( + store, + provider, + force_rebuild=True, + progress=lambda done, size: _advance_progress( + bar, done, size + ), + ) + _json(store.index_status(principal)) elif args.command == "add-resource": _add_resource(args, store, principal) elif args.command == "task": - _json(asdict(worker.get_job(principal, args.job_id))) + _status( + asdict(worker.get_job(principal, args.job_id)), + json_output=args.json, + ) elif args.command == "worker": if args.once: job = worker.run_once() @@ -399,6 +670,52 @@ def main(argv: list[str] | None = None) -> int: ) for hit in hits: _json(hit) + elif args.command == "search": + state = store.db.execute( + "SELECT model_id FROM vector_state WHERE singleton=1" + ).fetchone() + mode = args.mode + if mode == "auto": + mode = "hybrid" if state is not None else "lexical" + if args.image is not None and mode == "lexical": + raise ValueError("image search needs a vector index") + if mode in {"vector", "hybrid"}: + if state is None: + raise ValueError( + "run 'dc index vector rebuild' before vector search" + ) + store.require_exclusive_vector_scope(principal) + retriever = Retriever( + store, embedder=stored_embedding_provider(state["model_id"]) + ) + else: + retriever = Retriever(store) + result = retriever.search( + principal, + args.query, + image=args.image.read_bytes() if args.image is not None else None, + scope=SearchScope( + uri=_prefix(args.uri), + xpath=args.xpath, + document_uri=args.document, + tiers=tuple(args.tier) if args.tier else (0, 1, 2), + ), + k=args.k, + mode=mode, + max_tokens=args.max_tokens, + per_result_tokens=args.per_result_tokens, + ) + if args.json: + for retrieval_hit in result.hits: + _json(asdict(retrieval_hit)) + else: + print(_search_table(result.hits)) + if result.total_hits is None: + print(f"Showing {len(result.hits)} ranked nodes.") + else: + print( + f"Showing {len(result.hits)} of {result.total_hits} matching nodes." + ) except (OSError, ValueError, RuntimeError, KeyError, PermissionError) as exc: print(f"dc: {exc}", file=sys.stderr) return 1 diff --git a/src/docling_context/embeddings.py b/src/docling_context/embeddings.py new file mode 100644 index 0000000..5f0a257 --- /dev/null +++ b/src/docling_context/embeddings.py @@ -0,0 +1,184 @@ +"""Local embedding adapters with optional image and mixed-input support.""" + +from __future__ import annotations + +import math +import platform +import sys +import tempfile +from importlib.util import find_spec +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .vectors import EmbeddingProvider + +DEFAULT_MODEL = "BAAI/bge-small-en-v1.5" +CLIP_MODEL = "Qdrant/clip-ViT-B-32" +_CLIP_TEXT = "Qdrant/clip-ViT-B-32-text" +_CLIP_IMAGE = "Qdrant/clip-ViT-B-32-vision" +_MAX_IMAGE_BYTES = 20 * 1024 * 1024 +_MLX_MODELS = { + "BAAI/bge-small-en-v1.5": "bge-small", + "BAAI/bge-base-en-v1.5": "bge-base", + "BAAI/bge-large-en-v1.5": "bge-large", +} +_MLX_PREFIX = "mlx:" +_BGE_QUERY_INSTRUCTION = "Represent this sentence for searching relevant passages: " + + +def _apple_silicon() -> bool: + return sys.platform == "darwin" and platform.machine() == "arm64" + + +def embed_text(provider: EmbeddingProvider, text: str) -> list[float]: + """Embed a passage with any text-capable provider.""" + method = getattr(provider, "embed_text", None) + return method(text) if callable(method) else provider.embed([text])[0] + + +def embed_image(provider: EmbeddingProvider, image: bytes) -> list[float] | None: + """Return None when the selected provider has no image encoder.""" + method = getattr(provider, "embed_image", None) + return method(image) if callable(method) else None + + +def embed_text_image( + provider: EmbeddingProvider, text: str, image: bytes +) -> list[float] | None: + """Combine modalities when supported; otherwise use available text.""" + method = getattr(provider, "embed_text_image", None) + if callable(method): + return method(text, image) + if text.strip(): + return embed_text(provider, text) + return embed_image(provider, image) + + +def _unit(vector: list[float]) -> list[float]: + norm = math.sqrt(sum(value * value for value in vector)) + if not math.isfinite(norm) or norm <= 1e-10: + raise ValueError("embedding has zero or invalid norm") + return [value / norm for value in vector] + + +class FastEmbedProvider: + """FastEmbed text models, with paired CLIP encoders for multimodal input.""" + + def __init__(self, model_id: str = DEFAULT_MODEL, *, offline: bool = False): + try: + from fastembed import TextEmbedding # type: ignore[import-not-found] + except ImportError as exc: + raise RuntimeError( + "install docling-context[vectors] for local embedding models" + ) from exc + text_model = _CLIP_TEXT if model_id == CLIP_MODEL else model_id + supported = {entry["model"] for entry in TextEmbedding.list_supported_models()} + if text_model not in supported: + raise ValueError(f"unsupported local text embedding model: {model_id}") + self.model_id = model_id + self.supports_images = model_id == CLIP_MODEL + self.dimensions = TextEmbedding.get_embedding_size(text_model) + self._text = TextEmbedding(model_name=text_model, local_files_only=offline) + self._image: Any = None + self._offline = offline + + def embed(self, texts: list[str]) -> list[list[float]]: + return [vector.tolist() for vector in self._text.passage_embed(texts)] + + def embed_query(self, query: str) -> list[float]: + return next(iter(self._text.query_embed(query))).tolist() + + def embed_text(self, text: str) -> list[float]: + return self.embed([text])[0] + + def embed_image(self, image: bytes) -> list[float] | None: + if self.model_id != CLIP_MODEL: + return None + if not image or len(image) > _MAX_IMAGE_BYTES: + raise ValueError("image input is empty or too large") + if self._image is None: + from fastembed import ImageEmbedding # type: ignore[import-not-found] + + self._image = ImageEmbedding( + model_name=_CLIP_IMAGE, local_files_only=self._offline + ) + with tempfile.TemporaryDirectory(prefix="dc-embedding-") as directory: + path = Path(directory) / "image" + path.write_bytes(image) + return next(iter(self._image.embed([str(path)]))).tolist() + + def embed_text_image(self, text: str, image: bytes) -> list[float] | None: + if self.model_id != CLIP_MODEL: + return self.embed_text(text) if text.strip() else None + visual = self.embed_image(image) + if not text.strip(): + return visual + assert visual is not None + passage = self.embed_text(text) + return _unit( + [a + b for a, b in zip(_unit(passage), _unit(visual), strict=True)] + ) + + +class MlxEmbeddingProvider: + """Apple Silicon text embeddings with explicit backend identity.""" + + def __init__(self, model_name: str): + if model_name not in _MLX_MODELS: + raise ValueError(f"MLX does not support this model: {model_name}") + if not _apple_silicon(): + raise RuntimeError("MLX embeddings require Apple Silicon") + try: + import mlx_embedding_models.registry as mlx_registry # type: ignore[import-not-found,import-untyped] + from mlx_embedding_models.embedding import ( # type: ignore[import-not-found,import-untyped] + EmbeddingModel, + ) + except ImportError as exc: + raise RuntimeError( + "install docling-context[vectors] for MLX embeddings" + ) from exc + registry_name = _MLX_MODELS[model_name] + self.model_id = f"{_MLX_PREFIX}{model_name}" + self.supports_images = False + self.dimensions = int(mlx_registry.registry[registry_name]["ndim"]) + self._model = EmbeddingModel.from_registry(registry_name) + + def embed(self, texts: list[str]) -> list[list[float]]: + if not texts: + return [] + return self._model.encode(texts, show_progress=False).tolist() + + def embed_query(self, query: str) -> list[float]: + return self.embed([_BGE_QUERY_INSTRUCTION + query])[0] + + +def local_embedding_provider( + model_name: str = DEFAULT_MODEL, + *, + backend: str = "auto", + offline: bool = False, +) -> EmbeddingProvider: + """Choose MLX where available, retaining ONNX for other models and hosts.""" + if backend not in {"auto", "mlx", "onnx"}: + raise ValueError("backend must be auto, mlx, or onnx") + if backend == "mlx": + if offline: + raise ValueError("--offline is not supported with the MLX backend") + return MlxEmbeddingProvider(model_name) + if ( + backend == "auto" + and not offline + and model_name in _MLX_MODELS + and _apple_silicon() + and find_spec("mlx_embedding_models") is not None + ): + return MlxEmbeddingProvider(model_name) + return FastEmbedProvider(model_name, offline=offline) + + +def stored_embedding_provider(model_id: str) -> EmbeddingProvider: + """Open the same backend that wrote a vector generation.""" + if model_id.startswith(_MLX_PREFIX): + return MlxEmbeddingProvider(model_id[len(_MLX_PREFIX) :]) + return FastEmbedProvider(model_id) diff --git a/src/docling_context/local.py b/src/docling_context/local.py index 498798d..bec8b46 100644 --- a/src/docling_context/local.py +++ b/src/docling_context/local.py @@ -5,7 +5,7 @@ import json import sqlite3 import uuid -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import contextmanager from datetime import UTC, datetime from pathlib import Path @@ -26,6 +26,7 @@ load_package, read_node, ) +from .retrieval import index_package, remove_index_records from .uri import AccessDenied, authorize_uri, parse_uri MAX_TREE_LIMIT = 1_000 @@ -70,7 +71,7 @@ def __exit__(self, *_: object) -> None: def _migrate(self) -> None: version = self.db.execute("PRAGMA user_version").fetchone()[0] - if version > 2: + if version > 6: raise RuntimeError("database schema is newer than this library") if version == 0: self.db.executescript( @@ -164,6 +165,171 @@ def _migrate(self) -> None: if self.db.in_transaction: self.db.execute("ROLLBACK") raise + if version < 3: + try: + self.db.execute("CREATE VIRTUAL TABLE temp.fts_probe USING fts5(text)") + except sqlite3.OperationalError: + fts_module = "fts3" + else: + fts_module = "fts5" + self.db.execute("DROP TABLE temp.fts_probe") + self.db.executescript( + f"""BEGIN IMMEDIATE; + CREATE TABLE retrieval_units ( + vector_id INTEGER PRIMARY KEY AUTOINCREMENT, + tenant_id TEXT NOT NULL, uri TEXT NOT NULL, + document_id TEXT NOT NULL, revision_id TEXT NOT NULL, + xpath TEXT NOT NULL, parent_xpath TEXT, + tier INTEGER NOT NULL, page INTEGER, bbox_json TEXT, + text TEXT NOT NULL, updated_at TEXT NOT NULL, + model_id TEXT, dimensions INTEGER, normalization TEXT, + input_hash TEXT, index_generation TEXT, embedding BLOB, + UNIQUE (tenant_id, uri, revision_id, xpath) + ); + CREATE INDEX retrieval_scope ON retrieval_units + (tenant_id, uri, revision_id, tier, xpath); + CREATE VIRTUAL TABLE retrieval_fts USING {fts_module}(text); + CREATE TABLE vector_state ( + singleton INTEGER PRIMARY KEY CHECK (singleton=1), + generation TEXT NOT NULL, model_id TEXT NOT NULL, + dimensions INTEGER NOT NULL, normalization TEXT NOT NULL, + outbox_offset INTEGER NOT NULL, snapshot_hash TEXT NOT NULL + ); + PRAGMA user_version = 3; + COMMIT; + """ + ) + if version < 4: + self.db.executescript( + """BEGIN IMMEDIATE; + CREATE TABLE IF NOT EXISTS vector_members ( + generation TEXT NOT NULL, tenant_id TEXT NOT NULL, + uri TEXT NOT NULL, vector_id INTEGER NOT NULL, + PRIMARY KEY (generation, vector_id) + ); + CREATE INDEX IF NOT EXISTS vector_members_uri ON vector_members + (generation, tenant_id, uri); + PRAGMA user_version = 4; + COMMIT; + """ + ) + if version < 5: + columns = { + row["name"] + for row in self.db.execute("PRAGMA table_info(retrieval_units)") + } + if "asset_path" not in columns: + self.db.execute( + "ALTER TABLE retrieval_units ADD COLUMN asset_path TEXT" + ) + self.db.execute("PRAGMA user_version = 5") + if version < 6: + columns = { + row["name"] + for row in self.db.execute("PRAGMA table_info(retrieval_units)") + } + if "section_xpath" not in columns: + self.db.execute( + "ALTER TABLE retrieval_units ADD COLUMN section_xpath TEXT" + ) + self.db.execute("PRAGMA user_version = 6") + self._backfill_retrieval() + + def _backfill_retrieval(self) -> None: + rows = self.db.execute( + """SELECT d.* FROM documents d WHERE d.available=1 AND NOT EXISTS ( + SELECT 1 FROM retrieval_units r WHERE r.tenant_id=d.tenant_id + AND r.uri=d.uri AND r.revision_id=d.revision_id)""" + ).fetchall() + for row in rows: + try: + package = self.packages.get(row["package_hash"]) + with self.transaction(): + index_package(self.db, self._record(row), package) + self.db.execute("UPDATE vector_state SET outbox_offset=-1") + except (PackageMissing, PackageError): + continue + + def index_status(self, principal: Principal) -> dict[str, int | str | None]: + """Report the current, authorized retrieval projection.""" + rows = self.db.execute( + "SELECT uri,revision_id FROM documents WHERE tenant_id=? AND available=1", + (principal.tenant_id,), + ).fetchall() + visible = [row for row in rows if self._can_access(principal, row["uri"])] + indexed = units = embedded = 0 + for row in visible: + count, with_embeddings = self.db.execute( + """SELECT COUNT(*),COUNT(embedding) FROM retrieval_units + WHERE tenant_id=? AND uri=? AND revision_id=?""", + (principal.tenant_id, row["uri"], row["revision_id"]), + ).fetchone() + indexed += count > 0 + units += count + embedded += with_embeddings + state = self.db.execute( + "SELECT model_id,outbox_offset FROM vector_state WHERE singleton=1" + ).fetchone() + offset = self.db.execute( + "SELECT COALESCE(MAX(sequence),0) FROM outbox" + ).fetchone()[0] + return { + "documents": len(visible), + "indexed_documents": indexed, + "indexed_nodes": units, + "embedded_nodes": embedded, + "vector_model": state["model_id"] if state else None, + "vector_snapshot_current": bool(state and state["outbox_offset"] == offset), + } + + def rebuild_index( + self, + principal: Principal, + *, + progress: Callable[[int, int], None] | None = None, + ) -> dict[str, int | str | None]: + """Recreate authorized retrieval rows from stored DCLX packages.""" + rows = self.db.execute( + "SELECT * FROM documents WHERE tenant_id=? AND available=1 ORDER BY uri", + (principal.tenant_id,), + ).fetchall() + visible = [row for row in rows if self._can_access(principal, row["uri"])] + rebuilt = 0 + if progress is not None: + progress(0, len(visible)) + for row in visible: + package = self.packages.get(row["package_hash"]) + document = load_package(package) + with self.transaction(): + index_package(self.db, self._record(row), package, document=document) + self.db.execute("UPDATE vector_state SET outbox_offset=-1") + rebuilt += 1 + if progress is not None: + progress(rebuilt, len(visible)) + return {"rebuilt_documents": rebuilt, **self.index_status(principal)} + + def require_exclusive_vector_scope(self, principal: Principal) -> None: + """Prevent a store-wide vector build from reading another principal's data.""" + rows = self.db.execute( + "SELECT tenant_id,uri FROM documents WHERE available=1" + ).fetchall() + if any( + row["tenant_id"] != principal.tenant_id + or not self._can_access(principal, row["uri"]) + for row in rows + ): + raise PermissionError( + "vector indexing needs a store containing only documents " + "accessible to the selected tenant and user" + ) + + @staticmethod + def _can_access(principal: Principal, uri: str) -> bool: + try: + authorize_uri(principal, parse_uri(uri)) + except AccessDenied: + return False + return True def _queue_collection( self, principal: Principal, uri: str, revision_id: str, event: int, now: str @@ -244,6 +410,7 @@ def put( canonical = address.value source_json = json.dumps(source or {}, sort_keys=True, ensure_ascii=False) digest, size = self.packages.put(package) + indexed = load_package(package) now = _now() with self.transaction(): current = self.db.execute( @@ -293,6 +460,22 @@ def put( updated_at=excluded.updated_at, available=1""", values, ) + index_package( + self.db, + DocumentRecord( + canonical, + principal.tenant_id, + document_id, + revision_id, + PackageRef(digest, size), + address.parent, + source or {}, + created_at, + now, + ), + package, + document=indexed, + ) cursor = self.db.execute( "INSERT INTO outbox (tenant_id,uri,document_id,revision_id,operation,occurred_at) " "VALUES (?,?,?,?,?,?)", @@ -353,6 +536,7 @@ def delete(self, principal: Principal, uri: str, *, expected_revision: str) -> N "DELETE FROM documents WHERE tenant_id=? AND uri=?", (principal.tenant_id, address.value), ) + remove_index_records(self.db, principal.tenant_id, address.value) cursor = self.db.execute( "INSERT INTO outbox (tenant_id,uri,document_id,revision_id,operation,occurred_at) " "VALUES (?,?,?,?,?,?)", diff --git a/src/docling_context/package.py b/src/docling_context/package.py index 5fe256f..ae80fda 100644 --- a/src/docling_context/package.py +++ b/src/docling_context/package.py @@ -7,7 +7,7 @@ import tempfile from pathlib import Path -from doclang import ArchiveLimits, DocLangXDocument +from doclang import ArchiveLimits, DoclangDocument, DocLangXDocument from doclang.types import DoclangNodeRecord from .models import NodeAddress, NodeContent @@ -38,7 +38,7 @@ def _limits() -> ArchiveLimits: def bounded_nodes( - document: DocLangXDocument, + document: DoclangDocument, *, xpath: str | None = None, limit: int = 1_000, diff --git a/src/docling_context/retrieval.py b/src/docling_context/retrieval.py new file mode 100644 index 0000000..b886b21 --- /dev/null +++ b/src/docling_context/retrieval.py @@ -0,0 +1,585 @@ +"""Revision-aware lexical retrieval and bounded DocLang context assembly.""" + +from __future__ import annotations + +import hashlib +import json +import re +import sqlite3 +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from doclang import DocLangXDocument + +from .models import DocumentRecord, Principal +from .package import MAX_NODE_TEXT_CHARS, bounded_nodes, load_package +from .uri import authorize_uri, parse_uri + +if TYPE_CHECKING: + from .local import LocalContextStore + from .vectors import EmbeddingProvider, VectorIndex + +_SEMANTIC_NAMES = { + "text", + "paragraph", + "caption", + "table", + "formula", + "code", + "list-item", + "item", + "title", + "footnote", + "description", +} +_MAX_RESULTS = 100 +_MAX_CANDIDATES = 10_000 + + +def remove_index_records(db: sqlite3.Connection, tenant_id: str, uri: str) -> None: + ids = db.execute( + "SELECT vector_id FROM retrieval_units WHERE tenant_id=? AND uri=?", + (tenant_id, uri), + ).fetchall() + db.executemany("DELETE FROM retrieval_fts WHERE rowid=?", ids) + db.execute( + "DELETE FROM retrieval_units WHERE tenant_id=? AND uri=?", (tenant_id, uri) + ) + + +def index_package( + db: sqlite3.Connection, + record: DocumentRecord, + package: bytes, + *, + document: DocLangXDocument | None = None, +) -> None: + """Replace one document's projection inside the caller's transaction.""" + document = document or load_package(package) + remove_index_records(db, record.tenant_id, record.uri) + units: list[ + tuple[str, str | None, str | None, int, int | None, str | None, str, str | None] + ] = [] + archive_paths = set(document.archive_paths()) + picture_assets = [] + for element in ET.fromstring(document.xml()).iter(): + if element.tag.rsplit("}", 1)[-1] != "picture": + continue + source = next( + (child for child in element if child.tag.rsplit("}", 1)[-1] == "src"), + None, + ) + path = source.get("uri") if source is not None else None + picture_assets.append( + path + if path and path in archive_paths and path.startswith("assets/") + else None + ) + picture_index = 0 + summary = document.summary() + if summary is not None: + for node in bounded_nodes(summary, limit=100, text_bytes=4_096): + if node["name"] in {"text", "paragraph"} and node["text"].strip(): + units.append( + ( + f"@summary:{node['xpath']}", + None, + None, + 0, + None, + None, + node["text"], + None, + ) + ) + break + toc = document.toc() + if toc is not None: + for node in bounded_nodes(toc, limit=512, text_bytes=1_024): + if node["name"] == "description" and node["text"].strip(): + units.append( + ( + f"@toc:{node['xpath']}", + None, + None, + 1, + None, + None, + node["text"], + None, + ) + ) + section_xpath = None + for node in bounded_nodes(document, limit=10_000, text_bytes=8_192): + name = node["name"].lower() + if name == "heading": + section_xpath = node["xpath"] + asset_path = None + if name == "picture": + asset_path = ( + picture_assets[picture_index] + if picture_index < len(picture_assets) + else None + ) + picture_index += 1 + if not node["text"].strip() and asset_path is None: + continue + if name == "heading": + tier = 1 + elif name in _SEMANTIC_NAMES or asset_path is not None: + tier = 2 + else: + continue + units.append( + ( + node["xpath"], + node["parent_xpath"], + section_xpath, + tier, + node["page"] if node["page"] > 0 else None, + json.dumps(node["bbox"]) if node["bbox"] is not None else None, + node["text"], + asset_path, + ) + ) + for xpath, parent, section, tier, page, bbox, content, asset_path in units: + asset = document.get_part_bytes(asset_path) if asset_path else None + digest = hashlib.sha256(content.encode("utf-8") + (asset or b"")).hexdigest() + cursor = db.execute( + """INSERT INTO retrieval_units + (tenant_id,uri,document_id,revision_id,xpath,parent_xpath,section_xpath, + tier,page,bbox_json,text,updated_at,input_hash,asset_path) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", + ( + record.tenant_id, + record.uri, + record.document_id, + record.revision_id, + xpath, + parent, + section, + tier, + page, + bbox, + content, + record.updated_at, + digest, + asset_path, + ), + ) + db.execute( + "INSERT INTO retrieval_fts(rowid,text) VALUES (?,?)", + (cursor.lastrowid, content), + ) + + +@dataclass(frozen=True, slots=True) +class SearchScope: + uri: str = "docling://resources" + xpath: str | None = None + document_uri: str | None = None + tiers: tuple[int, ...] = (0, 1, 2) + + +@dataclass(frozen=True, slots=True) +class RetrievalHit: + uri: str + document_id: str + revision_id: str + xpath: str + parent_xpath: str | None + section_xpath: str | None + tier: int + page: int | None + bbox: tuple[tuple[float, float], tuple[float, float]] | None + text: str + asset_path: str | None + lexical_score: float + vector_score: float + tier_score: float + freshness_score: float + proximity_score: float + score: float + truncated: bool + + +@dataclass(frozen=True, slots=True) +class ContextResult: + hits: tuple[RetrievalHit, ...] + tokens_used: int + truncated: bool + total_hits: int | None = None + + +def _query_terms(query: str, operator: str = "OR") -> str: + terms = re.findall(r"\w+", query, flags=re.UNICODE)[:16] + if not terms: + raise ValueError("search query must contain a word") + return f" {operator} ".join(f'"{term}"' for term in terms) + + +def _token_prefix(text: str, limit: int) -> tuple[str, int, bool]: + spans = list(re.finditer(r"\S+", text)) + if len(spans) <= limit: + return text, len(spans), False + return text[: spans[limit - 1].end()], limit, True + + +class Retriever: + """Search an authorized SQLite projection and cite current DCLX bytes.""" + + def __init__( + self, + store: LocalContextStore, + *, + embedder: EmbeddingProvider | None = None, + vector_index: VectorIndex | None = None, + ): + self.store = store + self.embedder = embedder + if embedder is None and vector_index is not None: + raise ValueError("vector index needs an embedding provider") + if embedder is not None and vector_index is None: + from .vectors import TurbovecIndex + + vector_index = TurbovecIndex(store, embedder) + self.vector_index = vector_index + + def _scope_conditions( + self, principal: Principal, scope: SearchScope + ) -> tuple[str, list[object]]: + address = parse_uri(scope.uri.rstrip("/"), prefix=True) + authorize_uri(principal, address) + base = address.value + if scope.document_uri is not None: + doc = parse_uri(scope.document_uri) + authorize_uri(principal, doc) + if doc.value != base and not doc.value.startswith(base + "/"): + raise ValueError("document is outside URI scope") + if scope.xpath is not None and ( + not scope.xpath.startswith("/doclang[1]") or len(scope.xpath) > 2_048 + ): + raise ValueError("invalid XPath scope") + if not scope.tiers or set(scope.tiers) - {0, 1, 2}: + raise ValueError("tiers must be a nonempty subset of 0, 1, 2") + clauses = [ + "r.tenant_id=?", + "d.available=1", + "d.revision_id=r.revision_id", + "(r.uri=? OR substr(r.uri,1,?)=?)", + ] + prefix = base + "/" + params: list[object] = [principal.tenant_id, base, len(prefix), prefix] + if scope.document_uri is not None: + clauses.append("r.uri=?") + params.append(doc.value) + if scope.xpath is not None: + child = scope.xpath + "/" + clauses.append("(r.xpath=? OR substr(r.xpath,1,?)=?)") + params.extend((scope.xpath, len(child), child)) + clauses.append(f"r.tier IN ({','.join('?' for _ in scope.tiers)})") + params.extend(scope.tiers) + return " AND ".join(clauses), params + + def _allowed( + self, principal: Principal, scope: SearchScope, *, terms: str | None = None + ) -> list[sqlite3.Row]: + where, params = self._scope_conditions(principal, scope) + if terms is not None: + source = """retrieval_fts JOIN retrieval_units r + ON r.vector_id=retrieval_fts.rowid""" + where = "retrieval_fts MATCH ? AND " + where + params = [terms, *params] + else: + source = "retrieval_units r" + rows = self.store.db.execute( + "SELECT r.* FROM " + + source + + " JOIN documents d ON d.tenant_id=r.tenant_id AND d.uri=r.uri WHERE " + + where + + " ORDER BY r.vector_id LIMIT ?", + (*params, _MAX_CANDIDATES), + ).fetchall() + return [ + row + for row in rows + if parse_uri(row["uri"]).namespace != "users" + or _user_allowed(principal, row["uri"]) + ] + + def _lexical_count( + self, principal: Principal, scope: SearchScope, terms: str + ) -> int: + where, params = self._scope_conditions(principal, scope) + return self.store.db.execute( + """SELECT COUNT(*) FROM retrieval_units r + JOIN documents d ON d.tenant_id=r.tenant_id AND d.uri=r.uri + WHERE r.vector_id IN ( + SELECT rowid FROM retrieval_fts WHERE retrieval_fts MATCH ? + ) AND """ + + where, + (terms, *params), + ).fetchone()[0] + + def search( + self, + principal: Principal, + query: str, + *, + image: bytes | None = None, + scope: SearchScope | None = None, + k: int = 10, + mode: str = "hybrid", + max_tokens: int = 2_000, + per_result_tokens: int = 500, + ) -> ContextResult: + if ( + not 1 <= k <= _MAX_RESULTS + or not 1 <= per_result_tokens <= max_tokens <= 20_000 + ): + raise ValueError("retrieval limits are out of bounds") + if mode not in {"lexical", "vector", "hybrid"}: + raise ValueError("mode must be lexical, vector, or hybrid") + if mode == "vector" and self.vector_index is None: + raise ValueError("vector search needs an embedding provider and index") + if not query.strip() and (image is None or mode != "vector"): + raise ValueError("search needs query text or a vector-mode image") + if self.vector_index is not None: + refresh = getattr(self.vector_index, "refresh", None) + if callable(refresh): + refresh() + terms = _query_terms(query) if query.strip() else None + effective_scope = scope or SearchScope() + lexical_only = mode == "lexical" or ( + mode == "hybrid" and self.vector_index is None + ) + if lexical_only and terms is not None and len(query.split()) > 1: + conjunctive = _query_terms(query, "AND") + focused = self._allowed(principal, effective_scope, terms=conjunctive) + if focused: + terms = conjunctive + rows = focused + else: + rows = self._allowed(principal, effective_scope, terms=terms) + else: + rows = self._allowed( + principal, effective_scope, terms=terms if lexical_only else None + ) + total_hits = ( + self._lexical_count(principal, effective_scope, terms) + if lexical_only and terms is not None + else None + ) + by_id = {row["vector_id"]: row for row in rows} + if not by_id: + return ContextResult((), 0, False, total_hits) + scores: dict[int, list[float]] = {} + if mode in {"lexical", "hybrid"} and terms is not None: + query_words = re.findall(r"\w+", query.casefold()) + if lexical_only: + for identifier, row in by_id.items(): + content = row["text"].casefold() + frequency = sum(content.count(term) for term in query_words) + scores[identifier] = [min(1.0, frequency / 4), 0.0] + else: + identifiers = list(by_id) + for start in range(0, len(identifiers), 500): + batch = identifiers[start : start + 500] + marks = ",".join("?" for _ in batch) + fts_rows = self.store.db.execute( + "SELECT rowid FROM retrieval_fts WHERE retrieval_fts MATCH ? " + f"AND rowid IN ({marks})", + (terms, *batch), + ).fetchall() + for row in fts_rows: + content = by_id[row["rowid"]]["text"].casefold() + frequency = sum(content.count(term) for term in query_words) + scores[row["rowid"]] = [min(1.0, frequency / 4), 0.0] + if mode in {"vector", "hybrid"} and self.vector_index is not None: + assert self.embedder is not None + if image is not None: + from .embeddings import embed_image, embed_text_image + + query_vector = ( + embed_text_image(self.embedder, query, image) + if query.strip() + else embed_image(self.embedder, image) + ) + else: + query_embed = getattr(self.embedder, "embed_query", None) + query_vector = ( + query_embed(query) + if callable(query_embed) + else self.embedder.embed([query])[0] + ) + found = ( + self.vector_index.search(query_vector, set(by_id), min(k * 4, 100)) + if query_vector is not None + else [] + ) + for vector_id, score in found: + if vector_id in by_id: + scores.setdefault(vector_id, [0.0, 0.0])[1] = score + timestamps = sorted({by_id[identifier]["updated_at"] for identifier in scores}) + fresh_rank = {value: rank for rank, value in enumerate(timestamps)} + siblings: dict[tuple[str, str], int] = {} + for identifier in scores: + row = by_id[identifier] + section = row["section_xpath"] or row["parent_xpath"] + if section and section != "/doclang[1]": + key = (row["uri"], section) + siblings[key] = siblings.get(key, 0) + 1 + + def components(identifier: int) -> tuple[float, float, float]: + row = by_id[identifier] + tier = 0.06 if row["tier"] == 0 else 0.03 if row["tier"] == 1 else 0.0 + freshness = ( + 0.02 * fresh_rank[row["updated_at"]] / (len(timestamps) - 1) + if len(timestamps) > 1 + else 0.0 + ) + proximity = ( + 0.02 + if siblings.get( + (row["uri"], row["section_xpath"] or row["parent_xpath"]), 0 + ) + > 1 + else 0.0 + ) + return tier, freshness, proximity + + ranked = sorted( + scores, + key=lambda identifier: ( + by_id[identifier]["tier"], + -( + scores[identifier][0] + + max(0.0, scores[identifier][1]) + + sum(components(identifier)) + ), + -identifier, + ), + ) + hits: list[RetrievalHit] = [] + documents: dict[tuple[str, str], DocLangXDocument] = {} + used = 0 + truncated = False + selected_paths: set[tuple[str, str]] = set() + for identifier in ranked: + if len(hits) >= k or used >= max_tokens: + truncated = True + break + row = by_id[identifier] + path = (row["uri"], row["xpath"]) + if any( + prior_uri == path[0] + and ( + path[1].startswith(prior_xpath + "/") + or prior_xpath.startswith(path[1] + "/") + ) + for prior_uri, prior_xpath in selected_paths + ): + continue + try: + current = self.store.get_record(principal, row["uri"]) + if current.revision_id != row["revision_id"]: + continue + text, source_truncated = self._exact_text(principal, row, documents) + if ( + self.store.get_record(principal, row["uri"]).revision_id + != row["revision_id"] + ): + continue + except (KeyError, OSError, ValueError): + continue + available = min(per_result_tokens, max_tokens - used) + excerpt, count, shortened = _token_prefix(text, available) + if count == 0 and row["asset_path"] is None: + continue + used += count + selected_paths.add(path) + lexical, vector = scores[identifier] + tier_score, freshness_score, proximity_score = components(identifier) + bbox_value = json.loads(row["bbox_json"]) if row["bbox_json"] else None + bbox = ( + ( + (float(bbox_value[0][0]), float(bbox_value[0][1])), + (float(bbox_value[1][0]), float(bbox_value[1][1])), + ) + if bbox_value + else None + ) + hits.append( + RetrievalHit( + row["uri"], + row["document_id"], + row["revision_id"], + row["xpath"], + row["parent_xpath"], + row["section_xpath"], + row["tier"], + row["page"], + bbox, + excerpt, + row["asset_path"], + lexical, + vector, + tier_score, + freshness_score, + proximity_score, + lexical + + max(0.0, vector) + + tier_score + + freshness_score + + proximity_score, + shortened or source_truncated, + ) + ) + truncated |= shortened or source_truncated + return ContextResult(tuple(hits), used, truncated, total_hits) + + def _exact_text( + self, + principal: Principal, + row: sqlite3.Row, + documents: dict[tuple[str, str], DocLangXDocument], + ) -> tuple[str, bool]: + key = (row["uri"], row["revision_id"]) + document = documents.get(key) + if document is None: + document = self.store.get_document(principal, row["uri"]) + documents[key] = document + if row["xpath"].startswith(("@summary:", "@toc:")): + sidecar_name, xpath = row["xpath"].split(":", 1) + sidecar = ( + document.summary() if sidecar_name == "@summary" else document.toc() + ) + if sidecar is None: + raise KeyError(row["xpath"]) + nodes = bounded_nodes( + sidecar, xpath=xpath, limit=1, text_bytes=MAX_NODE_TEXT_CHARS + ) + if not nodes or nodes[0]["xpath"] != xpath: + raise KeyError(xpath) + return nodes[0]["text"], nodes[0]["truncated"] + nodes = bounded_nodes( + document, + xpath=row["xpath"], + limit=1, + text_bytes=MAX_NODE_TEXT_CHARS * 4, + ) + if not nodes or nodes[0]["xpath"] != row["xpath"]: + raise KeyError(row["xpath"]) + value = nodes[0]["text"] + return value[:MAX_NODE_TEXT_CHARS], ( + nodes[0]["truncated"] or len(value) > MAX_NODE_TEXT_CHARS + ) + + +def _user_allowed(principal: Principal, uri: str) -> bool: + try: + authorize_uri(principal, parse_uri(uri)) + except PermissionError: + return False + return True diff --git a/src/docling_context/vectors.py b/src/docling_context/vectors.py new file mode 100644 index 0000000..aaa79af --- /dev/null +++ b/src/docling_context/vectors.py @@ -0,0 +1,452 @@ +"""Optional, rebuildable local vector projection with stable external IDs.""" + +from __future__ import annotations + +import hashlib +import math +import uuid +from collections.abc import Callable +from pathlib import Path +from typing import TYPE_CHECKING, Any, Protocol + +from .package import load_package + +if TYPE_CHECKING: + from .local import LocalContextStore + + +class EmbeddingProvider(Protocol): + model_id: str + dimensions: int + + def embed(self, texts: list[str]) -> list[list[float]]: ... + + +class VectorIndex(Protocol): + def upsert(self, identifier: int, vector: list[float]) -> None: ... + def remove(self, identifier: int) -> None: ... + def search( + self, query: list[float], allowed_ids: set[int], k: int + ) -> list[tuple[int, float]]: ... + def checkpoint(self) -> None: ... + def rebuild(self) -> None: ... + + +class TurbovecIndex: + """IdMapIndex snapshot; SQLite embeddings and DCLX remain authoritative.""" + + def __init__( + self, + store: LocalContextStore, + provider: EmbeddingProvider, + *, + force_rebuild: bool = False, + progress: Callable[[int, int], None] | None = None, + ): + try: + import numpy as np + from turbovec import IdMapIndex # type: ignore[import-untyped] + except ImportError as exc: + raise RuntimeError( + "install docling-context[vectors] for vector search" + ) from exc + if ( + not provider.model_id + or not 8 <= provider.dimensions <= 16_384 + or provider.dimensions % 8 + ): + raise ValueError("embedding dimensions must be a multiple of 8 up to 16384") + self.store = store + self.provider = provider + self.np = np + self.index_type = IdMapIndex + self.index = IdMapIndex(dim=provider.dimensions, bit_width=4) + self.generation = "" + self._progress = progress + if force_rebuild: + self.rebuild() + else: + self._restore_or_rebuild() + + def _offset(self) -> int: + return self.store.db.execute( + "SELECT COALESCE(MAX(sequence),0) FROM outbox" + ).fetchone()[0] + + def _path(self, generation: str) -> Path: + return self.store.root / f"vectors-{generation}.tvim" + + def _restore_or_rebuild(self) -> None: + state = self.store.db.execute( + "SELECT * FROM vector_state WHERE singleton=1" + ).fetchone() + if state is not None and ( + state["model_id"] == self.provider.model_id + and state["dimensions"] == self.provider.dimensions + and state["normalization"] == "l2" + ): + path = self._path(state["generation"]) + try: + if ( + hashlib.sha256(path.read_bytes()).hexdigest() + == state["snapshot_hash"] + ): + loaded = self.index_type.load(str(path)) + if loaded.dim == self.provider.dimensions: + members = self.store.db.execute( + "SELECT COUNT(*) FROM vector_members WHERE generation=?", + (state["generation"],), + ).fetchone()[0] + if members or ( + state["outbox_offset"] == self._offset() + and not self.store.db.execute( + """SELECT 1 FROM retrieval_units + WHERE index_generation=? AND embedding IS NOT NULL + LIMIT 1""", + (state["generation"],), + ).fetchone() + ): + if state["outbox_offset"] == self._offset(): + self.index = loaded + self.generation = state["generation"] + return + if state["outbox_offset"] >= 0: + self._replay(state, loaded) + return + except (OSError, ValueError, RuntimeError): + pass + self.rebuild() + + def _replay(self, state, loaded) -> None: + """Apply changed document URIs to a compatible saved projection.""" + db = self.store.db + offset = self._offset() + changed = db.execute( + "SELECT DISTINCT tenant_id,uri FROM outbox WHERE sequence>? AND sequence<=?", + (state["outbox_offset"], offset), + ).fetchall() + old_ids: set[int] = set() + rows = [] + for event in changed: + old_ids.update( + row[0] + for row in db.execute( + """SELECT vector_id FROM vector_members + WHERE generation=? AND tenant_id=? AND uri=?""", + (state["generation"], event["tenant_id"], event["uri"]), + ) + ) + rows.extend( + db.execute( + """SELECT r.*, d.package_hash FROM retrieval_units r JOIN documents d + ON d.tenant_id=r.tenant_id AND d.uri=r.uri + AND d.revision_id=r.revision_id AND d.available=1 + WHERE r.tenant_id=? AND r.uri=? ORDER BY r.vector_id""", + (event["tenant_id"], event["uri"]), + ).fetchall() + ) + for identifier in old_ids: + loaded.remove(identifier) + updates: list[tuple[str, int, str, bytes, int]] = [] + member_rows = [] + for start in range(0, len(rows), 32): + batch = rows[start : start + 32] + generated = self._generate(batch) + eligible = [row for row in batch if row["vector_id"] in generated] + if eligible: + loaded.add_with_ids( + self.np.ascontiguousarray( + [generated[row["vector_id"]] for row in eligible], + dtype=self.np.float32, + ), + self.np.asarray( + [row["vector_id"] for row in eligible], dtype=self.np.uint64 + ), + ) + member_rows.extend(eligible) + updates.extend( + ( + self.provider.model_id, + self.provider.dimensions, + "l2", + generated[row["vector_id"]].tobytes(), + row["vector_id"], + ) + for row in eligible + ) + generation = uuid.uuid4().hex + path = self._path(generation) + loaded.write(str(path)) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + try: + with self.store.transaction(): + if self._offset() != offset: + raise RuntimeError("documents changed while replaying vector index") + db.executemany( + """UPDATE retrieval_units SET model_id=?,dimensions=?,normalization=?, + embedding=? WHERE vector_id=?""", + updates, + ) + db.execute( + "UPDATE retrieval_units SET index_generation=? WHERE index_generation=?", + (generation, state["generation"]), + ) + db.executemany( + "UPDATE retrieval_units SET index_generation=? WHERE vector_id=?", + ((generation, row["vector_id"]) for row in member_rows), + ) + db.execute( + """INSERT INTO vector_members (generation,tenant_id,uri,vector_id) + SELECT ?,tenant_id,uri,vector_id FROM vector_members + WHERE generation=?""", + (generation, state["generation"]), + ) + db.executemany( + "DELETE FROM vector_members WHERE generation=? AND vector_id=?", + ((generation, identifier) for identifier in old_ids), + ) + db.executemany( + "INSERT INTO vector_members VALUES (?,?,?,?)", + ( + (generation, row["tenant_id"], row["uri"], row["vector_id"]) + for row in member_rows + ), + ) + db.execute( + """UPDATE vector_state SET generation=?,outbox_offset=?,snapshot_hash=? + WHERE singleton=1 AND generation=?""", + (generation, offset, digest, state["generation"]), + ) + db.execute( + "DELETE FROM vector_members WHERE generation=?", + (state["generation"],), + ) + except BaseException: + path.unlink(missing_ok=True) + raise + self.index = loaded + self.generation = generation + self._path(state["generation"]).unlink(missing_ok=True) + + def refresh(self) -> None: + state = self.store.db.execute( + "SELECT outbox_offset,generation FROM vector_state WHERE singleton=1" + ).fetchone() + if ( + state is None + or state["outbox_offset"] != self._offset() + or state["generation"] != self.generation + ): + self._restore_or_rebuild() + + def _normal(self, values: list[float]): + if len(values) != self.provider.dimensions or not all( + math.isfinite(value) and abs(value) < 1e16 for value in values + ): + raise ValueError("embedding has invalid dimensions or coordinates") + array = self.np.ascontiguousarray(values, dtype=self.np.float32) + magnitude = float(self.np.linalg.norm(array)) + if not math.isfinite(magnitude) or magnitude <= 1e-10: + raise ValueError("embedding has zero or invalid norm") + return self.np.ascontiguousarray(array / magnitude, dtype=self.np.float32) + + def _generate(self, rows) -> dict[int, Any]: + """Embed text in batches and attached images only when supported.""" + from .embeddings import embed_image, embed_text_image + + supports_images = getattr( + self.provider, + "supports_images", + callable(getattr(self.provider, "embed_image", None)), + ) + text_rows = [ + row + for row in rows + if row["text"].strip() and (not row["asset_path"] or not supports_images) + ] + result: dict[int, Any] = {} + if text_rows: + values = self.provider.embed([row["text"] for row in text_rows]) + if len(values) != len(text_rows): + raise ValueError( + "embedding provider returned the wrong number of vectors" + ) + result.update( + (row["vector_id"], self._normal(vector)) + for row, vector in zip(text_rows, values, strict=True) + ) + if supports_images: + packages: dict[str, Any] = {} + for row in rows: + if not row["asset_path"]: + continue + package_hash = row["package_hash"] + if package_hash not in packages: + packages[package_hash] = load_package( + self.store.packages.get(package_hash) + ) + image = packages[package_hash].get_part_bytes(row["asset_path"]) + if image is None: + continue + vector = ( + embed_text_image(self.provider, row["text"], image) + if row["text"].strip() + else embed_image(self.provider, image) + ) + if vector is not None: + result[row["vector_id"]] = self._normal(vector) + return result + + def upsert(self, identifier: int, vector: list[float]) -> None: + normalized = self._normal(vector).reshape(1, -1) + self.index.remove(identifier) + self.index.add_with_ids( + normalized, self.np.asarray([identifier], dtype=self.np.uint64) + ) + + def remove(self, identifier: int) -> None: + self.index.remove(identifier) + + def search( + self, query: list[float], allowed_ids: set[int], k: int + ) -> list[tuple[int, float]]: + if not allowed_ids or k < 1: + return [] + permitted = sorted( + identifier for identifier in allowed_ids if identifier in self.index + ) + if not permitted: + return [] + vector = self._normal(query).reshape(1, -1) + scores, identifiers = self.index.search( + vector, + k=min(k, len(permitted)), + allowlist=self.np.asarray(permitted, dtype=self.np.uint64), + ) + return [ + (int(identifier), float(score)) + for identifier, score in zip(identifiers[0], scores[0], strict=True) + ] + + def checkpoint(self) -> None: + if not self.generation: + raise RuntimeError("vector generation is not set") + path = self._path(self.generation) + self.index.write(str(path)) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + with self.store.transaction(): + self.store.db.execute( + "UPDATE vector_state SET snapshot_hash=? WHERE singleton=1 AND generation=?", + (digest, self.generation), + ) + + def rebuild(self) -> None: + """Build a new model generation and switch it only after its snapshot exists.""" + db = self.store.db + previous = db.execute( + "SELECT generation FROM vector_state WHERE singleton=1" + ).fetchone() + rows = db.execute( + """SELECT r.*, d.package_hash FROM retrieval_units r JOIN documents d + ON d.tenant_id=r.tenant_id AND d.uri=r.uri + AND d.revision_id=r.revision_id AND d.available=1 + ORDER BY r.vector_id""" + ).fetchall() + if self._progress is not None: + self._progress(0, len(rows)) + offset = self._offset() + index = self.index_type(dim=self.provider.dimensions, bit_width=4) + updates = [] + member_rows = [] + for start in range(0, len(rows), 32): + batch = rows[start : start + 32] + missing = [ + row + for row in batch + if row["model_id"] != self.provider.model_id + or row["dimensions"] != self.provider.dimensions + or row["normalization"] != "l2" + or row["embedding"] is None + or len(row["embedding"]) != 4 * self.provider.dimensions + ] + missing_ids = {row["vector_id"] for row in missing} + new_vectors = self._generate(missing) if missing else {} + vectors = [] + eligible = [] + for row in batch: + vector = new_vectors.get(row["vector_id"]) + if vector is None and row["vector_id"] not in missing_ids: + vector = self.np.frombuffer(row["embedding"], dtype=self.np.float32) + if vector is None: + continue + updates.append( + ( + self.provider.model_id, + self.provider.dimensions, + "l2", + vector.tobytes(), + row["vector_id"], + ) + ) + vectors.append(vector) + eligible.append(row) + if eligible: + index.add_with_ids( + self.np.ascontiguousarray(vectors, dtype=self.np.float32), + self.np.asarray( + [row["vector_id"] for row in eligible], dtype=self.np.uint64 + ), + ) + member_rows.extend(eligible) + if self._progress is not None: + self._progress(min(start + len(batch), len(rows)), len(rows)) + generation = uuid.uuid4().hex + path = self._path(generation) + index.write(str(path)) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + with self.store.transaction(): + if self._offset() != offset: + path.unlink(missing_ok=True) + raise RuntimeError("documents changed while rebuilding vector index") + db.execute( + """UPDATE retrieval_units SET model_id=NULL,dimensions=NULL, + normalization=NULL,embedding=NULL,index_generation=NULL""" + ) + db.executemany( + """UPDATE retrieval_units SET model_id=?,dimensions=?,normalization=?, + embedding=? WHERE vector_id=?""", + updates, + ) + db.execute( + "UPDATE retrieval_units SET index_generation=? WHERE embedding IS NOT NULL", + (generation,), + ) + db.execute("DELETE FROM vector_members") + db.executemany( + "INSERT INTO vector_members VALUES (?,?,?,?)", + ( + (generation, row["tenant_id"], row["uri"], row["vector_id"]) + for row in member_rows + ), + ) + db.execute( + """INSERT INTO vector_state + (singleton,generation,model_id,dimensions,normalization,outbox_offset,snapshot_hash) + VALUES (1,?,?,?,?,?,?) ON CONFLICT(singleton) DO UPDATE SET + generation=excluded.generation,model_id=excluded.model_id, + dimensions=excluded.dimensions,normalization=excluded.normalization, + outbox_offset=excluded.outbox_offset,snapshot_hash=excluded.snapshot_hash""", + ( + generation, + self.provider.model_id, + self.provider.dimensions, + "l2", + offset, + digest, + ), + ) + old = previous["generation"] if previous is not None else self.generation + self.index = index + self.generation = generation + if old: + self._path(old).unlink(missing_ok=True) diff --git a/tests/test_cli.py b/tests/test_cli.py index 5747735..d701d72 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -133,10 +133,13 @@ def test_cli_import_worker_inspection_and_lexical_lookup(tmp_path, capsys): assert added["uri"] == "docling://resources/manuals/guide" assert added["task_id"] assert main(base + ["task", "status", added["task_id"]]) == 0 + task_table = capsys.readouterr().out + assert "Field" in task_table and "Status" in task_table and "queued" in task_table + assert main(base + ["task", "status", added["task_id"], "--json"]) == 0 assert json.loads(capsys.readouterr().out)["status"] == "queued" assert main(base + ["worker", "--once"]) == 0 capsys.readouterr() - assert main(base + ["task", "status", added["task_id"]]) == 0 + assert main(base + ["task", "status", added["task_id"], "--json"]) == 0 assert json.loads(capsys.readouterr().out)["status"] == "complete" assert main(base + ["ls", "docling://resources/"]) == 0 assert "docling://resources/manuals" in capsys.readouterr().out @@ -149,6 +152,9 @@ def test_cli_import_worker_inspection_and_lexical_lookup(tmp_path, capsys): assert main(base + ["grep", "blue", "--uri", "docling://resources/manuals"]) == 0 assert "Blue glaciers" in capsys.readouterr().out assert main(base + ["status"]) == 0 + store_table = capsys.readouterr().out + assert "Field" in store_table and "Documents" in store_table + assert main(base + ["status", "--json"]) == 0 assert json.loads(capsys.readouterr().out)["documents"] == 2 diff --git a/tests/test_embeddings.py b/tests/test_embeddings.py new file mode 100644 index 0000000..7d2c45d --- /dev/null +++ b/tests/test_embeddings.py @@ -0,0 +1,147 @@ +"""Direct text, image, and mixed-input embedding behavior.""" + +from __future__ import annotations + +import sys +from types import ModuleType, SimpleNamespace + +import pytest + +from docling_context import ( + CLIP_MODEL, + DEFAULT_MODEL, + FastEmbedProvider, + embed_image, + embed_text, + embed_text_image, + embeddings, +) + + +class TextOnly: + model_id = "fixture" + dimensions = 8 + + def embed(self, texts: list[str]) -> list[list[float]]: + return [[float(len(text)), *([0.0] * 7)] for text in texts] + + +def test_text_only_provider_skips_unsupported_images(): + provider = TextOnly() + assert embed_text(provider, "rock") == [4.0, *([0.0] * 7)] + assert embed_image(provider, b"image") is None + assert embed_text_image(provider, "rock", b"image") == embed_text(provider, "rock") + assert embed_text_image(provider, "", b"image") is None + + +def test_fastembed_provider_keeps_query_and_passage_paths_separate(monkeypatch): + class FakeVector(list): + def tolist(self): + return list(self) + + class FakeTextEmbedding: + @staticmethod + def list_supported_models(): + return [ + {"model": DEFAULT_MODEL}, + {"model": "Qdrant/clip-ViT-B-32-text"}, + ] + + @staticmethod + def get_embedding_size(_model): + return 8 + + def __init__(self, **_kwargs): + pass + + def passage_embed(self, texts): + return iter(FakeVector([1.0, *([0.0] * 7)]) for _ in texts) + + def query_embed(self, _text): + return iter([FakeVector([0.0, 1.0, *([0.0] * 6)])]) + + class FakeImageEmbedding: + def __init__(self, **_kwargs): + pass + + def embed(self, paths): + assert list(paths) + return iter([FakeVector([0.0, 1.0, *([0.0] * 6)])]) + + monkeypatch.setitem( + sys.modules, + "fastembed", + SimpleNamespace( + TextEmbedding=FakeTextEmbedding, ImageEmbedding=FakeImageEmbedding + ), + ) + text_provider = FastEmbedProvider() + assert text_provider.embed_text("rock")[0] == 1.0 + assert text_provider.embed_query("rock")[1] == 1.0 + assert text_provider.embed_image(b"image") is None + assert text_provider.embed_text_image("rock", b"image") == text_provider.embed_text( + "rock" + ) + assert text_provider.embed_text_image("", b"image") is None + mixed = FastEmbedProvider(CLIP_MODEL) + assert mixed.embed_image(b"image")[1] == 1.0 + vector = mixed.embed_text_image("rock", b"image") + assert vector is not None and vector[0] > 0 and vector[1] > 0 + + +def test_mlx_auto_selection_and_saved_backend_identity(monkeypatch): + encoded: list[list[str]] = [] + + class FakeResult: + def __init__(self, texts): + self.texts = texts + + def tolist(self): + return [[1.0, *([0.0] * 383)] for _ in self.texts] + + class FakeModel: + @classmethod + def from_registry(cls, name): + assert name == "bge-small" + return cls() + + def encode(self, texts, *, show_progress): + assert show_progress is False + encoded.append(texts) + return FakeResult(texts) + + package = ModuleType("mlx_embedding_models") + package.__path__ = [] + embedding_module = ModuleType("mlx_embedding_models.embedding") + embedding_module.EmbeddingModel = FakeModel + registry_module = ModuleType("mlx_embedding_models.registry") + registry_module.registry = {"bge-small": {"ndim": 384}} + monkeypatch.setitem(sys.modules, "mlx_embedding_models", package) + monkeypatch.setitem(sys.modules, "mlx_embedding_models.embedding", embedding_module) + monkeypatch.setitem(sys.modules, "mlx_embedding_models.registry", registry_module) + monkeypatch.setattr(embeddings, "_apple_silicon", lambda: True) + monkeypatch.setattr(embeddings, "find_spec", lambda _name: object()) + + class FakeOnnx: + def __init__(self, model_name, *, offline=False): + self.model_id = model_name + self.offline = offline + + monkeypatch.setattr(embeddings, "FastEmbedProvider", FakeOnnx) + provider = embeddings.local_embedding_provider(DEFAULT_MODEL) + assert provider.model_id == "mlx:BAAI/bge-small-en-v1.5" + assert provider.dimensions == 384 + assert provider.embed(["rock"])[0][0] == 1.0 + provider.embed_query("rock") + assert encoded[-1][0].startswith("Represent this sentence") + assert ( + embeddings.stored_embedding_provider(provider.model_id).model_id + == provider.model_id + ) + assert embeddings.stored_embedding_provider(DEFAULT_MODEL).model_id == DEFAULT_MODEL + assert embeddings.local_embedding_provider(CLIP_MODEL).model_id == CLIP_MODEL + assert embeddings.local_embedding_provider(DEFAULT_MODEL, offline=True).offline + monkeypatch.setattr(embeddings, "find_spec", lambda _name: None) + assert embeddings.local_embedding_provider(DEFAULT_MODEL).model_id == DEFAULT_MODEL + with pytest.raises(ValueError, match="does not support"): + embeddings.local_embedding_provider(CLIP_MODEL, backend="mlx") diff --git a/tests/test_step2.py b/tests/test_ingestion_jobs.py similarity index 97% rename from tests/test_step2.py rename to tests/test_ingestion_jobs.py index 7663af8..6764859 100644 --- a/tests/test_step2.py +++ b/tests/test_ingestion_jobs.py @@ -282,13 +282,15 @@ def fail(*_args, **_kwargs): assert worker.status(PRINCIPAL, COLLECTION).stale -def test_existing_step1_database_upgrades_without_losing_documents(tmp_path): +def test_existing_database_upgrades_without_losing_documents(tmp_path): with LocalContextStore(tmp_path) as store: original = store.put(PRINCIPAL, URI, dclx()) store.db.executescript( - "DROP TABLE jobs; DROP TABLE collection_status; PRAGMA user_version=1;" + "DROP TABLE jobs; DROP TABLE collection_status; " + "DROP TABLE retrieval_fts; DROP TABLE retrieval_units; " + "DROP TABLE vector_state; PRAGMA user_version=1;" ) with LocalContextStore(tmp_path) as store: assert store.get_record(PRINCIPAL, URI).revision_id == original.revision_id - assert store.db.execute("PRAGMA user_version").fetchone()[0] == 2 + assert store.db.execute("PRAGMA user_version").fetchone()[0] == 6 assert len(CollectionWorker(store).jobs(PRINCIPAL, COLLECTION)) == 1 diff --git a/tests/test_retrieval.py b/tests/test_retrieval.py new file mode 100644 index 0000000..033d956 --- /dev/null +++ b/tests/test_retrieval.py @@ -0,0 +1,723 @@ +"""Scoped retrieval, citation, and vector projection lifecycle.""" + +from __future__ import annotations + +import hashlib +import json + +import pytest +from doclang import DocLangXDocument + +from docling_context import ( + Ingestor, + LocalContextStore, + Principal, + Retriever, + SearchScope, + TurbovecIndex, +) +from docling_context.cli import _search_excerpt, main + + +def _package(body: str) -> bytes: + document = DocLangXDocument() + assert document.read_xml(body) + return document.write_bytes() + + +def _image_package() -> bytes: + document = DocLangXDocument() + assert document.read_xml( + '' + 'Rock diagram' + "" + ) + document.set_part_bytes("assets/one.png", b"first image", "image/png") + document.set_part_bytes("assets/two.png", b"second image", "image/png") + return document.write_bytes() + + +class WordVectors: + model_id = "word-hash-1" + dimensions = 8 + + def embed(self, texts: list[str]) -> list[list[float]]: + output = [] + for text in texts: + values = [0.0] * 8 + for word in text.casefold().split(): + index = hashlib.sha256(word.encode()).digest()[0] % 8 + values[index] += 1.0 + output.append(values) + return output + + +def test_lexical_scope_exact_citations_and_budget(tmp_path): + alice = Principal("north", "alice") + bob = Principal("south", "bob") + with LocalContextStore(tmp_path) as store: + record = Ingestor(store).add_resource( + alice, + "docling://resources/geology/one", + _package( + "Glacier report" + "Blue glacier advances in winter" + "VolcanoRed lava flows" + ), + filename="one.dclx", + ) + store.put( + alice, + "docling://resources/geology-sibling/two", + _package("glacier sibling"), + ) + store.put( + bob, + "docling://resources/geology/three", + _package("glacier other tenant"), + ) + retrieval = Retriever(store) + result = retrieval.search( + alice, + "glacier", + scope=SearchScope( + uri="docling://resources/geology", + xpath="/doclang[1]/text[1]", + ), + mode="lexical", + max_tokens=3, + per_result_tokens=3, + ) + assert len(result.hits) == 1 + hit = result.hits[0] + assert hit.uri == record.uri + assert hit.revision_id == record.revision_id + assert hit.xpath == "/doclang[1]/text[1]" + assert hit.text == "Blue glacier advances" + assert hit.truncated and result.truncated + assert result.tokens_used == 3 + assert hit.lexical_score > 0 + assert hit.vector_score == 0 + sidecars = retrieval.search( + alice, + "Glacier report", + scope=SearchScope(uri="docling://resources/geology"), + mode="lexical", + k=10, + ).hits + assert any(hit.xpath.startswith("@summary:") for hit in sidecars) + assert any(hit.xpath.startswith("@toc:") for hit in sidecars) + assert ( + retrieval.search( + alice, + "sibling", + scope=SearchScope(uri="docling://resources/geology"), + mode="lexical", + ).hits + == () + ) + with pytest.raises(PermissionError): + retrieval.search( + alice, + "glacier", + scope=SearchScope(uri="docling://users/south/bob/memories"), + ) + + +def test_replacement_delete_and_stale_row_are_never_cited(tmp_path): + principal = Principal("north", "alice") + uri = "docling://resources/geology/one" + with LocalContextStore(tmp_path) as store: + old = store.put( + principal, + uri, + _package("old glacier"), + ) + new = store.put( + principal, + uri, + _package("new basalt"), + expected_revision=old.revision_id, + ) + cursor = store.db.execute( + """INSERT INTO retrieval_units + (tenant_id,uri,document_id,revision_id,xpath,tier,text,updated_at) + VALUES (?,?,?,?,?,2,?,?)""", + ( + principal.tenant_id, + uri, + old.document_id, + old.revision_id, + "/doclang[1]/text[1]", + "old glacier", + old.updated_at, + ), + ) + store.db.execute( + "INSERT INTO retrieval_fts(rowid,text) VALUES (?,?)", + (cursor.lastrowid, "old glacier"), + ) + retrieval = Retriever(store) + assert retrieval.search(principal, "old", mode="lexical").hits == () + assert ( + retrieval.search(principal, "basalt", mode="lexical").hits[0].revision_id + == new.revision_id + ) + store.delete(principal, uri, expected_revision=new.revision_id) + assert retrieval.search(principal, "basalt", mode="lexical").hits == () + + +def test_cli_search_and_schema_backfill(tmp_path, capsys): + principal = Principal("north", "alice") + uri = "docling://resources/notes/one" + with LocalContextStore(tmp_path) as store: + store.put( + principal, uri, _package("amber fossil") + ) + store.db.execute("DELETE FROM retrieval_fts") + store.db.execute("DELETE FROM retrieval_units") + with LocalContextStore(tmp_path) as store: + assert Retriever(store).search(principal, "fossil", mode="lexical").hits + assert ( + main( + [ + "--store", + str(tmp_path), + "--tenant", + "north", + "--user", + "alice", + "search", + "fossil", + "--uri", + "docling://resources/notes", + "--json", + ] + ) + == 0 + ) + assert json.loads(capsys.readouterr().out)["xpath"] == "/doclang[1]/text[1]" + assert ( + main( + [ + "--store", + str(tmp_path), + "--tenant", + "north", + "--user", + "alice", + "search", + "fossil", + "--uri", + "docling://resources/notes", + ] + ) + == 0 + ) + table = capsys.readouterr().out + assert "Score" in table and "Document ID" in table and "Excerpt" in table + assert "amber fossil" in table and "/doclang[1]/text[1]" in table + assert "revision_id" not in table + + +def test_search_excerpt_does_not_overlap_or_wrap(): + first = "A" * 64 + last = "B" * 64 + assert _search_excerpt(first + last) == first + last + assert _search_excerpt(first + "X" + last) == first + " ... " + last + assert _search_excerpt("first\n second") == "first second" + + +def test_search_reports_matching_nodes_before_result_limit(tmp_path, capsys): + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + for number in range(3): + store.put( + principal, + f"docling://resources/notes/{number}", + _package(f"fossil {number}"), + ) + store.put( + principal, + "docling://resources/other/four", + _package("fossil elsewhere"), + ) + result = Retriever(store).search( + principal, + "fossil", + scope=SearchScope(uri="docling://resources/notes"), + mode="lexical", + k=1, + ) + assert len(result.hits) == 1 + assert result.total_hits == 3 + command = [ + "--store", + str(tmp_path), + "--tenant", + "north", + "--user", + "alice", + "search", + "fossil", + "--uri", + "docling://resources/notes", + "-k", + "1", + ] + assert main(command) == 0 + assert "Showing 1 of 3 matching nodes." in capsys.readouterr().out + assert main(command[:7] + ["missing"] + command[8:]) == 0 + assert "Showing 0 of 0 matching nodes." in capsys.readouterr().out + + +def test_index_status_and_rebuild_restore_search_records(tmp_path, capsys): + alice = Principal("north", "alice") + bob = Principal("north", "bob") + uri = "docling://resources/notes/one" + with LocalContextStore(tmp_path) as store: + store.put(alice, uri, _package("amber fossil")) + store.put( + bob, + "docling://users/north/bob/memories/secret", + _package("private fossil"), + ) + identifier = store.db.execute( + "SELECT vector_id FROM retrieval_units WHERE tenant_id=? AND uri=? AND text=?", + (alice.tenant_id, uri, "amber fossil"), + ).fetchone()[0] + store.db.execute("DELETE FROM retrieval_fts WHERE rowid=?", (identifier,)) + store.db.execute( + """INSERT INTO vector_state + (singleton,generation,model_id,dimensions,normalization,outbox_offset,snapshot_hash) + VALUES (1,'old','fixture',8,'l2',0,'old')""" + ) + command = ["--store", str(tmp_path), "--tenant", "north", "--user", "alice"] + assert main([*command, "index", "status"]) == 0 + assert "Indexed nodes" in capsys.readouterr().out + assert main([*command, "index", "status", "--json"]) == 0 + status = json.loads(capsys.readouterr().out) + assert status["documents"] == 1 + assert status["indexed_documents"] == 1 + assert status["embedded_nodes"] == 0 + assert status["vector_snapshot_current"] is False + with LocalContextStore(tmp_path) as store: + assert not Retriever(store).search(alice, "fossil", mode="lexical").hits + assert main([*command, "index", "lex", "status"]) == 0 + assert "Indexed documents" in capsys.readouterr().out + assert main([*command, "index", "lex", "status", "--json"]) == 0 + assert "embedded_nodes" not in json.loads(capsys.readouterr().out) + assert main([*command, "index", "lex", "rebuild"]) == 0 + rebuilt = json.loads(capsys.readouterr().out) + assert rebuilt["rebuilt_documents"] == 1 + assert rebuilt["indexed_nodes"] >= 1 + with LocalContextStore(tmp_path) as store: + assert Retriever(store).search(alice, "fossil", mode="lexical").hits + assert ( + store.db.execute( + "SELECT outbox_offset FROM vector_state WHERE singleton=1" + ).fetchone()[0] + == -1 + ) + + +def test_lexical_search_filters_before_candidate_limit(tmp_path, monkeypatch): + monkeypatch.setattr("docling_context.retrieval._MAX_CANDIDATES", 1) + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + store.put( + principal, + "docling://resources/notes/first", + _package("quartz"), + ) + store.put( + principal, + "docling://resources/notes/second", + _package("fossil"), + ) + result = Retriever(store).search(principal, "fossil", mode="lexical") + assert result.total_hits == 1 + assert len(result.hits) == 1 + assert result.hits[0].uri.endswith("/second") + + +def test_vector_snapshot_restart_corruption_and_generation(tmp_path): + pytest.importorskip("turbovec") + principal = Principal("north", "alice") + uri = "docling://resources/geology/one" + provider = WordVectors() + with LocalContextStore(tmp_path) as store: + first = store.put( + principal, + uri, + _package("glacier ice"), + ) + store.put( + Principal("south", "bob"), + uri, + _package("glacier private south"), + ) + retrieval = Retriever(store, embedder=provider) + assert isinstance(retrieval.vector_index, TurbovecIndex) + result = retrieval.search(principal, "glacier", mode="vector") + assert result.hits[0].revision_id == first.revision_id + assert all("private south" not in hit.text for hit in result.hits) + cited_text = result.hits[0].text + generation = retrieval.vector_index.generation + snapshot = tmp_path / f"vectors-{generation}.tvim" + assert snapshot.exists() + with LocalContextStore(tmp_path) as store: + retrieval = Retriever(store, embedder=provider) + assert retrieval.vector_index.generation == generation + assert retrieval.search(principal, "glacier", mode="vector").hits + snapshot.write_bytes(b"bad snapshot") + with LocalContextStore(tmp_path) as store: + retrieval = Retriever(store, embedder=provider) + assert retrieval.vector_index.generation != generation + assert ( + retrieval.search(principal, "glacier", mode="vector").hits[0].text + == cited_text + ) + new = store.put( + principal, + uri, + _package("basalt lava"), + expected_revision=first.revision_id, + ) + assert ( + retrieval.search(principal, "glacier", mode="vector").hits[0].revision_id + == new.revision_id + ) + store.delete(principal, uri, expected_revision=new.revision_id) + assert retrieval.search(principal, "glacier", mode="vector").hits == () + with LocalContextStore(tmp_path) as store: + assert ( + Retriever(store, embedder=provider) + .search(principal, "glacier", mode="vector") + .hits + == () + ) + + class NewModel(WordVectors): + model_id = "word-hash-2" + + changed = Retriever(store, embedder=NewModel()) + state = store.db.execute( + "SELECT model_id FROM vector_state WHERE singleton=1" + ).fetchone() + assert state["model_id"] == "word-hash-2" + assert changed.search(principal, "glacier", mode="vector").hits == () + + +def test_vector_outbox_replay_only_embeds_changed_document(tmp_path, monkeypatch): + pytest.importorskip("turbovec") + + class CountingVectors(WordVectors): + def __init__(self): + self.texts: list[str] = [] + + def embed(self, texts: list[str]) -> list[list[float]]: + self.texts.extend(texts) + return super().embed(texts) + + principal = Principal("north", "alice") + provider = CountingVectors() + with LocalContextStore(tmp_path) as store: + first = store.put( + principal, + "docling://resources/notes/one", + _package("glacier ice"), + ) + store.put( + principal, + "docling://resources/notes/two", + _package("basalt lava"), + ) + retriever = Retriever(store, embedder=provider) + initial = retriever.vector_index.generation + monkeypatch.setattr( + retriever.vector_index, + "rebuild", + lambda: pytest.fail("compatible outbox changes must replay"), + ) + provider.texts.clear() + store.put( + principal, + first.uri, + _package("quartz crystal"), + expected_revision=first.revision_id, + ) + assert retriever.search(principal, "quartz", mode="vector").hits + assert provider.texts == ["quartz crystal", "quartz"] + assert retriever.vector_index.generation != initial + assert all( + hit.uri != first.uri + for hit in retriever.search(principal, "ice", mode="lexical").hits + ) + provider.texts.clear() + current = store.get_record(principal, first.uri) + store.delete(principal, first.uri, expected_revision=current.revision_id) + assert retriever.search(principal, "quartz", mode="vector").hits + assert provider.texts == ["quartz"] + assert not store.db.execute( + "SELECT 1 FROM vector_members WHERE uri=?", (first.uri,) + ).fetchone() + + +def test_legacy_vector_snapshot_rebuilds_membership_on_upgrade(tmp_path): + pytest.importorskip("turbovec") + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + store.put( + principal, + "docling://resources/notes/one", + _package("glacier ice"), + ) + old_generation = Retriever( + store, embedder=WordVectors() + ).vector_index.generation + store.db.executescript("DROP TABLE vector_members; PRAGMA user_version=3;") + with LocalContextStore(tmp_path) as store: + retriever = Retriever(store, embedder=WordVectors()) + assert retriever.vector_index.generation != old_generation + assert retriever.search(principal, "glacier", mode="vector").hits + assert store.db.execute("SELECT COUNT(*) FROM vector_members").fetchone()[0] > 0 + + +def test_existing_retrieval_index_adds_picture_column_on_upgrade(tmp_path): + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + store.put( + principal, + "docling://resources/notes/one", + _package("glacier ice"), + ) + store.db.executescript( + "ALTER TABLE retrieval_units DROP COLUMN asset_path; PRAGMA user_version=4;" + ) + with LocalContextStore(tmp_path) as store: + assert store.db.execute("PRAGMA user_version").fetchone()[0] == 6 + assert "asset_path" in { + row["name"] + for row in store.db.execute("PRAGMA table_info(retrieval_units)") + } + assert Retriever(store).search(principal, "glacier", mode="lexical").hits + + +def test_section_xpath_tracks_heading_without_changing_xml_parent(tmp_path): + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + store.put( + principal, + "docling://resources/notes/one", + _package( + "IceGlacier snow" + "RockGlacier stone" + ), + ) + rows = store.db.execute( + """SELECT xpath,parent_xpath,section_xpath FROM retrieval_units + WHERE xpath IN ('/doclang[1]/text[1]','/doclang[1]/text[2]') + ORDER BY xpath""" + ).fetchall() + assert [row["parent_xpath"] for row in rows] == [ + "/doclang[1]", + "/doclang[1]", + ] + assert [row["section_xpath"] for row in rows] == [ + "/doclang[1]/heading[1]", + "/doclang[1]/heading[2]", + ] + + +def test_matching_outline_precedes_body_when_budget_allows(tmp_path): + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + Ingestor(store).add_resource( + principal, + "docling://resources/notes/outline", + _package( + "Glacier overview" + "Glacier details with glacier glacier glacier examples" + "" + ), + filename="outline.dclx", + ) + hits = Retriever(store).search(principal, "glacier", mode="lexical", k=4).hits + assert hits + assert [hit.tier for hit in hits] == sorted(hit.tier for hit in hits) + + +def test_search_decodes_each_cited_package_once(tmp_path, monkeypatch): + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + Ingestor(store).add_resource( + principal, + "docling://resources/notes/one", + _package( + "Glacier overview" + "Glacier iceGlacier snow" + ), + filename="one.dclx", + ) + original = store.get_document + calls = [] + + def counted(*args, **kwargs): + calls.append(args[1]) + return original(*args, **kwargs) + + monkeypatch.setattr(store, "get_document", counted) + hits = Retriever(store).search(principal, "glacier", mode="lexical", k=10).hits + assert len(hits) >= 3 + assert len(calls) == 1 + + +def test_multiword_lexical_search_uses_all_words_then_falls_back(tmp_path): + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + store.put( + principal, + "docling://resources/notes/one", + _package("glacier ice"), + ) + store.put( + principal, + "docling://resources/notes/two", + _package("glacier rock"), + ) + retriever = Retriever(store) + focused = retriever.search(principal, "glacier ice", mode="lexical") + assert focused.total_hits == 1 + assert [hit.uri for hit in focused.hits] == ["docling://resources/notes/one"] + fallback = retriever.search(principal, "glacier snow", mode="lexical") + assert fallback.total_hits == 2 + + +def test_picture_assets_are_vector_units_when_provider_supports_images(tmp_path): + pytest.importorskip("turbovec") + + class ImageVectors(WordVectors): + model_id = "image-test-1" + supports_images = True + + def __init__(self): + self.calls = [] + + def embed_image(self, image: bytes): + self.calls.append(("image", image)) + return [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + + def embed_text_image(self, text: str, image: bytes): + self.calls.append(("mixed", text, image)) + return [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + store.put(principal, "docling://resources/images/one", _image_package()) + provider = ImageVectors() + retriever = Retriever(store, embedder=provider) + assert ("image", b"first image") in provider.calls + assert ("mixed", "Rock diagram", b"second image") in provider.calls + hit = retriever.search(principal, "", image=b"query", mode="vector", k=1).hits[ + 0 + ] + assert hit.xpath == "/doclang[1]/picture[1]" + assert hit.asset_path == "assets/one.png" + assert hit.text == "" + text_only = Retriever(store, embedder=WordVectors()) + assert text_only.search(principal, "", image=b"query", mode="vector").hits == () + assert ( + store.db.execute( + "SELECT embedding FROM retrieval_units WHERE xpath=?", + ("/doclang[1]/picture[1]",), + ).fetchone()[0] + is None + ) + + +def test_rebuild_index_refreshes_existing_vector_retriever(tmp_path): + pytest.importorskip("turbovec") + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + store.put( + principal, + "docling://resources/notes/one", + _package("glacier ice"), + ) + retriever = Retriever(store, embedder=WordVectors()) + first = retriever.search(principal, "glacier", mode="vector") + assert first.hits and first.hits[0].vector_score > 0 + old_generation = retriever.vector_index.generation + lex_progress: list[tuple[int, int]] = [] + status = store.rebuild_index( + principal, + progress=lambda done, total: lex_progress.append((done, total)), + ) + assert lex_progress[0] == (0, 1) + assert lex_progress[-1] == (1, 1) + assert status["embedded_nodes"] == 0 + assert status["vector_snapshot_current"] is False + second = retriever.search(principal, "glacier", mode="vector") + assert second.hits and second.hits[0].vector_score > 0 + assert retriever.vector_index.generation != old_generation + assert store.index_status(principal)["vector_snapshot_current"] is True + vector_progress: list[tuple[int, int]] = [] + TurbovecIndex( + store, + WordVectors(), + force_rebuild=True, + progress=lambda done, total: vector_progress.append((done, total)), + ) + assert vector_progress[0][0] == 0 + assert vector_progress[-1][0] == vector_progress[-1][1] + assert vector_progress[-1][1] > 0 + + +def test_cli_vector_rebuild_search_and_scope_guard(tmp_path, capsys, monkeypatch): + pytest.importorskip("turbovec") + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + store.put( + principal, + "docling://resources/notes/one", + _package("glacier ice"), + ) + selected_backends: list[str] = [] + + def provider_for_cli(_model, *, backend, offline): + assert offline is False + selected_backends.append(backend) + return WordVectors() + + monkeypatch.setattr( + "docling_context.cli.local_embedding_provider", provider_for_cli + ) + monkeypatch.setattr( + "docling_context.cli.stored_embedding_provider", + lambda *_args, **_kwargs: WordVectors(), + ) + command = ["--store", str(tmp_path), "--tenant", "north", "--user", "alice"] + assert main([*command, "index", "vector", "rebuild", "--backend", "onnx"]) == 0 + assert selected_backends == ["onnx"] + assert json.loads(capsys.readouterr().out)["embedded_nodes"] > 0 + assert main([*command, "index", "vector", "status"]) == 0 + vector_table = capsys.readouterr().out + assert "Vector model" in vector_table and "word-hash-1" in vector_table + assert main([*command, "index", "vector", "status", "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["vector_model"] == "word-hash-1" + assert main([*command, "search", "glacier"]) == 0 + output = capsys.readouterr().out + assert "Showing" in output and "ranked nodes" in output + with LocalContextStore(tmp_path) as store: + result = Retriever(store, embedder=WordVectors()).search( + principal, "", image=b"unsupported", mode="vector" + ) + assert result.hits == () + store.put( + Principal("north", "bob"), + "docling://users/north/bob/memories/private", + _package("private ice"), + ) + assert main([*command, "index", "vector", "rebuild"]) == 1 + assert "accessible" in capsys.readouterr().err diff --git a/tests/test_step1.py b/tests/test_storage.py similarity index 99% rename from tests/test_step1.py rename to tests/test_storage.py index f437d6d..5dc7097 100644 --- a/tests/test_step1.py +++ b/tests/test_storage.py @@ -1,4 +1,4 @@ -"""Step 1 storage and native API contract checks.""" +"""Storage, URI, revision, and native package contracts.""" from __future__ import annotations diff --git a/uv.lock b/uv.lock index 603eeef..3ba31f1 100644 --- a/uv.lock +++ b/uv.lock @@ -8,9 +8,12 @@ resolution-markers = [ "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version == '3.14.*' and sys_platform == 'emscripten'", "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version < '3.14' and sys_platform == 'win32'", - "python_full_version < '3.14' and sys_platform == 'emscripten'", - "python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version < '3.13' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] [[package]] @@ -18,7 +21,8 @@ name = "accelerate" version = "1.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub" }, + { name = "huggingface-hub", version = "0.36.2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "huggingface-hub", version = "1.33.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, { name = "numpy" }, { name = "packaging" }, { name = "psutil" }, @@ -77,6 +81,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "awkward" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "awkward-cpp", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "fsspec", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "numpy", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "packaging", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1e/d7/ff39e4a334de35f46e3555b974e59834ed564ac987acfda0eb34c69a5746/awkward-2.14.0.tar.gz", hash = "sha256:f5bce2ddfd4b5ad5277462b01adf94fd6ca90c33a56949fb8c00490c30725737", size = 6508333, upload-time = "2026-09-17T20:00:34.352Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/42/8a9bb42d63524c15698a1c672b1e39b368e663acb1ada69e04a0828d3f1a/awkward-2.14.0-py3-none-any.whl", hash = "sha256:bbdcecf16c8358bac2eb20285659325fc7a71e439779f1259277693357689efd", size = 1000180, upload-time = "2026-09-17T20:00:32.141Z" }, +] + +[[package]] +name = "awkward-cpp" +version = "57" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/20/be33e3d81b165a0c59e406423b12d2310fb25c8afac1255d98df45cf4a43/awkward_cpp-57.tar.gz", hash = "sha256:df2fff584d72b9942db6eb700fa24fe898bc65e369eecbcd35bcf81269aa41d2", size = 1502719, upload-time = "2026-09-17T19:56:47.318Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/e3/cb615d80059cac60ddb32301322d49b394e3f908510a4a8f2222aa1cb41f/awkward_cpp-57-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:761bdfaf4b277c2b1bb4b1a21f4be5e1032692ee611b7c7f2446de8203d05d1b", size = 581269, upload-time = "2026-09-17T19:55:14.149Z" }, + { url = "https://files.pythonhosted.org/packages/3f/b5/ff27462cd25386900c9cb2374b96605f68c710c98b3a9fb32fd2f1805069/awkward_cpp-57-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:38f46062b1406c79b197e5065ca4efd275f357374921c3327416befcbfdad2dc", size = 581309, upload-time = "2026-09-17T19:55:28.197Z" }, + { url = "https://files.pythonhosted.org/packages/fd/ac/503c871e9c0333e48d2fcba8483de61c99bfb73c97c5eae3a8d4b8ce834d/awkward_cpp-57-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4831449298d847fcd1ac34366564f0e09f25cc5ab78ca9ed33bfcb95b24b59bf", size = 581957, upload-time = "2026-09-17T19:55:44.412Z" }, + { url = "https://files.pythonhosted.org/packages/24/91/f2644b813b7aeadaa89e09196716ec576895cf2fa8407eef84262f3f2b57/awkward_cpp-57-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1134d6d97398539f398378f9130c9a889e5ce8d551a46ce970fc977790114c55", size = 594828, upload-time = "2026-09-17T19:56:01.06Z" }, + { url = "https://files.pythonhosted.org/packages/e3/fb/9ed1c736e0d2bd6cc53c1b11c47ffe1405794dbe682f9455c2d927d0ec19/awkward_cpp-57-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:e15f2001ca97aea2290d82022382b4ffb64c65316f1006dd318a1c6fe0a46031", size = 582003, upload-time = "2026-09-17T19:56:16.443Z" }, + { url = "https://files.pythonhosted.org/packages/25/74/f00f70462b4aaec3c5b82846deebc7e525f41ee4bcd4c7e1b4d83bb906ee/awkward_cpp-57-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:d8215f8b9cac781eacb112f37e0073ae43e490098a3799d64a5d29cfb6bb2170", size = 594718, upload-time = "2026-09-17T19:56:34.819Z" }, +] + [[package]] name = "beautifulsoup4" version = "4.15.0" @@ -484,12 +520,21 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "doclang" }, + { name = "tabulate" }, + { name = "tqdm" }, ] [package.optional-dependencies] conversion = [ { name = "docling", extra = ["rapidocr"] }, ] +vectors = [ + { name = "fastembed" }, + { name = "mlx-embedding-models", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" }, + { name = "numpy" }, + { name = "transformers", version = "4.57.6", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" }, + { name = "turbovec" }, +] [package.dev-dependencies] ci = [ @@ -509,8 +554,15 @@ dev = [ requires-dist = [ { name = "doclang", git = "https://github.com/doclang-project/doclang?rev=92d299026f2f985328b52424483c711e19b2f306" }, { name = "docling", extras = ["rapidocr"], marker = "extra == 'conversion'", specifier = ">=2.134.0,<3" }, + { name = "fastembed", marker = "extra == 'vectors'", specifier = ">=0.8.1,<1" }, + { name = "mlx-embedding-models", marker = "platform_machine == 'arm64' and sys_platform == 'darwin' and extra == 'vectors'", specifier = "==0.0.11" }, + { name = "numpy", marker = "extra == 'vectors'", specifier = ">=1.26" }, + { name = "tabulate", specifier = ">=0.10,<1" }, + { name = "tqdm", specifier = ">=4.66,<5" }, + { name = "transformers", marker = "platform_machine == 'arm64' and sys_platform == 'darwin' and extra == 'vectors'", specifier = ">=4.47,<5" }, + { name = "turbovec", marker = "extra == 'vectors'", specifier = ">=1,<2" }, ] -provides-extras = ["conversion"] +provides-extras = ["conversion", "vectors"] [package.metadata.requires-dev] ci = [ @@ -554,7 +606,8 @@ wheels = [ [package.optional-dependencies] chunking = [ { name = "semchunk" }, - { name = "transformers" }, + { name = "transformers", version = "4.57.6", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "transformers", version = "5.18.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, { name = "tree-sitter" }, { name = "tree-sitter-c" }, { name = "tree-sitter-javascript" }, @@ -567,13 +620,15 @@ name = "docling-ibm-models" version = "4.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub" }, + { name = "huggingface-hub", version = "0.36.2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "huggingface-hub", version = "1.33.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, { name = "numpy" }, { name = "pillow" }, { name = "safetensors", extra = ["torch"] }, { name = "torch" }, { name = "torchvision" }, - { name = "transformers" }, + { name = "transformers", version = "4.57.6", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "transformers", version = "5.18.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/69/ee/7dd19487bf333320f0fc73eddcb42347d9f266310ebb1a605f120987f7c1/docling_ibm_models-4.0.3.tar.gz", hash = "sha256:509e89af75e06b48500977dca11ecb6022e81b1526c90dd54587a60ccc82bdb6", size = 77649, upload-time = "2026-09-18T16:15:36.731Z" } wheels = [ @@ -642,7 +697,8 @@ standard = [ { name = "docling-ibm-models" }, { name = "docling-parse" }, { name = "httpx" }, - { name = "huggingface-hub" }, + { name = "huggingface-hub", version = "0.36.2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "huggingface-hub", version = "1.33.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, { name = "mail-parser" }, { name = "marko" }, { name = "numpy" }, @@ -686,6 +742,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/d2/58beb2b8355f89b483fa21b99fe9934ee7312e3ccf228bb7cefe9e8d979c/faker-40.40.0-py3-none-any.whl", hash = "sha256:cd45ebdd1363f92a45740ac49945e49fa18f7e10771884a83c796a235550d7b7", size = 2066663, upload-time = "2026-09-29T00:56:17.952Z" }, ] +[[package]] +name = "fastembed" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub", version = "0.36.2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "huggingface-hub", version = "1.33.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "loguru" }, + { name = "mmh3" }, + { name = "numpy" }, + { name = "onnxruntime" }, + { name = "pillow" }, + { name = "py-rust-stemmers" }, + { name = "requests" }, + { name = "tokenizers", version = "0.22.2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "tokenizers", version = "0.23.2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/84/09d221078b6e2e2169fab52506dc2e0a6c7a9a4f0699000e9b9df55b42b2/fastembed-0.8.1.tar.gz", hash = "sha256:ad815f14873dce918f755a99f16929a1384775f91ca8a14645993a165b5f65ba", size = 82392, upload-time = "2026-09-22T20:01:24.201Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/76/209f9f4c421853f729176c5a6ca99a9cd92b3b593390a20f9414fd333ad7/fastembed-0.8.1-py3-none-any.whl", hash = "sha256:b4f4043080af36ee820d22d2d3034df635c1a8b9764e5d2a642bc7aeb1d0e749", size = 129755, upload-time = "2026-09-22T20:01:22.789Z" }, +] + [[package]] name = "filelock" version = "4.0.12" @@ -782,20 +861,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "huggingface-hub" +version = "0.36.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "filelock", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "fsspec", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "hf-xet", marker = "(platform_machine == 'aarch64' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'amd64' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'arm64' and sys_platform != 'emscripten' and sys_platform != 'win32') or (platform_machine == 'x86_64' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "packaging", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "pyyaml", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "requests", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "tqdm", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "typing-extensions", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7c/b7/8cb61d2eece5fb05a83271da168186721c450eb74e3c31f7ef3169fa475b/huggingface_hub-0.36.2.tar.gz", hash = "sha256:1934304d2fb224f8afa3b87007d58501acfda9215b334eed53072dd5e815ff7a", size = 649782, upload-time = "2026-02-06T09:24:13.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/af/48ac8483240de756d2438c380746e7130d1c6f75802ef22f3c6d49982787/huggingface_hub-0.36.2-py3-none-any.whl", hash = "sha256:48f0c8eac16145dfce371e9d2d7772854a4f591bcb56c9cf548accf531d54270", size = 566395, upload-time = "2026-02-06T09:24:11.133Z" }, +] + [[package]] name = "huggingface-hub" version = "1.33.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version < '3.13' and sys_platform == 'emscripten'", +] dependencies = [ - { name = "click" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, - { name = "httpx" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "tqdm" }, - { name = "typing-extensions" }, + { name = "click", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "filelock", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "fsspec", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "hf-xet", marker = "(platform_machine == 'AMD64' and sys_platform == 'emscripten') or (platform_machine == 'aarch64' and sys_platform == 'emscripten') or (platform_machine == 'amd64' and sys_platform == 'emscripten') or (platform_machine == 'arm64' and sys_platform == 'emscripten') or (platform_machine == 'x86_64' and sys_platform == 'emscripten') or (platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'win32') or (platform_machine == 'amd64' and sys_platform == 'win32') or (platform_machine == 'arm64' and sys_platform == 'win32') or (platform_machine == 'x86_64' and sys_platform == 'win32')" }, + { name = "httpx", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "packaging", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "pyyaml", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "tqdm", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "typing-extensions", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/25/2a/484d112c0d8fc5f665d7b65137ac9cdb2953c982391598c3597968a12ee7/huggingface_hub-1.33.0.tar.gz", hash = "sha256:367be21a201db9523eddf8aeac7048f2602c1b308691c97640d5e72ed188007e", size = 1043548, upload-time = "2026-09-24T09:49:29.971Z" } wheels = [ @@ -987,6 +1101,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/69/7a5d10ac409c4da0355e054a14371871da9b5557fcc42772cd00181c6cce/librt-0.16.0-cp315-cp315t-win_arm64.whl", hash = "sha256:8ceafb70f2a4f0826f11031942e59c0728fd98da112dc346d4352bde1e486866", size = 116391, upload-time = "2026-09-29T00:55:07.484Z" }, ] +[[package]] +name = "loguru" +version = "0.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "win32-setctime", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, +] + [[package]] name = "lxml" version = "6.1.3" @@ -1249,6 +1376,190 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "mlx" +version = "0.32.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mlx-metal", marker = "sys_platform == 'darwin'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/8c/2f8b796885b4c02ab9644916bebf656cd1d6d366df94d174b046903ea250/mlx-0.32.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:7aecc3d4a904279492dae58e28378a3c4bec4533e7066e64ae5c78c7a5511f22", size = 651041, upload-time = "2026-09-29T00:22:36.481Z" }, + { url = "https://files.pythonhosted.org/packages/6e/06/aacd94119d4a999d43f07f24b02904d7f895ee074ed4678b28456c319ea3/mlx-0.32.3-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:a6241ea8f56a08a6c3a06bbaad7d7fc2fc1c05f23757c61fe0a8797325ac6d46", size = 651042, upload-time = "2026-09-29T00:22:38.071Z" }, + { url = "https://files.pythonhosted.org/packages/4a/c1/13ff85d72f7cf01239c1031688b11cf1c7a3f0c7380636728976e510bfbd/mlx-0.32.3-cp312-cp312-macosx_26_0_arm64.whl", hash = "sha256:72711cb23cc6dccc5a460f63f39c11931a00b2c6dcd6fba253921090072e21bc", size = 651118, upload-time = "2026-09-29T00:22:39.503Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e1/0039c25cb2e40e605de6409e6ede52b01b7c79d6a0727e0c103ec60d88b2/mlx-0.32.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f5716b51fea3d06352d4d8c851fb6c5141b1a7de5a5164d44de5a6d368d84a7e", size = 651142, upload-time = "2026-09-29T00:22:48.119Z" }, + { url = "https://files.pythonhosted.org/packages/29/dc/01ac6c32fb405a542be80f021bc978125af51d2ee5ae1db624bfd954e756/mlx-0.32.3-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:656df5e36b9bfa862fd633d55dee4da072d292def9f27f22134c594e03ff28a7", size = 651143, upload-time = "2026-09-29T00:22:49.702Z" }, + { url = "https://files.pythonhosted.org/packages/53/a9/70bf746c2cd13a198a429e33244d2f072ecd0c89508299a636529d3317cb/mlx-0.32.3-cp313-cp313-macosx_26_0_arm64.whl", hash = "sha256:75333afef55819afd2a31e87ffbed0bf59f379bca21771e6b13a95a675e4c85b", size = 651075, upload-time = "2026-09-29T00:22:51.636Z" }, + { url = "https://files.pythonhosted.org/packages/68/73/f1511b38c9b200fc84bff84470ff1e4303f2f7b47f614b14a2d3f4cd6226/mlx-0.32.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:65122a7eeb718df56641d7ac6f7fdb0ada573fd841d1fb0a61a6a443e33f11b5", size = 675637, upload-time = "2026-09-29T00:22:59.92Z" }, + { url = "https://files.pythonhosted.org/packages/3e/40/20733670ac7a5571e10d80fb510bc0fbed6d0917a8e34cdf6c3de52b5a8e/mlx-0.32.3-cp313-cp313t-macosx_15_0_arm64.whl", hash = "sha256:e13f5ce391293dab946edf18e7ae7bb55768beef1140deb509f67a2b14edcf7c", size = 675638, upload-time = "2026-09-29T00:23:01.529Z" }, + { url = "https://files.pythonhosted.org/packages/15/ac/65a1b9484f3eedb57dc73589af213d03821d13d99caa76430ad849be6114/mlx-0.32.3-cp313-cp313t-macosx_26_0_arm64.whl", hash = "sha256:15e8578c9d10b4fd4728243cd6c926bc46e1da267c6a849cdbceca21d51caa51", size = 675600, upload-time = "2026-09-29T00:23:03.249Z" }, + { url = "https://files.pythonhosted.org/packages/74/91/4d7dc37533d5439e27e57b09d2200b1a7988e433259d9beac8f8241197e4/mlx-0.32.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:2dc1085f5c2eba18cdac095d7fb0b68ddfadd51bf695b18a1102eda15e77e76d", size = 652868, upload-time = "2026-09-29T00:23:10.986Z" }, + { url = "https://files.pythonhosted.org/packages/ed/50/b79bf60cad0cf3c6a0040c18963e682a628ad12bdd83ff0f45e9b5893b2d/mlx-0.32.3-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:712ca29119d9e2aae04af8f8e416bfb0097538d9ff73e6fab8b028053fb45ff8", size = 652868, upload-time = "2026-09-29T00:23:12.509Z" }, + { url = "https://files.pythonhosted.org/packages/d6/34/3bccf8ee413afb055b739215be3b74cb810ec655e508dc90bb3c79e4d16b/mlx-0.32.3-cp314-cp314-macosx_26_0_arm64.whl", hash = "sha256:452c621862684e8769be93c1517420c8ba2fd6e8b01a1c970c8ba8069022f0ac", size = 652880, upload-time = "2026-09-29T00:23:14.967Z" }, + { url = "https://files.pythonhosted.org/packages/ae/31/254d11d662a0f68157f659ab7eb90216d51a71acabeb38e61e54c6927623/mlx-0.32.3-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:d17f0c33789009a3b3e609a52af34dc687ed39816836e20ab466aab669c14e6c", size = 675632, upload-time = "2026-09-29T00:23:24.175Z" }, + { url = "https://files.pythonhosted.org/packages/3f/94/32e6f4c3e24635ec79fb751fccecd624d9f40d6c15e013acb40973501443/mlx-0.32.3-cp314-cp314t-macosx_15_0_arm64.whl", hash = "sha256:5d12de01f23d2fae739ecfecd9c1d4c07c1f9b778a3729f7c0884ea1168ec14d", size = 675635, upload-time = "2026-09-29T00:23:25.823Z" }, + { url = "https://files.pythonhosted.org/packages/c5/c2/b94c58c0d634907fed225abe81bacc73901b5f1a9fb1b976d7232f03afcb/mlx-0.32.3-cp314-cp314t-macosx_26_0_arm64.whl", hash = "sha256:f2909f683ffc5a2720f6c8099398a64fd1be53f1b6538e4e728ee55816ce699b", size = 675564, upload-time = "2026-09-29T00:23:27.737Z" }, +] + +[[package]] +name = "mlx-embedding-models" +version = "0.0.11" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "awkward", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "mlx", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "numpy", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "scipy", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "torch", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "tqdm", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "transformers", version = "4.57.6", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a4/09/ff384fc20f602dca2877b823bff15dbec2259edab146bd18992d3e388ebc/mlx_embedding_models-0.0.11.tar.gz", hash = "sha256:9dcb01462736cca9e9014a032f5df2fb58cad6f1a9d6f70e321103cb67cb136c", size = 13128, upload-time = "2024-09-27T18:02:13.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/4f/5c09805363daa711368c6c3b86d1a24839b2512f3e75406fe4d75cd16f33/mlx_embedding_models-0.0.11-py3-none-any.whl", hash = "sha256:518c8d02773bf8afadc832e41ae473bd7433d609084b4930f3ce6ee47fbc6a6e", size = 15427, upload-time = "2024-09-27T18:02:12.426Z" }, +] + +[[package]] +name = "mlx-metal" +version = "0.32.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/ed/8510985bcb24a6eae359a5441ce4cedcee13b84258a010cd0e515e9fb0bc/mlx_metal-0.32.3-py3-none-macosx_14_0_arm64.whl", hash = "sha256:fdba88ed9131448cf163be2c7ad6374746d8818205f28657aa91b6c69050f691", size = 42205858, upload-time = "2026-09-29T00:14:27.819Z" }, + { url = "https://files.pythonhosted.org/packages/3c/dc/4f852295fbe357bb9d3a2bfd55361a520b94ded873b72dc58b34a7f507cd/mlx_metal-0.32.3-py3-none-macosx_15_0_arm64.whl", hash = "sha256:feea99fe4d01f9e9f33ca73d869f988219a1725de1c18870baa7d5cd5aef6cb3", size = 42187333, upload-time = "2026-09-29T00:14:33.861Z" }, + { url = "https://files.pythonhosted.org/packages/ee/38/cb985ca86979ca6f954a2a45eb6e3f9b787c5d55227564a04f87953b3d56/mlx_metal-0.32.3-py3-none-macosx_26_0_arm64.whl", hash = "sha256:34ae9b83ad2f0ccdd3e5d48ec35176e7119f57069eef187122916dc941a4ae1f", size = 67661845, upload-time = "2026-09-29T00:14:41.248Z" }, +] + +[[package]] +name = "mmh3" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/3c/eb1d82a87c504259dac5ce1c7de7587b68ffac841b55d23f8ea2c9df8422/mmh3-5.3.1.tar.gz", hash = "sha256:bd86d0c86b52332319d981d03781ff77811a29db544a69902dc06b5506bb3e19", size = 33354, upload-time = "2026-09-30T17:38:09.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/2a/01734f735587e44b110fa7c44d3fa2fcd59db1cec2aea5ce0eb3002ebde6/mmh3-5.3.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:6ca2e4296573e67fbf4e4a52af029e6f8f7c947ec275fcc584f07d46d5149a13", size = 55355, upload-time = "2026-09-30T17:35:21.289Z" }, + { url = "https://files.pythonhosted.org/packages/a1/d9/4087514f8edc559d9f4a5e1cee258c18cab40e13a1ad4abba5f08c17a184/mmh3-5.3.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d0a3b185866b964b5c8c60cd644cabf6bd01509a38a29ba74c5bd34088e12b89", size = 39960, upload-time = "2026-09-30T17:35:22.735Z" }, + { url = "https://files.pythonhosted.org/packages/c2/85/31af9d6b280f04164eb493c0b2e716f8a8d681b0d2e0b6e5a5bbfd3fccc5/mmh3-5.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bd928feed4a6f28ea8d2b48c1a41eb5a35fb62cfd1e0f06c3335378cc59b6c4d", size = 39563, upload-time = "2026-09-30T17:35:24.037Z" }, + { url = "https://files.pythonhosted.org/packages/74/8b/bb4f0da4a0f8a117e01cb9ef90039b754ab25eaf591a964155c1d2fae133/mmh3-5.3.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:34744ba81a0111010e72639c5f677ca89393ba7950540596e856dd1ed8b2a5d9", size = 97087, upload-time = "2026-09-30T17:35:25.15Z" }, + { url = "https://files.pythonhosted.org/packages/d5/20/f2f5877cb22ee3c26c52f4be8737c7cb94e23e6a5e47bf05654b189ed0af/mmh3-5.3.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b037280edc7a609a987fa7661a1132a3f6d721f46b299ed5f9f641b35ab415a", size = 103108, upload-time = "2026-09-30T17:35:26.471Z" }, + { url = "https://files.pythonhosted.org/packages/d4/c3/00480ddfd4e00213089c4a50801b685e344086948d8d0075e6533dd81979/mmh3-5.3.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a48db69e7e7d40d18c24519a11b7d7d21a8b6b4af60fd2194d9ee514fa4354c", size = 106002, upload-time = "2026-09-30T17:35:27.705Z" }, + { url = "https://files.pythonhosted.org/packages/9a/10/f84fe70878ad89e059066a9977ff9f36116eac58f2480dd1046e4daef638/mmh3-5.3.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:334f2d7273bfc2ffd85f9b1a75d39d59da3158da94ca42a4e273120fcfc25edf", size = 112838, upload-time = "2026-09-30T17:35:29.006Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f5/a37d77d505a4a1598dae775b965c226de014af5c389f5c2bfa505bb8e159/mmh3-5.3.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b78173b6b9bc69a8a455c63b380d892add205efe531e8206ef71d600324048bb", size = 120486, upload-time = "2026-09-30T17:35:30.227Z" }, + { url = "https://files.pythonhosted.org/packages/12/c0/93581e98cd76df75962fbf8f2be9a9dc6d4e1c63dc6dc2b85598bb1f513e/mmh3-5.3.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b1e950308111308f54c12d12a223bbc2882b75b59892858463a16508b375fc56", size = 103111, upload-time = "2026-09-30T17:35:31.704Z" }, + { url = "https://files.pythonhosted.org/packages/f5/17/3480de8e4bb66f7019bb02fc7454e28d721a31e0dfce6b2dc6f72973e871/mmh3-5.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:010dcd7406c2f77b978beaadfeb7a01d4f7868ce862f6273b1c37bd902267394", size = 98890, upload-time = "2026-09-30T17:35:32.936Z" }, + { url = "https://files.pythonhosted.org/packages/d0/6b/d5a0287c85ef2284a88c0150d6d3262f034390cee82854844f8bede1ad90/mmh3-5.3.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4b87fe04af53cc9c2492f90a6d0287f52c94fef517df3de75c18346e0e119682", size = 98368, upload-time = "2026-09-30T17:35:34.502Z" }, + { url = "https://files.pythonhosted.org/packages/17/95/19efb8536b7cde6cd46abdc3d1c38354233b14288549b56434589e9b3fca/mmh3-5.3.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a847c3d57c64a76af48ed4f4e9abe8d3d966c577de1e39258595e5b38ddf6eb3", size = 106240, upload-time = "2026-09-30T17:35:35.763Z" }, + { url = "https://files.pythonhosted.org/packages/c6/03/9a715610de3f9350de45b2933221947460e3c421909801f65b0e96bf14ae/mmh3-5.3.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:814a69f39a3a3106eee1b870acb5ff09c436334a5a962df380c22988528cebd5", size = 100977, upload-time = "2026-09-30T17:35:37.108Z" }, + { url = "https://files.pythonhosted.org/packages/d9/6a/0f889bfbc7abcde5ec08eaff2381c093f7d00db63ca3071ab705bfb372c2/mmh3-5.3.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:d6c9a5cc1c19257b135874fe67b7ffcefad1eb7babd09ba9a2a4d9fb576e1a6a", size = 109754, upload-time = "2026-09-30T17:35:38.392Z" }, + { url = "https://files.pythonhosted.org/packages/37/7b/e3b441543a0e86f8635b6007ef8b7101442a7c90fae345a43538cd85c36c/mmh3-5.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a8ffd80966acfaf4f786699ce37b75121c8252bb65636c0ba2f0cd9c6bb276de", size = 97303, upload-time = "2026-09-30T17:35:39.679Z" }, + { url = "https://files.pythonhosted.org/packages/06/03/bbb91c0c094e7131fb5f622ff5a079a25c125b92c7ece2ac8b3e38e1992d/mmh3-5.3.1-cp312-cp312-win32.whl", hash = "sha256:d3a3b8afadf1196566750aed853dd91e358447b8c1f39ce8625aabf590e3e686", size = 40340, upload-time = "2026-09-30T17:35:40.91Z" }, + { url = "https://files.pythonhosted.org/packages/8a/94/41c97ce26200a1a9242d159c2c5499844ec4a688fd4e69df044044f9e012/mmh3-5.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b69e9f1d9c960236106f22bad1b3bf0a1110971decb0c1554d591c699e39a970", size = 41735, upload-time = "2026-09-30T17:35:41.972Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d8/5b173bb7b4682dd9e707a523ce24a91234f25deb02d2790f02a1f5ddc2d6/mmh3-5.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:cd7e7e54d8f90076059a15c3751e16b46211398af76a48db9e82143375f3a86c", size = 40098, upload-time = "2026-09-30T17:35:43.069Z" }, + { url = "https://files.pythonhosted.org/packages/e4/4c/c6faef1d29aa00a1f71d3a109547c86b029835b55ad19482dc625c98011c/mmh3-5.3.1-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:4b2b6d135aafc93a666056ae87cf11dce93e11a3ee9b938d46076d93074699bf", size = 40911, upload-time = "2026-09-30T17:35:44.698Z" }, + { url = "https://files.pythonhosted.org/packages/7b/23/a35e5090c3685c3bd22f07586c4efa428710ad1404d6cf3fd47ad654e711/mmh3-5.3.1-cp313-cp313-android_24_x86_64.whl", hash = "sha256:98c6373ec81d4e74305d8d13d5de3aacf0e53e78dcb4a43dd74f6f3ff8452967", size = 42024, upload-time = "2026-09-30T17:35:45.814Z" }, + { url = "https://files.pythonhosted.org/packages/b5/59/350d214e1a37e5c2c92182750c06c671348346d52eb455bceaa861801349/mmh3-5.3.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:bf65874ed7c948281719632b6960f4eb572aa33e1a093a6a1d31bf064b0e540d", size = 38668, upload-time = "2026-09-30T17:35:46.993Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c4/0a3d4e54549fd8edd6fa54cef0529dea7316666066f3bfa23c810d0c2b5e/mmh3-5.3.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d1f3f35b97adfcf4545a4def9e0fb17e61eed8a06c29137a02829a67232e1588", size = 39344, upload-time = "2026-09-30T17:35:48.097Z" }, + { url = "https://files.pythonhosted.org/packages/80/b8/e96e8da1b8d52f62c15a8acb33cfd180778c18d71ed63e30a2085e35cf9c/mmh3-5.3.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:39bbc0665b064e63a0e64e9efab9a97a1f0535b0e1ffd8e43e23aef82e41ca21", size = 39852, upload-time = "2026-09-30T17:35:49.227Z" }, + { url = "https://files.pythonhosted.org/packages/3e/28/c657ba46881ba84b2e1d260545c141b0794bb579c981fd71a0f7e5c15a73/mmh3-5.3.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5cc32468caf0071882c3682b9ab04f45d756231059b4e36cccc94eb972f8c192", size = 55334, upload-time = "2026-09-30T17:35:50.704Z" }, + { url = "https://files.pythonhosted.org/packages/12/5b/cbff42a3248d0869a940eebef0eefe7feb948f6eeed2f242f0098e2892a0/mmh3-5.3.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8cb9941e2613b22ed4901faa29338c134194b2dec501023e6433b7e62161e329", size = 39946, upload-time = "2026-09-30T17:35:51.944Z" }, + { url = "https://files.pythonhosted.org/packages/5a/0a/67d5082ad1fe184c4c928ca6be61d775947590863d52de0a9d9aa7d525b2/mmh3-5.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c25a6d4b6ff31d801ff6f1ad5ce003271bceabf21c3e9ffcf04a47774354e956", size = 39561, upload-time = "2026-09-30T17:35:53.331Z" }, + { url = "https://files.pythonhosted.org/packages/de/2c/948789af3824e81621f01183c1a2a017229bd7884e243641628682ea9ea5/mmh3-5.3.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9ba38fef5eeed0668a27f8b5a002a5e30c789dd11fa058495b307f76226a4662", size = 97120, upload-time = "2026-09-30T17:35:54.528Z" }, + { url = "https://files.pythonhosted.org/packages/7f/46/88420e1561f1a5cda72e23581b9cbbde336cc4a4a8deb1259559bf57b8d2/mmh3-5.3.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:931d9d86c66f306e91414e95509af05e5c79bfcbac78218c2ee55c9734000053", size = 103129, upload-time = "2026-09-30T17:35:55.78Z" }, + { url = "https://files.pythonhosted.org/packages/bb/97/064d5c9eed7afe9b2087c164ab4b11a9d4cd0cb1d8d826804df72d7e0e17/mmh3-5.3.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae367d0cf6cb40f3ec60ebdb572022f3cc875bcf4c661d345f3dbf24571e7aa3", size = 106044, upload-time = "2026-09-30T17:35:57.277Z" }, + { url = "https://files.pythonhosted.org/packages/39/b4/c4e968be21d55aead9ef78b6ac6fc0e4a455cfbbaff9ef62bcdaee40b26a/mmh3-5.3.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:59dab80b8124998406c168ddc9d6cbcede1c117dd0aed3db80e16e43ad71ef82", size = 112890, upload-time = "2026-09-30T17:35:58.596Z" }, + { url = "https://files.pythonhosted.org/packages/17/e9/b3f3da18b38bd08d39eb24c142ba3e9217b8975d3ca8468143dd3c63aafc/mmh3-5.3.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:803ba415427118e00ffccefbacc41b03df8ac60403cd9cf2dfd461ef56072002", size = 120520, upload-time = "2026-09-30T17:35:59.924Z" }, + { url = "https://files.pythonhosted.org/packages/fe/03/c7dc6eb152425dd2fba09b299a186be37bd53910a28531ab12f475d9bf99/mmh3-5.3.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d856a44ef94204820338b0e3312c02a6a4df8040ad06102c050d5005dbc601c", size = 103089, upload-time = "2026-09-30T17:36:01.266Z" }, + { url = "https://files.pythonhosted.org/packages/25/c5/1192cf2db35390b0ca1f54eae2699c62235fce57992eda605e675af06b9f/mmh3-5.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bf1fa41b7587477c86ffe4e69b854ef688e243f9feb97eebc08666031bde71e0", size = 98905, upload-time = "2026-09-30T17:36:02.545Z" }, + { url = "https://files.pythonhosted.org/packages/18/3a/9af0d1f08e3e03cd8b52e5d53fd3be74345993c0d6a7cf61b02e454c4daa/mmh3-5.3.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:86c1593ebec4bd8a7b1e0f28fce5f220e5bc0b2d5f9ba48c34224c04d9f63b8f", size = 98457, upload-time = "2026-09-30T17:36:04.006Z" }, + { url = "https://files.pythonhosted.org/packages/b1/66/ab879d60e7f2cd69e69a7f46613108d9d904c29acaaa1adb345a3a479fcd/mmh3-5.3.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:427f2ba51baf54ce25f32beb6edd2db70bdc95ac9746067eea0a2b2ca484fd10", size = 106276, upload-time = "2026-09-30T17:36:05.366Z" }, + { url = "https://files.pythonhosted.org/packages/cc/58/cd805eabd1fc01ad36861d3cbf4eb25df822bb0e72c8ee8b3ffd47c71225/mmh3-5.3.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:01159489255615d4be76a9ebb07cb0c9b0345f544197aa64ab18a7cfa5579a28", size = 100971, upload-time = "2026-09-30T17:36:06.74Z" }, + { url = "https://files.pythonhosted.org/packages/c2/d0/20d98b665deca070ce5e19df678d476ddab9652462f1e5bf636fc82265a2/mmh3-5.3.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24627cb76ff1e7870a07d7520cf5f3099b1767390236e46d38656dbea5cc6ad0", size = 109795, upload-time = "2026-09-30T17:36:08.14Z" }, + { url = "https://files.pythonhosted.org/packages/09/51/be441d264a38c390582b3b3382f629e66e10847cbcaa60f560d517237b1b/mmh3-5.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8d83f27143c8ae4e78781306ce002ee466219d334346609d0d7f675c8664aef4", size = 97326, upload-time = "2026-09-30T17:36:09.548Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c8/240446abf409338e93d9c8c2e31b47133a08306cbd4346abb0064736f20c/mmh3-5.3.1-cp313-cp313-win32.whl", hash = "sha256:4836a024fe923605d85049f887aacca98add969c8d4932aed5d0d3884cdaa682", size = 40331, upload-time = "2026-09-30T17:36:10.898Z" }, + { url = "https://files.pythonhosted.org/packages/06/5b/b63154d3d8d3dab42a6713df71da40c2952c4e640973a70ebd897df5508c/mmh3-5.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:6759c43a90729a963ab5503779e07cd000c372ebd2d80196f78da2bf2d4101f1", size = 41754, upload-time = "2026-09-30T17:36:12.084Z" }, + { url = "https://files.pythonhosted.org/packages/be/67/b03f7b39d5f22cbfe72e6d73374c820829a69813a7489ba2a4a8d252391b/mmh3-5.3.1-cp313-cp313-win_arm64.whl", hash = "sha256:78219f6b1cf27872295dd4548e862f317b48ef1e1b2c9e0143069ac3a8b822d7", size = 40103, upload-time = "2026-09-30T17:36:13.425Z" }, + { url = "https://files.pythonhosted.org/packages/32/b6/815e83303e366cc831e81820d92304d5e84068a6c5cd96d65b6fadf91c7b/mmh3-5.3.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:cabd413b4d6017b5117112a1036e3f980aa9a75ab345e48b4d2eb73b1bfad99c", size = 40908, upload-time = "2026-09-30T17:36:14.591Z" }, + { url = "https://files.pythonhosted.org/packages/b7/fc/10a374e021d7a531e668535a3defaea9ad3f92fe6b16c747fe560389f5cf/mmh3-5.3.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:2ca9402d9dc406f62094c271602629a6ab8b853d8052b89e8ee991dd122ce36c", size = 42038, upload-time = "2026-09-30T17:36:15.753Z" }, + { url = "https://files.pythonhosted.org/packages/01/4a/8427a3deeb561a32ab3c71e2fc91569d1ca4569d14dcc71605fe04c29b03/mmh3-5.3.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:2cb13fc23e8c3a3a2f59327d4456b7edb9a3d3f5b6936ca3aab3ace9e3675f50", size = 38674, upload-time = "2026-09-30T17:36:16.933Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6a/ca4ddc0cd6dd07030211c39f1863a164b357adf740d6a8713bbaff3189c9/mmh3-5.3.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:92292c047b82624ef972e10e54a1cec0c7651cbe86be7d346353456e682a58f4", size = 39361, upload-time = "2026-09-30T17:36:18.125Z" }, + { url = "https://files.pythonhosted.org/packages/0e/36/e4c1ef8bc22ccbad4967e8a637fe12dbcb740616a023bf50a3e551963d9c/mmh3-5.3.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:8f4626de7b5bcf922eb66f1d02eb4f62f6dc99e4f2b1519dccfd9e89fcf8ba8f", size = 39866, upload-time = "2026-09-30T17:36:19.404Z" }, + { url = "https://files.pythonhosted.org/packages/10/2a/880d51fa8368727ad428368b062a28dd735ac913392eb7d5760a7c0c8915/mmh3-5.3.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:71fb7fd092f2b4e3af00579a715dbe3c25bd8ee295acc56c9c60003a7c8cfd98", size = 55364, upload-time = "2026-09-30T17:36:20.723Z" }, + { url = "https://files.pythonhosted.org/packages/87/33/f1902c23f6a25c198d18045d6aee389b9442c68134d2b05ec26345b6f33d/mmh3-5.3.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1033b4943bdf401f9c402ed517fb64ece6307bc22cff2a83bc7e5b85f731eed5", size = 40023, upload-time = "2026-09-30T17:36:22.026Z" }, + { url = "https://files.pythonhosted.org/packages/24/30/c279c95e3dadbec8c389306cf08468bb68d43cdd7e26fe6107bcd00f157f/mmh3-5.3.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:741d1201ccc7716ec61140096dca084590da2eaf6c62b09b64813b6234ac58af", size = 39570, upload-time = "2026-09-30T17:36:23.222Z" }, + { url = "https://files.pythonhosted.org/packages/72/3b/c748fb11c98b3c3fa48249d8d36e7dbeb57669b3ce767f51c94d43a6cde9/mmh3-5.3.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fb6a472fb487e37556344fd2a4894eb5eb1896cb2eb03536e8f56dce2c5423f1", size = 97115, upload-time = "2026-09-30T17:36:24.487Z" }, + { url = "https://files.pythonhosted.org/packages/5d/bc/65eb32da2c7a7e03bf91f1c3ca6e6e2c8004c64515828a3fd5b6ce51669d/mmh3-5.3.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fd3bbb5dc3c3a045605c1a618bfccb2de6d65ae89b99ff9a7f54556cf81175b7", size = 103119, upload-time = "2026-09-30T17:36:25.807Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ca/eb318655aa79059ab7af2b658adba17eda27d0be0f49703e7b0e6086b72d/mmh3-5.3.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:66297ad16d75ffa14335ca23a40c23deed09fa0d832341fda29ad60a8a2a91ab", size = 106064, upload-time = "2026-09-30T17:36:27.162Z" }, + { url = "https://files.pythonhosted.org/packages/59/ee/491e1fec15a83fbf152e482e7f9c91c738bced77457f0f64e6f5792206e8/mmh3-5.3.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c7418304490de50c985a95416b62f1acac616d9bc9b259622072b487093f926d", size = 112935, upload-time = "2026-09-30T17:36:28.555Z" }, + { url = "https://files.pythonhosted.org/packages/5d/44/24f2bb97fc731ae4bf171588a9c2732a76309f4fb4366fa62ff3b1943262/mmh3-5.3.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:aeb4ac9626c89c9d093930abecd3cea40e6eb7f21fc60870b8c3495748158624", size = 120504, upload-time = "2026-09-30T17:36:29.872Z" }, + { url = "https://files.pythonhosted.org/packages/a8/43/e64fb48dbb9bfd5295d87b4b9a0b95a41139b6b623d7535ef864af98d75a/mmh3-5.3.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5e7eee7a8174ac340530bfd3500086972452a86adc8eec7d879501bcbef7b2b7", size = 103219, upload-time = "2026-09-30T17:36:31.317Z" }, + { url = "https://files.pythonhosted.org/packages/c3/08/261419201b69dede3368b1e4ff92183f603e32c565e1124d59de73c14043/mmh3-5.3.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ee868a903f387a192bb6fe2205d709156ab7dc5b2dda6fe75d2cfbba9b6a0e26", size = 99012, upload-time = "2026-09-30T17:36:32.693Z" }, + { url = "https://files.pythonhosted.org/packages/34/c6/6ef12522aa92d722c8f756719c870667a3d3aacb1563008e4f7331ee3de5/mmh3-5.3.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a212a14648107bb83c55ece4a85bfe0671fbba81059e20d48e25456eeea1fcb0", size = 98475, upload-time = "2026-09-30T17:36:34.14Z" }, + { url = "https://files.pythonhosted.org/packages/55/f4/7a89df34615cefcf705d66f54ba26f00425fbae5ed4f61f1ba86b2372648/mmh3-5.3.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:de8fb9ee364d7ad62cf6e6807b77937e1ec8e69396a46aa36b322aa772e774ed", size = 106299, upload-time = "2026-09-30T17:36:35.497Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e4/4cd526b1d7c424485d770b7ffc2976452087f68e8639421e27da93dd5baa/mmh3-5.3.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d25d161d01b428cd4a12a4489e62595989f826ca780197844f806cc984dacde5", size = 101112, upload-time = "2026-09-30T17:36:36.989Z" }, + { url = "https://files.pythonhosted.org/packages/23/f0/a245308671c6ce430e1347b6ea100def44599d808f1d841dd24f399ec5e0/mmh3-5.3.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:1769d9a4f4a54a383ff65ac79bd85b76f17b2f1f9a71bc57b46df9a556970ec8", size = 109779, upload-time = "2026-09-30T17:36:38.385Z" }, + { url = "https://files.pythonhosted.org/packages/48/24/869fbd94037a046ae7a286eacaf4108a48ce497dd087b0c5b4f980774773/mmh3-5.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5e06d41da4c6b2fad157db2d524e1134f0077d7bfaa645c114b7968433ea1b5b", size = 97327, upload-time = "2026-09-30T17:36:39.83Z" }, + { url = "https://files.pythonhosted.org/packages/c9/01/8c629d29655dd670819b236fafc484ec4f9b7711333fbd35c983edfa3941/mmh3-5.3.1-cp314-cp314-win32.whl", hash = "sha256:f86a308bd396fa69013c360abf98111e9d0fa534a7d75b9613ca4145fa63bff4", size = 40873, upload-time = "2026-09-30T17:36:41.194Z" }, + { url = "https://files.pythonhosted.org/packages/11/44/cf8e89595f64a2666db42b8bfa02e98c53244b7741cf6cd6b742901c0c73/mmh3-5.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:156152ea77713eecffed175e4e384aa47bf24fedcf1a9f30773dd2737a8b8c76", size = 42299, upload-time = "2026-09-30T17:36:42.623Z" }, + { url = "https://files.pythonhosted.org/packages/fb/68/3b001dacbf9f6a5e3a4837fa870efff34504ca21b9aae0df78bf3ebe7957/mmh3-5.3.1-cp314-cp314-win_arm64.whl", hash = "sha256:41082b86c3f24f41e1e8af97c80724ff7c23ebc33dff7e37d7e3b1caa4eaf183", size = 40652, upload-time = "2026-09-30T17:36:44.251Z" }, + { url = "https://files.pythonhosted.org/packages/1c/8d/b9778e43fd4332124ef07c194a84cae8bf9e424e42400b0919fb4974c62d/mmh3-5.3.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:7cd757dbf0f177555c1544c37aec5b1230cf5f4d1b79e900357684437608c1d4", size = 57052, upload-time = "2026-09-30T17:36:45.884Z" }, + { url = "https://files.pythonhosted.org/packages/66/b8/5cdbf15f818dae35ed95e09f3a182d24f35acdda441c1140bfcbe61b4ac3/mmh3-5.3.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:be5dab268537f7db00cca56a07b1301060351fd8ab41d6b3fc5cddd3bd207448", size = 40826, upload-time = "2026-09-30T17:36:47.496Z" }, + { url = "https://files.pythonhosted.org/packages/65/37/93b48f89a8173d8a0aa48567e5fff51569c9620854766bfa782214351d3a/mmh3-5.3.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:588c41c36378be5b62da340279300ad6e93d819edaeca0a89e38f1fc8a5b683d", size = 40344, upload-time = "2026-09-30T17:36:48.796Z" }, + { url = "https://files.pythonhosted.org/packages/c3/13/ee07da9f987e61a5cf95b3bfe49f59d9b95ee0e086243527248aa9d54912/mmh3-5.3.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4274ed59160e51c970553b6e3d28fdcf8ddb833c924cd305619ab06dda265c19", size = 109471, upload-time = "2026-09-30T17:36:50.42Z" }, + { url = "https://files.pythonhosted.org/packages/43/fd/58ad751dc961f38af8ea16594b8cfb2afeaaaa9fa2fe814bfeec9689d33c/mmh3-5.3.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2c5f055fec5901cfbd6951d2ff52352bfdb08e0f9acafae1b235aa835f4d80f9", size = 117238, upload-time = "2026-09-30T17:36:51.836Z" }, + { url = "https://files.pythonhosted.org/packages/76/c3/e43c6abc3ea4e91da55391a2b59c95834f61972e1b6a3e003a2f1a2b08fd/mmh3-5.3.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7670edd6751f21d7bd038b45f4eb671fc8d6441b0088ee01d25f5a23424a87c2", size = 120236, upload-time = "2026-09-30T17:36:53.269Z" }, + { url = "https://files.pythonhosted.org/packages/c8/f2/397bc3d1e934443c0b621392fa56fdeaee1974e167e1411b887fbc223316/mmh3-5.3.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b7a029b0d8a273dd746d7cabc13134fb6f08d866ed83d7619a7aa4fad9ae9946", size = 125811, upload-time = "2026-09-30T17:36:54.696Z" }, + { url = "https://files.pythonhosted.org/packages/1d/43/d7e4f9ed76d20114ad97d6d7b11d2c99a148208279cfeeab34408b68e936/mmh3-5.3.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b890e9fe7330f104dbda1b6a8dcd0a28e948db08eb9faf3001bc55d164402ff8", size = 135028, upload-time = "2026-09-30T17:36:56.42Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3b/6184b0de23d841e1cb1f36f7f93c8157065c70ec4072b18d5d84d6cccb90/mmh3-5.3.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:23ab950642fd0c29a7187a9073e70a203135282b91fd257153de8a8efc87fdbe", size = 113452, upload-time = "2026-09-30T17:36:57.947Z" }, + { url = "https://files.pythonhosted.org/packages/6e/31/3ff78b891c5b1251ddeab97b4c6e97fdda7a16e160892dace74578b26a55/mmh3-5.3.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:325e990de3fb60b0ee460be96096ec7cb0775d16f4a2ad20b613111d3cc608f9", size = 109896, upload-time = "2026-09-30T17:36:59.66Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c0/664278780e7ee5758717674f55a6d7447a3e0b862fe57678422431155729/mmh3-5.3.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:1936c40c171979cf230ccf9c9acad51dbd2b01de233a4829e307e4c71331f765", size = 110985, upload-time = "2026-09-30T17:37:00.924Z" }, + { url = "https://files.pythonhosted.org/packages/f3/5e/127ce3d2815ab311ef03fed126bd46533f79cef4639636b78b2d6682c99b/mmh3-5.3.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:196a15b6dbe96ed77e03887ea338e0df0278576143a9e66404074b6d9b10eb2c", size = 116708, upload-time = "2026-09-30T17:37:02.36Z" }, + { url = "https://files.pythonhosted.org/packages/4c/ac/962e120a6642f6eca9686b101b07e889466b13caef34524720d511448ac6/mmh3-5.3.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:21e68810f51f6e9b96da10073b8efdafefbd9f71ef644a9541372b8de0c02137", size = 112004, upload-time = "2026-09-30T17:37:03.737Z" }, + { url = "https://files.pythonhosted.org/packages/5c/54/82829ed9ab1272416bc87b923d51cd364de1fa05e83a81278030599efd97/mmh3-5.3.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:1c61932e7c9f9e6fad2b6cfa325d600792546028b2efa05a66e0c0a8ca5be1b6", size = 122863, upload-time = "2026-09-30T17:37:05.112Z" }, + { url = "https://files.pythonhosted.org/packages/31/93/55889a172c4b2aeab7220955a9765a5cf8ef81b183627c0a85a952689d5e/mmh3-5.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:de2074dbcc822f26f97aa83c6fba6ff54998e68b834f45a2a35775cb27b48cea", size = 110027, upload-time = "2026-09-30T17:37:06.409Z" }, + { url = "https://files.pythonhosted.org/packages/35/83/558d9c034a568f0edeee91b4f770d99eeee41300155e2be0eefef2466fcb/mmh3-5.3.1-cp314-cp314t-win32.whl", hash = "sha256:0fe225c870d34d08a0cebdddcb1c1062ecfb9655f1b844a76817ea17bcbc8316", size = 41326, upload-time = "2026-09-30T17:37:07.607Z" }, + { url = "https://files.pythonhosted.org/packages/fb/ed/e76d25bcc95b3b8ef8df663c99542056c9cd393bd70a6723e5d4f2e13b1d/mmh3-5.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:52fbda3c48e74f7c533964d91610e512e1971d1e88a264e1024f710b3a990af8", size = 43270, upload-time = "2026-09-30T17:37:08.827Z" }, + { url = "https://files.pythonhosted.org/packages/73/29/efd48025b974c588a5da623abc5916d3caba073fa7f658826c42dd673435/mmh3-5.3.1-cp314-cp314t-win_arm64.whl", hash = "sha256:d6d03f2e97225476a4ebacb7b26bdf58847f2879ed6f51c9aec296dd17a40537", size = 41270, upload-time = "2026-09-30T17:37:10.545Z" }, + { url = "https://files.pythonhosted.org/packages/74/44/7346139e65dcd6c0fb1fd810095311f3c83bfad406875db80b905dfe2795/mmh3-5.3.1-cp315-cp315-android_24_arm64_v8a.whl", hash = "sha256:8e9be3c05553ff265064c364e2f75c38a0a768c5e77378e68bba3cbdddbb3534", size = 40908, upload-time = "2026-09-30T17:37:11.848Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/f310775faa92f81a8e4d33a57f8871df049d343b9ad85ebf9df65f2c32fa/mmh3-5.3.1-cp315-cp315-android_24_x86_64.whl", hash = "sha256:409ab810e88d94152e0e17f184c5c0a0cc9d5cbc35d0064117977360cdfaa6d5", size = 42038, upload-time = "2026-09-30T17:37:13.191Z" }, + { url = "https://files.pythonhosted.org/packages/d7/f5/5297ec53642c1f3783c8e9fdaec247b8a154189a4270591f34233caf25dd/mmh3-5.3.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:a564dcbacf2fa7fbab47a061c69dd8e21882de7a4e7a0d5c129c3e4853f528d0", size = 38709, upload-time = "2026-09-30T17:37:15.061Z" }, + { url = "https://files.pythonhosted.org/packages/d1/f1/51085537195290833f46be8f6b3dd2985ee4d15490eb18b537057216076a/mmh3-5.3.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:e7618907f87ce29af7da06e014f9acabb90de2dbe37b4c4a4c06ae6d2166caab", size = 39416, upload-time = "2026-09-30T17:37:16.444Z" }, + { url = "https://files.pythonhosted.org/packages/b0/cd/5a749d76cc15fee759a8fa0febd939167476db22165b27bb0eb82239e187/mmh3-5.3.1-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:50a62eec3de8a3f608f5531901b7d7106231a5cb6cd294bb5f7fbe9e89e71b7b", size = 39867, upload-time = "2026-09-30T17:37:17.808Z" }, + { url = "https://files.pythonhosted.org/packages/59/43/e0c704b08d4adbf22cd9e698f492fb7d4ad7e3a1c4aa5497d0b0da4d7027/mmh3-5.3.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:d227fe27ac054c3999793a6e79cb70d8698d575cbedff216656757b4bdf2513b", size = 55425, upload-time = "2026-09-30T17:37:19.154Z" }, + { url = "https://files.pythonhosted.org/packages/5a/59/ef17c22d2de4dce1a32e443e1de7a3cb8789f60459a45c130ba9794e67ab/mmh3-5.3.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:c97deaabf7d9d49c56433c4c52292fd03425c29589a4b71d3d5f905285ed4038", size = 40029, upload-time = "2026-09-30T17:37:20.64Z" }, + { url = "https://files.pythonhosted.org/packages/f9/a1/93c1f18d5105493f4e46259080441d6b7fde971d628fdc7965b586937440/mmh3-5.3.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:c6badf50b4fe0e001a2ab5f6a4347aad3d10ba182a3ef64eacfb8a6cb581393c", size = 39628, upload-time = "2026-09-30T17:37:22.029Z" }, + { url = "https://files.pythonhosted.org/packages/0f/b7/ce9c92967bb98c69856ed1c3a8af1b05e1a14fc44f9ff28a7f6a59a17b10/mmh3-5.3.1-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:08d59963e361381b8052f57f48607c695e4ecf15e1fedcd28e003fd7a189579b", size = 98463, upload-time = "2026-09-30T17:37:23.699Z" }, + { url = "https://files.pythonhosted.org/packages/69/a5/43e7fcca7b892649d060717c595095a80d6468afbb57bd554ec347fff766/mmh3-5.3.1-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:714e150e76987dedef08ee370c597c8aabc11b8ac7796bc43876a0b8830e1ffa", size = 103974, upload-time = "2026-09-30T17:37:25.591Z" }, + { url = "https://files.pythonhosted.org/packages/7c/b7/0573655c5b8cc2440ff6113f3dbdd7c196561612059720d20fa745b7b1e5/mmh3-5.3.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f2fe7c4fb48c7b69877ae28a1fe4df4e1bfb53fad467673c100035f0d4210d3e", size = 106431, upload-time = "2026-09-30T17:37:26.908Z" }, + { url = "https://files.pythonhosted.org/packages/b2/de/60b68e00b8675cd16117a3429342acdcafcc4ab26a061db9dfc8bfa36f71/mmh3-5.3.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a209b121065c821358e965f481eba1a0cde6172d9d982bae6bf7ad026498ad6e", size = 113141, upload-time = "2026-09-30T17:37:28.334Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b1/78be5bda72dcf0e2bc9920ae26b801e101eb68c7c0b62e75d1c301b6adbb/mmh3-5.3.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cd0e2f3571fcf0b434929d28a2fc61e90215a03aa65368b70ca1faf89e36da25", size = 121217, upload-time = "2026-09-30T17:37:29.813Z" }, + { url = "https://files.pythonhosted.org/packages/dc/22/36c326cf3ed6350fdd9f9bd5261ff05e6c9d37fff2b9944f7d6c06054791/mmh3-5.3.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c2e6bd1b4757ee488283e035755da12da831509a2ff40fdd9733b0f34e9072da", size = 104071, upload-time = "2026-09-30T17:37:31.176Z" }, + { url = "https://files.pythonhosted.org/packages/41/30/70cb86cfde918fbcda9df7d6424963691a1978fef781b898d9c4d854c3d8/mmh3-5.3.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:53dde4944acc0be5198dfd0a1b2654db0b88ae82640f8299bf0f0414e075643d", size = 99434, upload-time = "2026-09-30T17:37:32.524Z" }, + { url = "https://files.pythonhosted.org/packages/a2/ab/f0b914fce03026c1c697a26b5cdfb012368f64fb1592744acfd191c76b07/mmh3-5.3.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:37d39bcc4a554d3f4c9868287686a0b342857ea6df01e532fa0fb722654be253", size = 99432, upload-time = "2026-09-30T17:37:33.954Z" }, + { url = "https://files.pythonhosted.org/packages/d8/c7/2673b90301ae8ebacc0785bd3a17ad47273ba170adff0398abf0505690a9/mmh3-5.3.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:b66b40b78a4dd5757ec2ce449db0d505f24ec6a901bffd6667dd60df0eb27a9e", size = 106446, upload-time = "2026-09-30T17:37:35.338Z" }, + { url = "https://files.pythonhosted.org/packages/e6/de/3d35d6a5cf606129c456f5fa250c5155f2c9b3e680e99243e13a9649f5a8/mmh3-5.3.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:429453058c769d9cffae36fcaa352241ca59f66aed41aa021bc66565213977f5", size = 102116, upload-time = "2026-09-30T17:37:36.838Z" }, + { url = "https://files.pythonhosted.org/packages/48/0d/73cc09401c50a3f04a79330bda8cabbac05ebd2655e8de785f3356339e42/mmh3-5.3.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:3b475f5b5a5f813a5f7c35f257b7c72b7a6f3da16fd564e8466c26bcbd4cacd7", size = 110094, upload-time = "2026-09-30T17:37:38.154Z" }, + { url = "https://files.pythonhosted.org/packages/9e/6c/95deb09624751abce5e60b119d0539314ec2b3536b54fc771a6d918322e9/mmh3-5.3.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:9184d09183b74a78815358c9dc5cc24cb94556779c279f6f2ce698da6a1766a8", size = 98065, upload-time = "2026-09-30T17:37:39.544Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2a/b748aa5ab7a389b3720164f207edb2ec2d7eb21785ad7ec9aa9d55685bdc/mmh3-5.3.1-cp315-cp315-win32.whl", hash = "sha256:0cf5a30de9df754c6977bb90637510ce1c9a2039d4402e1226cf584ba82c367f", size = 40877, upload-time = "2026-09-30T17:37:41.048Z" }, + { url = "https://files.pythonhosted.org/packages/63/15/47f2945f6d4f23c5cf21937a6a818c33875ff0d52578437419d29712df6d/mmh3-5.3.1-cp315-cp315-win_amd64.whl", hash = "sha256:0d7953b08712fb5bb894757568d62692db2e7b235c921eab6471c728ad51a728", size = 42294, upload-time = "2026-09-30T17:37:42.481Z" }, + { url = "https://files.pythonhosted.org/packages/bd/79/5f349380e9cc4722eef51bd47da78fe7e71408a7cc670cce435eac0e1451/mmh3-5.3.1-cp315-cp315-win_arm64.whl", hash = "sha256:5e837386acd3387d67c6821b7fa584a197748ae74e336de15d9f731fb3d9c1e4", size = 40663, upload-time = "2026-09-30T17:37:43.709Z" }, + { url = "https://files.pythonhosted.org/packages/82/0c/1303c58d814ad868815a3a88f8e3719970c4b5acba549f556369d8b82a36/mmh3-5.3.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:245665a94be009e874a9d290ddebb67a253805a2b4d87c2e6e042e16a3d1c298", size = 57090, upload-time = "2026-09-30T17:37:44.971Z" }, + { url = "https://files.pythonhosted.org/packages/73/cf/ae0b095688d03c6c1ebc2f9319d393b80e7ef6fb896a47b8cc4b8ac3bd80/mmh3-5.3.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:f4539cf0d522eb19c88c30a52ef4d1625840391dde7e07b3f69481b7c61cb6cc", size = 40831, upload-time = "2026-09-30T17:37:46.707Z" }, + { url = "https://files.pythonhosted.org/packages/34/c9/f2e2863e06f63024159cded330e4dc3df544179e45b7b31fc53071d010c4/mmh3-5.3.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:63a072d514e1762b823cb1129f826289839dd7d71d56cc81eb72223aeb730ce5", size = 40374, upload-time = "2026-09-30T17:37:47.905Z" }, + { url = "https://files.pythonhosted.org/packages/17/4c/1ccd1eadf3caa62b074005c31ad423a5d982f6b35e81305ea9ade8865a94/mmh3-5.3.1-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9358751ae6cd260e3595662e1503fa8791f31dfc69bfe3034c55a278cc5ab78b", size = 108164, upload-time = "2026-09-30T17:37:49.373Z" }, + { url = "https://files.pythonhosted.org/packages/f7/82/13305aaab0528f64204377db1ae4c5886a8b4bf1a3a032406a9d8d3c7798/mmh3-5.3.1-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6e9936eb3cef7e4fe7dc0c7fe51039da7ce12dc691b291ea4dc6d8ed28b9b990", size = 114119, upload-time = "2026-09-30T17:37:50.737Z" }, + { url = "https://files.pythonhosted.org/packages/4e/fc/97ab3b905aa27e2bd1886fc0c4ab5e6b05db697f2168300928d6b2184786/mmh3-5.3.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc25cc785743cdfc10b3f671fde6bea760689518372ffd355bda03f9c07dc9ec", size = 117054, upload-time = "2026-09-30T17:37:52.331Z" }, + { url = "https://files.pythonhosted.org/packages/3b/d7/911384e5a69b817074afa6a8e236a875e601c2b856f6ba520208f357401f/mmh3-5.3.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e885b5f0102899227456a6568585295e599f5f90872933a62274746e94c244dd", size = 122287, upload-time = "2026-09-30T17:37:53.683Z" }, + { url = "https://files.pythonhosted.org/packages/d2/8a/ec962c0bf2b720c2e1d8d2410e25f699df421ac8f77bbf4463068f103bc0/mmh3-5.3.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fc7a4ab919a66b9db72c1425d610f69a8480dce22aa4c6ad2218e4acf41f8c01", size = 130866, upload-time = "2026-09-30T17:37:54.951Z" }, + { url = "https://files.pythonhosted.org/packages/f4/a9/5dff431289844d190484a8dbb734320e1a19781a29c37372510e1d816799/mmh3-5.3.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0bd3b13c6ab9c851c6e74a06fde305c9661d985ceadf91ed60507a27ee1ab4cb", size = 110756, upload-time = "2026-09-30T17:37:56.459Z" }, + { url = "https://files.pythonhosted.org/packages/a0/74/15099e020e8d02e674078b1bdedf0db61f7f447dd684f004d59f5b250361/mmh3-5.3.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:bc3605b2db1dfcea3d0ea481ed32ac798753dd68e942fe3859374021d1668edb", size = 107650, upload-time = "2026-09-30T17:37:57.891Z" }, + { url = "https://files.pythonhosted.org/packages/7a/cb/f00da63de039f9891914d2969bae2d590a5f410c8a9ee93fe6e5ad7d35db/mmh3-5.3.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:ea031c00ffbfc38e54cc070426ec2d136e573cbae24cc34e7ee558399ac71ce0", size = 108729, upload-time = "2026-09-30T17:37:59.229Z" }, + { url = "https://files.pythonhosted.org/packages/17/7d/d6b33bb23a1c56d8af2571eaac999d526a09ac59e6eca35603d74bfe2c1a/mmh3-5.3.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:b4a40cd05113233d30b3035054f9d4939e25466bb63e75d158f116b030618e55", size = 114172, upload-time = "2026-09-30T17:38:00.596Z" }, + { url = "https://files.pythonhosted.org/packages/09/8d/bc670057ebd529792feebd6f8e6ac5463df3c8c9d8a4fd1530327d71a74f/mmh3-5.3.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:1c6593adfb734776dd93bd16b30512211be5d9a96aec0925b4a3f081edb6ad91", size = 109063, upload-time = "2026-09-30T17:38:01.947Z" }, + { url = "https://files.pythonhosted.org/packages/63/02/c9229c198ce251bbcdfd3ee96d35c711fc3dcb5d05b9330c55b2a63b5db7/mmh3-5.3.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:d3883cbc3da65076bf0972e7b76f4c972d9fe5810757030cd01c29cbe0ccde41", size = 119758, upload-time = "2026-09-30T17:38:03.264Z" }, + { url = "https://files.pythonhosted.org/packages/01/b9/25cde1ae2118d1962fb68bdf399d2676a242216a18cc9e89a8b34df19ba0/mmh3-5.3.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:5b5d7b1141ad7ad0091199f8440f3034ce775ea8c470a418bf6e2c371de08ec5", size = 107392, upload-time = "2026-09-30T17:38:04.611Z" }, + { url = "https://files.pythonhosted.org/packages/3b/93/ce57c57a62c97c4bceaafb4df29e5bfdc827f3cea08b27444dcf3f55ecb6/mmh3-5.3.1-cp315-cp315t-win32.whl", hash = "sha256:2d76a78ab3af4be19d31840ca4f69b429e602e719d0b351f20d96304dbfb155e", size = 41534, upload-time = "2026-09-30T17:38:05.901Z" }, + { url = "https://files.pythonhosted.org/packages/63/ea/fee223d57ef44e743e71216805b873fd452b1de034a5a6bf78011deb7e9d/mmh3-5.3.1-cp315-cp315t-win_amd64.whl", hash = "sha256:4c061c1072dc2f32ef7e6a57a92da3de217dc47114b2857da31b34bc79ac795b", size = 43019, upload-time = "2026-09-30T17:38:07.154Z" }, + { url = "https://files.pythonhosted.org/packages/c2/6f/727d4255c2ca4957f87400852449f22f148b4278529701c67c6c65dadafb/mmh3-5.3.1-cp315-cp315t-win_arm64.whl", hash = "sha256:c90b3503675892e7496bec63e2e18e413798a062e5d54e6cf220c2bf0bb7fe69", size = 41323, upload-time = "2026-09-30T17:38:08.405Z" }, +] + [[package]] name = "mpire" version = "2.10.2" @@ -1881,6 +2192,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, ] +[[package]] +name = "py-rust-stemmers" +version = "0.1.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/c1/9763f9fb1cd73f9c317a83feeed6e0d4af320c6bbddab47b4a94f3a47d0c/py_rust_stemmers-0.1.8.tar.gz", hash = "sha256:6b0f6f48bc54d607aed802de872fcd5a71bae969a6760976dc78ce55e8eaf3da", size = 9732, upload-time = "2026-05-22T11:00:24.358Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/6a/39080bc8f4a441a35378c0faeeb834fb27974997f40d51342574e70f9662/py_rust_stemmers-0.1.8-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6a9a4b8733d0b307bd0879ab7e321aa8a0bfd054a75a5cb23c647df5ca7d17c3", size = 290230, upload-time = "2026-05-22T10:59:44.551Z" }, + { url = "https://files.pythonhosted.org/packages/73/15/ae60b9010924adac465f418822d9c514690aba6846edd67b6e2b5c227745/py_rust_stemmers-0.1.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:51d0042d2a92ef0f7048bfc06b6c2a02306af31ea47f09d24b34e4b7e63c4e80", size = 275449, upload-time = "2026-05-22T10:59:45.547Z" }, + { url = "https://files.pythonhosted.org/packages/ec/7c/94be8b932179823d66e0d2be03a94706132a7d16a640d5e5710de1cb1b8f/py_rust_stemmers-0.1.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89d3d34094b9b6078a8ea6fe1c7044e5fd32f14e76c94818c5008f49ae075f08", size = 316676, upload-time = "2026-05-22T10:59:46.522Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a4/8bd5c9f31207136830457d819e3f98bb21c54c0cdc40d6f1845ce4efdf7c/py_rust_stemmers-0.1.8-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:40c86be90cee4a709ad84fde4db7f11ca44d65630a56b77ec86fe84c23adfc09", size = 319458, upload-time = "2026-05-22T10:59:47.914Z" }, + { url = "https://files.pythonhosted.org/packages/f9/95/95da2b353b164a3a2b8a1c799866a58060693be4f1dc21065663dc67dc17/py_rust_stemmers-0.1.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:515884bcfb47b10335146648f276930d0c1201ae5e8b7b400fb46d8ea05c0ec2", size = 323541, upload-time = "2026-05-22T10:59:48.894Z" }, + { url = "https://files.pythonhosted.org/packages/3e/ce/f34403b68808519dfa3220e1d94a40f26d5025f27e28893e2388ab9cfde5/py_rust_stemmers-0.1.8-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:fa42f5f8feb694aaaa869eedf477fcaf66f67a192cd64d94302d06920c33864a", size = 323873, upload-time = "2026-05-22T10:59:49.872Z" }, + { url = "https://files.pythonhosted.org/packages/57/01/fb8527f6474d576975415405c985a97260e0403829e062103d334230b7d2/py_rust_stemmers-0.1.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2e86ad68fe297a6652f0f0390625ea81858b6f27862fd4c5ee1214bf5af29b9d", size = 494761, upload-time = "2026-05-22T10:59:51.021Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ac/73816237dbec20a7299abf901e2f7b6061d238754e033b48e423603f5336/py_rust_stemmers-0.1.8-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4b90fc81411943b114e8eb4988a876ba3b12bd2d20741559803eddc4131575dc", size = 596141, upload-time = "2026-05-22T10:59:52.122Z" }, + { url = "https://files.pythonhosted.org/packages/52/0a/dd48debf386a206ee1c6ad75a0827eac89428441291c90d98bc3803fccf1/py_rust_stemmers-0.1.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:56cc2c2df742fa6529285b7d204720f34b7da789ed78eb578442f93c6de97d89", size = 541633, upload-time = "2026-05-22T10:59:53.18Z" }, + { url = "https://files.pythonhosted.org/packages/92/ca/ebb707ab280636b8f46d040ccb051d1a9ddbc1f1ca2d90cdba626872f405/py_rust_stemmers-0.1.8-cp312-cp312-win_amd64.whl", hash = "sha256:dd967eea2f808a1e73aa71ecccef0f4925a4cca4eb02ced94057afe3303153ef", size = 212134, upload-time = "2026-05-22T10:59:54.245Z" }, + { url = "https://files.pythonhosted.org/packages/c2/98/f078f3930311e7b6154ccdf9166c4e30a416c7d199e136b5f09265d58a35/py_rust_stemmers-0.1.8-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5bd15b89203ecd886960e237124d1aa6e55498d76418c36c967d3b12168d43dc", size = 290427, upload-time = "2026-05-22T10:59:55.316Z" }, + { url = "https://files.pythonhosted.org/packages/c9/46/21d784a3f1db6a23051ffd5826d8ee667d26a64587c1cfbda0443ed87fff/py_rust_stemmers-0.1.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6c92733b020534470ca5a0d7fe8b85c85622ff383d4f37fec75a1c677aa84921", size = 275628, upload-time = "2026-05-22T10:59:56.687Z" }, + { url = "https://files.pythonhosted.org/packages/57/d5/701c73a4f6a7fecfd96a6588f0cafe98d6b0acde93adf8a2e45535f3d1d5/py_rust_stemmers-0.1.8-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ab605a86c950ba7e8ab1392cf91296c0bec3084babb897a4aecf90a10c82395", size = 316656, upload-time = "2026-05-22T10:59:57.67Z" }, + { url = "https://files.pythonhosted.org/packages/9d/0d/c58fe98153cfdb6abf4dfb6ac335c923000d4af4e736080c3a3045b7aea7/py_rust_stemmers-0.1.8-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:21ed8055cec1f78d666afad8ffd7a51775ba419d2c615b8a1df7b32ca7f33e2b", size = 319377, upload-time = "2026-05-22T10:59:58.664Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d7/e60d04849e90aa3ad457211cc4999c30401f433341f9a5588c12b81f9877/py_rust_stemmers-0.1.8-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae773e1d01e9aa328d175f461475d0cd7074a82bfcc71de6dc5765e51f1cc9f7", size = 323719, upload-time = "2026-05-22T10:59:59.845Z" }, + { url = "https://files.pythonhosted.org/packages/6a/48/c0e4fb955db784cc354e0756354602f7043ff4c10fcbd9d901a2f8fe3239/py_rust_stemmers-0.1.8-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5cc8fab9d0f1b274a26935a632362b8278f03e81b65e8b8644d5ca3f62a5a1a4", size = 324110, upload-time = "2026-05-22T11:00:01.26Z" }, + { url = "https://files.pythonhosted.org/packages/48/eb/981b26baff37cf7a26ee206763cc4d2fb3e1db8f0f86ec030074431fae05/py_rust_stemmers-0.1.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:35570098da02eb439afcd7270a12bf850bbe874b85cb912e0fb2d87a6e703920", size = 494645, upload-time = "2026-05-22T11:00:02.737Z" }, + { url = "https://files.pythonhosted.org/packages/6d/af/f16e805b7aefc2257b192b83a89300c8360b0fdffd3dfefa92dee4ec9b15/py_rust_stemmers-0.1.8-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:0a68745d4b3c7f5abc778ca967e8711df6154873abcfe4e62a6631fa2363cc32", size = 596124, upload-time = "2026-05-22T11:00:04.499Z" }, + { url = "https://files.pythonhosted.org/packages/76/8c/e7a2c940ba00e0792ae346aed5e755d51d37cf6d6853f6b141e5380e285d/py_rust_stemmers-0.1.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7cc0cc0b8eb45d2158c28ea43e2f338c110aad63052ad3bd00bc7446a595e12f", size = 541771, upload-time = "2026-05-22T11:00:06.081Z" }, + { url = "https://files.pythonhosted.org/packages/c2/a0/dd7c5fc6ade6d2a2a49e49937f06f2d488511454e8ab1b313d277ee8c3b1/py_rust_stemmers-0.1.8-cp313-cp313-win_amd64.whl", hash = "sha256:15af4e12e1288de2e5241eec375afc6ad6be4c125a28ca010599d9f92db23f01", size = 212438, upload-time = "2026-05-22T11:00:07.244Z" }, + { url = "https://files.pythonhosted.org/packages/b0/7e/f4346adfd44acbd7eaedcbd7d21b7f40ec9712e6c699e71fddad8dae6f8d/py_rust_stemmers-0.1.8-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:526b58958c6ffa36c4a805326cfb624ecbd665d16ba435027dbed0bcbcaa09d2", size = 290379, upload-time = "2026-05-22T11:00:08.192Z" }, + { url = "https://files.pythonhosted.org/packages/c2/d8/988fc3f5dc0dbbd4bf5909f50ff953ab55ee8b5f79a835d00e57847d3123/py_rust_stemmers-0.1.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2b607f0b270951fb66479baf4b68716cc63a981585cbd898b0b6b5c359efde7e", size = 275458, upload-time = "2026-05-22T11:00:09.522Z" }, + { url = "https://files.pythonhosted.org/packages/f4/94/e04c8b6a8364bca1b368785cef143755dd2d1ffe74df8f8b47b075bb1043/py_rust_stemmers-0.1.8-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b0327b151ab8a338fb54fdac114ba34394327fc1e2c4c425ad1caf2013e5de3", size = 314711, upload-time = "2026-05-22T11:00:10.878Z" }, + { url = "https://files.pythonhosted.org/packages/4f/cb/f59f9a80caa099cb6625a46c9a8e6e7e80bb3ed284f17e80245c8240a66e/py_rust_stemmers-0.1.8-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dadd0e369703817fc7026987b3093f461f9f58d8dde74e689d546184bc8f3451", size = 319370, upload-time = "2026-05-22T11:00:11.961Z" }, + { url = "https://files.pythonhosted.org/packages/06/59/8211cd0f56e53f7770debd9a78de37985fb5662ae66e3b7b380f4c79888b/py_rust_stemmers-0.1.8-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:245e2c61c52e073341893a9682cd1396b61047154548aee30bb1af3d8ed4b4cc", size = 321373, upload-time = "2026-05-22T11:00:13.213Z" }, + { url = "https://files.pythonhosted.org/packages/10/72/fe33e614c114264d1ba54d39da4b5a4abeb6aedd0d26e5a8fd0637d6ddba/py_rust_stemmers-0.1.8-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:451ee1c02a3f5cf1e161b46ba9032cdda4ba10a8b03ff9ee61c1d34d42a0bc81", size = 321707, upload-time = "2026-05-22T11:00:14.177Z" }, + { url = "https://files.pythonhosted.org/packages/91/f9/3cd18902fe2fa54557d3fe9132552256372d381c7aca71346163055d78b1/py_rust_stemmers-0.1.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d396dd25c473c1bc4248c79cd223f4b36356b55a124652f015c6a001547f81ac", size = 492457, upload-time = "2026-05-22T11:00:15.245Z" }, + { url = "https://files.pythonhosted.org/packages/90/d7/32c6d3995e7036b73683389de2771f4dbbf40de192b7efe73c2528ee1eb5/py_rust_stemmers-0.1.8-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:479c77c32d8be692f3cfcde7e19273f02ac81d6f45c6aef49887ef95cab7abbb", size = 596085, upload-time = "2026-05-22T11:00:16.404Z" }, + { url = "https://files.pythonhosted.org/packages/00/8c/e68fa5d862ea6a27fced3535c25ea4eaa26ba1ce00dfef5841924c74b167/py_rust_stemmers-0.1.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c786235275c5c2abb7f206b8236aee3ca0bc53c7497daf7fb7b01d3491469547", size = 539747, upload-time = "2026-05-22T11:00:17.414Z" }, + { url = "https://files.pythonhosted.org/packages/44/48/aa584cf3772e01231641c95dc1aa73327a7d986c562639d78d0013733acf/py_rust_stemmers-0.1.8-cp314-cp314-win_amd64.whl", hash = "sha256:931d13570962b093417e5443a9d1bd63d73fa239ebb81e5b1d346663571403e4", size = 209636, upload-time = "2026-05-22T11:00:18.662Z" }, +] + [[package]] name = "pyclipper" version = "1.4.0" @@ -2759,28 +3108,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, ] +[[package]] +name = "tokenizers" +version = "0.22.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "huggingface-hub", version = "0.36.2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, + { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, + { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, + { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, + { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, + { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, + { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, + { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, + { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, + { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, +] + [[package]] name = "tokenizers" version = "0.23.2" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version < '3.13' and sys_platform == 'emscripten'", +] dependencies = [ - { name = "huggingface-hub" }, + { name = "huggingface-hub", version = "1.33.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/18/1e/bc6587c5ab643b2e17776cace9070a2ae73549c86bffac9934a600bf3c31/tokenizers-0.23.2.tar.gz", hash = "sha256:7f0f085686b9de0d0079e6f874ae053600db64c5d13049e0bbc0119926d25aac", size = 385745, upload-time = "2026-09-03T08:55:42.89Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/ed/8a443528baa6fac8dfe8c3b75b038c63ac92bb539bcabe311e227c718173/tokenizers-0.23.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:85a9a357a3764aecc904ee76bdaf8cf1ad8e5a67a1b929a487c4a39b49ed0e90", size = 3148852, upload-time = "2026-09-03T08:55:30.874Z" }, - { url = "https://files.pythonhosted.org/packages/67/49/22da045a91732384d3a3771816bf188dc5a1f702c32e635afa7c679c0bef/tokenizers-0.23.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:986670e43691469dcee610ea0f846f91a8f84e91fc6f7a48d4c064414c0ec2bf", size = 3101593, upload-time = "2026-09-03T08:55:28.587Z" }, - { url = "https://files.pythonhosted.org/packages/2e/4d/8f569ed49372a3ed8e57099bd515055fd48d7c95912c4307cda6973c2168/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a37039b5dfc4af84eb3ef0a92f4307e28936c8f9adccba2629d36f652e9bf7a2", size = 3516830, upload-time = "2026-09-03T08:55:14.741Z" }, - { url = "https://files.pythonhosted.org/packages/2a/de/e2f14c8919d5bf51874051d00d6c7b7e0e8bde6c6a2dbeddda7f642896ff/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7b7e37ba198f24150f523e1242e83c4970de4a525480586be5dcc24d9add32c5", size = 3407975, upload-time = "2026-09-03T08:55:16.842Z" }, - { url = "https://files.pythonhosted.org/packages/c5/bd/93c69152d02ef06ce47aed8b2bf4952dcf733c935a62791873932b2934d9/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43e4f2071e3cc8d5d86421c874aebc82659bb51a68bcdef5a0da75ee89511ccb", size = 3748165, upload-time = "2026-09-03T08:55:24.769Z" }, - { url = "https://files.pythonhosted.org/packages/2d/b7/56b84b80bc96942bba8eb23751a9e8a1fce4faaf4390425e7083f721c98c/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:325fee2e0418a9dc6c9ecf736a5f5f0db7875183ace9549ae339da76f7a1fbb7", size = 4024165, upload-time = "2026-09-03T08:55:18.806Z" }, - { url = "https://files.pythonhosted.org/packages/9b/8a/0175e216f005c2fe08238292663aa41e4c802b216e71047a69a0e9fc6fa3/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:950d7c9426fa72406a0ffeacdbc0bb9985f5db20eb8b263f29c79aaf83105703", size = 3591899, upload-time = "2026-09-03T08:55:22.752Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ca/ca6b93c7820df123b2662a9469e8facc826ccc94e98fdd0d615f6431e73a/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41c2f84d172449b4dadb9cdc508e3e364076613c35b16e76ecfe47a60d1e3305", size = 3386843, upload-time = "2026-09-03T08:55:26.584Z" }, - { url = "https://files.pythonhosted.org/packages/e9/a4/4f9106d317b14a80aefea9f0e3a8d07ef25f856a7607eb7f5ab894281fcb/tokenizers-0.23.2-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:12f0835dc2ee694746a76adf7b1567d4346a4a502ebe93fb1f5f80ea49799b78", size = 3577314, upload-time = "2026-09-03T08:55:20.825Z" }, - { url = "https://files.pythonhosted.org/packages/8d/6a/1552b70fb0d9ab074fd3fc961435d01364e79c9058481822c3af6e8d402c/tokenizers-0.23.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:eb2f9c8a24da020ea8c11a01a19c1c2547912d92121ae4a01cfbca46125dee40", size = 9967367, upload-time = "2026-09-03T08:55:33.188Z" }, - { url = "https://files.pythonhosted.org/packages/06/01/3ccb3a956c7528b2507b8a9714155c4baf86af593039db6ea375dd0c96c3/tokenizers-0.23.2-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:f486f402f6f9abee5bb032553736813af0c710a86b2e0ca592634c55cea1f835", size = 9811886, upload-time = "2026-09-03T08:55:35.642Z" }, - { url = "https://files.pythonhosted.org/packages/fa/73/7038e612d48bda1599457f712f6bd3854eae1a9dc9c13aa47f835349db48/tokenizers-0.23.2-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:bef235815a067b2648caf6dcc7a71091b0b0fff9ee8057f6451eb9335fae52ef", size = 10146224, upload-time = "2026-09-03T08:55:38.391Z" }, - { url = "https://files.pythonhosted.org/packages/b5/d8/8e9e4e0b287a338d8f88976729628c9d22e8a54cfaf9777018a7f7cb58a0/tokenizers-0.23.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5c56bda1511921587789163e524d196ed8284174ac23abd7685d5ea8da6c4718", size = 10256304, upload-time = "2026-09-03T08:55:40.977Z" }, { url = "https://files.pythonhosted.org/packages/f3/1f/c79a01f671a49728ebb0b61f7ff9ea45663b66cab40bc0858e9859b25c16/tokenizers-0.23.2-cp310-abi3-win32.whl", hash = "sha256:debf978920d93ba9c219bd67cc4bbfaf912c9039e41e7a28b91ec15e3728c95a", size = 2592809, upload-time = "2026-09-03T08:55:48.02Z" }, { url = "https://files.pythonhosted.org/packages/db/f7/0a69ac6b82dbccf3f71add938a161c497952749294b8dd6dfe03a819dc40/tokenizers-0.23.2-cp310-abi3-win_amd64.whl", hash = "sha256:2e96f5699d5249c9c64aa8412e044f727aae3a4098cf830f9901ec1afc361cde", size = 2863236, upload-time = "2026-09-03T08:55:46.193Z" }, { url = "https://files.pythonhosted.org/packages/d7/b0/dee84cb44175be1b4c35bd2f770727494e78f0bb38e571a623ade94dbebb/tokenizers-0.23.2-cp310-abi3-win_arm64.whl", hash = "sha256:e49c394456dd9985787fec76132438ba3fb8911f857b1bf3d40119f9292d41aa", size = 2729352, upload-time = "2026-09-03T08:55:44.345Z" }, @@ -2865,20 +3240,57 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a7/03/921a3d3c75785aca9ebfbfcabfbc3a1be12e2ab5265deb026d55a5a3f83e/tqdm-4.70.1-py3-none-any.whl", hash = "sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73", size = 80199, upload-time = "2026-09-11T07:25:14.599Z" }, ] +[[package]] +name = "transformers" +version = "4.57.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "filelock", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "huggingface-hub", version = "0.36.2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "numpy", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "packaging", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "pyyaml", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "regex", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "requests", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "safetensors", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "tokenizers", version = "0.22.2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "tqdm", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/35/67252acc1b929dc88b6602e8c4a982e64f31e733b804c14bc24b47da35e6/transformers-4.57.6.tar.gz", hash = "sha256:55e44126ece9dc0a291521b7e5492b572e6ef2766338a610b9ab5afbb70689d3", size = 10134912, upload-time = "2026-01-16T10:38:39.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/b8/e484ef633af3887baeeb4b6ad12743363af7cce68ae51e938e00aaa0529d/transformers-4.57.6-py3-none-any.whl", hash = "sha256:4c9e9de11333ddfe5114bc872c9f370509198acf0b87a832a0ab9458e2bd0550", size = 11993498, upload-time = "2026-01-16T10:38:31.289Z" }, +] + [[package]] name = "transformers" version = "5.18.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version < '3.13' and sys_platform == 'emscripten'", +] dependencies = [ - { name = "huggingface-hub" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "regex" }, - { name = "safetensors" }, - { name = "tokenizers" }, - { name = "tqdm" }, - { name = "typer" }, + { name = "huggingface-hub", version = "1.33.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "numpy", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "packaging", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "pyyaml", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "regex", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "safetensors", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "tokenizers", version = "0.23.2", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "tqdm", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "typer", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9e/6e/5a50ca8aff5fdd4ff21271198d2fb9e1f11bcd0ea279576199c0caa34cba/transformers-5.18.0.tar.gz", hash = "sha256:d89c206e42e841af7cd314b3d2b21c03e977d819cdb4aef65ada048769a644df", size = 10028745, upload-time = "2026-09-30T16:46:00.575Z" } wheels = [ @@ -2995,6 +3407,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fe/d1/aa8a3e935c37efee7945984fdb64d7e0851bf6d920afd97b2d21f9d23360/triton-3.8.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74217bb56ed8692759227758e4c4b3bd2d608a209c1a7a081bf361fb4c2c1bf9", size = 248077577, upload-time = "2026-08-28T15:56:24.94Z" }, ] +[[package]] +name = "turbovec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/53/2e07e0801fbc260dfe4859f4c925a5888b28e65892396bc03bd689dec39d/turbovec-1.1.1.tar.gz", hash = "sha256:a51571d1d6912f2d22a136374121cad6e34df2609600f643a429cd00fb957855", size = 764872, upload-time = "2026-10-04T21:24:06.93Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/41/6efa40e3100d6f56cc0fcc48acd8bcebefcb34ddf363f5a814487c8ed549/turbovec-1.1.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:e96eb34568def2b167798bc1f9691c4d953cb9d842b8e37cc6395a62832ca7c0", size = 772772, upload-time = "2026-10-04T21:24:00.934Z" }, + { url = "https://files.pythonhosted.org/packages/49/06/fb4b2737a8ba96845d99adc88cf21758e0681545180f90b9bd776e579c4c/turbovec-1.1.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d0fd9951321ce9fc700d5197ccb086ab50a8a4a4f5aa59f1c062f694d7290953", size = 812053, upload-time = "2026-10-04T21:24:02.488Z" }, + { url = "https://files.pythonhosted.org/packages/33/0c/09fca6d9db53db8ad6b497e883c090e5ab4f2412c8f28b588ff7e0805042/turbovec-1.1.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5c55f9839df926abb1fe89054b13ca3c0a5937a6a69582a1a03b7603791aa3b1", size = 831463, upload-time = "2026-10-04T21:24:03.981Z" }, + { url = "https://files.pythonhosted.org/packages/60/52/adee7678182faa272a31158d7a8e6cd3918cef79e62010ee6ebb88016a45/turbovec-1.1.1-cp39-abi3-win_amd64.whl", hash = "sha256:673de627a8ee6e81271305c1f11fee43d809282e1713cc382c6673d3336ea652", size = 724868, upload-time = "2026-10-04T21:24:05.292Z" }, +] + [[package]] name = "typer" version = "0.26.8" @@ -3126,6 +3553,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/4d/2d0d67834092e354d2b0498f014a41249a89556bc406cf86f3e1557bb463/websockets-16.1.1-py3-none-any.whl", hash = "sha256:6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3", size = 173814, upload-time = "2026-07-17T22:51:04.184Z" }, ] +[[package]] +name = "win32-setctime" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/705086c9d734d3b663af0e9bb3d4de6578d08f46b1b101c2442fd9aecaa2/win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0", size = 4867, upload-time = "2024-12-07T15:28:28.314Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, +] + [[package]] name = "xlsxwriter" version = "3.2.9"