From 03da39758af45ccb0f79597d85b37f1439a8dd1b Mon Sep 17 00:00:00 2001 From: Raphael Malikian Date: Mon, 28 Sep 2026 10:18:00 -0700 Subject: [PATCH 1/2] feat: support huggingface-hub 2.x Allow huggingface-hub>=2.0 (2.0.0 released): hub 2.x moved from httpx to httpx2, which fastembed's _hf_transport_errors() shim already handles via huggingface_hub.utils.httpx. All other used hub APIs (snapshot_download, model_info, list_repo_tree, repo_folder_name, RepoFile, utils helpers, constants) verified unchanged in 2.0.0. Fixes #742 Signed-off-by: Raphael Malikian --- pyproject.toml | 2 +- tests/test_model_management.py | 128 +++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 70df993bd..f83d299e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ onnxruntime = [ tqdm = "^4.66" requests = "^2.31" tokenizers = ">=0.15,<1.0" -huggingface-hub = ">=0.20,<2.0" +huggingface-hub = ">=0.20,<3.0" loguru = "^0.7.2" pillow = [ { version = ">=10.3.0,<13.0", python = ">=3.10,<3.13" }, diff --git a/tests/test_model_management.py b/tests/test_model_management.py index 4a5e7c44f..36164a8b0 100644 --- a/tests/test_model_management.py +++ b/tests/test_model_management.py @@ -90,3 +90,131 @@ def test_offline_probe_tolerates_auxiliary_file_drift( local_files_only=True, ) assert result == str(snapshot) + + +"""huggingface-hub version-compatibility tests (issue #742). + +These tests run fully offline and lock in support for huggingface-hub 2.x: +the import surface fastembed uses, the transport-error shim resolving to the +hub's own HTTP library, and the metadata-verified download flow with a hub +2.x-style RepoFile. +""" + + +def _make_repo_file(path: str, size: int, blob_id: str): + """Build a RepoFile the way the installed huggingface_hub does. + + hub 2.x constructs RepoFile from API payload keys (``oid`` instead of + ``blob_id``); older versions take ``blob_id=`` directly. + """ + from fastembed.common import model_management + + try: + return model_management.RepoFile(path=path, size=size, blob_id=blob_id) + except (TypeError, KeyError): + return model_management.RepoFile(path=path, size=size, oid=blob_id) + + +def test_hf_import_surface_resolves() -> None: + """Every huggingface_hub name model_management imports must exist.""" + import huggingface_hub + from huggingface_hub import constants + from huggingface_hub.file_download import repo_folder_name + from huggingface_hub.utils import ( + HFValidationError, + RepositoryNotFoundError, + disable_progress_bars, + enable_progress_bars, + ) + + from fastembed.common import model_management + + assert huggingface_hub.__version__ + assert constants.REPO_ID_SEPARATOR == "--" + assert isinstance(constants.HF_HUB_ETAG_TIMEOUT, (int, float)) + assert callable(model_management.snapshot_download) + assert callable(model_management.model_info) + assert callable(model_management.list_repo_tree) + assert callable(repo_folder_name) + for name in ( + HFValidationError, + RepositoryNotFoundError, + disable_progress_bars, + enable_progress_bars, + ): + assert name is not None + + repo_file = _make_repo_file("onnx/model.onnx", 10, "deadbeef") + assert (repo_file.path, repo_file.size, repo_file.blob_id) == ( + "onnx/model.onnx", + 10, + "deadbeef", + ) + + +def test_transport_error_shim_catches_hub_http_errors() -> None: + """_hf_transport_errors() must cover the hub's own HTTP library's errors.""" + import pytest + + from fastembed.common import model_management + + errors = model_management._hf_transport_errors() + assert isinstance(errors, tuple) + assert all(isinstance(err, type) and issubclass(err, Exception) for err in errors) + assert OSError in model_management._HF_DOWNLOAD_ERRORS + assert ValueError in model_management._HF_DOWNLOAD_ERRORS + from huggingface_hub.utils import RepositoryNotFoundError + + assert RepositoryNotFoundError in model_management._HF_DOWNLOAD_ERRORS + + if errors: + # hub >= 1.0 (httpx) and 2.x (httpx2) raise TransportError, not OSError, + # on refused connections / DNS failures / timeouts. + with pytest.raises(model_management._HF_DOWNLOAD_ERRORS): + raise errors[0]("simulated transport failure") + + +def test_online_download_flow_mocked(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Full download path works with hub 2.x-style RepoFile metadata, offline. + + model_info/list_repo_tree/snapshot_download are mocked so no network is + needed; the test exercises the metadata collection and verification logic + against a RepoFile built the way hub 2.x builds one. + """ + from types import SimpleNamespace + + from fastembed.common import model_management + + cache = tmp_path / "cache" + blobs = {"model.onnx": b"x" * 10, "config.json": b"y" * 20} + snapshot = _seed_cache(cache, files=blobs, metadata={}) + # no pre-existing metadata: the online flow must collect it from the repo files + (snapshot / ModelManagement.METADATA_FILE).unlink() + + monkeypatch.setattr( + model_management, + "model_info", + lambda repo_id, **kwargs: SimpleNamespace(sha=REVISION), + ) + monkeypatch.setattr( + model_management, + "list_repo_tree", + lambda repo_id, **kwargs: [ + _make_repo_file("model.onnx", len(blobs["model.onnx"]), "modelblob"), + _make_repo_file("config.json", len(blobs["config.json"]), "configblob"), + ], + ) + monkeypatch.setattr(model_management, "snapshot_download", lambda **kwargs: str(snapshot)) + + result = ModelManagement.download_files_from_huggingface( + "qdrant/fake-onnx", + cache_dir=str(cache), + extra_patterns=["model.onnx"], + local_files_only=False, + ) + assert result == str(snapshot) + + # the download wrote back the collected metadata, verified against the repo files + metadata = json.loads((snapshot / ModelManagement.METADATA_FILE).read_text()) + assert metadata["model.onnx"] == {"size": 10, "blob_id": "modelblob"} + assert metadata["config.json"] == {"size": 20, "blob_id": "configblob"} From 1f5987c8f73647f4d7ef2c279356abb978199b37 Mon Sep 17 00:00:00 2001 From: Raphael Malikian Date: Mon, 28 Sep 2026 15:17:31 -0700 Subject: [PATCH 2/2] update from working tree --- pyproject.toml | 4 +++- tests/test_model_management.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f83d299e9..7504a4642 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,9 @@ onnxruntime = [ tqdm = "^4.66" requests = "^2.31" tokenizers = ">=0.15,<1.0" -huggingface-hub = ">=0.20,<3.0" +huggingface-hub = ">=0.20,<3.0" # NB: tokenizers still pins huggingface-hub<2.0 +# (verified through tokenizers 1.0.0rc2), so the relaxed bound becomes +# installable only once tokenizers lifts its own pin. loguru = "^0.7.2" pillow = [ { version = ">=10.3.0,<13.0", python = ">=3.10,<3.13" }, diff --git a/tests/test_model_management.py b/tests/test_model_management.py index 36164a8b0..09a079d05 100644 --- a/tests/test_model_management.py +++ b/tests/test_model_management.py @@ -173,6 +173,18 @@ def test_transport_error_shim_catches_hub_http_errors() -> None: with pytest.raises(model_management._HF_DOWNLOAD_ERRORS): raise errors[0]("simulated transport failure") + # The shim prefers the hub's own re-exported HTTP library + # (huggingface_hub.utils.httpx: httpx for hub 1.x, httpx2 for 2.x), + # falling back to a direct httpx import on hub 1.0-1.29. When the + # re-export exists its TransportError must be among the caught errors; + # the errors[0] assertion above covers the older-hub fallback. + try: + from huggingface_hub.utils import httpx as hub_httpx + except ImportError: + hub_httpx = None + if hub_httpx is not None: + assert hub_httpx.TransportError in model_management._HF_DOWNLOAD_ERRORS + def test_online_download_flow_mocked(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Full download path works with hub 2.x-style RepoFile metadata, offline.