feat: support huggingface-hub 2.x

pyproject.toml

@@ -28,7 +28,9 @@ onnxruntime = [
 tqdm = "^4.66"
 requests = "^2.31"
 tokenizers = ">=0.15,<1.0"
-huggingface-hub = ">=0.20,<2.0"
+huggingface-hub = ">=0.20,<3.0"  # NB: tokenizers still pins huggingface-hub<2.0
+# (verified through tokenizers 1.0.0rc2), so the relaxed bound becomes
+# installable only once tokenizers lifts its own pin.
 loguru = "^0.7.2"
 pillow = [
     { version = ">=10.3.0,<13.0", python = ">=3.10,<3.13" },

tests/test_model_management.py

@@ -90,3 +90,143 @@ def test_offline_probe_tolerates_auxiliary_file_drift(
         local_files_only=True,
     )
     assert result == str(snapshot)
+
+
+"""huggingface-hub version-compatibility tests (issue #742).
+
+These tests run fully offline and lock in support for huggingface-hub 2.x:
+the import surface fastembed uses, the transport-error shim resolving to the
+hub's own HTTP library, and the metadata-verified download flow with a hub
+2.x-style RepoFile.
+"""
+
+
+def _make_repo_file(path: str, size: int, blob_id: str):
+    """Build a RepoFile the way the installed huggingface_hub does.
+
+    hub 2.x constructs RepoFile from API payload keys (``oid`` instead of
+    ``blob_id``); older versions take ``blob_id=`` directly.
+    """
+    from fastembed.common import model_management
+
+    try:
+        return model_management.RepoFile(path=path, size=size, blob_id=blob_id)
+    except (TypeError, KeyError):
+        return model_management.RepoFile(path=path, size=size, oid=blob_id)
+
+
+def test_hf_import_surface_resolves() -> None:
+    """Every huggingface_hub name model_management imports must exist."""
+    import huggingface_hub
+    from huggingface_hub import constants
+    from huggingface_hub.file_download import repo_folder_name
+    from huggingface_hub.utils import (
+        HFValidationError,
+        RepositoryNotFoundError,
+        disable_progress_bars,
+        enable_progress_bars,
+    )
+
+    from fastembed.common import model_management
+
+    assert huggingface_hub.__version__
+    assert constants.REPO_ID_SEPARATOR == "--"
+    assert isinstance(constants.HF_HUB_ETAG_TIMEOUT, (int, float))
+    assert callable(model_management.snapshot_download)
+    assert callable(model_management.model_info)
+    assert callable(model_management.list_repo_tree)
+    assert callable(repo_folder_name)
+    for name in (
+        HFValidationError,
+        RepositoryNotFoundError,
+        disable_progress_bars,
+        enable_progress_bars,
+    ):
+        assert name is not None
+
+    repo_file = _make_repo_file("onnx/model.onnx", 10, "deadbeef")
+    assert (repo_file.path, repo_file.size, repo_file.blob_id) == (
+        "onnx/model.onnx",
+        10,
+        "deadbeef",
+    )
+
+
+def test_transport_error_shim_catches_hub_http_errors() -> None:
+    """_hf_transport_errors() must cover the hub's own HTTP library's errors."""
+    import pytest
+
+    from fastembed.common import model_management
+
+    errors = model_management._hf_transport_errors()
+    assert isinstance(errors, tuple)
+    assert all(isinstance(err, type) and issubclass(err, Exception) for err in errors)
+    assert OSError in model_management._HF_DOWNLOAD_ERRORS
+    assert ValueError in model_management._HF_DOWNLOAD_ERRORS
+    from huggingface_hub.utils import RepositoryNotFoundError
+
+    assert RepositoryNotFoundError in model_management._HF_DOWNLOAD_ERRORS
+
+    if errors:
+        # hub >= 1.0 (httpx) and 2.x (httpx2) raise TransportError, not OSError,
+        # on refused connections / DNS failures / timeouts.
+        with pytest.raises(model_management._HF_DOWNLOAD_ERRORS):
+            raise errors[0]("simulated transport failure")
+
+    # The shim prefers the hub's own re-exported HTTP library
+    # (huggingface_hub.utils.httpx: httpx for hub 1.x, httpx2 for 2.x),
+    # falling back to a direct httpx import on hub 1.0-1.29. When the
+    # re-export exists its TransportError must be among the caught errors;
+    # the errors[0] assertion above covers the older-hub fallback.
+    try:
+        from huggingface_hub.utils import httpx as hub_httpx
+    except ImportError:
+        hub_httpx = None
+    if hub_httpx is not None:
+        assert hub_httpx.TransportError in model_management._HF_DOWNLOAD_ERRORS
+
+
+def test_online_download_flow_mocked(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    """Full download path works with hub 2.x-style RepoFile metadata, offline.
+
+    model_info/list_repo_tree/snapshot_download are mocked so no network is
+    needed; the test exercises the metadata collection and verification logic
+    against a RepoFile built the way hub 2.x builds one.
+    """
+    from types import SimpleNamespace
+
+    from fastembed.common import model_management
+
+    cache = tmp_path / "cache"
+    blobs = {"model.onnx": b"x" * 10, "config.json": b"y" * 20}
+    snapshot = _seed_cache(cache, files=blobs, metadata={})
+    # no pre-existing metadata: the online flow must collect it from the repo files
+    (snapshot / ModelManagement.METADATA_FILE).unlink()
+
+    monkeypatch.setattr(
+        model_management,
+        "model_info",
+        lambda repo_id, **kwargs: SimpleNamespace(sha=REVISION),
+    )
+    monkeypatch.setattr(
+        model_management,
+        "list_repo_tree",
+        lambda repo_id, **kwargs: [
+            _make_repo_file("model.onnx", len(blobs["model.onnx"]), "modelblob"),
+            _make_repo_file("config.json", len(blobs["config.json"]), "configblob"),
+        ],
+    )
+    monkeypatch.setattr(model_management, "snapshot_download", lambda **kwargs: str(snapshot))
+
+    result = ModelManagement.download_files_from_huggingface(
+        "qdrant/fake-onnx",
+        cache_dir=str(cache),
+        extra_patterns=["model.onnx"],
+        local_files_only=False,
+    )
+    assert result == str(snapshot)
+
+    # the download wrote back the collected metadata, verified against the repo files
+    metadata = json.loads((snapshot / ModelManagement.METADATA_FILE).read_text())
+    assert metadata["model.onnx"] == {"size": 10, "blob_id": "modelblob"}
+    assert metadata["config.json"] == {"size": 20, "blob_id": "configblob"}