fix: don't depend on the hub's case-normalizing redirect for model sources

fastembed/common/model_management.py

@@ -13,8 +13,10 @@
 
 import requests
 from huggingface_hub import constants, snapshot_download, model_info, list_repo_tree
+from huggingface_hub.file_download import repo_folder_name
 from huggingface_hub.hf_api import RepoFile
 from huggingface_hub.utils import (
+    HFValidationError,
     RepositoryNotFoundError,
     disable_progress_bars,
     enable_progress_bars,
@@ -157,6 +159,41 @@ def download_file_from_gcs(cls, url: str, output_path: str, show_progress: bool
                         file.write(chunk)
         return output_path
 
+    @classmethod
+    def _find_legacy_cased_source(cls, cache_dir: str, hf_source_repo: str) -> str | None:
+        """Looks for a cached snapshot of the same repo spelled with a different casing.
+
+        Built-in sources used to be lowercase and were canonicalized once it turned out that
+        relying on the hub's normalizing redirect breaks proxies. Both the hub and fastembed
+        derive the cache directory from the source verbatim, so on a case-sensitive filesystem
+        an offline load would otherwise miss a model an older version had already cached.
+
+        Args:
+            cache_dir (str): The path to the cache directory.
+            hf_source_repo (str): Name of the model on HuggingFace Hub.
+
+        Returns:
+            str | None: The differently cased source found in the cache, None if there is none.
+        """
+        separator = constants.REPO_ID_SEPARATOR
+        try:
+            expected = repo_folder_name(repo_id=hf_source_repo, repo_type="model")
+            entries = list(Path(cache_dir).iterdir())
+        except (HFValidationError, OSError):
+            return None
+
+        # the repo id sits at the tail of the folder name, with every "/" replaced
+        offset = len(expected) - len(hf_source_repo.replace("/", separator))
+
+        for entry in entries:
+            if entry.name == expected or entry.name.lower() != expected.lower():
+                continue
+            if not entry.is_dir():
+                continue
+            # the hub forbids the separator inside a repo id, so it only marks the split
+            return entry.name[offset:].replace(separator, "/")
+        return None
+
     @classmethod
     def download_files_from_huggingface(
         cls,
@@ -169,7 +206,7 @@ def download_files_from_huggingface(
         """
         Downloads a model from HuggingFace Hub.
         Args:
-            hf_source_repo (str): Name of the model on HuggingFace Hub, e.g. "qdrant/all-MiniLM-L6-v2-onnx".
+            hf_source_repo (str): Name of the model on HuggingFace Hub, e.g. "Qdrant/all-MiniLM-L6-v2-onnx".
             cache_dir (Optional[str]): The path to the cache directory.
             extra_patterns (list[str]): extra patterns to allow in the snapshot download, typically
                 includes the required model files.
@@ -238,26 +275,51 @@ def _save_file_metadata(model_dir: Path, meta: dict[str, dict[str, int | str]])
 
         allow_patterns.extend(extra_patterns)
 
-        snapshot_dir = Path(cache_dir) / f"models--{hf_source_repo.replace('/', '--')}"
+        snapshot_dir = Path(cache_dir) / repo_folder_name(
+            repo_id=hf_source_repo, repo_type="model"
+        )
         metadata_file = snapshot_dir / cls.METADATA_FILE
 
         if local_files_only:
             disable_progress_bars()
-            if metadata_file.exists():
-                metadata = json.loads(metadata_file.read_text())
-                verified = _verify_files_from_metadata(snapshot_dir, metadata, repo_files=[])
-                if not verified:
-                    logger.warning(
-                        "Local file sizes do not match the metadata."
-                    )  # do not raise, still make an attempt to load the model
-            result = snapshot_download(
-                repo_id=hf_source_repo,
-                allow_patterns=allow_patterns,
-                cache_dir=cache_dir,
-                local_files_only=local_files_only,
-                **kwargs,
-            )
-            return result
+            # a canonical directory can exist yet hold no usable snapshot, e.g. when an
+            # earlier download was interrupted, so fall back on failure rather than on
+            # the directory being absent
+            sources = [hf_source_repo]
+            legacy_source = cls._find_legacy_cased_source(cache_dir, hf_source_repo)
+            if legacy_source is not None:
+                sources.append(legacy_source)
+
+            for index, source in enumerate(sources):
+                snapshot_dir = Path(cache_dir) / repo_folder_name(
+                    repo_id=source, repo_type="model"
+                )
+                metadata_file = snapshot_dir / cls.METADATA_FILE
+                if metadata_file.exists():
+                    metadata = json.loads(metadata_file.read_text())
+                    verified = _verify_files_from_metadata(snapshot_dir, metadata, repo_files=[])
+                    if not verified:
+                        logger.warning(
+                            "Local file sizes do not match the metadata."
+                        )  # do not raise, still make an attempt to load the model
+                try:
+                    # a legacy source is only ever resolved against the cache: sending it
+                    # to the hub would ask for the very redirect this casing avoids
+                    return snapshot_download(
+                        repo_id=source,
+                        allow_patterns=allow_patterns,
+                        cache_dir=cache_dir,
+                        local_files_only=True,
+                        **kwargs,
+                    )
+                except _HF_DOWNLOAD_ERRORS:
+                    if index == len(sources) - 1:
+                        raise
+                    logger.info(
+                        f"{source} is not usable in {cache_dir}, loading "
+                        f"{sources[index + 1]}, cached from the same repo by an older "
+                        "fastembed version."
+                    )
 
         # hub sends this request with no timeout unless given one, so an endpoint that accepts
         # the connection but never answers would block here for good, before download_model can
@@ -480,7 +542,7 @@ def download_model(cls, model: T, cache_dir: str, retries: int = 3, **kwargs: An
                     "description": "Base English model, v1.5",
                     "size_in_GB": 0.44,
                     "sources": {
-                        "hf": "qdrant/bge-base-en-v1.5-onnx-q",
+                        "hf": "Qdrant/bge-base-en-v1.5-onnx-Q",
                     }
                 }
                 ```

fastembed/late_interaction/token_embeddings.py

@@ -19,7 +19,7 @@
         " Prefixes for queries/documents: not necessary, 2023 year.",
         license="apache-2.0",
         size_in_GB=0.12,
-        sources=ModelSource(hf="xenova/jina-embeddings-v2-small-en"),
+        sources=ModelSource(hf="Xenova/jina-embeddings-v2-small-en"),
         model_file="onnx/model.onnx",
     ),
 ]

fastembed/sparse/sparse_text_embedding.py

@@ -43,7 +43,7 @@ def list_supported_models(cls) -> list[dict[str, Any]]:
                         "license": "apache-2.0",
                         "size_in_GB": 0.532,
                         "sources": {
-                            "hf": "qdrant/SPLADE_PP_en_v1",
+                            "hf": "Qdrant/Splade_PP_en_v1",
                         },
                     }
                 ]

fastembed/text/onnx_embedding.py

@@ -47,7 +47,7 @@
         ),
         license="mit",
         size_in_GB=1.20,
-        sources=ModelSource(hf="qdrant/bge-large-en-v1.5-onnx"),
+        sources=ModelSource(hf="Qdrant/bge-large-en-v1.5-onnx"),
         model_file="model.onnx",
     ),
     DenseModelDescription(
@@ -113,7 +113,7 @@
         ),
         license="apache-2.0",
         size_in_GB=0.09,
-        sources=ModelSource(hf="snowflake/snowflake-arctic-embed-xs"),
+        sources=ModelSource(hf="Snowflake/snowflake-arctic-embed-xs"),
         model_file="onnx/model.onnx",
     ),
     DenseModelDescription(
@@ -125,7 +125,7 @@
         ),
         license="apache-2.0",
         size_in_GB=0.13,
-        sources=ModelSource(hf="snowflake/snowflake-arctic-embed-s"),
+        sources=ModelSource(hf="Snowflake/snowflake-arctic-embed-s"),
         model_file="onnx/model.onnx",
     ),
     DenseModelDescription(
@@ -149,7 +149,7 @@
         ),
         license="apache-2.0",
         size_in_GB=0.54,
-        sources=ModelSource(hf="snowflake/snowflake-arctic-embed-m-long"),
+        sources=ModelSource(hf="Snowflake/snowflake-arctic-embed-m-long"),
         model_file="onnx/model.onnx",
     ),
     DenseModelDescription(
@@ -161,7 +161,7 @@
         ),
         license="apache-2.0",
         size_in_GB=1.02,
-        sources=ModelSource(hf="snowflake/snowflake-arctic-embed-l"),
+        sources=ModelSource(hf="Snowflake/snowflake-arctic-embed-l"),
         model_file="onnx/model.onnx",
     ),
     DenseModelDescription(

fastembed/text/pooled_embedding.py

@@ -55,7 +55,7 @@
         ),
         license="apache-2.0",
         size_in_GB=0.22,
-        sources=ModelSource(hf="qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q"),
+        sources=ModelSource(hf="Qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q"),
         model_file="model_optimized.onnx",
     ),
     DenseModelDescription(
@@ -67,7 +67,7 @@
         ),
         license="apache-2.0",
         size_in_GB=1.00,
-        sources=ModelSource(hf="xenova/paraphrase-multilingual-mpnet-base-v2"),
+        sources=ModelSource(hf="Xenova/paraphrase-multilingual-mpnet-base-v2"),
         model_file="onnx/model.onnx",
     ),
     DenseModelDescription(
@@ -80,7 +80,7 @@
         license="mit",
         size_in_GB=2.24,
         sources=ModelSource(
-            hf="qdrant/multilingual-e5-large-onnx",
+            hf="Qdrant/multilingual-e5-large-onnx",
             _deprecated_tar_struct=True,
         ),
         model_file="model.onnx",

fastembed/text/pooled_normalized_embedding.py

@@ -19,7 +19,7 @@
         license="apache-2.0",
         size_in_GB=0.09,
         sources=ModelSource(
-            hf="qdrant/all-MiniLM-L6-v2-onnx",
+            hf="Qdrant/all-MiniLM-L6-v2-onnx",
             _deprecated_tar_struct=True,
         ),
         model_file="model.onnx",
@@ -33,7 +33,7 @@
         ),
         license="apache-2.0",
         size_in_GB=0.52,
-        sources=ModelSource(hf="xenova/jina-embeddings-v2-base-en"),
+        sources=ModelSource(hf="Xenova/jina-embeddings-v2-base-en"),
         model_file="onnx/model.onnx",
     ),
     DenseModelDescription(
@@ -45,7 +45,7 @@
         ),
         license="apache-2.0",
         size_in_GB=0.12,
-        sources=ModelSource(hf="xenova/jina-embeddings-v2-small-en"),
+        sources=ModelSource(hf="Xenova/jina-embeddings-v2-small-en"),
         model_file="onnx/model.onnx",
     ),
     DenseModelDescription(
@@ -117,7 +117,7 @@
         ),
         license="mit",
         size_in_GB=1.20,
-        sources=ModelSource(hf="qdrant/gte-large-onnx"),
+        sources=ModelSource(hf="Qdrant/gte-large-onnx"),
         model_file="model.onnx",
     ),
 ]