diff --git a/fastembed/common/model_management.py b/fastembed/common/model_management.py index 5974535e..6c85dffe 100644 --- a/fastembed/common/model_management.py +++ b/fastembed/common/model_management.py @@ -13,8 +13,10 @@ import requests from huggingface_hub import constants, snapshot_download, model_info, list_repo_tree +from huggingface_hub.file_download import repo_folder_name from huggingface_hub.hf_api import RepoFile from huggingface_hub.utils import ( + HFValidationError, RepositoryNotFoundError, disable_progress_bars, enable_progress_bars, @@ -157,6 +159,41 @@ def download_file_from_gcs(cls, url: str, output_path: str, show_progress: bool file.write(chunk) return output_path + @classmethod + def _find_legacy_cased_source(cls, cache_dir: str, hf_source_repo: str) -> str | None: + """Looks for a cached snapshot of the same repo spelled with a different casing. + + Built-in sources used to be lowercase and were canonicalized once it turned out that + relying on the hub's normalizing redirect breaks proxies. Both the hub and fastembed + derive the cache directory from the source verbatim, so on a case-sensitive filesystem + an offline load would otherwise miss a model an older version had already cached. + + Args: + cache_dir (str): The path to the cache directory. + hf_source_repo (str): Name of the model on HuggingFace Hub. + + Returns: + str | None: The differently cased source found in the cache, None if there is none. + """ + separator = constants.REPO_ID_SEPARATOR + try: + expected = repo_folder_name(repo_id=hf_source_repo, repo_type="model") + entries = list(Path(cache_dir).iterdir()) + except (HFValidationError, OSError): + return None + + # the repo id sits at the tail of the folder name, with every "/" replaced + offset = len(expected) - len(hf_source_repo.replace("/", separator)) + + for entry in entries: + if entry.name == expected or entry.name.lower() != expected.lower(): + continue + if not entry.is_dir(): + continue + # the hub forbids the separator inside a repo id, so it only marks the split + return entry.name[offset:].replace(separator, "/") + return None + @classmethod def download_files_from_huggingface( cls, @@ -169,7 +206,7 @@ def download_files_from_huggingface( """ Downloads a model from HuggingFace Hub. Args: - hf_source_repo (str): Name of the model on HuggingFace Hub, e.g. "qdrant/all-MiniLM-L6-v2-onnx". + hf_source_repo (str): Name of the model on HuggingFace Hub, e.g. "Qdrant/all-MiniLM-L6-v2-onnx". cache_dir (Optional[str]): The path to the cache directory. extra_patterns (list[str]): extra patterns to allow in the snapshot download, typically includes the required model files. @@ -238,26 +275,51 @@ def _save_file_metadata(model_dir: Path, meta: dict[str, dict[str, int | str]]) allow_patterns.extend(extra_patterns) - snapshot_dir = Path(cache_dir) / f"models--{hf_source_repo.replace('/', '--')}" + snapshot_dir = Path(cache_dir) / repo_folder_name( + repo_id=hf_source_repo, repo_type="model" + ) metadata_file = snapshot_dir / cls.METADATA_FILE if local_files_only: disable_progress_bars() - if metadata_file.exists(): - metadata = json.loads(metadata_file.read_text()) - verified = _verify_files_from_metadata(snapshot_dir, metadata, repo_files=[]) - if not verified: - logger.warning( - "Local file sizes do not match the metadata." - ) # do not raise, still make an attempt to load the model - result = snapshot_download( - repo_id=hf_source_repo, - allow_patterns=allow_patterns, - cache_dir=cache_dir, - local_files_only=local_files_only, - **kwargs, - ) - return result + # a canonical directory can exist yet hold no usable snapshot, e.g. when an + # earlier download was interrupted, so fall back on failure rather than on + # the directory being absent + sources = [hf_source_repo] + legacy_source = cls._find_legacy_cased_source(cache_dir, hf_source_repo) + if legacy_source is not None: + sources.append(legacy_source) + + for index, source in enumerate(sources): + snapshot_dir = Path(cache_dir) / repo_folder_name( + repo_id=source, repo_type="model" + ) + metadata_file = snapshot_dir / cls.METADATA_FILE + if metadata_file.exists(): + metadata = json.loads(metadata_file.read_text()) + verified = _verify_files_from_metadata(snapshot_dir, metadata, repo_files=[]) + if not verified: + logger.warning( + "Local file sizes do not match the metadata." + ) # do not raise, still make an attempt to load the model + try: + # a legacy source is only ever resolved against the cache: sending it + # to the hub would ask for the very redirect this casing avoids + return snapshot_download( + repo_id=source, + allow_patterns=allow_patterns, + cache_dir=cache_dir, + local_files_only=True, + **kwargs, + ) + except _HF_DOWNLOAD_ERRORS: + if index == len(sources) - 1: + raise + logger.info( + f"{source} is not usable in {cache_dir}, loading " + f"{sources[index + 1]}, cached from the same repo by an older " + "fastembed version." + ) # hub sends this request with no timeout unless given one, so an endpoint that accepts # the connection but never answers would block here for good, before download_model can @@ -480,7 +542,7 @@ def download_model(cls, model: T, cache_dir: str, retries: int = 3, **kwargs: An "description": "Base English model, v1.5", "size_in_GB": 0.44, "sources": { - "hf": "qdrant/bge-base-en-v1.5-onnx-q", + "hf": "Qdrant/bge-base-en-v1.5-onnx-Q", } } ``` diff --git a/fastembed/late_interaction/token_embeddings.py b/fastembed/late_interaction/token_embeddings.py index 55d3cd94..f94109d6 100644 --- a/fastembed/late_interaction/token_embeddings.py +++ b/fastembed/late_interaction/token_embeddings.py @@ -19,7 +19,7 @@ " Prefixes for queries/documents: not necessary, 2023 year.", license="apache-2.0", size_in_GB=0.12, - sources=ModelSource(hf="xenova/jina-embeddings-v2-small-en"), + sources=ModelSource(hf="Xenova/jina-embeddings-v2-small-en"), model_file="onnx/model.onnx", ), ] diff --git a/fastembed/sparse/sparse_text_embedding.py b/fastembed/sparse/sparse_text_embedding.py index 273e08f0..a604d2d3 100644 --- a/fastembed/sparse/sparse_text_embedding.py +++ b/fastembed/sparse/sparse_text_embedding.py @@ -43,7 +43,7 @@ def list_supported_models(cls) -> list[dict[str, Any]]: "license": "apache-2.0", "size_in_GB": 0.532, "sources": { - "hf": "qdrant/SPLADE_PP_en_v1", + "hf": "Qdrant/Splade_PP_en_v1", }, } ] diff --git a/fastembed/text/onnx_embedding.py b/fastembed/text/onnx_embedding.py index 6d14e845..0f6cec9f 100644 --- a/fastembed/text/onnx_embedding.py +++ b/fastembed/text/onnx_embedding.py @@ -47,7 +47,7 @@ ), license="mit", size_in_GB=1.20, - sources=ModelSource(hf="qdrant/bge-large-en-v1.5-onnx"), + sources=ModelSource(hf="Qdrant/bge-large-en-v1.5-onnx"), model_file="model.onnx", ), DenseModelDescription( @@ -113,7 +113,7 @@ ), license="apache-2.0", size_in_GB=0.09, - sources=ModelSource(hf="snowflake/snowflake-arctic-embed-xs"), + sources=ModelSource(hf="Snowflake/snowflake-arctic-embed-xs"), model_file="onnx/model.onnx", ), DenseModelDescription( @@ -125,7 +125,7 @@ ), license="apache-2.0", size_in_GB=0.13, - sources=ModelSource(hf="snowflake/snowflake-arctic-embed-s"), + sources=ModelSource(hf="Snowflake/snowflake-arctic-embed-s"), model_file="onnx/model.onnx", ), DenseModelDescription( @@ -149,7 +149,7 @@ ), license="apache-2.0", size_in_GB=0.54, - sources=ModelSource(hf="snowflake/snowflake-arctic-embed-m-long"), + sources=ModelSource(hf="Snowflake/snowflake-arctic-embed-m-long"), model_file="onnx/model.onnx", ), DenseModelDescription( @@ -161,7 +161,7 @@ ), license="apache-2.0", size_in_GB=1.02, - sources=ModelSource(hf="snowflake/snowflake-arctic-embed-l"), + sources=ModelSource(hf="Snowflake/snowflake-arctic-embed-l"), model_file="onnx/model.onnx", ), DenseModelDescription( diff --git a/fastembed/text/pooled_embedding.py b/fastembed/text/pooled_embedding.py index a99c7c42..ba9a26a2 100644 --- a/fastembed/text/pooled_embedding.py +++ b/fastembed/text/pooled_embedding.py @@ -55,7 +55,7 @@ ), license="apache-2.0", size_in_GB=0.22, - sources=ModelSource(hf="qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q"), + sources=ModelSource(hf="Qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q"), model_file="model_optimized.onnx", ), DenseModelDescription( @@ -67,7 +67,7 @@ ), license="apache-2.0", size_in_GB=1.00, - sources=ModelSource(hf="xenova/paraphrase-multilingual-mpnet-base-v2"), + sources=ModelSource(hf="Xenova/paraphrase-multilingual-mpnet-base-v2"), model_file="onnx/model.onnx", ), DenseModelDescription( @@ -80,7 +80,7 @@ license="mit", size_in_GB=2.24, sources=ModelSource( - hf="qdrant/multilingual-e5-large-onnx", + hf="Qdrant/multilingual-e5-large-onnx", _deprecated_tar_struct=True, ), model_file="model.onnx", diff --git a/fastembed/text/pooled_normalized_embedding.py b/fastembed/text/pooled_normalized_embedding.py index 63709602..b107b05c 100644 --- a/fastembed/text/pooled_normalized_embedding.py +++ b/fastembed/text/pooled_normalized_embedding.py @@ -19,7 +19,7 @@ license="apache-2.0", size_in_GB=0.09, sources=ModelSource( - hf="qdrant/all-MiniLM-L6-v2-onnx", + hf="Qdrant/all-MiniLM-L6-v2-onnx", _deprecated_tar_struct=True, ), model_file="model.onnx", @@ -33,7 +33,7 @@ ), license="apache-2.0", size_in_GB=0.52, - sources=ModelSource(hf="xenova/jina-embeddings-v2-base-en"), + sources=ModelSource(hf="Xenova/jina-embeddings-v2-base-en"), model_file="onnx/model.onnx", ), DenseModelDescription( @@ -45,7 +45,7 @@ ), license="apache-2.0", size_in_GB=0.12, - sources=ModelSource(hf="xenova/jina-embeddings-v2-small-en"), + sources=ModelSource(hf="Xenova/jina-embeddings-v2-small-en"), model_file="onnx/model.onnx", ), DenseModelDescription( @@ -117,7 +117,7 @@ ), license="mit", size_in_GB=1.20, - sources=ModelSource(hf="qdrant/gte-large-onnx"), + sources=ModelSource(hf="Qdrant/gte-large-onnx"), model_file="model.onnx", ), ]