From 217491299153dabbb5f42f9d0ca0dcbc19200770 Mon Sep 17 00:00:00 2001 From: tastelikefeet Date: Mon, 2 Nov 2026 17:13:53 +0800 Subject: [PATCH 1/6] Kernels subpackage and huggingface_hub patcher migration - Add modelscope_hub.kernels: a drop-in wrapper around the kernels library that loads optimized compute kernels from ModelScope (factory swap on kernels.hf_hub._get_hf_api, v1 fallback for branch-less mirrors, trusted-publisher allowlist via MS_KERNELS_TRUSTED_PUBLISHERS). - Add modelscope_hub.compat.hf_hub_patcher: monkey-patches the huggingface_hub module functions (hf_hub_download, file_exists, whoami, create_repo, upload_file, upload_folder, create_commit, repocard) onto ModelScope-backed implementations, with the legacy compatibility contract kept stable (whoami dict, RepoUrl-shaped str, commit_url shape, master default revision, Add-only create_commit) and idempotent markers shared with the legacy modelscope patcher. - Uploads go through the commit API (resumable, LFS-routed) instead of a git clone + push; non-model repo_type now fails fast instead of silently uploading to a model repository. - New extras: kernels (kernels>=0.17), hf (huggingface-hub>=0.19, verified against 0.19.4 source audit and 1.32 test suite). - Export patch helpers from modelscope_hub.compat; document both features in the README. --- README.md | 55 +++ pyproject.toml | 15 +- src/modelscope_hub/compat/__init__.py | 4 + src/modelscope_hub/compat/hf_hub_patcher.py | 443 ++++++++++++++++++++ src/modelscope_hub/constants.py | 19 + src/modelscope_hub/kernels/__init__.py | 43 ++ src/modelscope_hub/kernels/_hub.py | 266 ++++++++++++ src/modelscope_hub/kernels/_load.py | 185 ++++++++ src/modelscope_hub/kernels/_patch.py | 115 +++++ tests/test_hf_hub_patcher.py | 353 ++++++++++++++++ tests/test_kernels.py | 388 +++++++++++++++++ 11 files changed, 1885 insertions(+), 1 deletion(-) create mode 100644 src/modelscope_hub/compat/hf_hub_patcher.py create mode 100644 src/modelscope_hub/kernels/__init__.py create mode 100644 src/modelscope_hub/kernels/_hub.py create mode 100644 src/modelscope_hub/kernels/_load.py create mode 100644 src/modelscope_hub/kernels/_patch.py create mode 100644 tests/test_hf_hub_patcher.py create mode 100644 tests/test_kernels.py diff --git a/README.md b/README.md index 113bb60..a5c0b56 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,55 @@ path = api.download_file("Qwen/Qwen3-0.6B", "model", "config.json", local_files_ Downloads validate available size and SHA-256 metadata before replacing files, with safe resume, bounded retries, timeouts, and cancellation. Incomplete snapshots raise `FileDownloadError` (CLI exit code `1`), preserving completed files for subsequent retries. +### Load kernels (`modelscope_hub.kernels`) + +`modelscope_hub.kernels` is a drop-in wrapper around the [`kernels`](https://github.com/huggingface/kernels) library that downloads optimized compute kernels from ModelScope instead of the Hugging Face Hub. A kernel repository is an ordinary ModelScope model repository mirroring the HF kernel layout (`build//...` plus `v1`-style version branches or tags): + +```bash +pip install "modelscope-hub[kernels]" +``` + +```python +from modelscope_hub.kernels import get_kernel + +# Download optimized kernels from the ModelScope hub +activation = get_kernel("kernels-community/activation", version=1) +``` + +Also available: `has_kernel()` (environment compatibility check), `get_kernel_variants()` (build-variant decisions), and `get_local_kernel()` (load from a local path, dependencies from ModelScope). + +Notes: + +- The kernel loading, variant resolution, and dependency logic of `kernels` are reused unchanged — only its Hub API factory is swapped for a ModelScope-backed shim while a `modelscope_hub.kernels` call is in flight, so a plain `import kernels` elsewhere keeps using the Hugging Face Hub. `patch_kernels_hub()` / `unpatch_kernels_hub()` / `patch_scope()` switch this explicitly if you need it globally; the `modelscope` package's `patch_hub()` builds on the same implementation. +- `version=N` resolves against the repository's `v` branches/tags. A mirror that only carries `master` exposes it as kernel version `v1` automatically. +- ModelScope has no trusted-publisher flag, so untrusted publishers are rejected by default: only `kernels-community` is trusted out of the box. Extend the allowlist with `MS_KERNELS_TRUSTED_PUBLISHERS` (comma-separated), or allow specific repositories / all repositories through the trust parameters of `get_kernel` (its docstring mirrors the kernel library's API). +- The mirror must contain kernels built for your exact Torch version and platform (`kernels` selects `build/torchXY-...--/`); a repository whose builds predate `kernels` 0.17 metadata cannot be loaded until it is rebuilt with a current `kernel-builder`. + +### Redirect huggingface_hub (`modelscope_hub.compat`) + +`modelscope_hub.compat.hf_hub_patcher` monkey-patches the `huggingface_hub` module-level functions that HF-ecosystem code calls — `hf_hub_download`, `file_exists`, `whoami`, `create_repo`, `upload_file`, `upload_folder`, `create_commit`, and the `repocard` helpers — onto ModelScope-backed implementations: + +```bash +pip install "modelscope-hub[hf]" +``` + +```python +from modelscope_hub.compat import hf_hub_patch_scope, patch_hf_hub, unpatch_hf_hub + +# Globally redirect huggingface_hub to ModelScope +patch_hf_hub() +... # any HF-ecosystem code now downloads/uploads via ModelScope +unpatch_hf_hub() + +# Or scoped to a block; nesting reuses the outer patch +with hf_hub_patch_scope(): + ... +``` + +- The `modelscope` framework's `patch_hub()` builds on this implementation (plus its transformers/peft/diffusers class patches); the compatibility contract is kept stable: `whoami` returns `{'name': ...}`, `create_repo` returns a `RepoUrl`-shaped str, upload functions return a commit info whose `commit_url` points at `{endpoint}/models/{repo_id}/files`, the default revision is `master`, and `create_commit` only supports `Add` operations. +- Patches are idempotent and marked with the historical origin attributes, so an already-applied legacy `modelscope` patcher and this module treat each other as already patched. +- Uploads go through the ModelScope commit API (resumable, LFS-routed) rather than a local git clone + push. + ### Continuous dataset downloads For migration jobs, `download_repo_report` continues past isolated file failures and returns an explicit `COMPLETE` or `PARTIAL` report. The original `download_repo` and `snapshot_download` remain strict by default. @@ -1107,6 +1156,12 @@ Capacity thresholds are advisory and emit warnings without blocking upload. Stru | `MODELSCOPE_LOG_LEVEL` | `INFO` | SDK log level (`DEBUG`/`INFO`/`WARNING`/`ERROR`) | | `MODELSCOPE_NO_DEPRECATION_WARNINGS` | — | Suppress deprecation warnings | +**Kernels:** + +| Variable | Default | Description | +|----------|---------|-------------| +| `MS_KERNELS_TRUSTED_PUBLISHERS` | — | Comma-separated kernel publishers trusted in addition to the built-in allowlist (see [Load kernels](#load-kernels-modelscope_hubkernels)) | + > Old variable names (e.g. `API_TIMEOUT`, `DOWNLOAD_RETRY_TIMES`, `UPLOAD_USE_CACHE`) are > deprecated and remain temporarily supported. They emit a `FutureWarning` and will be removed > in a future version. Run `ms-hub list --envs` to see which deprecated names are active in your diff --git a/pyproject.toml b/pyproject.toml index afb9ff3..4a9fb82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,19 @@ dev = [ "ruff", "mypy", ] +# Runtime extra for the `modelscope_hub.kernels` package: the upstream +# `kernels` library (with huggingface_hub as a transitive dependency) provides +# the kernel loading machinery that `modelscope_hub.kernels` redirects onto +# ModelScope. +kernels = [ + "kernels>=0.17", +] +# Runtime extra for the huggingface_hub monkey-patcher in +# `modelscope_hub.compat.hf_hub_patcher` (also a transitive dependency of the +# `kernels` extra). +hf = [ + "huggingface-hub>=0.19", +] [project.scripts] # This package owns every ModelScope console script. Keeping all four in one @@ -94,5 +107,5 @@ strict = false warn_return_any = false [[tool.mypy.overrides]] -module = ["tqdm.*", "modelscope.*"] +module = ["tqdm.*", "modelscope.*", "kernels.*", "huggingface_hub.*"] ignore_missing_imports = true diff --git a/src/modelscope_hub/compat/__init__.py b/src/modelscope_hub/compat/__init__.py index 64e34f3..f0dd95c 100644 --- a/src/modelscope_hub/compat/__init__.py +++ b/src/modelscope_hub/compat/__init__.py @@ -32,11 +32,15 @@ ModelVisibility_PUBLIC, ) from .file_download import dataset_file_download, model_file_download +from .hf_hub_patcher import hf_hub_patch_scope, patch_hf_hub, unpatch_hf_hub from .hub_api import LegacyHubApi from .snapshot_download import dataset_snapshot_download, snapshot_download __all__ = [ "LegacyHubApi", + "hf_hub_patch_scope", + "patch_hf_hub", + "unpatch_hf_hub", "dataset_file_download", "dataset_snapshot_download", "model_file_download", diff --git a/src/modelscope_hub/compat/hf_hub_patcher.py b/src/modelscope_hub/compat/hf_hub_patcher.py new file mode 100644 index 0000000..b8395d6 --- /dev/null +++ b/src/modelscope_hub/compat/hf_hub_patcher.py @@ -0,0 +1,443 @@ +"""Monkey-patching of the ``huggingface_hub`` module onto the ModelScope Hub. + +Adapted from ``modelscope/utils/hf_util/patcher.py`` (Apache-2.0, Alibaba), +re-targeted from the legacy ``modelscope`` SDK onto ``modelscope_hub``. +The transformers/peft/diffusers patches stay with the ``modelscope`` package; +this module must stay importable without transformers and only depends on +``huggingface_hub`` (the optional ``hf`` extra). + +:func:`patch_hf_hub` replaces the ``huggingface_hub`` module-level functions +that HF-ecosystem code calls — ``hf_hub_download``, ``file_exists``, +``whoami``, ``create_repo``, ``upload_file``, ``upload_folder``, +``create_commit``, and the ``repocard`` helpers — with ModelScope-backed +implementations. Patch markers reuse the historical attribute names +(``hf_api._hf_hub_download_origin`` etc.), so a legacy ``modelscope`` +patcher and this module treat each other's patches as already applied. + +Compatibility contract (kept identical to the legacy patcher): + +- ``whoami`` returns ``{'name': ...}``; +- ``create_repo`` returns a ``RepoUrl``-shaped str (``HfRepoUrl``); +- ``upload_folder`` / ``upload_file`` / ``create_commit`` return a commit + info with ``commit_url`` shaped like ``{endpoint}/models/{repo_id}/files``; +- the default revision is ``master`` (HF's ``main`` is normalised away); +- ``create_commit`` only supports ``Add`` operations. + +One deliberate behaviour change: uploads go through the ModelScope commit +API (resumable, LFS-routed) instead of a local git clone + push. + +Version support: verified against ``huggingface_hub`` 0.19.4 (source-level +symbol audit) and 1.32 (full test suite); the ``hf`` extra pins +``huggingface_hub>=0.19``. Note that ``kernels>=0.17`` itself requires +``huggingface_hub>=1.10``, so installing both extras resolves to a 1.x hub. +""" + +from __future__ import annotations + +import contextlib +import os +from dataclasses import dataclass +from pathlib import Path +from types import MethodType + +import modelscope_hub +from modelscope_hub import compat +from modelscope_hub.constants import DEFAULT_MODELSCOPE_DATA_ENDPOINT +from modelscope_hub.utils.logger import get_logger + +logger = get_logger() + +__all__ = ["patch_hf_hub", "unpatch_hf_hub", "hf_hub_patch_scope"] + + +def _ms_revision(revision: str | None) -> str: + """Translate an HF revision string into one ModelScope accepts.""" + return "master" if revision in (None, "main") else revision + + +def _token(token) -> str | None: + """HF tokens may be bool (``True``/``False``) or strings; keep strings only.""" + return token if isinstance(token, str) else None + + +def _hub_api(token=None) -> modelscope_hub.HubApi: + """A ``HubApi`` for this call; ``token=None`` falls back to stored credentials.""" + return modelscope_hub.HubApi(token=_token(token)) + + +@dataclass +class HfCommitInfo: + """Return shape of the ``upload_*`` / ``create_commit`` patches. + + Mirrors the legacy ``modelscope.utils.repo_utils.CommitInfo`` fields that + callers (transformers ``push_to_hub``, diffusers) actually consume. + """ + + commit_url: str + commit_message: str | None + commit_description: str | None + oid: str | None + + +class HfRepoUrl(str): + """Lenient ``RepoUrl`` replacement returned by the ``create_repo`` patch. + + The legacy patcher returned the ``modelscope.utils.repo_utils.RepoUrl`` + str subclass (url plus ``repo_type``/``repo_id`` attributes), not HF's + own class; recent ``huggingface_hub`` releases additionally reject + non-Hugging Face hosts in their ``RepoUrl`` constructor. This subclass + keeps the legacy duck-typing shape (``str``, ``.repo_id``, ``.repo_type``). + """ + + def __new__(cls, url: str, repo_type: str = "model", repo_id: str | None = None): + obj = super().__new__(cls, url) + obj.repo_type = repo_type # type: ignore[attr-defined] + obj.repo_id = repo_id # type: ignore[attr-defined] + return obj + + +def _commit_url(repo_id: str) -> str: + return f"{DEFAULT_MODELSCOPE_DATA_ENDPOINT}/models/{repo_id}/files" + + +# --------------------------------------------------------------------------- +# Download / metadata patches +# --------------------------------------------------------------------------- +def _hf_hub_download( + repo_id: str, + filename: str, + *, + subfolder: str | None = None, + repo_type: str | None = None, + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + token=None, + local_files_only: bool = False, + **kwargs, +) -> str: + """ModelScope-backed ``huggingface_hub.hf_hub_download``.""" + if kwargs: + logger.warning( + "The passed in %s are not used by modelscope_hub.", + ", ".join(sorted(kwargs)), + ) + if repo_type not in (None, "model", "dataset"): + raise ValueError(f"repo_type={repo_type} is not supported in ModelScope") + file_path = os.path.join(subfolder, filename) if subfolder else filename + if repo_type == "dataset": + return compat.dataset_file_download( + repo_id, + file_path, + revision=_ms_revision(revision), + cache_dir=str(cache_dir) if cache_dir else None, + local_dir=str(local_dir) if local_dir else None, + token=_token(token), + local_files_only=local_files_only, + ) + return compat.model_file_download( + repo_id, + file_path, + revision=_ms_revision(revision), + cache_dir=str(cache_dir) if cache_dir else None, + local_dir=str(local_dir) if local_dir else None, + token=_token(token), + local_files_only=local_files_only, + ) + + +def _file_exists( + repo_id: str, + filename: str, + *, + repo_type: str | None = None, + revision: str | None = None, + token=None, + **kwargs, +) -> bool: + """ModelScope-backed ``huggingface_hub.file_exists``.""" + if repo_type is not None: + logger.warning("repo_type is ignored by modelscope_hub; only model repos can be queried.") + infos = _hub_api(token).list_repo_files(repo_id, "model", revision=_ms_revision(revision)) + return any(info.path == filename for info in infos) + + +def _whoami(token=None) -> dict: + """ModelScope-backed ``huggingface_hub.whoami``.""" + user = _hub_api(token).whoami() + return {"name": getattr(user, "username", None) or "unknown"} + + +# --------------------------------------------------------------------------- +# Repository / upload patches +# --------------------------------------------------------------------------- +def _create_repo(repo_id: str, *, token=None, private: bool = False, **kwargs) -> HfRepoUrl: + """ModelScope-backed ``huggingface_hub.create_repo``; returns a ``RepoUrl``-shaped str.""" + compat.LegacyHubApi(token=_token(token)).create_repo( + repo_id, + visibility="private" if private else "public", + **kwargs, + ) + return HfRepoUrl( + f"{DEFAULT_MODELSCOPE_DATA_ENDPOINT}/models/{repo_id}", + repo_type="model", + repo_id=repo_id, + ) + + +def _upload_file( + *, + path_or_fileobj, + path_in_repo: str, + repo_id: str, + token=None, + repo_type: str | None = "model", + revision: str | None = "master", + commit_message: str | None = None, + commit_description: str | None = None, + **kwargs, +) -> None: + """ModelScope-backed ``huggingface_hub.upload_file`` (commit-API upload).""" + if repo_type not in (None, "model"): + raise ValueError(f"repo_type={repo_type} is not supported in ModelScope") + _hub_api(token).upload_file( + repo_id, + "model", + path_or_fileobj, + path_in_repo, + commit_message=commit_message, + commit_description=commit_description, + revision=_ms_revision(revision), + ) + + +def _upload_folder( + *, + repo_id: str, + folder_path: str | Path, + path_in_repo: str | None = None, + commit_message: str | None = None, + commit_description: str | None = None, + token=None, + repo_type: str | None = "model", + revision: str | None = "master", + ignore_patterns=None, + allow_patterns=None, + **kwargs, +) -> HfCommitInfo: + """ModelScope-backed ``huggingface_hub.upload_folder`` (commit-API upload).""" + if repo_type not in (None, "model"): + raise ValueError(f"repo_type={repo_type} is not supported in ModelScope") + _hub_api(token).upload_folder( + repo_id, + "model", + folder_path, + path_in_repo=path_in_repo or "", + commit_message=commit_message, + commit_description=commit_description, + revision=_ms_revision(revision), + ignore_patterns=ignore_patterns, + allow_patterns=allow_patterns, + ) + return HfCommitInfo( + commit_url=_commit_url(repo_id), + commit_message=commit_message, + commit_description=commit_description, + oid=None, + ) + + +def _create_commit( + repo_id: str, + operations, + *, + commit_message: str, + commit_description: str | None = None, + token=None, + repo_type: str | None = None, + revision: str | None = "master", + **kwargs, +) -> None: + """ModelScope-backed ``huggingface_hub.create_commit`` (Add operations only). + + Each ``CommitOperationAdd`` becomes one commit through the ModelScope + commit API, so a multi-operation batch produces several commits on the + Hub, unlike a single git push. + """ + from huggingface_hub import CommitOperationAdd + + if repo_type not in (None, "model"): + raise ValueError(f"repo_type={repo_type} is not supported in ModelScope") + api = _hub_api(token) + for op in operations: + if not isinstance(op, CommitOperationAdd): + raise ValueError("ModelScope create_commit only supports Add operations for now.") + api.upload_file( + repo_id, + "model", + op.path_or_fileobj, + op.path_in_repo, + commit_message=commit_message, + commit_description=commit_description, + revision=_ms_revision(revision), + ) + + +def _repocard_load(cls, repo_id_or_path, repo_type=None, token=None, ignore_metadata_errors=False): + """ModelScope-backed ``huggingface_hub.repocard.RepoCard.load``.""" + if os.path.exists(repo_id_or_path): + file_path = repo_id_or_path + elif repo_type == "dataset": + file_path = compat.dataset_file_download(repo_id_or_path, "README.md", token=_token(token)) + else: + file_path = compat.model_file_download(repo_id_or_path, "README.md", token=_token(token)) + + with open(file_path, encoding="utf-8") as f: + repo_card = cls(f.read(), ignore_metadata_errors=ignore_metadata_errors) + if not hasattr(repo_card.data, "tags"): + repo_card.data.tags = [] + return repo_card + + +# --------------------------------------------------------------------------- +# Patch / unpatch +# --------------------------------------------------------------------------- +def patch_hf_hub() -> None: + """Monkey-patch ``huggingface_hub`` to route Hub calls to ModelScope. + + Idempotent: each function is guarded by its historical origin marker, so + a legacy ``modelscope`` patcher that has already applied the same + attribute names is respected instead of being double-patched. + """ + import huggingface_hub + from huggingface_hub import hf_api, repocard + + if not hasattr(hf_api, "_hf_hub_download_origin"): + setattr(hf_api, "_hf_hub_download_origin", huggingface_hub.file_download.hf_hub_download) + huggingface_hub.hf_hub_download = _hf_hub_download + huggingface_hub.file_download.hf_hub_download = _hf_hub_download + + if not hasattr(hf_api, "_file_exists_origin"): + setattr(hf_api, "_file_exists_origin", hf_api.file_exists) + hf_api.file_exists = _file_exists + huggingface_hub.file_exists = _file_exists + huggingface_hub.hf_api.file_exists = _file_exists + + if not hasattr(hf_api, "_whoami_origin"): + setattr(hf_api, "_whoami_origin", hf_api.whoami) + hf_api.whoami = _whoami + huggingface_hub.whoami = _whoami + huggingface_hub.hf_api.whoami = _whoami + + if not hasattr(hf_api, "_create_repo_origin"): + setattr(hf_api, "_create_repo_origin", hf_api.create_repo) + hf_api.create_repo = _create_repo + huggingface_hub.create_repo = _create_repo + huggingface_hub.hf_api.create_repo = _create_repo + + if not hasattr(hf_api, "_upload_folder_origin"): + setattr(hf_api, "_upload_folder_origin", hf_api.upload_folder) + hf_api.upload_folder = _upload_folder + huggingface_hub.upload_folder = _upload_folder + huggingface_hub.hf_api.upload_folder = _upload_folder + + if not hasattr(hf_api, "_upload_file_origin"): + setattr(hf_api, "_upload_file_origin", hf_api.upload_file) + hf_api.upload_file = _upload_file + huggingface_hub.upload_file = _upload_file + huggingface_hub.hf_api.upload_file = _upload_file + repocard.upload_file = _upload_file + + if not hasattr(hf_api, "_create_commit_origin"): + setattr(hf_api, "_create_commit_origin", hf_api.create_commit) + hf_api.create_commit = _create_commit + huggingface_hub.create_commit = _create_commit + huggingface_hub.hf_api.create_commit = _create_commit + + if not hasattr(repocard.RepoCard, "_validate_origin"): + setattr(repocard.RepoCard, "_validate_origin", repocard.RepoCard.validate) + repocard.RepoCard.validate = lambda *args, **kwargs: None + setattr(repocard.RepoCard, "_load_origin", repocard.RepoCard.load) + repocard.RepoCard.load = MethodType(_repocard_load, repocard.RepoCard) + + +def unpatch_hf_hub() -> None: + """Undo :func:`patch_hf_hub` and restore the original huggingface_hub functions.""" + import huggingface_hub + from huggingface_hub import hf_api, repocard + + if hasattr(repocard.RepoCard, "_validate_origin"): + repocard.RepoCard.validate = repocard.RepoCard._validate_origin + delattr(repocard.RepoCard, "_validate_origin") + if hasattr(repocard.RepoCard, "_load_origin"): + repocard.RepoCard.load = repocard.RepoCard._load_origin + delattr(repocard.RepoCard, "_load_origin") + + if hasattr(hf_api, "_hf_hub_download_origin"): + origin = hf_api._hf_hub_download_origin + huggingface_hub.file_download.hf_hub_download = origin + huggingface_hub.hf_hub_download = origin + delattr(hf_api, "_hf_hub_download_origin") + + if hasattr(hf_api, "_file_exists_origin"): + origin = hf_api._file_exists_origin + hf_api.file_exists = origin + huggingface_hub.file_exists = origin + huggingface_hub.hf_api.file_exists = origin + delattr(hf_api, "_file_exists_origin") + + if hasattr(hf_api, "_whoami_origin"): + origin = hf_api._whoami_origin + hf_api.whoami = origin + huggingface_hub.whoami = origin + huggingface_hub.hf_api.whoami = origin + delattr(hf_api, "_whoami_origin") + + if hasattr(hf_api, "_create_repo_origin"): + origin = hf_api._create_repo_origin + hf_api.create_repo = origin + huggingface_hub.create_repo = origin + huggingface_hub.hf_api.create_repo = origin + delattr(hf_api, "_create_repo_origin") + + if hasattr(hf_api, "_upload_folder_origin"): + origin = hf_api._upload_folder_origin + hf_api.upload_folder = origin + huggingface_hub.upload_folder = origin + huggingface_hub.hf_api.upload_folder = origin + delattr(hf_api, "_upload_folder_origin") + + if hasattr(hf_api, "_upload_file_origin"): + origin = hf_api._upload_file_origin + hf_api.upload_file = origin + huggingface_hub.upload_file = origin + huggingface_hub.hf_api.upload_file = origin + repocard.upload_file = origin + delattr(hf_api, "_upload_file_origin") + + if hasattr(hf_api, "_create_commit_origin"): + origin = hf_api._create_commit_origin + hf_api.create_commit = origin + huggingface_hub.create_commit = origin + huggingface_hub.hf_api.create_commit = origin + delattr(hf_api, "_create_commit_origin") + + +@contextlib.contextmanager +def hf_hub_patch_scope(): + """Apply :func:`patch_hf_hub` for the duration of the ``with`` block. + + Nesting is safe: an already-applied patch (including one applied by the + legacy ``modelscope`` patcher) is reused, and only the outermost scope + restores the original functions. + """ + from huggingface_hub import hf_api + + if hasattr(hf_api, "_hf_hub_download_origin"): + yield + return + patch_hf_hub() + try: + yield + finally: + unpatch_hf_hub() diff --git a/src/modelscope_hub/constants.py b/src/modelscope_hub/constants.py index 7599c68..2b4be65 100644 --- a/src/modelscope_hub/constants.py +++ b/src/modelscope_hub/constants.py @@ -52,6 +52,7 @@ class EnvVar: "Download", "Upload", "Logging", + "Kernels", "Deprecated", ) @@ -1526,6 +1527,23 @@ def get_upload_ignore_file_pattern() -> str | None: "Core", ) +# --------------------------------------------------------------------------- +# Kernels (``modelscope_hub.kernels``) +# +# Publisher trust for HF-style kernel repositories loaded through +# :mod:`modelscope_hub.kernels`. ModelScope has no trusted-publisher flag, +# so the default trust decision falls back to an allowlist extended by this +# environment variable. +# --------------------------------------------------------------------------- +ENV_KERNELS_TRUSTED_PUBLISHERS: str = "MS_KERNELS_TRUSTED_PUBLISHERS" + +_env_register( + ENV_KERNELS_TRUSTED_PUBLISHERS, + "", + "Comma-separated kernel publishers trusted in addition to the built-in allowlist", + "Kernels", +) + __all__ = [ "AGENT_PLUGIN_TRUSTED_OWNERS", "API_CONNECT_TIMEOUT", @@ -1570,6 +1588,7 @@ def get_upload_ignore_file_pattern() -> str | None: "ENV_INTRA_CLOUD_ACCELERATION", "ENV_INTRA_CLOUD_REGION", "ENV_INTER_CLOUD_REGIONS", + "ENV_KERNELS_TRUSTED_PUBLISHERS", "ENV_MODELSCOPE_DOMAIN", "ENV_PREFER_AI_SITE", "ENV_REGISTRY", diff --git a/src/modelscope_hub/kernels/__init__.py b/src/modelscope_hub/kernels/__init__.py new file mode 100644 index 0000000..42de1f6 --- /dev/null +++ b/src/modelscope_hub/kernels/__init__.py @@ -0,0 +1,43 @@ +"""ModelScope-flavoured ``kernels``: load HF-style compute kernels from ModelScope. + +``modelscope_hub.kernels`` is a drop-in wrapper around the `kernels +`_ package that redirects every Hub +interaction (revision lookup, metadata download, build snapshot) from the +Hugging Face Hub to the ModelScope Hub. Kernel repositories on ModelScope are +ordinary model repositories (e.g. ``kernels-community/activation``) that +mirror the HF kernel repo layout: ``build//...`` files plus ``v1``-style +version branches or tags. + +The kernel loading, variant resolution, and dependency logic of ``kernels`` +are reused unchanged; only its Hub API factory is swapped for a ModelScope +backed shim while a ``modelscope_hub.kernels`` call is in flight. This is +also the implementation behind the ``kernels`` monkey-patching applied by the +``modelscope`` package's ``patch_hub()``. + +Requires the optional ``kernels`` dependency:: + + pip install "modelscope-hub[kernels]" + +Example: + ```python + from modelscope_hub.kernels import get_kernel + + # Download optimized kernels from the ModelScope hub + activation = get_kernel("kernels-community/activation", version=1) + ``` +""" + +from __future__ import annotations + +from modelscope_hub.kernels._load import get_kernel, get_kernel_variants, get_local_kernel, has_kernel +from modelscope_hub.kernels._patch import patch_kernels_hub, patch_scope, unpatch_kernels_hub + +__all__ = [ + "get_kernel", + "get_kernel_variants", + "get_local_kernel", + "has_kernel", + "patch_kernels_hub", + "patch_scope", + "unpatch_kernels_hub", +] diff --git a/src/modelscope_hub/kernels/_hub.py b/src/modelscope_hub/kernels/_hub.py new file mode 100644 index 0000000..88fca2b --- /dev/null +++ b/src/modelscope_hub/kernels/_hub.py @@ -0,0 +1,266 @@ +"""ModelScope-backed stand-in for the small ``HfApi`` surface used by ``kernels``. + +The upstream ``kernels`` library routes every Hub interaction through +``kernels.hf_hub._get_hf_api()``. The patched factory (see +:mod:`modelscope_hub.kernels._patch`) returns :class:`MsKernelApi`: a +duck-typed API client that forwards the handful of methods ``kernels`` +actually calls onto :class:`modelscope_hub.HubApi`, mapping HF kernel +repositories onto ModelScope model repositories. + +Method-to-method mapping: + +- ``snapshot_download`` -> :meth:`modelscope_hub.HubApi.download_repo` +- ``hf_hub_download`` -> :meth:`modelscope_hub.HubApi.download_file` +- ``list_repo_refs`` -> :meth:`modelscope_hub.legacy.list_revisions_detail` +- ``list_repo_tree`` -> :meth:`modelscope_hub.HubApi.list_repo_files` +- ``get_organization_overview`` -> local trusted-publisher allowlist + +``huggingface_hub`` is imported lazily inside the methods that need its +dataclasses/exceptions: it arrives as a transitive dependency of ``kernels`` +and this module must stay importable without it. +""" + +from __future__ import annotations + +import os +import re +from pathlib import Path +from types import SimpleNamespace + +import modelscope_hub +from modelscope_hub.constants import ENV_KERNELS_TRUSTED_PUBLISHERS + +#: Publisher organizations trusted when ``trust_remote_code=False``. ModelScope +#: has no equivalent of the Hub's ``trustedKernelPublisher`` flag, so the +#: default trust decision falls back to this allowlist (extend it via the +#: ``MS_KERNELS_TRUSTED_PUBLISHERS`` environment variable, comma-separated). +TRUSTED_KERNEL_PUBLISHERS: frozenset[str] = frozenset({"kernels-community"}) + +# Kernel-version refs (``v1``, ``v2``, ...) as created by the kernels Hub. +_VERSION_REF_RE = re.compile(r"v(\d+)") + + +def _ms_revision(revision: str | None) -> str: + """Translate an HF revision string into one ModelScope accepts.""" + return "master" if revision in (None, "main") else revision + + +def _ms_repo_type(repo_type: str | None) -> str: + """Map HF repo types onto ModelScope repo types (kernel repos are models).""" + return "model" if repo_type in (None, "kernel", "model") else repo_type + + +def _trusted_publishers() -> frozenset[str]: + publishers = set(TRUSTED_KERNEL_PUBLISHERS) + extra = os.environ.get(ENV_KERNELS_TRUSTED_PUBLISHERS, "") + publishers.update(part.strip() for part in extra.split(",") if part.strip()) + return frozenset(publishers) + + +def _user_agent(extra: str | dict | None) -> str | dict: + """Telemetry info appended to the SDK user agent on every download.""" + if extra: + return extra + return f"ms-kernels; modelscope_hub/{modelscope_hub.__version__}" + + +class MsKernelApi: + """Minimal ``HfApi`` look-alike that forwards to ModelScope. + + ``hub_api`` exists for dependency injection (tests); production callers get + a lazily constructed :class:`modelscope_hub.HubApi`. Only the methods that + ``kernels`` calls are implemented. + """ + + def __init__( + self, + hub_api: modelscope_hub.HubApi | None = None, + user_agent: str | dict | None = None, + ) -> None: + self._hub_api_instance = hub_api + self._user_agent = user_agent + + def _api(self) -> modelscope_hub.HubApi: + if self._hub_api_instance is None: + self._hub_api_instance = modelscope_hub.HubApi() + return self._hub_api_instance + + # ------------------------------------------------------------------ + # Downloads + # ------------------------------------------------------------------ + def snapshot_download( + self, + repo_id: str, + *, + repo_type: str | None = None, + allow_patterns: list[str] | None = None, + ignore_patterns: list[str] | None = None, + cache_dir: str | Path | None = None, + revision: str | None = None, + local_files_only: bool = False, + **kwargs, # HF-signature compatibility + ) -> Path: + """Download a repository snapshot, mirroring ``HfApi.snapshot_download``.""" + from huggingface_hub.errors import LocalEntryNotFoundError + + try: + return self._api().download_repo( + repo_id, + _ms_repo_type(repo_type), + revision=_ms_revision(revision), + cache_dir=Path(cache_dir) if cache_dir else None, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + local_files_only=local_files_only, + user_agent=_user_agent(self._user_agent), + ) + except modelscope_hub.CacheNotFound as e: + raise LocalEntryNotFoundError( + f"Cannot find a local snapshot for {repo_id} (revision: {_ms_revision(revision)}). " + "The kernel must be downloaded while online first." + ) from e + + def hf_hub_download( + self, + repo_id: str | None = None, + *, + repo_type: str | None = None, + filename: str | None = None, + cache_dir: str | Path | None = None, + revision: str | None = None, + local_files_only: bool = False, + **kwargs, # HF-signature compatibility + ) -> Path: + """Download a single repository file, mirroring ``HfApi.hf_hub_download``.""" + from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError + + if repo_id is None or filename is None: + raise ValueError("Both `repo_id` and `filename` must be provided.") + try: + return self._api().download_file( + repo_id, + _ms_repo_type(repo_type), + file_path=filename, + revision=_ms_revision(revision), + cache_dir=Path(cache_dir) if cache_dir else None, + local_files_only=local_files_only, + user_agent=_user_agent(self._user_agent), + ) + except modelscope_hub.CacheNotFound as e: + raise LocalEntryNotFoundError( + f"Cannot find the local cache entry for {repo_id}/{filename} " + f"(revision: {_ms_revision(revision)}). The file must be downloaded while online first." + ) from e + except modelscope_hub.NotExistError as e: + raise EntryNotFoundError( + f"File {filename} does not exist in {repo_id} (revision: {_ms_revision(revision)})." + ) from e + + # ------------------------------------------------------------------ + # Repository metadata + # ------------------------------------------------------------------ + def list_repo_refs(self, repo_id: str | None = None, *, repo_type: str | None = None, **kwargs): + """List repository branches and tags, mirroring ``HfApi.list_repo_refs``. + + ModelScope accepts branch/tag names as revisions, so ``target_commit`` + reuses the name: kernels passes it on as the revision for later calls. + When the repository carries no ``v`` refs at all (typical for a + plain mirror with only a ``master`` branch), ``master`` is exposed as + kernel version ``v1`` so that ``get_kernel(..., version=1)`` works + without the mirror having to create version branches. + """ + from huggingface_hub.hf_api import GitRefInfo + + if repo_id is None: + raise ValueError("`repo_id` must be provided.") + branches, tags = self._api().legacy.list_revisions_detail(repo_id, _ms_repo_type(repo_type)) + branch_refs = [ + GitRefInfo(name=ref["Revision"], ref=f"refs/heads/{ref['Revision']}", target_commit=ref["Revision"]) + for ref in branches or [] + ] + tag_refs = [ + GitRefInfo(name=ref["Revision"], ref=f"refs/tags/{ref['Revision']}", target_commit=ref["Revision"]) + for ref in tags or [] + ] + if not any(_VERSION_REF_RE.fullmatch(ref.name) for ref in branch_refs + tag_refs): + branch_refs.append(GitRefInfo(name="v1", ref="refs/heads/v1", target_commit="master")) + return SimpleNamespace(branches=branch_refs, tags=tag_refs, converts=[]) + + def list_repo_tree( + self, + repo_id: str | None = None, + *, + path_in_repo: str | None = None, + repo_type: str | None = None, + revision: str | None = None, + recursive: bool = False, # ModelScope always lists recursively + **kwargs, # HF-signature compatibility + ): + """List the direct children of a repository folder, mirroring ``HfApi.list_repo_tree``. + + ModelScope cannot list the direct children of a subfolder + non-recursively, so the full recursive tree is fetched and the direct + subfolders derived from it — the only thing ``kernels`` consumes. + """ + from huggingface_hub.hf_api import RepoFolder + + if repo_id is None: + raise ValueError("`repo_id` must be provided.") + root = path_in_repo or "" + prefix = f"{root}/" if root else "" + infos = self._api().list_repo_files( + repo_id, + _ms_repo_type(repo_type), + revision=_ms_revision(revision), + recursive=True, + ) + folders: set[str] = set() + for info in infos: + path = info.path + if prefix and not path.startswith(prefix): + continue + rest = path[len(prefix) :] + if not rest: + continue + first, _, remainder = rest.partition("/") + if remainder or (info.is_dir and not remainder): + # Either the parent of a nested entry, or an explicit + # tree entry at direct-child depth. + folders.add(first) + return [ + RepoFolder(path=f"{root}/{name}" if root else name, oid="", last_commit=None) for name in sorted(folders) + ] + + def file_exists( + self, + repo_id: str | None = None, + filename: str | None = None, + *, + repo_type: str | None = None, + revision: str | None = None, + **kwargs, # HF-signature compatibility + ) -> bool: + """Check whether ``filename`` exists in the repository.""" + if repo_id is None or filename is None: + raise ValueError("Both `repo_id` and `filename` must be provided.") + infos = self._api().list_repo_files( + repo_id, + _ms_repo_type(repo_type), + revision=_ms_revision(revision), + recursive=True, + ) + return any(info.path == filename for info in infos) + + # ------------------------------------------------------------------ + # Publisher trust + # ------------------------------------------------------------------ + def get_organization_overview(self, organization: str): + """Stand-in for the Hub's organization lookup. + + There is no ``trustedKernelPublisher`` equivalent on ModelScope, so + trust is decided against :data:`TRUSTED_KERNEL_PUBLISHERS` (plus the + ``MS_KERNELS_TRUSTED_PUBLISHERS`` environment variable). Untrusted + publishers cause kernels' usual ``ValueError`` with instructions to + pass ``trust_remote_code=True``. + """ + return SimpleNamespace(trustedKernelPublisher=organization in _trusted_publishers()) diff --git a/src/modelscope_hub/kernels/_load.py b/src/modelscope_hub/kernels/_load.py new file mode 100644 index 0000000..39f4d6d --- /dev/null +++ b/src/modelscope_hub/kernels/_load.py @@ -0,0 +1,185 @@ +"""Public entry points: scope-managed wrappers around the ``kernels`` library. + +Each wrapper applies :func:`modelscope_hub.kernels._patch.patch_scope` around +the corresponding ``kernels`` call, so Hub traffic is redirected to ModelScope +for that call only — ``import kernels`` elsewhere keeps using the Hugging +Face Hub. +""" + +from __future__ import annotations + +from pathlib import Path +from types import ModuleType + +from modelscope_hub.kernels._patch import patch_scope + +__all__ = ["get_kernel", "get_kernel_variants", "get_local_kernel", "has_kernel"] + + +def _require_kernels() -> ModuleType: + """Import ``kernels`` or raise an actionable ``ImportError``.""" + try: + import kernels + except ImportError as e: + raise ImportError( + "The `kernels` package is required by `modelscope_hub.kernels` but is not installed. " + 'Install it with: pip install "modelscope-hub[kernels]".' + ) from e + return kernels + + +def get_kernel( + repo_id: str, + *, + revision: str | None = None, + version: int | None = None, + backend: str | None = None, + user_agent: str | dict | None = None, + trust_remote_code: bool | list[str] = False, + check_arch: bool = True, +) -> ModuleType: + """Load a kernel from the ModelScope Hub. + + This function downloads a kernel to the local ModelScope cache directory + (if it was not downloaded before) and then loads the kernel. The kernel + repository must be a ModelScope model repository mirroring the HF kernel + repo layout (``build//...`` files plus ``v1``-style version + branches or tags). + + Args: + repo_id (`str`): + The ModelScope model repository containing the kernel, + e.g. ``"kernels-community/activation"``. + revision (`str`, *optional*): + The specific revision (branch, tag, or commit) to download. + Cannot be used together with `version`. + version (`int`, *optional*): + The kernel version to download (resolved against the repository's + ``v`` branches/tags). Cannot be used together with `revision`. + Either `version` or `revision` must be specified. + backend (`str`, *optional*): + The backend to load the kernel for. Can only be `cpu` or the + backend that Torch is compiled for. The backend is detected + automatically if not provided. + user_agent (`Union[str, dict]`, *optional*): + Extra telemetry info appended to the ModelScope SDK user agent. + trust_remote_code (`bool | list[str]`, *optional*, defaults to `False`): + Whether to allow loading kernels from untrusted organisations. + ModelScope has no trusted-publisher flag, so trust is decided + against the built-in allowlist (see + :data:`modelscope_hub.kernels._hub.TRUSTED_KERNEL_PUBLISHERS`); a + list of repository IDs allows those repositories in addition. + check_arch (`bool`, *optional*, defaults to `True`): + Whether to check that the kernel build supports the architecture + of the current device. `check_arch=False` skips the check. + + Returns: + `ModuleType`: The imported kernel module. + + Example: + ```python + import torch + from modelscope_hub.kernels import get_kernel + + activation = get_kernel("kernels-community/activation", version=1) + x = torch.randn(10, 20, device="cpu") + out = torch.empty_like(x) + result = activation.relu(out, x) + ``` + """ + kernels = _require_kernels() + with patch_scope(): + return kernels.get_kernel( + repo_id, + revision=revision, + version=version, + backend=backend, + user_agent=user_agent, + trust_remote_code=trust_remote_code, + check_arch=check_arch, + ) + + +def has_kernel( + repo_id: str, + *, + revision: str | None = None, + version: int | None = None, + backend: str | None = None, + trust_remote_code: bool | list[str] = False, + check_arch: bool = True, +) -> bool: + """Check whether a kernel build exists for the current environment. + + Arguments are the same as :func:`get_kernel` (minus ``user_agent``). + + Returns: + `bool`: `True` if a kernel is available for the current environment. + """ + kernels = _require_kernels() + with patch_scope(): + return kernels.has_kernel( + repo_id, + revision=revision, + version=version, + backend=backend, + trust_remote_code=trust_remote_code, + check_arch=check_arch, + ) + + +def get_kernel_variants( + repo_id: str, + *, + revision: str | None = None, + version: int | None = None, + backend: str | None = None, +) -> list: + """Resolve all build variants of a kernel against the current environment. + + Arguments are the same as :func:`get_kernel` (minus ``user_agent``). + + Returns: + `list`: One ``VariantAccepted`` or ``VariantRejected`` per build + variant in the repository, compatible variants first. + """ + kernels = _require_kernels() + with patch_scope(): + return kernels.get_kernel_variants( + repo_id, + revision=revision, + version=version, + backend=backend, + ) + + +def get_local_kernel( + repo_path: Path, + *, + backend: str | None = None, + trust_remote_code: bool | list[str] = False, +) -> ModuleType: + """Import a kernel from a local kernel repository path. + + If the kernel has any (transitive) dependencies, they are downloaded from + the ModelScope Hub. + + Args: + repo_path (`Path`): + The local path to the kernel repository. + backend (`str`, *optional*): + The backend to load the kernel for. + trust_remote_code (`bool | list[str]`, *optional*, defaults to `False`): + Whether to allow loading kernel dependencies from untrusted + organisations. + + Returns: + `ModuleType`: The imported kernel module. + """ + kernels = _require_kernels() + with patch_scope(): + return kernels.get_local_kernel( + repo_path, + backend=backend, + trust_remote_code=trust_remote_code, + ) diff --git a/src/modelscope_hub/kernels/_patch.py b/src/modelscope_hub/kernels/_patch.py new file mode 100644 index 0000000..30fb0a8 --- /dev/null +++ b/src/modelscope_hub/kernels/_patch.py @@ -0,0 +1,115 @@ +"""Monkey-patching of the ``kernels`` library onto the ModelScope Hub. + +Only ``kernels.hf_hub._get_hf_api`` is replaced; every download, file +listing, and revision lookup that ``kernels`` performs goes through +:class:`modelscope_hub.kernels._hub.MsKernelApi`, so the kernel loading, +variant resolution, and dependency logic stay untouched. + +``patch_scope()`` scopes the patch to a single call (used by the public +wrappers in :mod:`modelscope_hub.kernels._load`), so ``from kernels import +get_kernel`` keeps using the Hugging Face Hub unless the user explicitly +applies ``patch_kernels_hub()``. + +Adapted from ``modelscope/utils/hf_util/patcher.py`` (Apache-2.0, Alibaba), +re-targeted from the legacy ``modelscope`` SDK to ``modelscope_hub``. +""" + +from __future__ import annotations + +import contextlib +import sys + +from modelscope_hub.kernels._hub import MsKernelApi + +__all__ = ["patch_kernels_hub", "patch_scope", "unpatch_kernels_hub"] + + +def _import_kernels_api_module(): + """Return the module holding ``_get_hf_api`` in the installed ``kernels``. + + kernels >= 0.17 renamed ``kernels.utils`` to ``kernels.hf_hub`` (with + ``_get_hf_api`` moved along); support both layouts. Returns ``None`` + when ``kernels`` is not installed. + """ + try: + import kernels.hf_hub as kernels_api + except ImportError: + try: + import kernels.utils as kernels_api + except ImportError: + return None + return kernels_api + + +def _ms_api(user_agent=None): + """Replacement for ``kernels.hf_hub._get_hf_api``.""" + return MsKernelApi(user_agent=user_agent) + + +def patch_kernels_hub() -> None: + """Monkey-patch the ``kernels`` library to route Hub API calls to ModelScope. + + Only ``_get_hf_api`` is replaced (both in its home module and in the + submodules that bind it at import time, e.g. ``kernels.load``). Returns + silently when ``kernels`` is not installed or already patched. + """ + kernels_api = _import_kernels_api_module() + if kernels_api is None or hasattr(kernels_api, "_get_hf_api_origin"): + return + + kernels_api._get_hf_api_origin = kernels_api._get_hf_api + kernels_api._get_hf_api = _ms_api + # kernels >= 0.17 binds `_get_hf_api` at import time in submodules + # (e.g. `from kernels.hf_hub import _get_hf_api` in `kernels.load`); + # patch those already-bound references as well. + for name, module in list(sys.modules.items()): + if name != "kernels" and not name.startswith("kernels."): + continue + if module is kernels_api or hasattr(module, "_get_hf_api_origin"): + continue + if getattr(module, "_get_hf_api", None) is not None: + setattr(module, "_get_hf_api_origin", getattr(module, "_get_hf_api")) + setattr(module, "_get_hf_api", _ms_api) + + +def unpatch_kernels_hub() -> None: + """Undo :func:`patch_kernels_hub` and restore the original HfApi factory.""" + kernels_api = _import_kernels_api_module() + if kernels_api is None: + return + origin = getattr(kernels_api, "_get_hf_api_origin", None) + if origin is None: + return + patched = kernels_api._get_hf_api + for name, module in list(sys.modules.items()): + if name != "kernels" and not name.startswith("kernels."): + continue + if module is kernels_api: + continue + module_origin = getattr(module, "_get_hf_api_origin", None) + if module_origin is not None: + setattr(module, "_get_hf_api", module_origin) + delattr(module, "_get_hf_api_origin") + elif getattr(module, "_get_hf_api", None) is patched: + # Imported while the patch was active (kernels >= 0.17). + setattr(module, "_get_hf_api", origin) + kernels_api._get_hf_api = origin + del kernels_api._get_hf_api_origin + + +@contextlib.contextmanager +def patch_scope(): + """Apply :func:`patch_kernels_hub` for the duration of the ``with`` block. + + Nesting is safe: when an outer patch is already in effect the block simply + reuses it, so only the outermost scope restores the original factory. + """ + kernels_api = _import_kernels_api_module() + if kernels_api is None or hasattr(kernels_api, "_get_hf_api_origin"): + yield + return + patch_kernels_hub() + try: + yield + finally: + unpatch_kernels_hub() diff --git a/tests/test_hf_hub_patcher.py b/tests/test_hf_hub_patcher.py new file mode 100644 index 0000000..5517688 --- /dev/null +++ b/tests/test_hf_hub_patcher.py @@ -0,0 +1,353 @@ +"""Network-free tests for the huggingface_hub monkey-patcher in ``modelscope_hub.compat``. + +The patch functions are exercised with mocked ``HubApi``/``LegacyHubApi``/ +``compat`` downloads; the tests run only when the optional ``huggingface_hub`` +package is installed. +""" + +from __future__ import annotations + +from unittest import mock + +import pytest + +from modelscope_hub import FileInfo +from modelscope_hub.compat import hf_hub_patcher +from modelscope_hub.compat.hf_hub_patcher import ( + HfCommitInfo, + HfRepoUrl, + _create_commit, + _create_repo, + _file_exists, + _hf_hub_download, + _upload_file, + _upload_folder, + _whoami, + hf_hub_patch_scope, + patch_hf_hub, + unpatch_hf_hub, +) + +hf_api_mod = pytest.importorskip("huggingface_hub.hf_api") +hf_top = pytest.importorskip("huggingface_hub") +repocard = pytest.importorskip("huggingface_hub.repocard") + + +@pytest.fixture +def patched_hub(): + patch_hf_hub() + yield + unpatch_hf_hub() + + +# --------------------------------------------------------------------------- +# _hf_hub_download +# --------------------------------------------------------------------------- +def test_hf_hub_download_routes_model_file_download(): + with mock.patch("modelscope_hub.compat.model_file_download", return_value="/cache/x.json") as dl: + result = _hf_hub_download( + "foo/bar", + "config.json", + revision="main", + cache_dir="/tmp/cache", + local_dir="/tmp/out", + token="ms-token", + local_files_only=False, + ) + + assert result == "/cache/x.json" + assert dl.call_args.args == ("foo/bar", "config.json") + kwargs = dl.call_args.kwargs + # `main` is normalized to `master`; string tokens pass through. + assert kwargs["revision"] == "master" + assert kwargs["cache_dir"] == "/tmp/cache" + assert kwargs["local_dir"] == "/tmp/out" + assert kwargs["token"] == "ms-token" + assert kwargs["local_files_only"] is False + + +def test_hf_hub_download_dataset_routes_dataset_download(): + with mock.patch("modelscope_hub.compat.dataset_file_download", return_value="/cache/x.bin") as dl: + result = _hf_hub_download("foo/data", "train.bin", repo_type="dataset", revision="v2") + + assert result == "/cache/x.bin" + assert dl.call_args.args == ("foo/data", "train.bin") + assert dl.call_args.kwargs["revision"] == "v2" + + +def test_hf_hub_download_rejects_unsupported_repo_type(): + with pytest.raises(ValueError, match="repo_type=space"): + _hf_hub_download("foo/bar", "config.json", repo_type="space") + + +def test_hf_hub_download_bool_token_is_dropped(): + with mock.patch("modelscope_hub.compat.model_file_download", return_value="/cache/x") as dl: + _hf_hub_download("foo/bar", "config.json", token=True) + + assert dl.call_args.kwargs["token"] is None + + +def test_hf_hub_download_subfolder_is_joined(): + with mock.patch("modelscope_hub.compat.model_file_download", return_value="/cache/x") as dl: + _hf_hub_download("foo/bar", "nested.txt", subfolder="configs") + + assert dl.call_args.args == ("foo/bar", "configs/nested.txt") + + +# --------------------------------------------------------------------------- +# _file_exists / _whoami +# --------------------------------------------------------------------------- +def test_file_exists_uses_listing(): + fake = mock.MagicMock() + fake.list_repo_files.return_value = [ + FileInfo(path="README.md", size=1), + FileInfo(path="config.json", size=2), + ] + with mock.patch("modelscope_hub.HubApi", return_value=fake): + assert _file_exists("foo/bar", "config.json", revision="main") + assert not _file_exists("foo/bar", "missing.txt", revision="main") + + # `main` is normalized to `master`. + call = fake.list_repo_files.call_args + assert call.args == ("foo/bar", "model") + assert call.kwargs["revision"] == "master" + + +def test_whoami_returns_name_dict(): + fake = mock.MagicMock() + fake.whoami.return_value.username = "alice" + with mock.patch("modelscope_hub.HubApi", return_value=fake): + assert _whoami(token="ms-token") == {"name": "alice"} + + fake.whoami.assert_called_once() + + +# --------------------------------------------------------------------------- +# _create_repo / _upload_file / _upload_folder / _create_commit +# --------------------------------------------------------------------------- +def test_create_repo_returns_hf_repo_url(): + with mock.patch("modelscope_hub.compat.LegacyHubApi") as legacy: + result = _create_repo("foo/bar", token="ms-token", private=True) + + legacy.assert_called_once_with(token="ms-token") + legacy.return_value.create_repo.assert_called_once_with("foo/bar", visibility="private") + + # Legacy duck-typing shape: a str URL plus repo_id / repo_type attributes + # (hf's own RepoUrl rejects non-HF hosts on recent releases). + assert isinstance(result, HfRepoUrl) + assert "foo/bar" in result + assert result.repo_id == "foo/bar" + assert result.repo_type == "model" + + +def test_upload_file_forwards_to_hub_upload(): + fake = mock.MagicMock() + with mock.patch("modelscope_hub.HubApi", return_value=fake): + _upload_file( + path_or_fileobj="/tmp/w.safetensors", + path_in_repo="w.safetensors", + repo_id="foo/bar", + token="ms-token", + revision="main", + commit_message="add weights", + commit_description="tuned", + ) + + fake.upload_file.assert_called_once_with( + "foo/bar", + "model", + "/tmp/w.safetensors", + "w.safetensors", + commit_message="add weights", + commit_description="tuned", + revision="master", + ) + + +def test_upload_folder_returns_legacy_commit_info_shape(): + fake = mock.MagicMock() + with mock.patch("modelscope_hub.HubApi", return_value=fake): + result = _upload_folder( + repo_id="foo/bar", + folder_path="/tmp/out", + commit_message="sync", + ignore_patterns=["*.tmp"], + revision="v2", + ) + + fake.upload_folder.assert_called_once_with( + "foo/bar", + "model", + "/tmp/out", + path_in_repo="", + commit_message="sync", + commit_description=None, + revision="v2", + ignore_patterns=["*.tmp"], + allow_patterns=None, + ) + assert isinstance(result, HfCommitInfo) + assert result.commit_url.endswith("/models/foo/bar/files") + assert result.commit_message == "sync" + assert result.oid is None + + +@pytest.mark.parametrize("patch_fn", [_upload_file, _upload_folder]) +def test_upload_rejects_non_model_repo_type(patch_fn): + kwargs = ( + dict(path_or_fileobj="/tmp/a", path_in_repo="a", repo_id="foo/data") + if patch_fn is _upload_file + else dict(repo_id="foo/data", folder_path="/tmp/out") + ) + with mock.patch("modelscope_hub.HubApi"): + with pytest.raises(ValueError, match="repo_type=dataset"): + patch_fn(repo_type="dataset", **kwargs) + + +def test_create_commit_rejects_non_add_operations(): + from huggingface_hub import CommitOperationDelete + + fake = mock.MagicMock() + with mock.patch("modelscope_hub.HubApi", return_value=fake): + with pytest.raises(ValueError, match="only supports Add"): + _create_commit("foo/bar", [CommitOperationDelete("old.txt", False)], commit_message="x") + + fake.upload_file.assert_not_called() + + +def test_create_commit_maps_each_add_operation(tmp_path): + from huggingface_hub import CommitOperationAdd + + # hf >= 1.x verifies that str/Path sources exist on the local filesystem. + a = tmp_path / "a.txt" + a.write_text("a", encoding="utf-8") + b = tmp_path / "b.txt" + b.write_text("b", encoding="utf-8") + + fake = mock.MagicMock() + with mock.patch("modelscope_hub.HubApi", return_value=fake): + _create_commit( + "foo/bar", + [ + CommitOperationAdd(path_in_repo="a.txt", path_or_fileobj=str(a)), + CommitOperationAdd(path_in_repo="b.txt", path_or_fileobj=b"bytes"), + ], + commit_message="batch", + revision="main", + ) + + assert fake.upload_file.call_count == 2 + first, second = fake.upload_file.call_args_list + assert first.args == ("foo/bar", "model", str(a), "a.txt") + assert second.args == ("foo/bar", "model", b"bytes", "b.txt") + assert first.kwargs["commit_message"] == "batch" + assert first.kwargs["revision"] == "master" + + +def test_create_commit_rejects_non_model_repo_type(tmp_path): + from huggingface_hub import CommitOperationAdd + + a = tmp_path / "a.txt" + a.write_text("a", encoding="utf-8") + + with mock.patch("modelscope_hub.HubApi"): + with pytest.raises(ValueError, match="repo_type=dataset"): + _create_commit( + "foo/data", + [CommitOperationAdd(path_in_repo="a.txt", path_or_fileobj=str(a))], + commit_message="x", + repo_type="dataset", + ) + + +# --------------------------------------------------------------------------- +# repocard +# --------------------------------------------------------------------------- +def test_repocard_load_routes_to_compat_download(tmp_path): + from huggingface_hub.repocard import RepoCard + + with mock.patch( + "modelscope_hub.compat.model_file_download", + return_value=str(tmp_path / "README.md"), + ) as dl: + (tmp_path / "README.md").write_text("---\nlicense: apache-2.0\n---\nhello", encoding="utf-8") + card = hf_hub_patcher._repocard_load(RepoCard, "foo/bar") + + assert dl.call_args.args == ("foo/bar", "README.md") + assert card.data.tags == [] + + +# --------------------------------------------------------------------------- +# Patch round-trip +# --------------------------------------------------------------------------- +def test_patch_roundtrip_restores_all_symbols(): + original = hf_top.hf_hub_download + original_file_download = hf_top.file_download.hf_hub_download + + patch_hf_hub() + try: + assert hf_top.hf_hub_download is hf_hub_patcher._hf_hub_download + assert hf_top.file_exists is hf_hub_patcher._file_exists + assert hf_top.whoami is hf_hub_patcher._whoami + assert hf_top.create_repo is hf_hub_patcher._create_repo + assert hf_top.upload_file is hf_hub_patcher._upload_file + assert hf_top.upload_folder is hf_hub_patcher._upload_folder + assert hf_top.create_commit is hf_hub_patcher._create_commit + # repocard class patches are active while patched. + assert repocard.RepoCard.__dict__["load"].__func__ is hf_hub_patcher._repocard_load + finally: + unpatch_hf_hub() + + assert hf_top.hf_hub_download is original + assert hf_top.file_download.hf_hub_download is original_file_download + assert not hasattr(hf_api_mod, "_hf_hub_download_origin") + assert not hasattr(hf_api_mod, "_file_exists_origin") + assert not hasattr(hf_api_mod, "_whoami_origin") + assert not hasattr(hf_api_mod, "_create_repo_origin") + assert not hasattr(hf_api_mod, "_upload_folder_origin") + assert not hasattr(hf_api_mod, "_upload_file_origin") + assert not hasattr(hf_api_mod, "_create_commit_origin") + assert not hasattr(repocard.RepoCard, "_validate_origin") + assert not hasattr(repocard.RepoCard, "_load_origin") + + +def test_patch_is_idempotent(patched_hub): + first = hf_top.hf_hub_download + patch_hf_hub() + assert hf_top.hf_hub_download is first + + +def test_patch_respects_legacy_markers(): + """A pre-existing legacy patch (same marker names) must not be overwritten.""" + fake_legacy = mock.MagicMock(name="legacy_download") + with mock.patch.object(hf_top, "hf_hub_download", fake_legacy): + setattr(hf_api_mod, "_hf_hub_download_origin", fake_legacy) + try: + patch_hf_hub() + assert hf_top.hf_hub_download is fake_legacy + # The already-patched function stays; markers unchanged. + assert hf_api_mod._hf_hub_download_origin is fake_legacy + finally: + delattr(hf_api_mod, "_hf_hub_download_origin") + + +def test_patch_scope_restores_on_exit(): + original = hf_top.hf_hub_download + + with hf_hub_patch_scope(): + assert hf_top.hf_hub_download is hf_hub_patcher._hf_hub_download + with hf_hub_patch_scope(): + assert hf_top.hf_hub_download is hf_hub_patcher._hf_hub_download + # The inner scope must not undo the outer patch. + assert hf_top.hf_hub_download is hf_hub_patcher._hf_hub_download + + assert hf_top.hf_hub_download is original + + +def test_module_has_no_transformers_import(): + """The hub-side patcher must stay importable without transformers.""" + import inspect + + source = inspect.getsource(hf_hub_patcher) + assert "import transformers" not in source + assert "from transformers" not in source diff --git a/tests/test_kernels.py b/tests/test_kernels.py new file mode 100644 index 0000000..d391340 --- /dev/null +++ b/tests/test_kernels.py @@ -0,0 +1,388 @@ +"""Network-free tests for the ``modelscope_hub.kernels`` ModelScope-backed shim. + +``MsKernelApi`` is exercised with an injected fake ``HubApi``; the patch +round-trip tests run only when the optional ``kernels`` package (and its +``huggingface_hub`` dependency) is installed. +""" + +from __future__ import annotations + +from pathlib import Path +from unittest import mock + +import pytest + +from modelscope_hub import FileInfo +from modelscope_hub.errors import CacheNotFound, NotExistError +from modelscope_hub.kernels import get_kernel as kernels_get_kernel +from modelscope_hub.kernels._hub import MsKernelApi, _ms_repo_type, _ms_revision +from modelscope_hub.kernels._patch import _ms_api, patch_kernels_hub, patch_scope, unpatch_kernels_hub + + +# --------------------------------------------------------------------------- +# Revision / repo-type mapping +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + ("revision", "expected"), + [(None, "master"), ("main", "master"), ("master", "master"), ("v1", "v1"), ("deadbeef", "deadbeef")], +) +def test_ms_revision_maps_hf_defaults(revision, expected): + assert _ms_revision(revision) == expected + + +@pytest.mark.parametrize( + ("repo_type", "expected"), + [(None, "model"), ("kernel", "model"), ("model", "model"), ("dataset", "dataset")], +) +def test_ms_repo_type_maps_kernel_to_model(repo_type, expected): + assert _ms_repo_type(repo_type) == expected + + +# --------------------------------------------------------------------------- +# Downloads +# --------------------------------------------------------------------------- +def test_snapshot_download_forwards_to_download_repo(): + fake = mock.MagicMock() + fake.download_repo.return_value = Path("/cache/snapshots/abc") + api = MsKernelApi(hub_api=fake) + + result = api.snapshot_download( + "kernels-community/activation", + repo_type="kernel", + allow_patterns=["build/torch-cpu/*"], + ignore_patterns=["*.pyc"], + cache_dir="/tmp/kernels-cache", + revision="v1", + local_files_only=False, + ) + + assert result == Path("/cache/snapshots/abc") + fake.download_repo.assert_called_once_with( + "kernels-community/activation", + "model", + revision="v1", + cache_dir=Path("/tmp/kernels-cache"), + allow_patterns=["build/torch-cpu/*"], + ignore_patterns=["*.pyc"], + local_files_only=False, + user_agent=mock.ANY, + ) + + +def test_snapshot_download_cache_miss_raises_local_entry_not_found(): + hf_errors = pytest.importorskip("huggingface_hub.errors") + + fake = mock.MagicMock() + fake.download_repo.side_effect = CacheNotFound("no snapshot") + api = MsKernelApi(hub_api=fake) + + with pytest.raises(hf_errors.LocalEntryNotFoundError): + api.snapshot_download("kernels-community/activation", repo_type="kernel", revision="v1") + + +def test_hf_hub_download_forwards_to_download_file(): + fake = mock.MagicMock() + fake.download_file.return_value = Path("/cache/build/torch-cpu/metadata.json") + api = MsKernelApi(hub_api=fake) + + result = api.hf_hub_download( + "kernels-community/activation", + repo_type="kernel", + filename="build/torch-cpu/metadata.json", + cache_dir="/tmp/kernels-cache", + revision="v1", + local_files_only=False, + ) + + assert result == Path("/cache/build/torch-cpu/metadata.json") + fake.download_file.assert_called_once_with( + "kernels-community/activation", + "model", + file_path="build/torch-cpu/metadata.json", + revision="v1", + cache_dir=Path("/tmp/kernels-cache"), + local_files_only=False, + user_agent=mock.ANY, + ) + + +def test_hf_hub_download_missing_file_raises_entry_not_found(): + hf_errors = pytest.importorskip("huggingface_hub.errors") + + fake = mock.MagicMock() + fake.download_file.side_effect = NotExistError("no such file") + api = MsKernelApi(hub_api=fake) + + with pytest.raises(hf_errors.EntryNotFoundError): + api.hf_hub_download( + "kernels-community/activation", + repo_type="kernel", + filename="kernel-status.toml", + revision="v1", + ) + + +def test_hf_hub_download_cache_miss_raises_local_entry_not_found(): + hf_errors = pytest.importorskip("huggingface_hub.errors") + + fake = mock.MagicMock() + fake.download_file.side_effect = CacheNotFound("no cached file") + api = MsKernelApi(hub_api=fake) + + with pytest.raises(hf_errors.LocalEntryNotFoundError): + api.hf_hub_download( + "kernels-community/activation", + repo_type="kernel", + filename="kernel-status.toml", + revision="v1", + local_files_only=True, + ) + + +# --------------------------------------------------------------------------- +# Revision listing +# --------------------------------------------------------------------------- +def test_list_repo_refs_builds_hf_ref_objects(): + hf_api = pytest.importorskip("huggingface_hub.hf_api") + + fake = mock.MagicMock() + fake.legacy.list_revisions_detail.return_value = ( + [{"Revision": "v1"}, {"Revision": "v2"}, {"Revision": "master"}], + [{"Revision": "1.0"}], + ) + api = MsKernelApi(hub_api=fake) + + refs = api.list_repo_refs("kernels-community/activation", repo_type="kernel") + + fake.legacy.list_revisions_detail.assert_called_once_with("kernels-community/activation", "model") + assert [b.name for b in refs.branches] == ["v1", "v2", "master"] + assert [t.name for t in refs.tags] == ["1.0"] + # ModelScope accepts branch/tag names as revisions, so the name doubles + # as the target commit used by later kernels calls. + assert all(isinstance(b, hf_api.GitRefInfo) for b in refs.branches) + assert all(b.target_commit == b.name for b in refs.branches) + assert refs.converts == [] + + +def test_list_repo_refs_synthesizes_v1_when_no_version_refs(): + """A plain ModelScope mirror with only `master` exposes it as kernel version v1.""" + pytest.importorskip("huggingface_hub.hf_api") + + fake = mock.MagicMock() + fake.legacy.list_revisions_detail.return_value = ([{"Revision": "master"}], []) + api = MsKernelApi(hub_api=fake) + + refs = api.list_repo_refs("kernels-community/activation", repo_type="kernel") + + v1 = [b for b in refs.branches if b.name == "v1"] + assert len(v1) == 1 + # The synthetic v1 ref resolves downloads back onto `master`. + assert v1[0].target_commit == "master" + + +# --------------------------------------------------------------------------- +# Tree listing +# --------------------------------------------------------------------------- +def test_list_repo_tree_derives_direct_folders_from_blob_paths(): + hf_api = pytest.importorskip("huggingface_hub.hf_api") + + fake = mock.MagicMock() + fake.list_repo_files.return_value = [ + FileInfo(path="README.md", size=10), + FileInfo(path="build/torch-cpu/kernel.py", size=20), + FileInfo(path="build/torch-cpu/sub/deep.py", size=5), + FileInfo(path="build/torch215-cuda124-aarch64-linux/kernel.py", size=30), + ] + api = MsKernelApi(hub_api=fake) + + tree = api.list_repo_tree("kernels-community/activation", path_in_repo="build", repo_type="kernel", revision="v1") + + fake.list_repo_files.assert_called_once_with("kernels-community/activation", "model", revision="v1", recursive=True) + assert [item.path for item in tree] == [ + "build/torch-cpu", + "build/torch215-cuda124-aarch64-linux", + ] + assert all(isinstance(item, hf_api.RepoFolder) for item in tree) + + +def test_list_repo_tree_uses_explicit_tree_entries(): + pytest.importorskip("huggingface_hub.hf_api") + + fake = mock.MagicMock() + fake.list_repo_files.return_value = [ + FileInfo(path="build/torch-cpu", type="tree"), + FileInfo(path="build/torch-cpu/kernel.py", size=20), + ] + api = MsKernelApi(hub_api=fake) + + tree = api.list_repo_tree("kernels-community/activation", path_in_repo="build", revision="v1") + + assert [item.path for item in tree] == ["build/torch-cpu"] + + +def test_file_exists_checks_listing(): + fake = mock.MagicMock() + fake.list_repo_files.return_value = [ + FileInfo(path="README.md", size=10), + FileInfo(path="build/torch-cpu/kernel.py", size=20), + ] + api = MsKernelApi(hub_api=fake) + + assert api.file_exists("kernels-community/activation", "build/torch-cpu/kernel.py", revision="v1") + assert not api.file_exists("kernels-community/activation", "missing.py", revision="v1") + + +# --------------------------------------------------------------------------- +# Publisher trust +# --------------------------------------------------------------------------- +def test_trusted_publisher_allowlist(): + api = MsKernelApi(hub_api=mock.MagicMock()) + + assert api.get_organization_overview("kernels-community").trustedKernelPublisher is True + assert api.get_organization_overview("someone-else").trustedKernelPublisher is False + + +def test_trusted_publisher_allowlist_env_extension(monkeypatch): + monkeypatch.setenv("MS_KERNELS_TRUSTED_PUBLISHERS", "my-org, other-org") + api = MsKernelApi(hub_api=mock.MagicMock()) + + assert api.get_organization_overview("my-org").trustedKernelPublisher is True + assert api.get_organization_overview("other-org").trustedKernelPublisher is True + assert api.get_organization_overview("kernels-community").trustedKernelPublisher is True + + +# --------------------------------------------------------------------------- +# Patch round-trip (requires the optional `kernels` package) +# --------------------------------------------------------------------------- +@pytest.fixture +def kernels_modules(): + kernels = pytest.importorskip("kernels") + kernels_hf_hub = pytest.importorskip("kernels.hf_hub") + kernels_load = pytest.importorskip("kernels.load") + + return kernels, kernels_hf_hub, kernels_load + + +def test_patch_kernels_hub_roundtrip(kernels_modules): + _, kernels_hf_hub, kernels_load = kernels_modules + original = kernels_hf_hub._get_hf_api + original_load = kernels_load._get_hf_api + + patch_kernels_hub() + try: + assert kernels_hf_hub._get_hf_api is _ms_api + assert kernels_load._get_hf_api is _ms_api + assert isinstance(kernels_hf_hub._get_hf_api(), MsKernelApi) + finally: + unpatch_kernels_hub() + + assert kernels_hf_hub._get_hf_api is original + assert kernels_load._get_hf_api is original_load + + +def test_patch_kernels_hub_is_idempotent(kernels_modules): + kernels_hf_hub = kernels_modules[1] + original = kernels_hf_hub._get_hf_api + + patch_kernels_hub() + try: + first = kernels_hf_hub._get_hf_api + patch_kernels_hub() + assert kernels_hf_hub._get_hf_api is first + finally: + unpatch_kernels_hub() + + assert kernels_hf_hub._get_hf_api is original + + +def test_patch_scope_restores_on_exit_and_nests(kernels_modules): + kernels_hf_hub = kernels_modules[1] + original = kernels_hf_hub._get_hf_api + + with patch_scope(): + assert kernels_hf_hub._get_hf_api is _ms_api + with patch_scope(): + assert kernels_hf_hub._get_hf_api is _ms_api + # The inner scope must not undo the outer patch. + assert kernels_hf_hub._get_hf_api is _ms_api + + assert kernels_hf_hub._get_hf_api is original + + +def test_get_kernel_wrapper_scopes_the_patch(kernels_modules, monkeypatch): + kernels, kernels_hf_hub, _ = kernels_modules + original = kernels_hf_hub._get_hf_api + recorded = {} + + def fake_get_kernel(repo_id, **kwargs): + recorded["repo_id"] = repo_id + recorded["kwargs"] = kwargs + recorded["patched"] = kernels_hf_hub._get_hf_api is _ms_api + return "kernel-module" + + monkeypatch.setattr(kernels, "get_kernel", fake_get_kernel) + + result = kernels_get_kernel( + "kernels-community/activation", + version=1, + trust_remote_code=True, + ) + + assert result == "kernel-module" + assert recorded["repo_id"] == "kernels-community/activation" + assert recorded["kwargs"] == { + "revision": None, + "version": 1, + "backend": None, + "user_agent": None, + "trust_remote_code": True, + "check_arch": True, + } + # The patch is active for the duration of the call only. + assert recorded["patched"] is True + assert kernels_hf_hub._get_hf_api is original + + +def test_has_kernel_wrapper_scopes_the_patch(kernels_modules, monkeypatch): + from modelscope_hub.kernels import has_kernel + + kernels, kernels_hf_hub, _ = kernels_modules + original = kernels_hf_hub._get_hf_api + calls = {} + + def fake_has_kernel(repo_id, **kwargs): + calls["patched"] = kernels_hf_hub._get_hf_api is _ms_api + return True + + monkeypatch.setattr(kernels, "has_kernel", fake_has_kernel) + + assert has_kernel("kernels-community/activation", version=1) is True + assert calls["patched"] is True + assert kernels_hf_hub._get_hf_api is original + + +def test_scoped_call_inside_global_patch_keeps_global_patch(kernels_modules, monkeypatch): + """Dual entry points: a scoped wrapper call inside a global patch must not undo it.""" + kernels, kernels_hf_hub, _ = kernels_modules + original = kernels_hf_hub._get_hf_api + recorded = {} + + def fake_get_kernel(repo_id, **kwargs): + recorded["patched"] = kernels_hf_hub._get_hf_api is _ms_api + return "kernel-module" + + monkeypatch.setattr(kernels, "get_kernel", fake_get_kernel) + + patch_kernels_hub() + try: + assert kernels_hf_hub._get_hf_api is _ms_api + result = kernels_get_kernel("kernels-community/activation", version=1) + assert result == "kernel-module" + # The scoped call saw the (already active) patch... + assert recorded["patched"] is True + # ...and did not undo the global patch on exit. + assert kernels_hf_hub._get_hf_api is _ms_api + finally: + unpatch_kernels_hub() + + assert kernels_hf_hub._get_hf_api is original From 72423f296102aaf8f32ec196abe41a9d50c6617e Mon Sep 17 00:00:00 2001 From: tastelikefeet Date: Tue, 3 Nov 2026 11:44:27 +0800 Subject: [PATCH 2/6] Add install/lock kernel wrappers and a lockfile commit-hash diagnostic - New scoped entry points in modelscope_hub.kernels: install_kernel (cache-only download for CI pre-install / offline preparation), install_kernel_all_variants (every build variant plus the dependency tree; resolved from the kernels.install submodule since it is not exported on the kernels top level), get_locked_kernel (lockfile-pinned load), and load_kernel (import an already-installed locked kernel). - Kernel lockfiles record hub-specific commit hashes: a lockfile created against the Hugging Face Hub cannot resolve on ModelScope. This is a semantic constraint, not a defect, so it is documented in the wrapper docstrings and the README, and the shim now attaches a PEP 678 note (Python 3.11+, silently skipped otherwise) when a full commit-hash revision cannot be found, explaining how to re-lock in the ModelScope environment. Exception types are unchanged, so kernels' own except clauses keep working. --- README.md | 3 +- src/modelscope_hub/kernels/__init__.py | 15 ++- src/modelscope_hub/kernels/_hub.py | 29 +++++ src/modelscope_hub/kernels/_load.py | 149 ++++++++++++++++++++++++- tests/test_kernels.py | 143 ++++++++++++++++++++++++ 5 files changed, 336 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a5c0b56..731908c 100644 --- a/README.md +++ b/README.md @@ -234,12 +234,13 @@ from modelscope_hub.kernels import get_kernel activation = get_kernel("kernels-community/activation", version=1) ``` -Also available: `has_kernel()` (environment compatibility check), `get_kernel_variants()` (build-variant decisions), and `get_local_kernel()` (load from a local path, dependencies from ModelScope). +Also available: `has_kernel()` (environment compatibility check), `get_kernel_variants()` (build-variant decisions), `get_local_kernel()` (load from a local path, dependencies from ModelScope), `install_kernel()` (download to the local cache without importing — CI pre-installation / offline preparation), `install_kernel_all_variants()` (download every build variant plus the dependency tree), `get_locked_kernel()` (load a kernel pinned by a lockfile, downloading missing pieces), and `load_kernel()` (import an already-installed locked kernel; never downloads). Under `patch_kernels_hub()` every other networked `kernels` entry point (the `layer`/`kernelize` APIs, benchmarks, etc.) is redirected the same way, because the whole library routes through one Hub API factory. Notes: - The kernel loading, variant resolution, and dependency logic of `kernels` are reused unchanged — only its Hub API factory is swapped for a ModelScope-backed shim while a `modelscope_hub.kernels` call is in flight, so a plain `import kernels` elsewhere keeps using the Hugging Face Hub. `patch_kernels_hub()` / `unpatch_kernels_hub()` / `patch_scope()` switch this explicitly if you need it globally; the `modelscope` package's `patch_hub()` builds on the same implementation. - `version=N` resolves against the repository's `v` branches/tags. A mirror that only carries `master` exposes it as kernel version `v1` automatically. +- Kernel lockfiles record hub-specific commit hashes: a lockfile created against the Hugging Face Hub cannot resolve on ModelScope (the two hubs have different git histories). Create the lockfile in the ModelScope environment (`get_locked_kernel` / `install_kernel_all_variants`, or the `kernels` CLI under `patch_kernels_hub()`) so it records ModelScope commits; a commit-hash revision that cannot be found raises with a note explaining this. - ModelScope has no trusted-publisher flag, so untrusted publishers are rejected by default: only `kernels-community` is trusted out of the box. Extend the allowlist with `MS_KERNELS_TRUSTED_PUBLISHERS` (comma-separated), or allow specific repositories / all repositories through the trust parameters of `get_kernel` (its docstring mirrors the kernel library's API). - The mirror must contain kernels built for your exact Torch version and platform (`kernels` selects `build/torchXY-...--/`); a repository whose builds predate `kernels` 0.17 metadata cannot be loaded until it is rebuilt with a current `kernel-builder`. diff --git a/src/modelscope_hub/kernels/__init__.py b/src/modelscope_hub/kernels/__init__.py index 42de1f6..68cf8d3 100644 --- a/src/modelscope_hub/kernels/__init__.py +++ b/src/modelscope_hub/kernels/__init__.py @@ -29,7 +29,16 @@ from __future__ import annotations -from modelscope_hub.kernels._load import get_kernel, get_kernel_variants, get_local_kernel, has_kernel +from modelscope_hub.kernels._load import ( + get_kernel, + get_kernel_variants, + get_local_kernel, + get_locked_kernel, + has_kernel, + install_kernel, + install_kernel_all_variants, + load_kernel, +) from modelscope_hub.kernels._patch import patch_kernels_hub, patch_scope, unpatch_kernels_hub __all__ = [ @@ -37,6 +46,10 @@ "get_kernel_variants", "get_local_kernel", "has_kernel", + "install_kernel", + "install_kernel_all_variants", + "get_locked_kernel", + "load_kernel", "patch_kernels_hub", "patch_scope", "unpatch_kernels_hub", diff --git a/src/modelscope_hub/kernels/_hub.py b/src/modelscope_hub/kernels/_hub.py index 88fca2b..713b315 100644 --- a/src/modelscope_hub/kernels/_hub.py +++ b/src/modelscope_hub/kernels/_hub.py @@ -39,6 +39,31 @@ # Kernel-version refs (``v1``, ``v2``, ...) as created by the kernels Hub. _VERSION_REF_RE = re.compile(r"v(\d+)") +# Full git commit hashes (SHA-1); lockfiles pin revisions this way. +_COMMIT_HASH_RE = re.compile(r"[0-9a-f]{40}", re.IGNORECASE) + +_LOCKFILE_COMMIT_HINT = ( + "Revision {revision!r} looks like a commit hash. Commit hashes are " + "hub-specific: a kernel lockfile created against the Hugging Face Hub " + "cannot resolve on ModelScope. Re-create the lockfile in the ModelScope " + "environment (get_locked_kernel / install_kernel_all_variants from " + "modelscope_hub.kernels, or the kernels CLI under patch_kernels_hub) so " + "it records ModelScope commits." +) + + +def _note_commit_hash_hint(exc: BaseException, revision: str | None) -> None: + """Append a lockfile hint when a full commit-hash revision is not found. + + PEP 678 notes require Python 3.11+; on older interpreters the hint is + skipped silently (the original exception propagates unchanged either way). + """ + if revision is None or not _COMMIT_HASH_RE.fullmatch(str(revision)): + return + add_note = getattr(exc, "add_note", None) + if add_note is not None: + add_note(_LOCKFILE_COMMIT_HINT.format(revision=revision)) + def _ms_revision(revision: str | None) -> str: """Translate an HF revision string into one ModelScope accepts.""" @@ -119,6 +144,9 @@ def snapshot_download( f"Cannot find a local snapshot for {repo_id} (revision: {_ms_revision(revision)}). " "The kernel must be downloaded while online first." ) from e + except modelscope_hub.NotExistError as e: + _note_commit_hash_hint(e, revision) + raise def hf_hub_download( self, @@ -152,6 +180,7 @@ def hf_hub_download( f"(revision: {_ms_revision(revision)}). The file must be downloaded while online first." ) from e except modelscope_hub.NotExistError as e: + _note_commit_hash_hint(e, revision) raise EntryNotFoundError( f"File {filename} does not exist in {repo_id} (revision: {_ms_revision(revision)})." ) from e diff --git a/src/modelscope_hub/kernels/_load.py b/src/modelscope_hub/kernels/_load.py index 39f4d6d..530942f 100644 --- a/src/modelscope_hub/kernels/_load.py +++ b/src/modelscope_hub/kernels/_load.py @@ -13,7 +13,16 @@ from modelscope_hub.kernels._patch import patch_scope -__all__ = ["get_kernel", "get_kernel_variants", "get_local_kernel", "has_kernel"] +__all__ = [ + "get_kernel", + "get_kernel_variants", + "get_local_kernel", + "has_kernel", + "install_kernel", + "install_kernel_all_variants", + "get_locked_kernel", + "load_kernel", +] def _require_kernels() -> ModuleType: @@ -183,3 +192,141 @@ def get_local_kernel( backend=backend, trust_remote_code=trust_remote_code, ) + + +def install_kernel( + repo_id: str, + *, + revision: str | None = None, + version: int | None = None, + backend: str | None = None, + local_files_only: bool = False, + user_agent: str | dict | None = None, + trust_remote_code: bool | list[str] = False, +) -> Path: + """Download a kernel for the current environment into the local cache. + + Unlike :func:`get_kernel` the kernel is not imported; this is the entry + point for CI pre-installation and offline preparation. Arguments are the + same as :func:`get_kernel` (plus ``local_files_only``). + + Returns: + `Path`: The path to the downloaded variant directory. + """ + kernels = _require_kernels() + with patch_scope(): + return kernels.install_kernel( + repo_id, + revision=revision, + version=version, + backend=backend, + local_files_only=local_files_only, + user_agent=user_agent, + trust_remote_code=trust_remote_code, + ) + + +def install_kernel_all_variants( + repo_id: str, + *, + revision: str | None = None, + version: int | None = None, +) -> Path: + """Download every build variant of a kernel (and its dependencies). + + Uses the locking code path to resolve the full dependency tree, so the + ``revision``/``version`` recorded in the resulting lock state refers to + ModelScope commits. A lockfile produced against the Hugging Face Hub + cannot resolve here: the two hubs have different git histories, and + commit hashes recorded on one are meaningless on the other. + + Returns: + `Path`: The path to the ``build`` directory of the requested kernel. + """ + _require_kernels() + # Not exported on the `kernels` top level (0.17/0.18); live in the + # `kernels.install` submodule, which `kernels` itself imports at load time. + from kernels.install import install_kernel_all_variants as _install_all + + with patch_scope(): + return _install_all( + repo_id, + revision=revision, + version=version, + ) + + +def get_locked_kernel( + repo_id: str, + *, + lockfile: Path | None = None, + trust_remote_code: bool | list[str] = False, + user_agent: str | dict | None = None, +) -> ModuleType: + """Load a kernel pinned by a lockfile, downloading missing pieces. + + Important: kernel lockfiles record hub-specific commit hashes. A lockfile + created against the Hugging Face Hub (e.g. via the ``kernels`` CLI there) + cannot resolve on ModelScope — the recorded commits do not exist on the + ModelScope mirror. Create the lockfile in the ModelScope environment + (``get_locked_kernel`` / ``install_kernel_all_variants`` from this + package, or the ``kernels`` CLI under ``patch_kernels_hub()``) so it + records ModelScope commits. + + Args: + repo_id (`str`): + The ModelScope model repository containing the kernel. + lockfile (`Path`, *optional*): + Path to the lockfile. When not provided, the lockfile is loaded + from the caller's package metadata. + trust_remote_code (`bool | list[str]`, *optional*, defaults to `False`): + Whether to allow loading kernels from untrusted organisations. + user_agent (`Union[str, dict]`, *optional*): + Extra telemetry info appended to the ModelScope SDK user agent. + + Returns: + `ModuleType`: The imported kernel module. + """ + kernels = _require_kernels() + with patch_scope(): + return kernels.get_locked_kernel( + repo_id, + lockfile=lockfile, + trust_remote_code=trust_remote_code, + user_agent=user_agent, + ) + + +def load_kernel( + repo_id: str, + *, + lockfile: Path | None = None, + backend: str | None = None, +) -> ModuleType: + """Import a pre-downloaded, locked kernel from the local cache. + + Never downloads: the kernel must already have been installed locally (via + :func:`install_kernel`, :func:`install_kernel_all_variants`, or + :func:`get_kernel`). Raises when the kernel is not available locally. + The lockfile must have been created in the ModelScope environment (see + :func:`get_locked_kernel`). + + Args: + repo_id (`str`): + The kernel repository the lockfile pins. + lockfile (`Path`, *optional*): + Path to the lockfile. When not provided, the lockfile is loaded + from the caller's package metadata. + backend (`str`, *optional*): + The backend to load the kernel for. + + Returns: + `ModuleType`: The imported kernel module. + """ + kernels = _require_kernels() + with patch_scope(): + return kernels.load_kernel( + repo_id, + lockfile=lockfile, + backend=backend, + ) diff --git a/tests/test_kernels.py b/tests/test_kernels.py index d391340..b34efcf 100644 --- a/tests/test_kernels.py +++ b/tests/test_kernels.py @@ -386,3 +386,146 @@ def fake_get_kernel(repo_id, **kwargs): unpatch_kernels_hub() assert kernels_hf_hub._get_hf_api is original + + +# --------------------------------------------------------------------------- +# New scoped wrappers: install / lock entries +# --------------------------------------------------------------------------- +def test_install_kernel_wrapper_scopes_the_patch(kernels_modules, monkeypatch): + from modelscope_hub.kernels import install_kernel + + kernels, kernels_hf_hub, _ = kernels_modules + original = kernels_hf_hub._get_hf_api + calls = {} + + def fake_install(repo_id, **kwargs): + calls["patched"] = kernels_hf_hub._get_hf_api is _ms_api + calls["repo_id"] = repo_id + calls["kwargs"] = kwargs + return Path("/cache/variants/x") + + monkeypatch.setattr(kernels, "install_kernel", fake_install) + + result = install_kernel( + "kernels-community/activation", + version=1, + local_files_only=False, + trust_remote_code=True, + ) + assert result == Path("/cache/variants/x") + assert calls["patched"] is True + assert calls["repo_id"] == "kernels-community/activation" + assert calls["kwargs"]["version"] == 1 + assert calls["kwargs"]["trust_remote_code"] is True + assert kernels_hf_hub._get_hf_api is original + + +def test_install_kernel_all_variants_wrapper_scopes_the_patch(kernels_modules, monkeypatch): + from modelscope_hub.kernels import install_kernel_all_variants + + kernels, kernels_hf_hub, _ = kernels_modules + original = kernels_hf_hub._get_hf_api + calls = {} + + def fake_install_all(repo_id, **kwargs): + calls["patched"] = kernels_hf_hub._get_hf_api is _ms_api + return Path("/cache/build") + + # `install_kernel_all_variants` lives in the `kernels.install` submodule + # and is not exported on the `kernels` top level. + monkeypatch.setattr(kernels.install, "install_kernel_all_variants", fake_install_all) + + result = install_kernel_all_variants("kernels-community/activation", version=1) + assert result == Path("/cache/build") + assert calls["patched"] is True + assert kernels_hf_hub._get_hf_api is original + + +def test_get_locked_kernel_wrapper_scopes_the_patch(kernels_modules, monkeypatch, tmp_path): + from modelscope_hub.kernels import get_locked_kernel + + kernels, kernels_hf_hub, _ = kernels_modules + original = kernels_hf_hub._get_hf_api + calls = {} + + def fake_get_locked(repo_id, **kwargs): + calls["patched"] = kernels_hf_hub._get_hf_api is _ms_api + calls["kwargs"] = kwargs + return "kernel-module" + + monkeypatch.setattr(kernels, "get_locked_kernel", fake_get_locked) + + lockfile = tmp_path / "kernel-lock.toml" + result = get_locked_kernel("kernels-community/activation", lockfile=lockfile) + assert result == "kernel-module" + assert calls["patched"] is True + assert calls["kwargs"]["lockfile"] == lockfile + assert kernels_hf_hub._get_hf_api is original + + +def test_load_kernel_wrapper_scopes_the_patch(kernels_modules, monkeypatch, tmp_path): + from modelscope_hub.kernels import load_kernel + + kernels, kernels_hf_hub, _ = kernels_modules + original = kernels_hf_hub._get_hf_api + calls = {} + + def fake_load(repo_id, **kwargs): + calls["patched"] = kernels_hf_hub._get_hf_api is _ms_api + return "kernel-module" + + monkeypatch.setattr(kernels, "load_kernel", fake_load) + + lockfile = tmp_path / "kernel-lock.toml" + result = load_kernel("kernels-community/activation", lockfile=lockfile, backend="cpu") + assert result == "kernel-module" + assert calls["patched"] is True + assert kernels_hf_hub._get_hf_api is original + + +# --------------------------------------------------------------------------- +# Lockfile commit-hash hint +# --------------------------------------------------------------------------- +def test_hf_hub_download_commit_hash_failure_gets_lockfile_hint(): + hf_errors = pytest.importorskip("huggingface_hub.errors") + fake = mock.MagicMock() + fake.download_file.side_effect = NotExistError("no such revision") + api = MsKernelApi(hub_api=fake) + + commit = "a" * 40 + with pytest.raises(hf_errors.EntryNotFoundError) as excinfo: + api.hf_hub_download("kernels-community/activation", filename="build/x/metadata.json", revision=commit) + + cause = excinfo.value.__cause__ + assert cause is not None + # PEP 678 notes exist on Python 3.11+; the hint text is in the notes. + notes = list(getattr(cause, "__notes__", ())) + if notes: + assert any("lockfile" in note for note in notes) + assert commit in "".join(notes) + + +def test_snapshot_download_commit_hash_failure_propagates_with_hint(): + fake = mock.MagicMock() + fake.download_repo.side_effect = NotExistError("no such revision") + api = MsKernelApi(hub_api=fake) + + commit = "b" * 40 + with pytest.raises(NotExistError) as excinfo: + api.snapshot_download("kernels-community/activation", repo_type="kernel", revision=commit) + + notes = list(getattr(excinfo.value, "__notes__", ())) + if notes: + assert any("lockfile" in note for note in notes) + + +def test_hf_hub_download_branch_failure_gets_no_lockfile_hint(): + hf_errors = pytest.importorskip("huggingface_hub.errors") + fake = mock.MagicMock() + fake.download_file.side_effect = NotExistError("no such file") + api = MsKernelApi(hub_api=fake) + + with pytest.raises(hf_errors.EntryNotFoundError) as excinfo: + api.hf_hub_download("kernels-community/activation", filename="missing.json", revision="v1") + + assert not list(getattr(excinfo.value.__cause__, "__notes__", ())) From 32ed7b4a40e8581459c53739fb98c79cd12b9219 Mon Sep 17 00:00:00 2001 From: tastelikefeet Date: Tue, 3 Nov 2026 12:33:54 +0800 Subject: [PATCH 3/6] Bump version to 0.7.0.dev0 Development version for the upcoming 0.7.0 release (kernels subpackage and huggingface_hub patcher). A dev build on PyPI lets downstream requirements pin >=0.7.0.dev0 resolve before the final 0.7.0 tag. --- src/modelscope_hub/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modelscope_hub/version.py b/src/modelscope_hub/version.py index 02d16d8..4f8f459 100644 --- a/src/modelscope_hub/version.py +++ b/src/modelscope_hub/version.py @@ -1,3 +1,3 @@ """Version information for modelscope_hub.""" -__version__ = "0.6.1+main" +__version__ = "0.7.0.dev0" From 24de497e92bd0686664f037577a66babf0c37691 Mon Sep 17 00:00:00 2001 From: tastelikefeet Date: Tue, 3 Nov 2026 12:48:56 +0800 Subject: [PATCH 4/6] Serve synthetic OIDs as list_repo_refs target commits (kernels 0.18) kernels 0.18 validates ref target_commits with Oid.from_str (40/64 hex digits) and both kernels 0.17 and 0.18 use target_commit as the download revision, so filling in the branch name made version resolution fail on kernels 0.18 with "Invalid git object id ...: master". ModelScope's revision API does not expose git commit hashes, so list_repo_refs now hands out a deterministic synthetic OID per (repo, ref) and every download/list entry point (snapshot_download, hf_hub_download, list_repo_tree, file_exists) maps a known synthetic OID back to the real ModelScope branch/tag name before forwarding. Real branch/tag revisions pass through untouched, and an unknown hex (e.g. from a cross-hub lockfile or a stale cached ref) still fails with the existing diagnostic note. Verified against kernels 0.17.2 and 0.18.1 end to end on the kernels-community mirrors (42 variants resolved via the master->v1 fallback on both). --- src/modelscope_hub/kernels/_hub.py | 74 ++++++++++++++++++++++-------- tests/test_kernels.py | 73 +++++++++++++++++++++++++++-- 2 files changed, 124 insertions(+), 23 deletions(-) diff --git a/src/modelscope_hub/kernels/_hub.py b/src/modelscope_hub/kernels/_hub.py index 713b315..beededb 100644 --- a/src/modelscope_hub/kernels/_hub.py +++ b/src/modelscope_hub/kernels/_hub.py @@ -22,6 +22,7 @@ from __future__ import annotations +import hashlib import os import re from pathlib import Path @@ -70,6 +71,29 @@ def _ms_revision(revision: str | None) -> str: return "master" if revision in (None, "main") else revision +#: Synthetic OID -> ModelScope branch/tag name, filled by ``list_repo_refs``. +#: ModelScope's revision API does not expose git commit hashes, but kernels +#: 0.18 validates ``Oid.from_str(target_commit)`` (40/64 hex digits) and both +#: kernels 0.17 and 0.18 use ``target_commit`` as the download revision. The +#: shim therefore hands out a deterministic synthetic OID per (repo, ref) and +#: maps it back to the real ref name at every download/list entry point. +_synthetic_oids: dict[str, str] = {} + + +def _synthetic_oid(repo_id: str, ref_name: str) -> str: + """A stable 40-hex stand-in for the commit hash ModelScope does not expose.""" + oid = hashlib.sha1(f"{repo_id}:{ref_name}".encode()).hexdigest() + _synthetic_oids.setdefault(oid, ref_name) + return oid + + +def _resolve_revision(revision: str | None) -> str | None: + """Map a synthetic OID back to its ModelScope branch/tag name.""" + if revision is None: + return None + return _synthetic_oids.get(str(revision), revision) + + def _ms_repo_type(repo_type: str | None) -> str: """Map HF repo types onto ModelScope repo types (kernel repos are models).""" return "model" if repo_type in (None, "kernel", "model") else repo_type @@ -128,11 +152,12 @@ def snapshot_download( """Download a repository snapshot, mirroring ``HfApi.snapshot_download``.""" from huggingface_hub.errors import LocalEntryNotFoundError + ms_revision = _ms_revision(_resolve_revision(revision)) try: return self._api().download_repo( repo_id, _ms_repo_type(repo_type), - revision=_ms_revision(revision), + revision=ms_revision, cache_dir=Path(cache_dir) if cache_dir else None, allow_patterns=allow_patterns, ignore_patterns=ignore_patterns, @@ -141,7 +166,7 @@ def snapshot_download( ) except modelscope_hub.CacheNotFound as e: raise LocalEntryNotFoundError( - f"Cannot find a local snapshot for {repo_id} (revision: {_ms_revision(revision)}). " + f"Cannot find a local snapshot for {repo_id} (revision: {ms_revision}). " "The kernel must be downloaded while online first." ) from e except modelscope_hub.NotExistError as e: @@ -164,12 +189,13 @@ def hf_hub_download( if repo_id is None or filename is None: raise ValueError("Both `repo_id` and `filename` must be provided.") + ms_revision = _ms_revision(_resolve_revision(revision)) try: return self._api().download_file( repo_id, _ms_repo_type(repo_type), file_path=filename, - revision=_ms_revision(revision), + revision=ms_revision, cache_dir=Path(cache_dir) if cache_dir else None, local_files_only=local_files_only, user_agent=_user_agent(self._user_agent), @@ -177,13 +203,11 @@ def hf_hub_download( except modelscope_hub.CacheNotFound as e: raise LocalEntryNotFoundError( f"Cannot find the local cache entry for {repo_id}/{filename} " - f"(revision: {_ms_revision(revision)}). The file must be downloaded while online first." + f"(revision: {ms_revision}). The file must be downloaded while online first." ) from e except modelscope_hub.NotExistError as e: _note_commit_hash_hint(e, revision) - raise EntryNotFoundError( - f"File {filename} does not exist in {repo_id} (revision: {_ms_revision(revision)})." - ) from e + raise EntryNotFoundError(f"File {filename} does not exist in {repo_id} (revision: {ms_revision}).") from e # ------------------------------------------------------------------ # Repository metadata @@ -191,12 +215,16 @@ def hf_hub_download( def list_repo_refs(self, repo_id: str | None = None, *, repo_type: str | None = None, **kwargs): """List repository branches and tags, mirroring ``HfApi.list_repo_refs``. - ModelScope accepts branch/tag names as revisions, so ``target_commit`` - reuses the name: kernels passes it on as the revision for later calls. - When the repository carries no ``v`` refs at all (typical for a - plain mirror with only a ``master`` branch), ``master`` is exposed as - kernel version ``v1`` so that ``get_kernel(..., version=1)`` works - without the mirror having to create version branches. + ModelScope's revision API does not expose git commit hashes, so every + ref's ``target_commit`` is a deterministic synthetic OID for the ref + name (see :func:`_synthetic_oid`); the download entry points map it + back to the real name, so both kernels 0.17 (which downloads by + ``target_commit`` verbatim) and kernels 0.18 (which validates it with + ``Oid.from_str`` before fetching) work unchanged. When the repository + carries no ``v`` refs at all (typical for a plain mirror with only + a ``master`` branch), ``master`` is exposed as kernel version ``v1`` + so that ``get_kernel(..., version=1)`` works without the mirror having + to create version branches. """ from huggingface_hub.hf_api import GitRefInfo @@ -204,15 +232,25 @@ def list_repo_refs(self, repo_id: str | None = None, *, repo_type: str | None = raise ValueError("`repo_id` must be provided.") branches, tags = self._api().legacy.list_revisions_detail(repo_id, _ms_repo_type(repo_type)) branch_refs = [ - GitRefInfo(name=ref["Revision"], ref=f"refs/heads/{ref['Revision']}", target_commit=ref["Revision"]) + GitRefInfo( + name=ref["Revision"], + ref=f"refs/heads/{ref['Revision']}", + target_commit=_synthetic_oid(repo_id, ref["Revision"]), + ) for ref in branches or [] ] tag_refs = [ - GitRefInfo(name=ref["Revision"], ref=f"refs/tags/{ref['Revision']}", target_commit=ref["Revision"]) + GitRefInfo( + name=ref["Revision"], + ref=f"refs/tags/{ref['Revision']}", + target_commit=_synthetic_oid(repo_id, ref["Revision"]), + ) for ref in tags or [] ] if not any(_VERSION_REF_RE.fullmatch(ref.name) for ref in branch_refs + tag_refs): - branch_refs.append(GitRefInfo(name="v1", ref="refs/heads/v1", target_commit="master")) + branch_refs.append( + GitRefInfo(name="v1", ref="refs/heads/v1", target_commit=_synthetic_oid(repo_id, "master")) + ) return SimpleNamespace(branches=branch_refs, tags=tag_refs, converts=[]) def list_repo_tree( @@ -240,7 +278,7 @@ def list_repo_tree( infos = self._api().list_repo_files( repo_id, _ms_repo_type(repo_type), - revision=_ms_revision(revision), + revision=_ms_revision(_resolve_revision(revision)), recursive=True, ) folders: set[str] = set() @@ -275,7 +313,7 @@ def file_exists( infos = self._api().list_repo_files( repo_id, _ms_repo_type(repo_type), - revision=_ms_revision(revision), + revision=_ms_revision(_resolve_revision(revision)), recursive=True, ) return any(info.path == filename for info in infos) diff --git a/tests/test_kernels.py b/tests/test_kernels.py index b34efcf..7aca264 100644 --- a/tests/test_kernels.py +++ b/tests/test_kernels.py @@ -157,16 +157,19 @@ def test_list_repo_refs_builds_hf_ref_objects(): fake.legacy.list_revisions_detail.assert_called_once_with("kernels-community/activation", "model") assert [b.name for b in refs.branches] == ["v1", "v2", "master"] assert [t.name for t in refs.tags] == ["1.0"] - # ModelScope accepts branch/tag names as revisions, so the name doubles - # as the target commit used by later kernels calls. assert all(isinstance(b, hf_api.GitRefInfo) for b in refs.branches) - assert all(b.target_commit == b.name for b in refs.branches) + # target_commit is a synthetic hex OID (kernels 0.18 validates it with + # Oid.from_str); the download entry points map it back to the ref name. + import re as _re + + assert all(_re.fullmatch(r"[0-9a-f]{40}", b.target_commit) for b in refs.branches) assert refs.converts == [] def test_list_repo_refs_synthesizes_v1_when_no_version_refs(): """A plain ModelScope mirror with only `master` exposes it as kernel version v1.""" pytest.importorskip("huggingface_hub.hf_api") + import re as _re fake = mock.MagicMock() fake.legacy.list_revisions_detail.return_value = ([{"Revision": "master"}], []) @@ -176,8 +179,9 @@ def test_list_repo_refs_synthesizes_v1_when_no_version_refs(): v1 = [b for b in refs.branches if b.name == "v1"] assert len(v1) == 1 - # The synthetic v1 ref resolves downloads back onto `master`. - assert v1[0].target_commit == "master" + # The synthetic v1 ref carries a hex OID that resolves downloads back + # onto `master` (see test_synthetic_oid_resolved_back_to_branch_on_downloads). + assert _re.fullmatch(r"[0-9a-f]{40}", v1[0].target_commit) # --------------------------------------------------------------------------- @@ -529,3 +533,62 @@ def test_hf_hub_download_branch_failure_gets_no_lockfile_hint(): api.hf_hub_download("kernels-community/activation", filename="missing.json", revision="v1") assert not list(getattr(excinfo.value.__cause__, "__notes__", ())) + + +# --------------------------------------------------------------------------- +# Synthetic OIDs (kernels 0.18 validates target_commit as a git object id) +# --------------------------------------------------------------------------- +def test_list_repo_refs_returns_hex_target_commits(): + import re as _re + + fake = mock.MagicMock() + fake.legacy.list_revisions_detail.return_value = ([{"Revision": "master"}], []) + api = MsKernelApi(hub_api=fake) + + refs = api.list_repo_refs("kernels-community/activation") + # kernels 0.18 requires Oid.from_str(target_commit): 40/64 hex digits. + for ref in refs.branches + refs.tags: + assert _re.fullmatch(r"[0-9a-f]{40}", ref.target_commit), ref + # Master-only mirror: the synthesized v1 ref also carries a hex OID. + v1 = next(r for r in refs.branches if r.name == "v1") + assert _re.fullmatch(r"[0-9a-f]{40}", v1.target_commit) + # Deterministic: same repo/ref always maps to the same OID. + fake.legacy.list_revisions_detail.return_value = ([{"Revision": "master"}], []) + again = api.list_repo_refs("kernels-community/activation") + assert next(r for r in again.branches if r.name == "v1").target_commit == v1.target_commit + + +def test_synthetic_oid_resolved_back_to_branch_on_downloads(): + fake = mock.MagicMock() + fake.legacy.list_revisions_detail.return_value = ([{"Revision": "master"}], []) + api = MsKernelApi(hub_api=fake) + + refs = api.list_repo_refs("kernels-community/activation") + oid = refs.branches[0].target_commit # master's synthetic OID (0.17-style resolve) + v1_oid = next(r.target_commit for r in refs.branches if r.name == "v1") + + # kernels passes the OID as the download revision (both 0.17 and 0.18); + # the shim maps it back to the real ModelScope ref name. + api.snapshot_download("kernels-community/activation", repo_type="kernel", revision=oid) + assert fake.download_repo.call_args.kwargs["revision"] == "master" + + api.hf_hub_download("kernels-community/activation", filename="README.md", revision=v1_oid) + assert fake.download_file.call_args.kwargs["revision"] == "master" + + api.list_repo_tree("kernels-community/activation", revision=oid) + assert fake.list_repo_files.call_args.kwargs["revision"] == "master" + + api.file_exists("kernels-community/activation", "README.md", revision=oid) + assert fake.list_repo_files.call_args.kwargs["revision"] == "master" + + +def test_plain_revision_not_mapped(): + """A normal branch/tag revision passes through untouched.""" + fake = mock.MagicMock() + api = MsKernelApi(hub_api=fake) + + api.snapshot_download("kernels-community/activation", repo_type="kernel", revision="v2") + assert fake.download_repo.call_args.kwargs["revision"] == "v2" + + api.snapshot_download("kernels-community/activation", repo_type="kernel", revision=None) + assert fake.download_repo.call_args.kwargs["revision"] == "master" From 31b33fdcb8ff8cdde2d75855e6f171db04764f23 Mon Sep 17 00:00:00 2001 From: tastelikefeet Date: Tue, 3 Nov 2026 15:00:26 +0800 Subject: [PATCH 5/6] Drop the hf extra; check for huggingface_hub at call time modelscope-hub no longer declares any relationship to huggingface_hub in its metadata: the `hf` extra is removed. The patcher entry points (patch_hf_hub / hf_hub_patch_scope) now check for huggingface_hub when used and raise an actionable ImportError with installation instructions when it is missing; unpatch_hf_hub stays a silent no-op in minimal environments (nothing can have been patched). kernels remains untouched: its own dependency chain pulls huggingface_hub, and the kernels shim only runs inside that library. huggingface_hub>=0.19 remains the documented minimum in the module docstring and the README. --- README.md | 4 +- pyproject.toml | 6 --- src/modelscope_hub/compat/hf_hub_patcher.py | 53 +++++++++++++++++---- tests/test_hf_hub_patcher.py | 29 +++++++++++ 4 files changed, 75 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 731908c..0d860a8 100644 --- a/README.md +++ b/README.md @@ -246,10 +246,10 @@ Notes: ### Redirect huggingface_hub (`modelscope_hub.compat`) -`modelscope_hub.compat.hf_hub_patcher` monkey-patches the `huggingface_hub` module-level functions that HF-ecosystem code calls — `hf_hub_download`, `file_exists`, `whoami`, `create_repo`, `upload_file`, `upload_folder`, `create_commit`, and the `repocard` helpers — onto ModelScope-backed implementations: +`modelscope_hub.compat.hf_hub_patcher` monkey-patches the `huggingface_hub` module-level functions that HF-ecosystem code calls — `hf_hub_download`, `file_exists`, `whoami`, `create_repo`, `upload_file`, `upload_folder`, `create_commit`, and the `repocard` helpers — onto ModelScope-backed implementations. `huggingface_hub` is not a dependency of this SDK: install it yourself (`>=0.19` recommended) and the patch entry points check for it at call time, failing with an actionable error when it is missing: ```bash -pip install "modelscope-hub[hf]" +pip install huggingface_hub ``` ```python diff --git a/pyproject.toml b/pyproject.toml index 4a9fb82..878aec6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,12 +41,6 @@ dev = [ kernels = [ "kernels>=0.17", ] -# Runtime extra for the huggingface_hub monkey-patcher in -# `modelscope_hub.compat.hf_hub_patcher` (also a transitive dependency of the -# `kernels` extra). -hf = [ - "huggingface-hub>=0.19", -] [project.scripts] # This package owns every ModelScope console script. Keeping all four in one diff --git a/src/modelscope_hub/compat/hf_hub_patcher.py b/src/modelscope_hub/compat/hf_hub_patcher.py index b8395d6..448667a 100644 --- a/src/modelscope_hub/compat/hf_hub_patcher.py +++ b/src/modelscope_hub/compat/hf_hub_patcher.py @@ -3,8 +3,10 @@ Adapted from ``modelscope/utils/hf_util/patcher.py`` (Apache-2.0, Alibaba), re-targeted from the legacy ``modelscope`` SDK onto ``modelscope_hub``. The transformers/peft/diffusers patches stay with the ``modelscope`` package; -this module must stay importable without transformers and only depends on -``huggingface_hub`` (the optional ``hf`` extra). +this module must stay importable without transformers. ``huggingface_hub`` +itself is *not* a declared dependency of this SDK — the patch entry points +check for it at call time and raise an actionable ``ImportError`` when it is +not installed. :func:`patch_hf_hub` replaces the ``huggingface_hub`` module-level functions that HF-ecosystem code calls — ``hf_hub_download``, ``file_exists``, @@ -27,9 +29,9 @@ API (resumable, LFS-routed) instead of a local git clone + push. Version support: verified against ``huggingface_hub`` 0.19.4 (source-level -symbol audit) and 1.32 (full test suite); the ``hf`` extra pins -``huggingface_hub>=0.19``. Note that ``kernels>=0.17`` itself requires -``huggingface_hub>=1.10``, so installing both extras resolves to a 1.x hub. +symbol audit) and 1.32 (full test suite); ``huggingface_hub>=0.19`` is the +documented minimum. Note that ``kernels>=0.17`` itself requires +``huggingface_hub>=1.10``, so a ``kernels`` install resolves to a 1.x hub. """ from __future__ import annotations @@ -50,6 +52,24 @@ __all__ = ["patch_hf_hub", "unpatch_hf_hub", "hf_hub_patch_scope"] +def _require_hf_hub(): + """Import ``huggingface_hub`` or raise an actionable ``ImportError``. + + ``huggingface_hub`` is deliberately not a declared dependency of this + SDK; the huggingface_hub patches simply cannot work without it, so the + entry points fail with installation instructions at call time. + """ + try: + import huggingface_hub + except ImportError as e: + raise ImportError( + "The `huggingface_hub` package is required to redirect huggingface_hub " + "calls to ModelScope, but it is not installed. " + "Install it with: pip install huggingface_hub" + ) from e + return huggingface_hub + + def _ms_revision(revision: str | None) -> str: """Translate an HF revision string into one ModelScope accepts.""" return "master" if revision in (None, "main") else revision @@ -308,8 +328,11 @@ def patch_hf_hub() -> None: Idempotent: each function is guarded by its historical origin marker, so a legacy ``modelscope`` patcher that has already applied the same attribute names is respected instead of being double-patched. + + Raises: + ImportError: when ``huggingface_hub`` is not installed. """ - import huggingface_hub + huggingface_hub = _require_hf_hub() from huggingface_hub import hf_api, repocard if not hasattr(hf_api, "_hf_hub_download_origin"): @@ -362,9 +385,17 @@ def patch_hf_hub() -> None: def unpatch_hf_hub() -> None: - """Undo :func:`patch_hf_hub` and restore the original huggingface_hub functions.""" - import huggingface_hub - from huggingface_hub import hf_api, repocard + """Undo :func:`patch_hf_hub` and restore the original huggingface_hub functions. + + Without ``huggingface_hub`` nothing can have been patched, so this is a + silent no-op rather than an error (patch/unpatch stay symmetric in + minimal environments). + """ + try: + import huggingface_hub + from huggingface_hub import hf_api, repocard + except ImportError: + return if hasattr(repocard.RepoCard, "_validate_origin"): repocard.RepoCard.validate = repocard.RepoCard._validate_origin @@ -430,7 +461,11 @@ def hf_hub_patch_scope(): Nesting is safe: an already-applied patch (including one applied by the legacy ``modelscope`` patcher) is reused, and only the outermost scope restores the original functions. + + Raises: + ImportError: when ``huggingface_hub`` is not installed. """ + _require_hf_hub() from huggingface_hub import hf_api if hasattr(hf_api, "_hf_hub_download_origin"): diff --git a/tests/test_hf_hub_patcher.py b/tests/test_hf_hub_patcher.py index 5517688..b8af76a 100644 --- a/tests/test_hf_hub_patcher.py +++ b/tests/test_hf_hub_patcher.py @@ -351,3 +351,32 @@ def test_module_has_no_transformers_import(): source = inspect.getsource(hf_hub_patcher) assert "import transformers" not in source assert "from transformers" not in source + + +# --------------------------------------------------------------------------- +# huggingface_hub is not a declared dependency: fail at call time +# --------------------------------------------------------------------------- +def test_patch_raises_actionable_error_without_hf_hub(monkeypatch): + """No `hf` extra exists: patch_hf_hub fails with install instructions.""" + import sys + + monkeypatch.setitem(sys.modules, "huggingface_hub", None) + with pytest.raises(ImportError, match="pip install huggingface_hub"): + patch_hf_hub() + + +def test_unpatch_is_noop_without_hf_hub(monkeypatch): + """Nothing can have been patched, so unpatch stays a silent no-op.""" + import sys + + monkeypatch.setitem(sys.modules, "huggingface_hub", None) + unpatch_hf_hub() # must not raise + + +def test_patch_scope_raises_actionable_error_without_hf_hub(monkeypatch): + import sys + + monkeypatch.setitem(sys.modules, "huggingface_hub", None) + with pytest.raises(ImportError, match="pip install huggingface_hub"): + with hf_hub_patch_scope(): + pass From e1a405b822cf5c80defe17420e43c30b7bcc8ec6 Mon Sep 17 00:00:00 2001 From: tastelikefeet Date: Tue, 3 Nov 2026 19:55:18 +0800 Subject: [PATCH 6/6] Revert the hf_hub_patcher: huggingface_hub patching stays in modelscope Per the product decision, the huggingface_hub monkey-patcher is not migrated to modelscope_hub: compat/hf_hub_patcher.py, its exports and tests are removed, and the README section is dropped. modelscope_hub 0.7.0 now ships exactly one integration feature, the kernels package (modelscope_hub.kernels), which the modelscope package's patch_hub() keeps delegating to. The synthetic-OID ref handling, variant discovery, convenience wrappers and the kernels tests are unchanged. --- .gitignore | 1 + README.md | 25 - src/modelscope_hub/compat/__init__.py | 4 - src/modelscope_hub/compat/hf_hub_patcher.py | 478 -------------------- tests/test_hf_hub_patcher.py | 382 ---------------- 5 files changed, 1 insertion(+), 889 deletions(-) delete mode 100644 src/modelscope_hub/compat/hf_hub_patcher.py delete mode 100644 tests/test_hf_hub_patcher.py diff --git a/.gitignore b/.gitignore index 7e956c3..89e4ce7 100644 --- a/.gitignore +++ b/.gitignore @@ -133,3 +133,4 @@ ckpt/ # ast template ast_index_file.py +modelscope diff --git a/README.md b/README.md index 0d860a8..2fb8a7a 100644 --- a/README.md +++ b/README.md @@ -244,31 +244,6 @@ Notes: - ModelScope has no trusted-publisher flag, so untrusted publishers are rejected by default: only `kernels-community` is trusted out of the box. Extend the allowlist with `MS_KERNELS_TRUSTED_PUBLISHERS` (comma-separated), or allow specific repositories / all repositories through the trust parameters of `get_kernel` (its docstring mirrors the kernel library's API). - The mirror must contain kernels built for your exact Torch version and platform (`kernels` selects `build/torchXY-...--/`); a repository whose builds predate `kernels` 0.17 metadata cannot be loaded until it is rebuilt with a current `kernel-builder`. -### Redirect huggingface_hub (`modelscope_hub.compat`) - -`modelscope_hub.compat.hf_hub_patcher` monkey-patches the `huggingface_hub` module-level functions that HF-ecosystem code calls — `hf_hub_download`, `file_exists`, `whoami`, `create_repo`, `upload_file`, `upload_folder`, `create_commit`, and the `repocard` helpers — onto ModelScope-backed implementations. `huggingface_hub` is not a dependency of this SDK: install it yourself (`>=0.19` recommended) and the patch entry points check for it at call time, failing with an actionable error when it is missing: - -```bash -pip install huggingface_hub -``` - -```python -from modelscope_hub.compat import hf_hub_patch_scope, patch_hf_hub, unpatch_hf_hub - -# Globally redirect huggingface_hub to ModelScope -patch_hf_hub() -... # any HF-ecosystem code now downloads/uploads via ModelScope -unpatch_hf_hub() - -# Or scoped to a block; nesting reuses the outer patch -with hf_hub_patch_scope(): - ... -``` - -- The `modelscope` framework's `patch_hub()` builds on this implementation (plus its transformers/peft/diffusers class patches); the compatibility contract is kept stable: `whoami` returns `{'name': ...}`, `create_repo` returns a `RepoUrl`-shaped str, upload functions return a commit info whose `commit_url` points at `{endpoint}/models/{repo_id}/files`, the default revision is `master`, and `create_commit` only supports `Add` operations. -- Patches are idempotent and marked with the historical origin attributes, so an already-applied legacy `modelscope` patcher and this module treat each other as already patched. -- Uploads go through the ModelScope commit API (resumable, LFS-routed) rather than a local git clone + push. - ### Continuous dataset downloads For migration jobs, `download_repo_report` continues past isolated file failures and returns an explicit `COMPLETE` or `PARTIAL` report. The original `download_repo` and `snapshot_download` remain strict by default. diff --git a/src/modelscope_hub/compat/__init__.py b/src/modelscope_hub/compat/__init__.py index f0dd95c..64e34f3 100644 --- a/src/modelscope_hub/compat/__init__.py +++ b/src/modelscope_hub/compat/__init__.py @@ -32,15 +32,11 @@ ModelVisibility_PUBLIC, ) from .file_download import dataset_file_download, model_file_download -from .hf_hub_patcher import hf_hub_patch_scope, patch_hf_hub, unpatch_hf_hub from .hub_api import LegacyHubApi from .snapshot_download import dataset_snapshot_download, snapshot_download __all__ = [ "LegacyHubApi", - "hf_hub_patch_scope", - "patch_hf_hub", - "unpatch_hf_hub", "dataset_file_download", "dataset_snapshot_download", "model_file_download", diff --git a/src/modelscope_hub/compat/hf_hub_patcher.py b/src/modelscope_hub/compat/hf_hub_patcher.py deleted file mode 100644 index 448667a..0000000 --- a/src/modelscope_hub/compat/hf_hub_patcher.py +++ /dev/null @@ -1,478 +0,0 @@ -"""Monkey-patching of the ``huggingface_hub`` module onto the ModelScope Hub. - -Adapted from ``modelscope/utils/hf_util/patcher.py`` (Apache-2.0, Alibaba), -re-targeted from the legacy ``modelscope`` SDK onto ``modelscope_hub``. -The transformers/peft/diffusers patches stay with the ``modelscope`` package; -this module must stay importable without transformers. ``huggingface_hub`` -itself is *not* a declared dependency of this SDK — the patch entry points -check for it at call time and raise an actionable ``ImportError`` when it is -not installed. - -:func:`patch_hf_hub` replaces the ``huggingface_hub`` module-level functions -that HF-ecosystem code calls — ``hf_hub_download``, ``file_exists``, -``whoami``, ``create_repo``, ``upload_file``, ``upload_folder``, -``create_commit``, and the ``repocard`` helpers — with ModelScope-backed -implementations. Patch markers reuse the historical attribute names -(``hf_api._hf_hub_download_origin`` etc.), so a legacy ``modelscope`` -patcher and this module treat each other's patches as already applied. - -Compatibility contract (kept identical to the legacy patcher): - -- ``whoami`` returns ``{'name': ...}``; -- ``create_repo`` returns a ``RepoUrl``-shaped str (``HfRepoUrl``); -- ``upload_folder`` / ``upload_file`` / ``create_commit`` return a commit - info with ``commit_url`` shaped like ``{endpoint}/models/{repo_id}/files``; -- the default revision is ``master`` (HF's ``main`` is normalised away); -- ``create_commit`` only supports ``Add`` operations. - -One deliberate behaviour change: uploads go through the ModelScope commit -API (resumable, LFS-routed) instead of a local git clone + push. - -Version support: verified against ``huggingface_hub`` 0.19.4 (source-level -symbol audit) and 1.32 (full test suite); ``huggingface_hub>=0.19`` is the -documented minimum. Note that ``kernels>=0.17`` itself requires -``huggingface_hub>=1.10``, so a ``kernels`` install resolves to a 1.x hub. -""" - -from __future__ import annotations - -import contextlib -import os -from dataclasses import dataclass -from pathlib import Path -from types import MethodType - -import modelscope_hub -from modelscope_hub import compat -from modelscope_hub.constants import DEFAULT_MODELSCOPE_DATA_ENDPOINT -from modelscope_hub.utils.logger import get_logger - -logger = get_logger() - -__all__ = ["patch_hf_hub", "unpatch_hf_hub", "hf_hub_patch_scope"] - - -def _require_hf_hub(): - """Import ``huggingface_hub`` or raise an actionable ``ImportError``. - - ``huggingface_hub`` is deliberately not a declared dependency of this - SDK; the huggingface_hub patches simply cannot work without it, so the - entry points fail with installation instructions at call time. - """ - try: - import huggingface_hub - except ImportError as e: - raise ImportError( - "The `huggingface_hub` package is required to redirect huggingface_hub " - "calls to ModelScope, but it is not installed. " - "Install it with: pip install huggingface_hub" - ) from e - return huggingface_hub - - -def _ms_revision(revision: str | None) -> str: - """Translate an HF revision string into one ModelScope accepts.""" - return "master" if revision in (None, "main") else revision - - -def _token(token) -> str | None: - """HF tokens may be bool (``True``/``False``) or strings; keep strings only.""" - return token if isinstance(token, str) else None - - -def _hub_api(token=None) -> modelscope_hub.HubApi: - """A ``HubApi`` for this call; ``token=None`` falls back to stored credentials.""" - return modelscope_hub.HubApi(token=_token(token)) - - -@dataclass -class HfCommitInfo: - """Return shape of the ``upload_*`` / ``create_commit`` patches. - - Mirrors the legacy ``modelscope.utils.repo_utils.CommitInfo`` fields that - callers (transformers ``push_to_hub``, diffusers) actually consume. - """ - - commit_url: str - commit_message: str | None - commit_description: str | None - oid: str | None - - -class HfRepoUrl(str): - """Lenient ``RepoUrl`` replacement returned by the ``create_repo`` patch. - - The legacy patcher returned the ``modelscope.utils.repo_utils.RepoUrl`` - str subclass (url plus ``repo_type``/``repo_id`` attributes), not HF's - own class; recent ``huggingface_hub`` releases additionally reject - non-Hugging Face hosts in their ``RepoUrl`` constructor. This subclass - keeps the legacy duck-typing shape (``str``, ``.repo_id``, ``.repo_type``). - """ - - def __new__(cls, url: str, repo_type: str = "model", repo_id: str | None = None): - obj = super().__new__(cls, url) - obj.repo_type = repo_type # type: ignore[attr-defined] - obj.repo_id = repo_id # type: ignore[attr-defined] - return obj - - -def _commit_url(repo_id: str) -> str: - return f"{DEFAULT_MODELSCOPE_DATA_ENDPOINT}/models/{repo_id}/files" - - -# --------------------------------------------------------------------------- -# Download / metadata patches -# --------------------------------------------------------------------------- -def _hf_hub_download( - repo_id: str, - filename: str, - *, - subfolder: str | None = None, - repo_type: str | None = None, - revision: str | None = None, - cache_dir: str | Path | None = None, - local_dir: str | Path | None = None, - token=None, - local_files_only: bool = False, - **kwargs, -) -> str: - """ModelScope-backed ``huggingface_hub.hf_hub_download``.""" - if kwargs: - logger.warning( - "The passed in %s are not used by modelscope_hub.", - ", ".join(sorted(kwargs)), - ) - if repo_type not in (None, "model", "dataset"): - raise ValueError(f"repo_type={repo_type} is not supported in ModelScope") - file_path = os.path.join(subfolder, filename) if subfolder else filename - if repo_type == "dataset": - return compat.dataset_file_download( - repo_id, - file_path, - revision=_ms_revision(revision), - cache_dir=str(cache_dir) if cache_dir else None, - local_dir=str(local_dir) if local_dir else None, - token=_token(token), - local_files_only=local_files_only, - ) - return compat.model_file_download( - repo_id, - file_path, - revision=_ms_revision(revision), - cache_dir=str(cache_dir) if cache_dir else None, - local_dir=str(local_dir) if local_dir else None, - token=_token(token), - local_files_only=local_files_only, - ) - - -def _file_exists( - repo_id: str, - filename: str, - *, - repo_type: str | None = None, - revision: str | None = None, - token=None, - **kwargs, -) -> bool: - """ModelScope-backed ``huggingface_hub.file_exists``.""" - if repo_type is not None: - logger.warning("repo_type is ignored by modelscope_hub; only model repos can be queried.") - infos = _hub_api(token).list_repo_files(repo_id, "model", revision=_ms_revision(revision)) - return any(info.path == filename for info in infos) - - -def _whoami(token=None) -> dict: - """ModelScope-backed ``huggingface_hub.whoami``.""" - user = _hub_api(token).whoami() - return {"name": getattr(user, "username", None) or "unknown"} - - -# --------------------------------------------------------------------------- -# Repository / upload patches -# --------------------------------------------------------------------------- -def _create_repo(repo_id: str, *, token=None, private: bool = False, **kwargs) -> HfRepoUrl: - """ModelScope-backed ``huggingface_hub.create_repo``; returns a ``RepoUrl``-shaped str.""" - compat.LegacyHubApi(token=_token(token)).create_repo( - repo_id, - visibility="private" if private else "public", - **kwargs, - ) - return HfRepoUrl( - f"{DEFAULT_MODELSCOPE_DATA_ENDPOINT}/models/{repo_id}", - repo_type="model", - repo_id=repo_id, - ) - - -def _upload_file( - *, - path_or_fileobj, - path_in_repo: str, - repo_id: str, - token=None, - repo_type: str | None = "model", - revision: str | None = "master", - commit_message: str | None = None, - commit_description: str | None = None, - **kwargs, -) -> None: - """ModelScope-backed ``huggingface_hub.upload_file`` (commit-API upload).""" - if repo_type not in (None, "model"): - raise ValueError(f"repo_type={repo_type} is not supported in ModelScope") - _hub_api(token).upload_file( - repo_id, - "model", - path_or_fileobj, - path_in_repo, - commit_message=commit_message, - commit_description=commit_description, - revision=_ms_revision(revision), - ) - - -def _upload_folder( - *, - repo_id: str, - folder_path: str | Path, - path_in_repo: str | None = None, - commit_message: str | None = None, - commit_description: str | None = None, - token=None, - repo_type: str | None = "model", - revision: str | None = "master", - ignore_patterns=None, - allow_patterns=None, - **kwargs, -) -> HfCommitInfo: - """ModelScope-backed ``huggingface_hub.upload_folder`` (commit-API upload).""" - if repo_type not in (None, "model"): - raise ValueError(f"repo_type={repo_type} is not supported in ModelScope") - _hub_api(token).upload_folder( - repo_id, - "model", - folder_path, - path_in_repo=path_in_repo or "", - commit_message=commit_message, - commit_description=commit_description, - revision=_ms_revision(revision), - ignore_patterns=ignore_patterns, - allow_patterns=allow_patterns, - ) - return HfCommitInfo( - commit_url=_commit_url(repo_id), - commit_message=commit_message, - commit_description=commit_description, - oid=None, - ) - - -def _create_commit( - repo_id: str, - operations, - *, - commit_message: str, - commit_description: str | None = None, - token=None, - repo_type: str | None = None, - revision: str | None = "master", - **kwargs, -) -> None: - """ModelScope-backed ``huggingface_hub.create_commit`` (Add operations only). - - Each ``CommitOperationAdd`` becomes one commit through the ModelScope - commit API, so a multi-operation batch produces several commits on the - Hub, unlike a single git push. - """ - from huggingface_hub import CommitOperationAdd - - if repo_type not in (None, "model"): - raise ValueError(f"repo_type={repo_type} is not supported in ModelScope") - api = _hub_api(token) - for op in operations: - if not isinstance(op, CommitOperationAdd): - raise ValueError("ModelScope create_commit only supports Add operations for now.") - api.upload_file( - repo_id, - "model", - op.path_or_fileobj, - op.path_in_repo, - commit_message=commit_message, - commit_description=commit_description, - revision=_ms_revision(revision), - ) - - -def _repocard_load(cls, repo_id_or_path, repo_type=None, token=None, ignore_metadata_errors=False): - """ModelScope-backed ``huggingface_hub.repocard.RepoCard.load``.""" - if os.path.exists(repo_id_or_path): - file_path = repo_id_or_path - elif repo_type == "dataset": - file_path = compat.dataset_file_download(repo_id_or_path, "README.md", token=_token(token)) - else: - file_path = compat.model_file_download(repo_id_or_path, "README.md", token=_token(token)) - - with open(file_path, encoding="utf-8") as f: - repo_card = cls(f.read(), ignore_metadata_errors=ignore_metadata_errors) - if not hasattr(repo_card.data, "tags"): - repo_card.data.tags = [] - return repo_card - - -# --------------------------------------------------------------------------- -# Patch / unpatch -# --------------------------------------------------------------------------- -def patch_hf_hub() -> None: - """Monkey-patch ``huggingface_hub`` to route Hub calls to ModelScope. - - Idempotent: each function is guarded by its historical origin marker, so - a legacy ``modelscope`` patcher that has already applied the same - attribute names is respected instead of being double-patched. - - Raises: - ImportError: when ``huggingface_hub`` is not installed. - """ - huggingface_hub = _require_hf_hub() - from huggingface_hub import hf_api, repocard - - if not hasattr(hf_api, "_hf_hub_download_origin"): - setattr(hf_api, "_hf_hub_download_origin", huggingface_hub.file_download.hf_hub_download) - huggingface_hub.hf_hub_download = _hf_hub_download - huggingface_hub.file_download.hf_hub_download = _hf_hub_download - - if not hasattr(hf_api, "_file_exists_origin"): - setattr(hf_api, "_file_exists_origin", hf_api.file_exists) - hf_api.file_exists = _file_exists - huggingface_hub.file_exists = _file_exists - huggingface_hub.hf_api.file_exists = _file_exists - - if not hasattr(hf_api, "_whoami_origin"): - setattr(hf_api, "_whoami_origin", hf_api.whoami) - hf_api.whoami = _whoami - huggingface_hub.whoami = _whoami - huggingface_hub.hf_api.whoami = _whoami - - if not hasattr(hf_api, "_create_repo_origin"): - setattr(hf_api, "_create_repo_origin", hf_api.create_repo) - hf_api.create_repo = _create_repo - huggingface_hub.create_repo = _create_repo - huggingface_hub.hf_api.create_repo = _create_repo - - if not hasattr(hf_api, "_upload_folder_origin"): - setattr(hf_api, "_upload_folder_origin", hf_api.upload_folder) - hf_api.upload_folder = _upload_folder - huggingface_hub.upload_folder = _upload_folder - huggingface_hub.hf_api.upload_folder = _upload_folder - - if not hasattr(hf_api, "_upload_file_origin"): - setattr(hf_api, "_upload_file_origin", hf_api.upload_file) - hf_api.upload_file = _upload_file - huggingface_hub.upload_file = _upload_file - huggingface_hub.hf_api.upload_file = _upload_file - repocard.upload_file = _upload_file - - if not hasattr(hf_api, "_create_commit_origin"): - setattr(hf_api, "_create_commit_origin", hf_api.create_commit) - hf_api.create_commit = _create_commit - huggingface_hub.create_commit = _create_commit - huggingface_hub.hf_api.create_commit = _create_commit - - if not hasattr(repocard.RepoCard, "_validate_origin"): - setattr(repocard.RepoCard, "_validate_origin", repocard.RepoCard.validate) - repocard.RepoCard.validate = lambda *args, **kwargs: None - setattr(repocard.RepoCard, "_load_origin", repocard.RepoCard.load) - repocard.RepoCard.load = MethodType(_repocard_load, repocard.RepoCard) - - -def unpatch_hf_hub() -> None: - """Undo :func:`patch_hf_hub` and restore the original huggingface_hub functions. - - Without ``huggingface_hub`` nothing can have been patched, so this is a - silent no-op rather than an error (patch/unpatch stay symmetric in - minimal environments). - """ - try: - import huggingface_hub - from huggingface_hub import hf_api, repocard - except ImportError: - return - - if hasattr(repocard.RepoCard, "_validate_origin"): - repocard.RepoCard.validate = repocard.RepoCard._validate_origin - delattr(repocard.RepoCard, "_validate_origin") - if hasattr(repocard.RepoCard, "_load_origin"): - repocard.RepoCard.load = repocard.RepoCard._load_origin - delattr(repocard.RepoCard, "_load_origin") - - if hasattr(hf_api, "_hf_hub_download_origin"): - origin = hf_api._hf_hub_download_origin - huggingface_hub.file_download.hf_hub_download = origin - huggingface_hub.hf_hub_download = origin - delattr(hf_api, "_hf_hub_download_origin") - - if hasattr(hf_api, "_file_exists_origin"): - origin = hf_api._file_exists_origin - hf_api.file_exists = origin - huggingface_hub.file_exists = origin - huggingface_hub.hf_api.file_exists = origin - delattr(hf_api, "_file_exists_origin") - - if hasattr(hf_api, "_whoami_origin"): - origin = hf_api._whoami_origin - hf_api.whoami = origin - huggingface_hub.whoami = origin - huggingface_hub.hf_api.whoami = origin - delattr(hf_api, "_whoami_origin") - - if hasattr(hf_api, "_create_repo_origin"): - origin = hf_api._create_repo_origin - hf_api.create_repo = origin - huggingface_hub.create_repo = origin - huggingface_hub.hf_api.create_repo = origin - delattr(hf_api, "_create_repo_origin") - - if hasattr(hf_api, "_upload_folder_origin"): - origin = hf_api._upload_folder_origin - hf_api.upload_folder = origin - huggingface_hub.upload_folder = origin - huggingface_hub.hf_api.upload_folder = origin - delattr(hf_api, "_upload_folder_origin") - - if hasattr(hf_api, "_upload_file_origin"): - origin = hf_api._upload_file_origin - hf_api.upload_file = origin - huggingface_hub.upload_file = origin - huggingface_hub.hf_api.upload_file = origin - repocard.upload_file = origin - delattr(hf_api, "_upload_file_origin") - - if hasattr(hf_api, "_create_commit_origin"): - origin = hf_api._create_commit_origin - hf_api.create_commit = origin - huggingface_hub.create_commit = origin - huggingface_hub.hf_api.create_commit = origin - delattr(hf_api, "_create_commit_origin") - - -@contextlib.contextmanager -def hf_hub_patch_scope(): - """Apply :func:`patch_hf_hub` for the duration of the ``with`` block. - - Nesting is safe: an already-applied patch (including one applied by the - legacy ``modelscope`` patcher) is reused, and only the outermost scope - restores the original functions. - - Raises: - ImportError: when ``huggingface_hub`` is not installed. - """ - _require_hf_hub() - from huggingface_hub import hf_api - - if hasattr(hf_api, "_hf_hub_download_origin"): - yield - return - patch_hf_hub() - try: - yield - finally: - unpatch_hf_hub() diff --git a/tests/test_hf_hub_patcher.py b/tests/test_hf_hub_patcher.py deleted file mode 100644 index b8af76a..0000000 --- a/tests/test_hf_hub_patcher.py +++ /dev/null @@ -1,382 +0,0 @@ -"""Network-free tests for the huggingface_hub monkey-patcher in ``modelscope_hub.compat``. - -The patch functions are exercised with mocked ``HubApi``/``LegacyHubApi``/ -``compat`` downloads; the tests run only when the optional ``huggingface_hub`` -package is installed. -""" - -from __future__ import annotations - -from unittest import mock - -import pytest - -from modelscope_hub import FileInfo -from modelscope_hub.compat import hf_hub_patcher -from modelscope_hub.compat.hf_hub_patcher import ( - HfCommitInfo, - HfRepoUrl, - _create_commit, - _create_repo, - _file_exists, - _hf_hub_download, - _upload_file, - _upload_folder, - _whoami, - hf_hub_patch_scope, - patch_hf_hub, - unpatch_hf_hub, -) - -hf_api_mod = pytest.importorskip("huggingface_hub.hf_api") -hf_top = pytest.importorskip("huggingface_hub") -repocard = pytest.importorskip("huggingface_hub.repocard") - - -@pytest.fixture -def patched_hub(): - patch_hf_hub() - yield - unpatch_hf_hub() - - -# --------------------------------------------------------------------------- -# _hf_hub_download -# --------------------------------------------------------------------------- -def test_hf_hub_download_routes_model_file_download(): - with mock.patch("modelscope_hub.compat.model_file_download", return_value="/cache/x.json") as dl: - result = _hf_hub_download( - "foo/bar", - "config.json", - revision="main", - cache_dir="/tmp/cache", - local_dir="/tmp/out", - token="ms-token", - local_files_only=False, - ) - - assert result == "/cache/x.json" - assert dl.call_args.args == ("foo/bar", "config.json") - kwargs = dl.call_args.kwargs - # `main` is normalized to `master`; string tokens pass through. - assert kwargs["revision"] == "master" - assert kwargs["cache_dir"] == "/tmp/cache" - assert kwargs["local_dir"] == "/tmp/out" - assert kwargs["token"] == "ms-token" - assert kwargs["local_files_only"] is False - - -def test_hf_hub_download_dataset_routes_dataset_download(): - with mock.patch("modelscope_hub.compat.dataset_file_download", return_value="/cache/x.bin") as dl: - result = _hf_hub_download("foo/data", "train.bin", repo_type="dataset", revision="v2") - - assert result == "/cache/x.bin" - assert dl.call_args.args == ("foo/data", "train.bin") - assert dl.call_args.kwargs["revision"] == "v2" - - -def test_hf_hub_download_rejects_unsupported_repo_type(): - with pytest.raises(ValueError, match="repo_type=space"): - _hf_hub_download("foo/bar", "config.json", repo_type="space") - - -def test_hf_hub_download_bool_token_is_dropped(): - with mock.patch("modelscope_hub.compat.model_file_download", return_value="/cache/x") as dl: - _hf_hub_download("foo/bar", "config.json", token=True) - - assert dl.call_args.kwargs["token"] is None - - -def test_hf_hub_download_subfolder_is_joined(): - with mock.patch("modelscope_hub.compat.model_file_download", return_value="/cache/x") as dl: - _hf_hub_download("foo/bar", "nested.txt", subfolder="configs") - - assert dl.call_args.args == ("foo/bar", "configs/nested.txt") - - -# --------------------------------------------------------------------------- -# _file_exists / _whoami -# --------------------------------------------------------------------------- -def test_file_exists_uses_listing(): - fake = mock.MagicMock() - fake.list_repo_files.return_value = [ - FileInfo(path="README.md", size=1), - FileInfo(path="config.json", size=2), - ] - with mock.patch("modelscope_hub.HubApi", return_value=fake): - assert _file_exists("foo/bar", "config.json", revision="main") - assert not _file_exists("foo/bar", "missing.txt", revision="main") - - # `main` is normalized to `master`. - call = fake.list_repo_files.call_args - assert call.args == ("foo/bar", "model") - assert call.kwargs["revision"] == "master" - - -def test_whoami_returns_name_dict(): - fake = mock.MagicMock() - fake.whoami.return_value.username = "alice" - with mock.patch("modelscope_hub.HubApi", return_value=fake): - assert _whoami(token="ms-token") == {"name": "alice"} - - fake.whoami.assert_called_once() - - -# --------------------------------------------------------------------------- -# _create_repo / _upload_file / _upload_folder / _create_commit -# --------------------------------------------------------------------------- -def test_create_repo_returns_hf_repo_url(): - with mock.patch("modelscope_hub.compat.LegacyHubApi") as legacy: - result = _create_repo("foo/bar", token="ms-token", private=True) - - legacy.assert_called_once_with(token="ms-token") - legacy.return_value.create_repo.assert_called_once_with("foo/bar", visibility="private") - - # Legacy duck-typing shape: a str URL plus repo_id / repo_type attributes - # (hf's own RepoUrl rejects non-HF hosts on recent releases). - assert isinstance(result, HfRepoUrl) - assert "foo/bar" in result - assert result.repo_id == "foo/bar" - assert result.repo_type == "model" - - -def test_upload_file_forwards_to_hub_upload(): - fake = mock.MagicMock() - with mock.patch("modelscope_hub.HubApi", return_value=fake): - _upload_file( - path_or_fileobj="/tmp/w.safetensors", - path_in_repo="w.safetensors", - repo_id="foo/bar", - token="ms-token", - revision="main", - commit_message="add weights", - commit_description="tuned", - ) - - fake.upload_file.assert_called_once_with( - "foo/bar", - "model", - "/tmp/w.safetensors", - "w.safetensors", - commit_message="add weights", - commit_description="tuned", - revision="master", - ) - - -def test_upload_folder_returns_legacy_commit_info_shape(): - fake = mock.MagicMock() - with mock.patch("modelscope_hub.HubApi", return_value=fake): - result = _upload_folder( - repo_id="foo/bar", - folder_path="/tmp/out", - commit_message="sync", - ignore_patterns=["*.tmp"], - revision="v2", - ) - - fake.upload_folder.assert_called_once_with( - "foo/bar", - "model", - "/tmp/out", - path_in_repo="", - commit_message="sync", - commit_description=None, - revision="v2", - ignore_patterns=["*.tmp"], - allow_patterns=None, - ) - assert isinstance(result, HfCommitInfo) - assert result.commit_url.endswith("/models/foo/bar/files") - assert result.commit_message == "sync" - assert result.oid is None - - -@pytest.mark.parametrize("patch_fn", [_upload_file, _upload_folder]) -def test_upload_rejects_non_model_repo_type(patch_fn): - kwargs = ( - dict(path_or_fileobj="/tmp/a", path_in_repo="a", repo_id="foo/data") - if patch_fn is _upload_file - else dict(repo_id="foo/data", folder_path="/tmp/out") - ) - with mock.patch("modelscope_hub.HubApi"): - with pytest.raises(ValueError, match="repo_type=dataset"): - patch_fn(repo_type="dataset", **kwargs) - - -def test_create_commit_rejects_non_add_operations(): - from huggingface_hub import CommitOperationDelete - - fake = mock.MagicMock() - with mock.patch("modelscope_hub.HubApi", return_value=fake): - with pytest.raises(ValueError, match="only supports Add"): - _create_commit("foo/bar", [CommitOperationDelete("old.txt", False)], commit_message="x") - - fake.upload_file.assert_not_called() - - -def test_create_commit_maps_each_add_operation(tmp_path): - from huggingface_hub import CommitOperationAdd - - # hf >= 1.x verifies that str/Path sources exist on the local filesystem. - a = tmp_path / "a.txt" - a.write_text("a", encoding="utf-8") - b = tmp_path / "b.txt" - b.write_text("b", encoding="utf-8") - - fake = mock.MagicMock() - with mock.patch("modelscope_hub.HubApi", return_value=fake): - _create_commit( - "foo/bar", - [ - CommitOperationAdd(path_in_repo="a.txt", path_or_fileobj=str(a)), - CommitOperationAdd(path_in_repo="b.txt", path_or_fileobj=b"bytes"), - ], - commit_message="batch", - revision="main", - ) - - assert fake.upload_file.call_count == 2 - first, second = fake.upload_file.call_args_list - assert first.args == ("foo/bar", "model", str(a), "a.txt") - assert second.args == ("foo/bar", "model", b"bytes", "b.txt") - assert first.kwargs["commit_message"] == "batch" - assert first.kwargs["revision"] == "master" - - -def test_create_commit_rejects_non_model_repo_type(tmp_path): - from huggingface_hub import CommitOperationAdd - - a = tmp_path / "a.txt" - a.write_text("a", encoding="utf-8") - - with mock.patch("modelscope_hub.HubApi"): - with pytest.raises(ValueError, match="repo_type=dataset"): - _create_commit( - "foo/data", - [CommitOperationAdd(path_in_repo="a.txt", path_or_fileobj=str(a))], - commit_message="x", - repo_type="dataset", - ) - - -# --------------------------------------------------------------------------- -# repocard -# --------------------------------------------------------------------------- -def test_repocard_load_routes_to_compat_download(tmp_path): - from huggingface_hub.repocard import RepoCard - - with mock.patch( - "modelscope_hub.compat.model_file_download", - return_value=str(tmp_path / "README.md"), - ) as dl: - (tmp_path / "README.md").write_text("---\nlicense: apache-2.0\n---\nhello", encoding="utf-8") - card = hf_hub_patcher._repocard_load(RepoCard, "foo/bar") - - assert dl.call_args.args == ("foo/bar", "README.md") - assert card.data.tags == [] - - -# --------------------------------------------------------------------------- -# Patch round-trip -# --------------------------------------------------------------------------- -def test_patch_roundtrip_restores_all_symbols(): - original = hf_top.hf_hub_download - original_file_download = hf_top.file_download.hf_hub_download - - patch_hf_hub() - try: - assert hf_top.hf_hub_download is hf_hub_patcher._hf_hub_download - assert hf_top.file_exists is hf_hub_patcher._file_exists - assert hf_top.whoami is hf_hub_patcher._whoami - assert hf_top.create_repo is hf_hub_patcher._create_repo - assert hf_top.upload_file is hf_hub_patcher._upload_file - assert hf_top.upload_folder is hf_hub_patcher._upload_folder - assert hf_top.create_commit is hf_hub_patcher._create_commit - # repocard class patches are active while patched. - assert repocard.RepoCard.__dict__["load"].__func__ is hf_hub_patcher._repocard_load - finally: - unpatch_hf_hub() - - assert hf_top.hf_hub_download is original - assert hf_top.file_download.hf_hub_download is original_file_download - assert not hasattr(hf_api_mod, "_hf_hub_download_origin") - assert not hasattr(hf_api_mod, "_file_exists_origin") - assert not hasattr(hf_api_mod, "_whoami_origin") - assert not hasattr(hf_api_mod, "_create_repo_origin") - assert not hasattr(hf_api_mod, "_upload_folder_origin") - assert not hasattr(hf_api_mod, "_upload_file_origin") - assert not hasattr(hf_api_mod, "_create_commit_origin") - assert not hasattr(repocard.RepoCard, "_validate_origin") - assert not hasattr(repocard.RepoCard, "_load_origin") - - -def test_patch_is_idempotent(patched_hub): - first = hf_top.hf_hub_download - patch_hf_hub() - assert hf_top.hf_hub_download is first - - -def test_patch_respects_legacy_markers(): - """A pre-existing legacy patch (same marker names) must not be overwritten.""" - fake_legacy = mock.MagicMock(name="legacy_download") - with mock.patch.object(hf_top, "hf_hub_download", fake_legacy): - setattr(hf_api_mod, "_hf_hub_download_origin", fake_legacy) - try: - patch_hf_hub() - assert hf_top.hf_hub_download is fake_legacy - # The already-patched function stays; markers unchanged. - assert hf_api_mod._hf_hub_download_origin is fake_legacy - finally: - delattr(hf_api_mod, "_hf_hub_download_origin") - - -def test_patch_scope_restores_on_exit(): - original = hf_top.hf_hub_download - - with hf_hub_patch_scope(): - assert hf_top.hf_hub_download is hf_hub_patcher._hf_hub_download - with hf_hub_patch_scope(): - assert hf_top.hf_hub_download is hf_hub_patcher._hf_hub_download - # The inner scope must not undo the outer patch. - assert hf_top.hf_hub_download is hf_hub_patcher._hf_hub_download - - assert hf_top.hf_hub_download is original - - -def test_module_has_no_transformers_import(): - """The hub-side patcher must stay importable without transformers.""" - import inspect - - source = inspect.getsource(hf_hub_patcher) - assert "import transformers" not in source - assert "from transformers" not in source - - -# --------------------------------------------------------------------------- -# huggingface_hub is not a declared dependency: fail at call time -# --------------------------------------------------------------------------- -def test_patch_raises_actionable_error_without_hf_hub(monkeypatch): - """No `hf` extra exists: patch_hf_hub fails with install instructions.""" - import sys - - monkeypatch.setitem(sys.modules, "huggingface_hub", None) - with pytest.raises(ImportError, match="pip install huggingface_hub"): - patch_hf_hub() - - -def test_unpatch_is_noop_without_hf_hub(monkeypatch): - """Nothing can have been patched, so unpatch stays a silent no-op.""" - import sys - - monkeypatch.setitem(sys.modules, "huggingface_hub", None) - unpatch_hf_hub() # must not raise - - -def test_patch_scope_raises_actionable_error_without_hf_hub(monkeypatch): - import sys - - monkeypatch.setitem(sys.modules, "huggingface_hub", None) - with pytest.raises(ImportError, match="pip install huggingface_hub"): - with hf_hub_patch_scope(): - pass