From 8350a31fba7da4ce23e96093238f0b2e2ad8d52d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Tue, 1 Sep 2026 18:56:12 +0800 Subject: [PATCH 01/22] [Feature] Full OpenAPI coverage for MCP and Studios, with token permission tiers Audited the live OpenAPI document (1.1.0+master.20260813) against the client: of the 23 operations under the MCP and Studios tags, 8 were unimplemented and several implemented ones had drifted from the spec. Closes the gap and adds a guard so the next drift fails the suite instead of going unnoticed. Feature - Studios: list_studios, list_studio_hardware / _base_images / _sdk_versions, and the four plaintext-variable operations; exposed as `ms-hub studio list`, `studio variable`, `studio hardware|base-images|sdk-versions`, and `ms-hub list --repo-type studio`. - MCP: HubApi.list_operational_mcp_servers() plus `ms-hub mcp list --hosted`, which also unblocks the umbrella SDK delegating its own MCPApi here. - Studio visibility gains `protected` (app public, code hidden), previously inexpressible because visibility was folded into a `private` bool. - RepoInfo now carries the Studio runtime fields (sdk_type, sdk_version, base_image, hardware, mcp_support, runtime) instead of discarding them. - Token permission tiers: TokenScope annotations, a read-scoped token can log in (warning that writes need a higher tier), and a rejected write names the tier it requires. Fix - The compat layer forwarded `token` / `endpoint` into request bodies, so `HubApi().update_studio_settings(..., token=...)` serialised the caller's API token into the PATCH body. All Studio shims now split control kwargs from business fields. - `HubApi().get_studio_logs(..., token=...)` raised TypeError and could never succeed; a per-call `token=` was silently ignored on every Studio shim. - `cover_image` was renamed to `coverImage` on the way out and dropped by the server, so Studio cover images never applied. - 403 now separates insufficient permission from an exhausted quota (new QuotaExceededError, deliberately not retryable); 409 maps to AlreadyExistsError; the OpenAPI string error codes are recognised at all. - `get_studio` demanded a token for spaces the spec says are public. - `studio logs` read a `total` field the response never had, so its pagination footer never printed. Enhance - MCP discovery sends PUT first -- the only verb the spec defines -- and remembers which verb a deployment serves, so the loser is tried at most once. - Listing a Studio owner asks for every status: the endpoint keeps filtering to running spaces even with an owner set, so "list my spaces" returned nothing. Documentation - README: new Studio/MCP command surface, a Token permission levels section, and the SDK method reference. Testing - Vendored the spec under tests/data/ and check it against OPERATION_REGISTRY. - 723 unit tests (881 in mock mode) and 28 integration tests against the live service. --- README.md | 149 +- src/modelscope_hub/__init__.py | 6 +- src/modelscope_hub/_openapi.py | 492 +- src/modelscope_hub/api.py | 332 +- src/modelscope_hub/cli/mcp.py | 31 + src/modelscope_hub/cli/repo.py | 14 + src/modelscope_hub/cli/studio.py | 326 +- src/modelscope_hub/compat/hub_api.py | 99 +- src/modelscope_hub/constants.py | 51 + src/modelscope_hub/errors.py | 63 + src/modelscope_hub/types.py | 29 +- src/modelscope_hub/version.py | 2 +- tests/.env.example | 9 + tests/cli/test_mcp.py | 41 + tests/cli/test_openapi.py | 140 +- tests/cli/test_openapi_studio.py | 360 + tests/cli/test_repo.py | 20 +- tests/cli/test_studio.py | 269 + tests/data/README.md | 29 + tests/data/openapi.json | 10183 +++++++++++++++++++++++++ tests/integration/test_openapi.py | 174 + tests/test_compat_studio.py | 300 + tests/test_errors_openapi_codes.py | 165 + tests/test_login_scoped_token.py | 203 + tests/test_openapi_coverage.py | 128 + 25 files changed, 13476 insertions(+), 139 deletions(-) create mode 100644 tests/cli/test_openapi_studio.py create mode 100644 tests/data/README.md create mode 100644 tests/data/openapi.json create mode 100644 tests/test_compat_studio.py create mode 100644 tests/test_errors_openapi_codes.py create mode 100644 tests/test_login_scoped_token.py create mode 100644 tests/test_openapi_coverage.py diff --git a/README.md b/README.md index 7c27b19..103c228 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,19 @@ The official Python SDK & CLI for [ModelScope Hub](https://modelscope.cn) — do ## News +**v0.4.0** (2026-09-01) +- **Feature**: full OpenAPI coverage for MCP and Studios — `ms-hub studio list`, `ms-hub list --repo-type studio`, `ms-hub studio variable` (plaintext environment variables), and `ms-hub studio hardware` / `base-images` / `sdk-versions` so valid `--hardware` / `--base-image` / `--sdk-version` values no longer have to be guessed; `ms-hub mcp list --hosted` and `HubApi.list_operational_mcp_servers()` +- **Feature**: Studio `--visibility protected` (app public, code repository hidden), previously inexpressible; `RepoInfo` now carries the Studio runtime fields (`sdk_type`, `sdk_version`, `base_image`, `hardware`, `mcp_support`, `runtime`) instead of discarding them +- **Feature**: permission-tiered token support — a read-only token can now `login` (with a warning that write operations need a higher tier), and a rejected write reports which tier it requires +- **Fix**: the compat layer no longer forwards `token` / `endpoint` into request bodies — `HubApi().update_studio_settings(..., token=...)` was serialising your **API token into the `PATCH` body** +- **Fix**: `HubApi().get_studio_logs(..., token=...)` raised `TypeError` and could never succeed; a per-call `token=` on the Studio compat methods was silently ignored +- **Fix**: `cover_image` was renamed to `coverImage` on the way out and silently dropped by the server, so Studio cover images never applied (`ms-hub studio settings --cover-image`, `create_repo(cover_image=...)`) +- **Fix**: HTTP 403 now distinguishes insufficient permission (`PermissionDeniedError`) from an exhausted quota (new `QuotaExceededError`, never retried); 409 maps to `AlreadyExistsError`; the OpenAPI string error codes are recognised at last +- **Fix**: `ms-hub info --repo-type studio` works anonymously for public spaces; `ms-hub studio logs` prints its pagination footer again (it read a field the response never had) +- **Enhance**: MCP discovery sends `PUT` first — the only verb the spec defines — instead of spending a wasted round trip on a `GET` 404 every call; whichever verb a deployment serves is remembered, so the loser is attempted at most once +- **Enhance**: `ms-hub studio list --owner` / `ms-hub list --repo-type studio --owner` now ask for every status, because the endpoint keeps filtering to running spaces even with an owner set and "list my spaces" therefore answered with nothing +- **Quality**: the OpenAPI specification is vendored under `tests/data/` and checked against an operation registry, so a newly published MCP/Studios endpoint now fails the suite by name + **v0.2.0** (2026-08-01) - **Breaking**: `HubApi.login()` now raises `NetworkError` / `ServerError` / `RequestTimeoutError` for non-authentication failures instead of always raising `AuthenticationError` — widen `except AuthenticationError` to `HubError` if you catch *any* login failure - **Fix**: network, timeout and 5xx errors during login are no longer misreported as an invalid token; a failed login no longer deletes stored credentials; bare and upper-case endpoints are accepted (e.g. `--endpoint modelscope.ai`) @@ -264,6 +277,36 @@ ms-hub login --token $MY_TOKEN # non-interactive |--------|-------------| | `--token TOKEN` | API token; prompted interactively if omitted | +#### Token permission levels + +Tokens are issued with one of three permission levels, set when you create the +token at [modelscope.cn/my/myaccesstoken](https://modelscope.cn/my/myaccesstoken): + +| Level | Can do | +|-------|--------| +| `read` | Browse and download; list your Studios, secrets and variables | +| `write` | Everything above, plus create/upload/deploy and change settings | +| `admin` | Everything above, plus administrative operations | + +What this means in practice: + +- **A read-only token can log in.** `ms-hub login` succeeds and prints a warning + that git and session credentials were not issued, so pushes and uploads will + need a `write` token. Downloads and browsing work normally. +- **A rejected write is explicit.** Attempting a write with a read-only token + fails with `[E3002] Permission denied` and a suggestion naming the level the + operation requires, rather than leaving you to guess. +- **Quota exhaustion is not a permission problem.** The service reports both as + HTTP 403; the SDK separates them into `PermissionDeniedError` and + `QuotaExceededError` (the latter is never retried, since retrying an exhausted + quota only delays the error). +- **Public reads never fail because of a token.** For public endpoints the SDK + retries once without credentials, so a read-scoped or stale token cannot hide + data any anonymous caller can see. + +> The SDK cannot tell you which level *your* token has: the API does not report +> it. Check the token settings page if you are unsure. + ### `ms-hub whoami` Show the user associated with the current token. @@ -400,6 +443,7 @@ ms-hub create my-org/my-model --repo-type model --visibility private ms-hub create my-org/demo --repo-type studio --sdk-type gradio ms-hub info my-org/my-model --repo-type model ms-hub list --repo-type model --owner my-org --page-size 20 +ms-hub list --repo-type studio --owner my-org ms-hub delete my-org/my-model --repo-type model --yes ``` @@ -414,7 +458,7 @@ ms-hub delete my-org/my-model --repo-type model --yes |-------------------|----------|-------------| | `repo_id` | yes | Repository identifier | | `--repo-type` | yes | `model`, `dataset`, `studio`, or `skill` | -| `--visibility` | no | `public`, `private`, or `internal` | +| `--visibility` | no | `public`, `private` or `internal`. For Studios also `protected` (app public, code repository hidden) | | `--license` | no | SPDX license identifier (e.g. `apache-2.0`) | | `--chinese-name` | no | Display name in Chinese | | `--description` | no | Repository description | @@ -453,15 +497,24 @@ ms-hub stop my-org/chat-demo --repo-type studio -### `ms-hub secret` +### `ms-hub secret` / `ms-hub studio variable` -Manage secrets for Studio spaces (studio only, `--repo-type` defaults to `studio`). +Manage a Studio's environment variables. Two commands, because the disclosure +differs: a **secret**'s value is never returned by the API, a **variable**'s value +is publicly visible. ```bash +# Secrets — values are write-only ms-hub secret add my-org/demo API_KEY sk-xxx ms-hub secret list my-org/demo ms-hub secret update my-org/demo API_KEY sk-new ms-hub secret delete my-org/demo API_KEY --yes + +# Plaintext variables — values are readable by anyone +ms-hub studio variable add my-org/demo MODEL_NAME Qwen2.5-7B +ms-hub studio variable list my-org/demo +ms-hub studio variable update my-org/demo MODEL_NAME Qwen2.5-14B +ms-hub studio variable delete my-org/demo MODEL_NAME --yes ```
@@ -469,12 +522,64 @@ ms-hub secret delete my-org/demo API_KEY --yes | Subcommand | Arguments | Description | |------------|-----------|-------------| -| `add` | `repo_id key value` | Add a new secret | -| `list` | `repo_id` | List all secret keys | -| `update` | `repo_id key value` | Update a secret value | -| `delete` | `repo_id key [--yes]` | Delete a secret | +| `add` | `repo_id key value` | Add a new entry | +| `list` | `repo_id` | List entries (`secret` shows keys only; `variable` shows keys and values) | +| `update` | `repo_id key value` | Update a value | +| `delete` | `repo_id key [--yes]` | Delete an entry | + +`ms-hub secret` accepts `--repo-type` (default: `studio`, currently the only supported type). -All subcommands accept `--repo-type` (default: `studio`, currently the only supported type). +> **Put anything sensitive in a secret.** A plaintext variable's value is public. + +
+ +### `ms-hub studio` + +Browse Studio spaces and the runtime resources they can be configured with. + +```bash +# Browse +ms-hub studio list --search chat --sort likes --page-size 20 +ms-hub studio list --owner my-org --status all +ms-hub studio list --mcp-support --hardware-type xgpu + +# Discover valid values for --hardware / --base-image / --sdk-version +ms-hub studio hardware --sdk-type gradio +ms-hub studio hardware --studio my-org/demo +ms-hub studio base-images +ms-hub studio sdk-versions --sdk-type gradio + +# Runtime management (also available as the top-level deploy/stop/logs/settings) +ms-hub studio deploy my-org/demo +ms-hub studio logs my-org/demo --type build +ms-hub studio settings my-org/demo --visibility protected +``` + +
+Subcommands + +| Subcommand | Arguments | Key Options | +|------------|-----------|-------------| +| `list` | — | `--search`, `--owner`, `--sort {default,last_modified,view_num,likes}`, `--status {running,all}`, `--mcp-support`/`--no-mcp-support`, `--hardware-type {xgpu,amd}`, `--page`, `--page-size` | +| `hardware` | — | `--sdk-type`, `--studio owner/name` | +| `base-images` | — | — | +| `sdk-versions` | — | `--sdk-type` (default `gradio`; only gradio publishes versions) | +| `deploy` / `stop` | `studio_id` | — | +| `logs` | `studio_id` | `--type {run,build}`, `--keyword`, `--page`, `--page-size` (max 500) | +| `settings` | `studio_id [key=val...]` | `--display-name`, `--description`, `--license`, `--cover-image`, `--sdk-type`, `--sdk-version`, `--base-image`, `--hardware`, `--visibility`, `--private`/`--public` | +| `secret` | see above | — | +| `variable` | see above | — | + +`--visibility` accepts three values: `public` (code and app public), `protected` +(app public, code repository hidden) and `private`. `--private` / `--public` +remain as shorthands. + +Paid hardware tiers are selected as `--hardware paid/`; run +`ms-hub studio hardware` to see the available identifiers. + +`--status` defaults to `all` whenever `--owner` is given, so listing your own +spaces includes stopped ones. Without `--owner` the server's own default applies +(running spaces only).
@@ -484,6 +589,7 @@ Manage MCP (Model Context Protocol) servers. ```bash ms-hub mcp list --search weather --page-size 10 +ms-hub mcp list --hosted # only your own hosted servers, with live URLs ms-hub mcp info my-org/weather-mcp ms-hub mcp deploy my-org/weather-mcp ms-hub mcp undeploy my-org/weather-mcp @@ -494,11 +600,17 @@ ms-hub mcp undeploy my-org/weather-mcp | Subcommand | Arguments | Key Options | |------------|-----------|-------------| -| `list` | — | `--search`, `--page`, `--page-size` | +| `list` | — | `--search`, `--hosted`, `--page`, `--page-size` | | `info` | `server_id` | — | -| `deploy` | `server_id` | — | +| `deploy` | `server_id` | `--transport-type`, `--expiration-minutes`, `--auth-check`, `--env KEY=VALUE` | | `undeploy` | `server_id` | — | +`--hosted` lists the servers you currently have hosted along with their +`operational_urls`. It takes no search or paging options, because the underlying +endpoint accepts none. + +> Discovery is capped at `page * page_size <= 100` by the service. + ### `ms-hub cache` @@ -598,7 +710,7 @@ api = HubApi(token="...", endpoint="https://modelscope.ai") | | `whoami()` | Get current user info | | **Repo** | `create_repo(repo_id, repo_type, ...)` | Create a repository | | | `get_repo(repo_id, repo_type)` | Get repository metadata | -| | `list_repos(repo_type, ...)` | Paginated listing | +| | `list_repos(repo_type, ...)` | Paginated listing (`model`, `dataset`, `studio`, `skill`, `mcp`) | | | `delete_repo(repo_id, repo_type)` | Delete a repository *(deprecated — see note below)* | | | `repo_exists(repo_id, repo_type)` | Check existence | | **Files** | `upload_file(repo_id, repo_type, local, remote)` | Upload a single file | @@ -614,10 +726,18 @@ api = HubApi(token="...", endpoint="https://modelscope.ai") | | `get_repo_logs(repo_id, ...)` | Fetch logs | | | `update_repo_settings(repo_id, repo_type, ...)` | Update settings | | **Secrets** | `add_secret(repo_id, key, value)` | Add a secret | -| | `list_secrets(repo_id)` | List secrets | +| | `list_secrets(repo_id)` | List secret keys (values are never returned) | | | `update_secret(repo_id, key, value)` | Update a secret | | | `delete_secret(repo_id, key)` | Delete a secret | +| **Variables** | `add_variable(repo_id, key, value)` | Add a plaintext variable *(value is public)* | +| | `list_variables(repo_id)` | List variables with their values | +| | `update_variable(repo_id, key, value)` | Update a variable | +| | `delete_variable(repo_id, key)` | Delete a variable | +| **Studio resources** | `list_studio_hardware(...)` | Hardware tiers a Studio can run on | +| | `list_studio_base_images()` | Available base images | +| | `list_studio_sdk_versions(...)` | Available SDK versions | | **MCP** | `list_mcp_servers(...)` | List available MCP servers | +| | `list_operational_mcp_servers()` | List your hosted servers, with live URLs | | | `get_mcp_server(server_id)` | Get server details | | | `deploy_mcp_server(server_id)` | Deploy an MCP server | | | `undeploy_mcp_server(server_id)` | Undeploy an MCP server | @@ -628,6 +748,11 @@ api = HubApi(token="...", endpoint="https://modelscope.ai") > - `delete_repo` is deprecated for security reasons (emits `DeprecationWarning`). Will be restored with token-scoped auth. Use the [web console](https://modelscope.cn) instead. > - `delete_files` requires cookie-based session auth; API tokens (`ms-...`) may receive a 401 error. +> **Token permission levels:** every write method needs a token issued with +> `write` permission or higher. A rejected write raises `PermissionDeniedError` +> whose `suggestion` names the required level; an exhausted quota raises +> `QuotaExceededError` instead. See [Token permission levels](#token-permission-levels). + --- diff --git a/src/modelscope_hub/__init__.py b/src/modelscope_hub/__init__.py index 8b3641a..342d70d 100644 --- a/src/modelscope_hub/__init__.py +++ b/src/modelscope_hub/__init__.py @@ -12,7 +12,7 @@ from ._download import ProgressCallback, TqdmCallback from .api import HubApi from .config import HubConfig, get_default_config, set_default_config -from .constants import License, RepoType, Visibility +from .constants import License, RepoType, StudioVisibility, TokenScope, Visibility from .errors import ( APIError, AuthenticationError, @@ -28,6 +28,7 @@ NotSupportedError, PermissionDeniedError, PermissionError, + QuotaExceededError, RateLimitError, RequestTimeoutError, ServerError, @@ -58,6 +59,8 @@ # Enums "License", "RepoType", + "StudioVisibility", + "TokenScope", "Visibility", # Progress callbacks "ProgressCallback", @@ -85,6 +88,7 @@ "NotExistError", "NotSupportedError", "PermissionDeniedError", + "QuotaExceededError", "RateLimitError", "RequestTimeoutError", "ServerError", diff --git a/src/modelscope_hub/_openapi.py b/src/modelscope_hub/_openapi.py index 198007e..30592f7 100644 --- a/src/modelscope_hub/_openapi.py +++ b/src/modelscope_hub/_openapi.py @@ -6,11 +6,16 @@ Design goals ------------ -* A single :meth:`OpenAPIClient._request` chokepoint owns transport concerns — +* A single :meth:`OpenAPIClient._send` chokepoint owns transport concerns — URL composition, authentication injection, retry/back-off, error decoding - and ``data`` envelope unwrapping. -* Each of the 25 OpenAPI endpoints maps to exactly one method, named after the - resource it manipulates and grouped by section comments. + and ``data`` envelope unwrapping. :meth:`OpenAPIClient._request` wraps it with + the authorisation policy (required token tier, anonymous fallback) and is what + the endpoint methods call. +* Each OpenAPI endpoint maps to exactly one method, named after the resource it + manipulates and grouped by section comments. :data:`OPERATION_REGISTRY` records + that mapping for the tags this client covers in full, and a test checks it + against the vendored specification so a newly published endpoint cannot go + unnoticed. * Filter-style query parameters (``filter.task=...`` etc.) are accepted as a flat ``filters`` mapping and serialised transparently. """ @@ -27,7 +32,7 @@ import requests from .config import HubConfig, get_default_config -from .constants import API_CONNECT_TIMEOUT, API_MAX_RETRIES, API_TIMEOUT, OPENAPI_PREFIX +from .constants import API_CONNECT_TIMEOUT, API_MAX_RETRIES, API_TIMEOUT, OPENAPI_PREFIX, TokenScope from .errors import ( APIError, AuthenticationError, @@ -48,7 +53,7 @@ ) from .utils.logger import get_logger -__all__ = ["OpenAPIClient"] +__all__ = ["OPERATION_REGISTRY", "OpenAPIClient"] _logger = get_logger("openapi") @@ -65,11 +70,87 @@ RateLimitError, ) +# Where a user checks which permission tier their token was issued with. +_TOKEN_SETTINGS_URL = "https://modelscope.cn/my/myaccesstoken" + +# Closed value sets published by the Studios section of the specification. +# Validating client-side turns a silently-ignored typo into a named error. +_STUDIO_SORTS: tuple[str, ...] = ("default", "last_modified", "view_num", "likes") +_STUDIO_STATUS_FILTERS: tuple[str, ...] = ("running", "all") +_STUDIO_HARDWARE_TYPES: tuple[str, ...] = ("xgpu", "amd") +_STUDIO_SDK_TYPES: tuple[str, ...] = ("gradio", "streamlit", "docker", "static") +_STUDIO_LOG_TYPES: tuple[str, ...] = ("build", "run") + +# ``GET /studios/{owner}/{repo}/logs/{type}`` caps page_size at 500. +_STUDIO_LOG_MAX_PAGE_SIZE = 500 + JSON = dict[str, Any] QueryParams = list[tuple[str, str]] Filters = Mapping[str, str | int | float | bool] | None +def _validate_choice(name: str, value: str | None, allowed: tuple[str, ...]) -> None: + """Reject a value the endpoint's enum does not accept. + + The server ignores an unknown enum value rather than complaining, which turns + a typo in e.g. ``sort`` into silently wrong results. Failing here names the + parameter and lists what it accepts. + """ + if value is not None and value not in allowed: + raise InvalidParameter(f"{name} must be one of {', '.join(allowed)} (got {value!r}).") + + +def _as_wire_bool(value: bool | None) -> str | None: + """Render a tri-state flag the way query strings expect (``true``/``false``).""" + if value is None: + return None + return "true" if value else "false" + + +# --------------------------------------------------------------------------- +# Operation registry +# +# Maps each specification ``operationId`` to the method implementing it and the +# minimum token permission tier it needs. Two jobs: +# +# 1. ``tests/test_openapi_coverage.py`` checks it against the vendored spec, so a +# newly published operation fails the suite by name instead of going unnoticed +# -- which is exactly how the gaps this table closes accumulated. +# 2. It documents the required tier per operation in one place, since the spec +# itself declares no scopes. +# +# Only the tags the SDK claims to cover are listed; the rest are enumerated as +# deferred in that test. +# --------------------------------------------------------------------------- +OPERATION_REGISTRY: dict[str, tuple[str, TokenScope]] = { + # -- MCP ---------------------------------------------------------------- + "listMcpServers": ("list_mcp_servers", TokenScope.READ), + "listOperationalMcpServers": ("list_operational_mcp_servers", TokenScope.READ), + "getMcpServer": ("get_mcp_server", TokenScope.READ), + "deployMcpServer": ("deploy_mcp_server", TokenScope.WRITE), + "undeployMcpServer": ("undeploy_mcp_server", TokenScope.WRITE), + # -- Studios ------------------------------------------------------------ + "listStudios": ("list_studios", TokenScope.READ), + "createStudio": ("create_studio", TokenScope.WRITE), + "getStudio": ("get_studio", TokenScope.READ), + "updateStudioSettings": ("update_studio_settings", TokenScope.WRITE), + "deployStudio": ("deploy_studio", TokenScope.WRITE), + "stopStudio": ("stop_studio", TokenScope.WRITE), + "getStudioLogs": ("get_studio_logs", TokenScope.READ), + "listHardware": ("list_studio_hardware", TokenScope.READ), + "listBaseImages": ("list_studio_base_images", TokenScope.READ), + "listSdkVersions": ("list_studio_sdk_versions", TokenScope.READ), + "listStudioSecrets": ("list_studio_secrets", TokenScope.READ), + "addStudioSecret": ("add_studio_secret", TokenScope.WRITE), + "updateStudioSecret": ("update_studio_secret", TokenScope.WRITE), + "deleteStudioSecret": ("delete_studio_secret", TokenScope.WRITE), + "listStudioVariables": ("list_studio_variables", TokenScope.READ), + "addStudioVariable": ("add_studio_variable", TokenScope.WRITE), + "updateStudioVariable": ("update_studio_variable", TokenScope.WRITE), + "deleteStudioVariable": ("delete_studio_variable", TokenScope.WRITE), +} + + class OpenAPIClient: """Thin, typed wrapper around the public ``/openapi/v1`` endpoints. @@ -102,6 +183,9 @@ def __init__( float(timeout) if timeout is not None else (float(API_CONNECT_TIMEOUT), float(API_TIMEOUT)) ) self._max_retries = int(max_retries) if max_retries is not None else int(API_MAX_RETRIES) + # Tri-state cache for whether this deployment serves ``GET /mcp/servers``. + # ``None`` means "not probed yet"; see :meth:`list_mcp_servers`. + self._mcp_list_supports_get: bool | None = None # ------------------------------------------------------------------ # Lifecycle @@ -267,11 +351,95 @@ def _request( anonymous: bool = False, unwrap: bool = True, timeout: float | None = None, + required_scope: TokenScope | None = None, + anonymous_retry: bool = False, ) -> Any: - """Execute an HTTP request and return the unwrapped ``data`` payload. + """Execute a request, then interpret authorisation failures. + + Wraps :meth:`_send` (which owns transport, retries and envelope + decoding) with the two behaviours that depend on *why* a call was made + rather than *how*: + + *required_scope* + The minimum token permission tier the endpoint needs. Purely + advisory -- the Hub does not publish a token's tier, so nothing can + be pre-validated; on a 403 it turns "permission denied" into a + message that names the missing tier. + *anonymous_retry* + For endpoints whose content is public, retry once without + credentials when the token is rejected. A read-scoped or stale token + must not be able to hide data that any anonymous caller can see. + Never enable this for account-private data: degrading to anonymous + there would answer "empty" or "not found" and bury the real cause. + """ + try: + return self._send( + method, + path, + url=url, + params=params, + json_body=json_body, + data=data, + files=files, + headers=headers, + require_token=require_token, + anonymous=anonymous, + unwrap=unwrap, + timeout=timeout, + ) + except (AuthenticationError, PermissionDeniedError) as exc: + if anonymous_retry and not anonymous and self._resolve_token(): + _logger.debug( + "Credentials rejected for %s %s (%s); retrying anonymously", + method.upper(), + url or path, + exc.__class__.__name__, + ) + return self._send( + method, + path, + url=url, + params=params, + json_body=json_body, + data=data, + files=files, + headers=headers, + require_token=False, + anonymous=True, + unwrap=unwrap, + timeout=timeout, + ) + if required_scope is not None and isinstance(exc, PermissionDeniedError): + # Shadow the class-level suggestion on this instance only, so the + # error code, status, request id and traceback all stay intact. + exc.suggestion = ( + f"This operation requires a token with '{required_scope.value}' permission. " + f"Verify the token's permission level at {_TOKEN_SETTINGS_URL}" + ) + raise - The method centralises authentication, retries on transient errors, - and decoding of the standard ``{"success": ..., "data": ...}`` envelope. + def _send( + self, + method: str, + path: str = "", + *, + url: str | None = None, + params: Mapping[str, Any] | QueryParams | None = None, + json_body: Any | None = None, + data: Any | None = None, + files: Any | None = None, + headers: Mapping[str, str] | None = None, + require_token: bool = True, + anonymous: bool = False, + unwrap: bool = True, + timeout: float | None = None, + ) -> Any: + """Send one logical request and return the unwrapped ``data`` payload. + + Owns the transport concerns only: authentication injection, retries on + transient errors, and decoding of the standard + ``{"success": ..., "data": ...}`` envelope. Authorisation policy lives in + :meth:`_request`, which is what callers should use. When *url* is given, it is used as-is (absolute URL). Otherwise the final URL is derived from *path* via :meth:`_url`. @@ -584,13 +752,129 @@ def update_skill_settings( # ================================================================== # Studios # ================================================================== + def list_studios( + self, + *, + search: str | None = None, + owner: str | None = None, + sort: str | None = None, + page_number: int = 1, + page_size: int = 10, + status: str | None = None, + mcp_support: bool | None = None, + hardware_type: str | None = None, + ) -> JSON: + """``GET /studios`` — list Studio spaces with pagination and filters. + + Note on *status*: the server defaults to ``running`` for a plain search + but to ``all`` when ``owner`` is set. That switch is deliberately left to + the server rather than second-guessed here, so passing nothing yields + whichever default the endpoint considers correct for the query. + + Unlike the other list endpoints this one answers *without* the + ``{"success": ..., "data": ...}`` envelope -- ``studios`` sits at the top + level of the response body. + """ + if page_number * page_size > 3000: + raise InvalidParameter(f"page_number * page_size must be <= 3000 (got {page_number * page_size}).") + _validate_choice("sort", sort, _STUDIO_SORTS) + _validate_choice("status", status, _STUDIO_STATUS_FILTERS) + _validate_choice("hardware_type", hardware_type, _STUDIO_HARDWARE_TYPES) + params = self._merge_params( + { + "search": search, + "owner": owner, + "sort": sort, + "page_number": page_number, + "page_size": page_size, + "status": status, + "mcp_support": _as_wire_bool(mcp_support), + "hardware_type": hardware_type, + } + ) + return self._request( + "GET", + "/studios", + params=params, + require_token=False, + required_scope=TokenScope.READ, + anonymous_retry=True, + ) + + def list_studio_hardware( + self, + *, + sdk_type: str | None = None, + studio: str | None = None, + ) -> JSON: + """``GET /studios/hardware`` — hardware tiers available to the caller. + + Anonymous callers get the default free tier; an authenticated caller also + gets the paid tiers with prices. Pass *studio* (``owner/repo_name``) to + scope the free tiers to what that space may actually use. Paid resources + are selected as ``paid/`` in the ``hardware`` field of + :meth:`create_studio` / :meth:`update_studio_settings`. + """ + _validate_choice("sdk_type", sdk_type, _STUDIO_SDK_TYPES) + params = self._merge_params({"sdk_type": sdk_type, "studio": studio}) + return self._request( + "GET", + "/studios/hardware", + params=params, + require_token=False, + required_scope=TokenScope.READ, + anonymous_retry=True, + ) + + def list_studio_base_images(self) -> JSON: + """``GET /studios/base-images`` — base images available to Studio spaces.""" + return self._request( + "GET", + "/studios/base-images", + require_token=False, + required_scope=TokenScope.READ, + anonymous_retry=True, + ) + + def list_studio_sdk_versions(self, *, sdk_type: str | None = None) -> JSON: + """``GET /studios/sdk-versions`` — SDK versions available to Studio spaces. + + Only ``sdk_type="gradio"`` yields versions; every other SDK type (and + omitting it) returns an empty list. + """ + _validate_choice("sdk_type", sdk_type, _STUDIO_SDK_TYPES) + params = self._merge_params({"sdk_type": sdk_type}) + return self._request( + "GET", + "/studios/sdk-versions", + params=params, + require_token=False, + required_scope=TokenScope.READ, + anonymous_retry=True, + ) + def create_studio(self, payload: CreateStudioPayload | Mapping[str, Any]) -> JSON: """``POST /studios`` — create a new Studio space.""" - return self._request("POST", "/studios", json_body=dict(payload)) + return self._request( + "POST", + "/studios", + json_body=dict(payload), + required_scope=TokenScope.WRITE, + ) def get_studio(self, owner: str, repo_name: str) -> JSON: - """``GET /studios/{owner}/{repo_name}`` — fetch Studio metadata.""" - return self._request("GET", f"/studios/{owner}/{repo_name}") + """``GET /studios/{owner}/{repo_name}`` — fetch Studio metadata. + + Public and experience-public (``protected``) spaces are readable without + credentials, so no token is demanded up front. + """ + return self._request( + "GET", + f"/studios/{owner}/{repo_name}", + require_token=False, + required_scope=TokenScope.READ, + anonymous_retry=True, + ) def deploy_studio( self, @@ -598,16 +882,27 @@ def deploy_studio( repo_name: str, payload: Mapping[str, Any] | None = None, ) -> JSON: - """``POST /studios/{owner}/{repo_name}/deploy`` — trigger a deployment.""" + """``POST /studios/{owner}/{repo_name}/deploy`` — trigger a deployment. + + The specification defines no request body for this operation; *payload* + is forwarded when given only so that callers written against an older + server keep working. + """ return self._request( "POST", f"/studios/{owner}/{repo_name}/deploy", json_body=dict(payload) if payload else None, + required_scope=TokenScope.WRITE, ) def stop_studio(self, owner: str, repo_name: str) -> JSON: """``POST /studios/{owner}/{repo_name}/stop`` — stop a running Studio.""" - return self._request("POST", f"/studios/{owner}/{repo_name}/stop", json_body=None) + return self._request( + "POST", + f"/studios/{owner}/{repo_name}/stop", + json_body=None, + required_scope=TokenScope.WRITE, + ) def get_studio_logs( self, @@ -621,7 +916,14 @@ def get_studio_logs( start_timestamp: int | None = None, end_timestamp: int | None = None, ) -> JSON: - """``GET /studios/{owner}/{repo_name}/logs/{log_type}`` — paginated logs.""" + """``GET /studios/{owner}/{repo_name}/logs/{log_type}`` — paginated logs. + + The response payload carries ``logs``, ``page_num``, ``page_size``, + ``total_count`` and ``total_page_num``. + """ + _validate_choice("log_type", log_type, _STUDIO_LOG_TYPES) + if page_size > _STUDIO_LOG_MAX_PAGE_SIZE: + raise InvalidParameter(f"page_size must be <= {_STUDIO_LOG_MAX_PAGE_SIZE} (got {page_size}).") params = self._merge_params( { "page_num": page_num, @@ -635,11 +937,20 @@ def get_studio_logs( "GET", f"/studios/{owner}/{repo_name}/logs/{log_type}", params=params, + required_scope=TokenScope.READ, ) def list_studio_secrets(self, owner: str, repo_name: str) -> JSON: - """``GET /studios/{owner}/{repo_name}/secrets`` — list configured secrets.""" - return self._request("GET", f"/studios/{owner}/{repo_name}/secrets") + """``GET /studios/{owner}/{repo_name}/secrets`` — list secret keys. + + Only the keys are returned; values are never disclosed. Use + :meth:`list_studio_variables` for the plaintext counterpart. + """ + return self._request( + "GET", + f"/studios/{owner}/{repo_name}/secrets", + required_scope=TokenScope.READ, + ) def add_studio_secret(self, owner: str, repo_name: str, key: str, value: str) -> JSON: """``POST /studios/{owner}/{repo_name}/secrets`` — add a new secret.""" @@ -647,6 +958,7 @@ def add_studio_secret(self, owner: str, repo_name: str, key: str, value: str) -> "POST", f"/studios/{owner}/{repo_name}/secrets", json_body={"key": key, "value": value}, + required_scope=TokenScope.WRITE, ) def update_studio_secret(self, owner: str, repo_name: str, key: str, value: str) -> JSON: @@ -655,6 +967,7 @@ def update_studio_secret(self, owner: str, repo_name: str, key: str, value: str) "PUT", f"/studios/{owner}/{repo_name}/secrets", json_body={"key": key, "value": value}, + required_scope=TokenScope.WRITE, ) def delete_studio_secret(self, owner: str, repo_name: str, key: str) -> JSON: @@ -663,6 +976,53 @@ def delete_studio_secret(self, owner: str, repo_name: str, key: str) -> JSON: "DELETE", f"/studios/{owner}/{repo_name}/secrets", json_body={"key": key}, + required_scope=TokenScope.WRITE, + ) + + # -- Plaintext variables -------------------------------------------- + # Mirrors the secrets block above one-for-one. The only difference is + # disclosure: a variable's value is public, a secret's never is. + def list_studio_variables(self, owner: str, repo_name: str) -> JSON: + """``GET /studios/{owner}/{repo_name}/variables`` — list plaintext variables. + + Both keys and values are returned, because unlike secrets the values of + plaintext variables are publicly visible. + """ + return self._request( + "GET", + f"/studios/{owner}/{repo_name}/variables", + required_scope=TokenScope.READ, + ) + + def add_studio_variable(self, owner: str, repo_name: str, key: str, value: str) -> JSON: + """``POST /studios/{owner}/{repo_name}/variables`` — add a plaintext variable. + + Both key and value are publicly visible; use + :meth:`add_studio_secret` for anything sensitive. + """ + return self._request( + "POST", + f"/studios/{owner}/{repo_name}/variables", + json_body={"key": key, "value": value}, + required_scope=TokenScope.WRITE, + ) + + def update_studio_variable(self, owner: str, repo_name: str, key: str, value: str) -> JSON: + """``PUT /studios/{owner}/{repo_name}/variables`` — overwrite a plaintext variable.""" + return self._request( + "PUT", + f"/studios/{owner}/{repo_name}/variables", + json_body={"key": key, "value": value}, + required_scope=TokenScope.WRITE, + ) + + def delete_studio_variable(self, owner: str, repo_name: str, key: str) -> JSON: + """``DELETE /studios/{owner}/{repo_name}/variables`` — remove a variable by key.""" + return self._request( + "DELETE", + f"/studios/{owner}/{repo_name}/variables", + json_body={"key": key}, + required_scope=TokenScope.WRITE, ) def update_studio_settings( @@ -671,11 +1031,17 @@ def update_studio_settings( repo_name: str, settings: UpdateStudioSettingsPayload | Mapping[str, Any], ) -> JSON: - """``PATCH /studios/{owner}/{repo_name}/settings`` — update Studio settings.""" + """``PATCH /studios/{owner}/{repo_name}/settings`` — update Studio settings. + + Only the fields present in *settings* are modified. Changes to + ``sdk_type`` / ``sdk_version`` / ``base_image`` / ``hardware`` take effect + on the next deployment. + """ return self._request( "PATCH", f"/studios/{owner}/{repo_name}/settings", json_body=dict(settings), + required_scope=TokenScope.WRITE, ) # ================================================================== @@ -706,16 +1072,17 @@ def list_mcp_servers( *, search: str | None = None, page_number: int = 1, - page_size: int = 10, + page_size: int = 20, filter: Mapping[str, Any] | None = None, extra: Mapping[str, Any] | None = None, ) -> JSON: - """``GET /mcp/servers`` — discover MCP servers. + """``PUT /mcp/servers`` — discover MCP servers. - Falls back to the historical ``PUT /mcp/servers`` endpoint while the - service rolls out GET support. If an optional token is rejected by the - legacy PUT list endpoint, retry anonymously so readonly or stale tokens - do not block public MCP discovery. + ``PUT`` is the only method the specification defines for this route, so it + is tried first. A ``GET`` variant is probed only after a ``PUT`` failure, + because probing first cost every single call a wasted round trip to a 404. + Whichever verb a deployment turns out to serve is remembered for the life + of this client, so the loser is attempted at most once. Parameters ---------- @@ -723,6 +1090,9 @@ def list_mcp_servers( Nested filter object. Supported keys: ``category``, ``is_hosted``. """ if page_number * page_size > 100: + # The service enforces this itself, answering 403 QuotaLimitExceed with + # exactly this rule. Checking here spares the round trip and reports it + # as what it is -- a caller parameter mistake, not an exhausted quota. raise InvalidParameter( f"page_number * page_size must be <= 100 for MCP servers (got {page_number * page_size})." ) @@ -736,30 +1106,58 @@ def list_mcp_servers( if extra: body.update(extra) body = {k: v for k, v in body.items() if v is not None} - params = self._flatten_mcp_list_params(body) - has_token = self._resolve_token() is not None - try: - return self._request("GET", "/mcp/servers", params=params, require_token=False) - except APIError as exc: - if not self._is_method_or_route_unsupported(exc): - raise + + def _get() -> JSON: + return self._request( + "GET", + "/mcp/servers", + params=self._flatten_mcp_list_params(body), + require_token=False, + required_scope=TokenScope.READ, + anonymous_retry=True, + ) + + if self._mcp_list_supports_get: + # This deployment already answered GET and refused PUT, so leading + # with PUT again would just repeat a known 404 on every call. + return _get() try: - return self._request("PUT", "/mcp/servers", json_body=body, require_token=False) - except (AuthenticationError, PermissionDeniedError): - if not has_token: - raise return self._request( "PUT", "/mcp/servers", json_body=body, require_token=False, - anonymous=True, + required_scope=TokenScope.READ, + anonymous_retry=True, ) + except APIError as exc: + if self._mcp_list_supports_get is False or not self._is_method_or_route_unsupported(exc): + raise + + try: + data = _get() + except APIError as exc: + if self._is_method_or_route_unsupported(exc): + # Serves neither verb: stop probing GET on subsequent calls. + self._mcp_list_supports_get = False + raise + # Serves GET but not PUT: skip the PUT attempt from now on. + self._mcp_list_supports_get = True + return data def list_operational_mcp_servers(self) -> JSON: - """``GET /mcp/servers/operational`` — list servers deployed by the caller.""" - return self._request("GET", "/mcp/servers/operational") + """``GET /mcp/servers/operational`` — list servers deployed by the caller. + + Answers with the caller's own hosting, so no anonymous fallback: degrading + to an anonymous request would report "nothing deployed" for what is really + a credential problem. + """ + return self._request( + "GET", + "/mcp/servers/operational", + required_scope=TokenScope.READ, + ) def get_mcp_server( self, @@ -768,8 +1166,15 @@ def get_mcp_server( get_operational_url: bool | None = None, ) -> JSON: """``GET /mcp/servers/{id}`` — fetch a single MCP server's manifest.""" - params = self._merge_params({"get_operational_url": get_operational_url}) - return self._request("GET", f"/mcp/servers/{server_id}", params=params, require_token=False) + params = self._merge_params({"get_operational_url": _as_wire_bool(get_operational_url)}) + return self._request( + "GET", + f"/mcp/servers/{server_id}", + params=params, + require_token=False, + required_scope=TokenScope.READ, + anonymous_retry=True, + ) def deploy_mcp_server( self, @@ -785,11 +1190,16 @@ def deploy_mcp_server( "POST", f"/mcp/servers/{server_id}/deploy", json_body=body, + required_scope=TokenScope.WRITE, ) def undeploy_mcp_server(self, server_id: str | int) -> JSON: """``DELETE /mcp/servers/{id}/undeploy`` — tear down a deployed MCP server.""" - return self._request("DELETE", f"/mcp/servers/{server_id}/undeploy") + return self._request( + "DELETE", + f"/mcp/servers/{server_id}/undeploy", + required_scope=TokenScope.WRITE, + ) # Re-export iterables-of-strings helper for parity with other modules that may diff --git a/src/modelscope_hub/api.py b/src/modelscope_hub/api.py index 226953e..fd22be0 100644 --- a/src/modelscope_hub/api.py +++ b/src/modelscope_hub/api.py @@ -39,7 +39,7 @@ from ._openapi import OpenAPIClient from ._upload import UploadManager from .config import HubConfig, get_default_config -from .constants import DEFAULT_ENDPOINT, RepoType, Visibility +from .constants import DEFAULT_ENDPOINT, RepoType, StudioVisibility, Visibility from .errors import ( AuthenticationError, HubError, @@ -47,6 +47,7 @@ NetworkError, NotExistError, NotSupportedError, + PermissionDeniedError, ) from .types import CacheInfo, CacheVerification, FileInfo, PagedResult, RepoInfo, UserInfo from .utils.logger import get_logger @@ -87,8 +88,12 @@ } -_STUDIO_FIELD_RENAMES: dict[str, str] = { - "cover_image": "coverImage", +# Studio payload keys the SDK once renamed on the way out. The specification +# spells the field ``cover_image``, so the old outbound rename silently dropped +# every cover image the caller supplied; the mapping now runs the other way, to +# normalise callers still passing the camelCase spelling. +_STUDIO_FIELD_ALIASES: dict[str, str] = { + "coverImage": "cover_image", } # Reserved fields that are controlled by create_repo method parameters. @@ -96,6 +101,36 @@ _RESERVED_EXTRA_FIELDS: frozenset[str] = frozenset({"Path", "Owner", "Name"}) +def _unwrap_list(payload: Any, keys: tuple[str, ...]) -> list[dict]: + """Pull a list out of an OpenAPI payload, tolerating either shape. + + Some endpoints answer with the collection at the top level and others nest it + under a resource-specific key, so both are accepted rather than assuming one. + """ + if isinstance(payload, list): + return payload + if isinstance(payload, dict): + for key in keys: + if isinstance(payload.get(key), list): + return payload[key] + return [] + + +def _normalise_studio_fields(fields: Mapping[str, Any]) -> dict[str, Any]: + """Map legacy Studio field spellings onto the ones the endpoint accepts. + + Also expands ``visibility`` into the ``private`` companion flag so a caller + who reaches the Studio endpoints through ``**settings`` gets the same + treatment as one who passes ``visibility=`` explicitly. + """ + normalised = {_STUDIO_FIELD_ALIASES.get(key, key): value for key, value in fields.items()} + studio_visibility = StudioVisibility.parse(normalised.get("visibility")) + if studio_visibility is not None: + normalised["visibility"] = studio_visibility.value + normalised.setdefault("private", studio_visibility is StudioVisibility.PRIVATE) + return normalised + + class HubApi: """Unified client for ModelScope Hub operations. @@ -269,6 +304,8 @@ def _normalize_visibility(visibility: int | str | Visibility | None) -> int | No "models", "datasets", "skills", + "studios", + "variables", "servers", "mcp_server_list", "Models", @@ -367,7 +404,20 @@ def _repo_info_from_payload( # The OpenAPI surface uses ``private`` bool for visibility. # gated is orthogonal and does not affect visibility mapping. - if normalised.get("visibility") is None: + raw_visibility = normalised.get("visibility") + if isinstance(raw_visibility, str): + # Studio payloads report visibility as a string enum. ``public`` and + # ``private`` have integer equivalents; ``protected`` (app public, + # code hidden) does not, so it is preserved verbatim rather than + # forced into a value that would misrepresent it. + studio_visibility = StudioVisibility.parse(raw_visibility) + if studio_visibility is StudioVisibility.PUBLIC: + normalised["visibility"] = Visibility.PUBLIC + elif studio_visibility is StudioVisibility.PRIVATE: + normalised["visibility"] = Visibility.PRIVATE + elif studio_visibility is not None: + normalised["visibility"] = studio_visibility.value + elif raw_visibility is None: private_flag = normalised.get("private") if isinstance(private_flag, bool): if private_flag: @@ -499,6 +549,15 @@ def login(self, token: str) -> UserInfo: caller's only working one, so revoking it on failure would turn a mistyped token into an unintended logout. + Tokens are issued with a permission tier (read / write / admin). The + legacy login endpoint exists to mint git credentials, which a read-only + token is not entitled to -- yet such a token authenticates fine and is + all a caller needs in order to browse and download. So when that + endpoint rejects the credential, the token is re-checked against + ``GET /users/me`` before concluding it is bad: if that succeeds the + login completes without cookies or a git token, and warns that write + operations will need a higher tier. + Examples -------- >>> api = HubApi() @@ -520,6 +579,15 @@ def login(self, token: str) -> UserInfo: try: data, cookies = self.legacy.login(token) + except (AuthenticationError, PermissionDeniedError) as exc: + scoped = self._login_with_scoped_token(token) + if scoped is not None: + return scoped + self._restore_credential_state(previous_token, previous_logged_out) + explained = self._explain_login_failure(token, exc) + if explained is exc: + raise + raise explained from exc except HubError as exc: self._restore_credential_state(previous_token, previous_logged_out) explained = self._explain_login_failure(token, exc) @@ -534,11 +602,54 @@ def login(self, token: str) -> UserInfo: self._config.save_cookies(cookies) if git_token: self._config.save_git_token(git_token) + else: + logger.debug("Login returned no git access token; git-based operations will be unavailable.") if username: self._config.save_user_info(username, email or "") return self.whoami() + def _login_with_scoped_token(self, token: str) -> UserInfo | None: + """Complete a reduced-capability login, or return ``None`` if impossible. + + Called only when the legacy login endpoint refused the credential, at + which point *token* is already installed on the config -- so a successful + ``GET /users/me`` proves the token itself is valid, and the refusal was + about the permission tier rather than the token. Reporting that as + "invalid token" would send the caller after the wrong remedy. + + The token has to be persisted explicitly here. On the normal path it is + stored as a side effect of saving the server-issued session cookies, + which this tier does not receive -- without this the login would appear + to succeed yet leave nothing on disk for the next process to find. + + Like :meth:`_token_valid_on`, the probe is advisory: retries are disabled + so a dead network cannot stall the error report, and any exception simply + means "cannot confirm", which hands the original rejection back untouched. + """ + from .constants import API_CONNECT_TIMEOUT + + probe = OpenAPIClient(self._config, timeout=API_CONNECT_TIMEOUT, max_retries=0) + try: + payload = probe.get_current_user() + except Exception: # advisory only -- never mask the original failure + return None + finally: + probe.close() + + profile = UserInfo.from_dict(payload if isinstance(payload, dict) else {}) + if not profile.username: + return None + logger.warning( + "Token accepted with reduced permissions: signed in as %s, but git and session " + "credentials were not issued. Uploads, pushes and other write operations need a " + "token with 'write' permission or higher.", + profile.username, + ) + self._config.save_token(token) + self._config.save_user_info(profile.username, profile.email or "") + return profile + def _restore_credential_state(self, token: str | None, logged_out: bool) -> None: """Roll the in-memory credential back to its pre-login value. @@ -749,7 +860,8 @@ def create_repo( f"create_repo does not support repo_type={rt.value!r}. Supported types: {supported}." ) owner, name = self._parse_repo_id(repo_id) - vis = self._normalize_visibility(visibility) + studio_visibility = StudioVisibility.parse(visibility) if rt is RepoType.STUDIO else None + vis = None if studio_visibility is not None else self._normalize_visibility(visibility) if license is not None: license = _LICENSE_DISPLAY_TO_SPDX.get(license, license) @@ -760,7 +872,14 @@ def create_repo( "owner": owner, "repo_name": name, } - if vis is not None: + if studio_visibility is not None: + # ``visibility`` is the expressive form (it is the only way to + # ask for ``protected``); ``private`` is sent alongside it for + # servers that predate the tri-state field. The two are + # consistent by construction, so they cannot disagree. + payload["visibility"] = studio_visibility.value + payload["private"] = studio_visibility is StudioVisibility.PRIVATE + elif vis is not None: payload["private"] = is_private if chinese_name is not None: payload["display_name"] = chinese_name @@ -777,10 +896,10 @@ def create_repo( payload["license"] = license if description is not None: payload["description"] = description - for old_key, new_key in _STUDIO_FIELD_RENAMES.items(): - if old_key in extra: - extra[new_key] = extra.pop(old_key) - payload.update(extra) + # Only Studios get the Studio field normalisation: a Skill has no + # cover image and no visibility field, so rewriting those keys on a + # Skill payload would invent fields its endpoint does not accept. + payload.update(_normalise_studio_fields(extra) if rt is RepoType.STUDIO else extra) data = self.openapi.create_studio(payload) if rt is RepoType.STUDIO else self.openapi.create_skill(payload) return self._repo_info_from_payload(data, rt, owner_hint=owner, name_hint=name) @@ -915,8 +1034,7 @@ def list_repos( Parameters ---------- repo_type : str or RepoType - One of ``"model"``, ``"dataset"``, ``"skill"``, ``"mcp"``. - ``"studio"`` raises :class:`NotSupportedError` (no list endpoint). + One of ``"model"``, ``"dataset"``, ``"studio"``, ``"skill"``, ``"mcp"``. owner : str, optional Restrict results to repositories owned by this user/org. search : str, optional @@ -938,7 +1056,7 @@ def list_repos( Raises ------ NotSupportedError - When ``repo_type`` is ``"studio"`` (no list endpoint yet). + When ``repo_type`` is not one of the listed values. Examples -------- @@ -992,7 +1110,14 @@ def list_repos( filter=clean_filters or None, ) elif rt is RepoType.STUDIO: - raise NotSupportedError("Listing studios is not supported by the OpenAPI surface yet.") + payload = self.openapi.list_studios( + search=search, + owner=owner, + sort=sort, + page_number=page_number, + page_size=page_size, + **clean_filters, + ) else: # pragma: no cover - defensive raise NotSupportedError(f"list_repos not supported for {rt}") @@ -1007,6 +1132,7 @@ def list_repos( RepoType.MODEL: "models", RepoType.DATASET: "datasets", RepoType.SKILL: "skills", + RepoType.STUDIO: "studios", RepoType.MCP: "servers", } key = _COLLECTION_KEYS.get(rt, "items") @@ -1805,11 +1931,8 @@ def update_repo_settings( """ rt = self._normalize_repo_type(repo_type) owner, name = self._parse_repo_id(repo_id) - for old_key, new_key in _STUDIO_FIELD_RENAMES.items(): - if old_key in settings: - settings[new_key] = settings.pop(old_key) if rt is RepoType.STUDIO: - return self.openapi.update_studio_settings(owner, name, settings) + return self.openapi.update_studio_settings(owner, name, _normalise_studio_fields(settings)) if rt is RepoType.SKILL: return self.openapi.update_skill_settings(owner, name, settings) raise NotSupportedError(f"update_repo_settings is not supported for repo_type={rt.value!r}.") @@ -1818,25 +1941,22 @@ def update_repo_settings( # Secrets # ================================================================== def list_secrets(self, repo_id: str, repo_type: RepoTypeLike = RepoType.STUDIO) -> list[dict]: - """List secrets attached to a Studio. + """List the keys of the secrets attached to a Studio. + + Values are never disclosed by the endpoint. Use :meth:`list_variables` + for the plaintext counterpart, whose values are returned. Examples -------- >>> api.list_secrets("alice/chat-demo") - [{'key': 'OPENAI_API_KEY', 'updated_at': 1712345678}, ...] + [{'key': 'OPENAI_API_KEY'}, ...] """ rt = self._normalize_repo_type(repo_type) if rt is not RepoType.STUDIO: raise NotSupportedError(f"Secret management is only supported for studio (got {rt.value!r}).") owner, name = self._parse_repo_id(repo_id) data = self.openapi.list_studio_secrets(owner, name) - if isinstance(data, list): - return data - if isinstance(data, dict): - for key in ("items", "secrets", "list"): - if isinstance(data.get(key), list): - return data[key] - return [] + return _unwrap_list(data, ("secrets", "items", "list")) def add_secret( self, @@ -1894,6 +2014,144 @@ def delete_secret( owner, name = self._parse_repo_id(repo_id) return self.openapi.delete_studio_secret(owner, name, key) + # ================================================================== + # Plaintext variables + # + # The counterpart to the secrets block above. Same shape, different + # disclosure: a variable's value is publicly readable, a secret's is not. + # ================================================================== + def list_variables(self, repo_id: str, repo_type: RepoTypeLike = RepoType.STUDIO) -> list[dict]: + """List plaintext environment variables attached to a Studio. + + Unlike :meth:`list_secrets`, the values are returned too -- plaintext + variables are publicly visible by design. + + Examples + -------- + >>> api.list_variables("alice/chat-demo") + [{'key': 'MODEL_NAME', 'value': 'Qwen2.5-7B'}, ...] + """ + rt = self._normalize_repo_type(repo_type) + if rt is not RepoType.STUDIO: + raise NotSupportedError(f"Variable management is only supported for studio (got {rt.value!r}).") + owner, name = self._parse_repo_id(repo_id) + data = self.openapi.list_studio_variables(owner, name) + return _unwrap_list(data, ("variables", "items", "list")) + + def add_variable( + self, + repo_id: str, + key: str, + value: str, + repo_type: RepoTypeLike = RepoType.STUDIO, + ) -> dict: + """Add a plaintext environment variable to a Studio. + + Both key and value are publicly visible; use :meth:`add_secret` for + anything sensitive. + + Examples + -------- + >>> api.add_variable("alice/chat-demo", "MODEL_NAME", "Qwen2.5-7B") + """ + rt = self._normalize_repo_type(repo_type) + if rt is not RepoType.STUDIO: + raise NotSupportedError("Only studio variables are supported.") + owner, name = self._parse_repo_id(repo_id) + return self.openapi.add_studio_variable(owner, name, key, value) + + def update_variable( + self, + repo_id: str, + key: str, + value: str, + repo_type: RepoTypeLike = RepoType.STUDIO, + ) -> dict: + """Update an existing plaintext variable's value. + + Examples + -------- + >>> api.update_variable("alice/chat-demo", "MODEL_NAME", "Qwen2.5-14B") + """ + rt = self._normalize_repo_type(repo_type) + if rt is not RepoType.STUDIO: + raise NotSupportedError("Only studio variables are supported.") + owner, name = self._parse_repo_id(repo_id) + return self.openapi.update_studio_variable(owner, name, key, value) + + def delete_variable( + self, + repo_id: str, + key: str, + repo_type: RepoTypeLike = RepoType.STUDIO, + ) -> dict: + """Delete a plaintext variable from a Studio. + + Examples + -------- + >>> api.delete_variable("alice/chat-demo", "MODEL_NAME") + """ + rt = self._normalize_repo_type(repo_type) + if rt is not RepoType.STUDIO: + raise NotSupportedError("Only studio variables are supported.") + owner, name = self._parse_repo_id(repo_id) + return self.openapi.delete_studio_variable(owner, name, key) + + # ================================================================== + # Studio resource discovery + # ================================================================== + def list_studio_hardware( + self, + *, + sdk_type: str | None = None, + repo_id: str | None = None, + ) -> list[dict]: + """List the hardware tiers a Studio may be deployed on. + + Answers the question ``--hardware`` used to leave to guesswork. Anonymous + callers see the default free tier; authenticated callers also see paid + tiers with prices, which are selected as ``paid/``. + + Parameters + ---------- + sdk_type : str, optional + Keep only tiers supporting this SDK type. + repo_id : str, optional + Scope the free tiers to what this ``owner/name`` space may use. + + Examples + -------- + >>> [h["name"] for h in api.list_studio_hardware(sdk_type="gradio")] + ['CPU basic 2 vCPU 16GB', ...] + """ + data = self.openapi.list_studio_hardware(sdk_type=sdk_type, studio=repo_id) + return _unwrap_list(data, ("hardware", "items", "list")) + + def list_studio_base_images(self) -> list[dict]: + """List the base images available to Studio spaces. + + Examples + -------- + >>> api.list_studio_base_images() + [{'name': 'ubuntu22.04-py311-torch2.3.1', 'tag': '...'}, ...] + """ + data = self.openapi.list_studio_base_images() + return _unwrap_list(data, ("base_images", "items", "list")) + + def list_studio_sdk_versions(self, *, sdk_type: str | None = None) -> list[dict]: + """List the SDK versions available to Studio spaces. + + Only ``sdk_type="gradio"`` yields versions; the other SDK types (and + omitting it) return an empty list. + + Examples + -------- + >>> [v["version"] for v in api.list_studio_sdk_versions(sdk_type="gradio")] + ['4.44.1', ...] + """ + data = self.openapi.list_studio_sdk_versions(sdk_type=sdk_type) + return _unwrap_list(data, ("sdk_versions", "items", "list")) + # ================================================================== # MCP convenience wrappers # ================================================================== @@ -1942,6 +2200,26 @@ def list_mcp_servers( items, total, page, size = self._extract_paged(payload) return PagedResult(items=list(items), total_count=total, page_number=page, page_size=size) + def list_operational_mcp_servers(self) -> PagedResult[dict]: + """List the MCP servers the caller currently has hosted. + + Returns the same :class:`PagedResult` shape as :meth:`list_mcp_servers`, + so both can be consumed identically. Each entry additionally carries + ``operational_urls`` with the live endpoint(s). + + Unlike :meth:`list_mcp_servers` this always requires a valid token: the + answer is account-private, so there is no anonymous view to fall back on. + + Examples + -------- + >>> page = api.list_operational_mcp_servers() + >>> [s["id"] for s in page.items] + ['alice/weather-mcp', ...] + """ + payload = self.openapi.list_operational_mcp_servers() + items, total, page, size = self._extract_paged(payload) + return PagedResult(items=list(items), total_count=total, page_number=page, page_size=size) + def get_mcp_server( self, server_id: str, diff --git a/src/modelscope_hub/cli/mcp.py b/src/modelscope_hub/cli/mcp.py index d383dbf..70c501d 100644 --- a/src/modelscope_hub/cli/mcp.py +++ b/src/modelscope_hub/cli/mcp.py @@ -36,6 +36,12 @@ class _McpList(CLICommand): def register(subparsers: SubParsers) -> None: p = subparsers.add_parser("list", help="List MCP servers.") p.add_argument("--search", default=None) + p.add_argument( + "--hosted", + action="store_true", + default=False, + help="List only the servers you currently have hosted, with their live URLs.", + ) p.add_argument("--page", dest="page_number", type=int, default=1) p.add_argument("--page-size", dest="page_size", type=int, default=20) add_subcmd_token_endpoint(p) @@ -43,6 +49,9 @@ def register(subparsers: SubParsers) -> None: def execute(self) -> None: api = make_api(self.args) + if self.args.hosted: + self._list_hosted(api) + return result = api.list_mcp_servers( search=self.args.search, page_number=self.args.page_number, @@ -63,6 +72,28 @@ def execute(self) -> None: info(render_table(rows, headers=["id", "name", "status", "description"])) info(f"\npage {result.page_number} / total {result.total_count}") + @staticmethod + def _list_hosted(api) -> None: + """Render the caller's own hosted servers, including their endpoints. + + ``--search`` and paging are not forwarded: the endpoint takes neither, so + accepting them silently would imply a filter that never applied. + """ + result = api.list_operational_mcp_servers() + if not result.items: + info("(no hosted MCP servers)") + return + for item in result.items: + server_id = item.get("id") or "-" + name = item.get("name") or "-" + info(f"{server_id} ({name})") + for url in item.get("operational_urls") or []: + if not isinstance(url, dict): + continue + transport = url.get("transport_type") or "-" + info(f" {transport}: {url.get('url') or '-'}") + info(f"\ntotal {result.total_count}") + class _McpInfo(CLICommand): @staticmethod diff --git a/src/modelscope_hub/cli/repo.py b/src/modelscope_hub/cli/repo.py index c6ac7da..7072896 100644 --- a/src/modelscope_hub/cli/repo.py +++ b/src/modelscope_hub/cli/repo.py @@ -238,6 +238,7 @@ def _add_arguments(p) -> None: choices=[ RepoType.MODEL.value, RepoType.DATASET.value, + RepoType.STUDIO.value, RepoType.SKILL.value, RepoType.MCP.value, ], @@ -254,6 +255,17 @@ def _add_arguments(p) -> None: _MAX_PAGE_SIZE = 50 + def _extra_filters(self) -> dict: + """Per-type filters this generic listing has to supply. + + Studios are documented as switching their own status default to ``all`` + once ``owner`` is set, but the endpoint keeps filtering to running spaces, + so ``list --repo-type studio --owner me`` answered with nothing at all. + """ + if self.args.repo_type == RepoType.STUDIO.value and self.args.owner: + return {"status": "all"} + return {} + def execute(self) -> None: if self.args.envs: print_env_table() @@ -279,6 +291,7 @@ def execute(self) -> None: search=self.args.search, page_number=self.args.page_number, page_size=self.args.page_size, + **self._extra_filters(), ) if not result.items: info("(no repositories found)") @@ -297,6 +310,7 @@ def _fetch_all_pages(self, api) -> list[RepoInfo]: search=self.args.search, page_number=page_number, page_size=page_size, + **self._extra_filters(), ) all_items.extend(result.items) if not result.has_next or not result.items: diff --git a/src/modelscope_hub/cli/studio.py b/src/modelscope_hub/cli/studio.py index ba825bd..feb30c9 100644 --- a/src/modelscope_hub/cli/studio.py +++ b/src/modelscope_hub/cli/studio.py @@ -12,11 +12,14 @@ from argparse import SUPPRESS from typing import Any -from ..constants import RepoType -from .base import CLICommand, SubParsers, info, make_api, parse_kv_pairs, success +from ..constants import RepoType, StudioVisibility +from .base import CLICommand, SubParsers, info, make_api, parse_kv_pairs, render_table, success _LOG_TYPES = ("run", "build") _STUDIO_SDK_TYPES = ("gradio", "streamlit", "docker", "static") +_STUDIO_SORTS = ("default", "last_modified", "view_num", "likes") +_STUDIO_STATUS_FILTERS = ("running", "all") +_STUDIO_HARDWARE_TYPES = ("xgpu", "amd") _SETTINGS_FIELDS = ( "display_name", "description", @@ -26,6 +29,7 @@ "sdk_version", "base_image", "hardware", + "visibility", "private", ) @@ -33,6 +37,14 @@ class StudioCommand(CLICommand): """Top-level dispatcher for ``studio`` subcommands.""" + # The other built-in commands do not need this, because the parser name is + # the string handed to ``add_parser``. This one does: it is the single command + # that crosses the package boundary (the umbrella SDK re-exports it as + # ``StudioCMD``, and its plugin classes are all identified by ``name``), so + # plugin-collision detection can key off the class instead of the entry-point + # label it happens to be registered under. + name = "studio" + @staticmethod def register(subparsers: SubParsers) -> None: parser = subparsers.add_parser("studio", help="Manage ModelScope Studio spaces.") @@ -40,17 +52,28 @@ def register(subparsers: SubParsers) -> None: actions = parser.add_subparsers(dest="studio_action", metavar="ACTION") actions.required = True + _StudioList.register(actions) _StudioDeploy.register(actions) _StudioStop.register(actions) _StudioLogs.register(actions) _StudioSettings.register(actions) _StudioSecret.register(actions) + _StudioVariable.register(actions) + _StudioHardware.register(actions) + _StudioBaseImages.register(actions) + _StudioSdkVersions.register(actions) parser.set_defaults(_command=StudioCommand) def execute(self) -> None: leaf = getattr(self.args, "_studio_leaf", None) - if leaf is None: # pragma: no cover - argparse enforces this + if leaf is None: + # Namespaces built by hand -- as embedders and the umbrella SDK's + # tests do -- carry only the action name, because ``_studio_leaf`` is + # set by argparse during dispatch. Fall back to the action so a + # programmatic caller is not forced to know that internal detail. + leaf = _STUDIO_LEAVES.get(getattr(self.args, "studio_action", None) or "") + if leaf is None: raise SystemExit("No studio subcommand specified. Run 'modelscope studio --help'.") leaf(self.args).execute() @@ -75,6 +98,72 @@ def _add_leaf_auth_args(parser) -> None: parser.add_argument("--endpoint", dest="subcmd_endpoint", default=SUPPRESS, help=SUPPRESS) +class _StudioList(CLICommand): + @staticmethod + def register(subparsers: SubParsers) -> None: + p = subparsers.add_parser("list", help="List Studio spaces.") + p.add_argument("--search", default=None, help="Match name, display name or author.") + p.add_argument("--owner", default=None, help="Restrict to spaces owned by this user/org.") + p.add_argument("--sort", choices=_STUDIO_SORTS, default=None) + p.add_argument( + "--status", + choices=_STUDIO_STATUS_FILTERS, + default=None, + help="Runtime status filter. Defaults to 'all' when --owner is given, else to the server default.", + ) + mcp = p.add_mutually_exclusive_group() + mcp.add_argument("--mcp-support", dest="mcp_support", action="store_const", const=True, default=None) + mcp.add_argument("--no-mcp-support", dest="mcp_support", action="store_const", const=False) + p.add_argument("--hardware-type", dest="hardware_type", choices=_STUDIO_HARDWARE_TYPES, default=None) + p.add_argument("--page", dest="page_number", type=int, default=1) + p.add_argument("--page-size", dest="page_size", type=int, default=10) + _add_leaf_auth_args(p) + p.set_defaults(_command=StudioCommand, _studio_leaf=_StudioList) + + def execute(self) -> None: + api = make_api(self.args) + result = api.list_repos( + RepoType.STUDIO, + search=self.args.search, + owner=self.args.owner, + sort=self.args.sort, + page_number=self.args.page_number, + page_size=self.args.page_size, + status=self._status(), + mcp_support=self.args.mcp_support, + hardware_type=self.args.hardware_type, + ) + if not result.items: + info("(no studios found)") + return + rows = [ + ( + r.repo_id or "-", + _visibility_label(r.visibility), + r.sdk_type or "-", + r.hardware or "-", + _runtime_status(r), + r.likes, + ) + for r in result.items + ] + info(render_table(rows, headers=["repo_id", "visibility", "sdk_type", "hardware", "status", "likes"])) + info(f"\npage {result.page_number} / total {result.total_count} (page_size={result.page_size})") + + def _status(self) -> str | None: + """Resolve the status filter, defaulting to ``all`` when listing an owner. + + The endpoint is documented as switching its own default to ``all`` once + ``owner`` is set, but it does not: it keeps filtering to running spaces, + so "list my spaces" answered with nothing at all. Asking for ``all`` + explicitly gives the command the meaning a user expects, while leaving + the SDK faithful to whatever the server actually does. + """ + if self.args.status: + return str(self.args.status) + return "all" if self.args.owner else None + + class _StudioDeploy(CLICommand): @staticmethod def register(subparsers: SubParsers) -> None: @@ -146,6 +235,12 @@ def register(subparsers: SubParsers) -> None: p.add_argument("--base-image", dest="base_image", default=None) p.add_argument("--hardware", default=None) visibility = p.add_mutually_exclusive_group() + visibility.add_argument( + "--visibility", + choices=[v.value for v in StudioVisibility], + default=None, + help="public (code and app public), protected (app public, code hidden) or private.", + ) visibility.add_argument("--private", dest="private", action="store_const", const=True, default=None) visibility.add_argument("--public", dest="private", action="store_const", const=False) p.set_defaults(_command=StudioCommand, _studio_leaf=_StudioSettings) @@ -155,7 +250,8 @@ def execute(self) -> None: if not settings: raise ValueError( "No setting specified. Provide key=value or one of: --display-name, --description, --license, " - "--cover-image, --sdk-type, --sdk-version, --base-image, --hardware, --private/--public." + "--cover-image, --sdk-type, --sdk-version, --base-image, --hardware, --visibility, " + "--private/--public." ) api = make_api(self.args) data = api.update_repo_settings(self.args.studio_id, RepoType.STUDIO, **settings) @@ -218,6 +314,159 @@ def execute(self) -> None: raise ValueError(f"Unknown secret subcommand: {action}") +class _StudioVariable(CLICommand): + """``studio variable`` — plaintext environment variables. + + Deliberately separate from ``studio secret``: a variable's value is publicly + visible, so ``list`` prints it, whereas a secret's value is never disclosed. + """ + + @staticmethod + def register(subparsers: SubParsers) -> None: + parser = subparsers.add_parser( + "variable", + help="Manage Studio plaintext environment variables (values are public).", + ) + actions = parser.add_subparsers(dest="variable_action", metavar="ACTION") + actions.required = True + + list_p = actions.add_parser("list", help="List variable keys and values.") + _add_studio_id(list_p) + list_p.set_defaults(_command=StudioCommand, _studio_leaf=_StudioVariable) + + add_p = actions.add_parser("add", help="Add a plaintext variable.") + _add_studio_id(add_p) + add_p.add_argument("key") + add_p.add_argument("value") + add_p.set_defaults(_command=StudioCommand, _studio_leaf=_StudioVariable) + + update_p = actions.add_parser("update", help="Update an existing plaintext variable.") + _add_studio_id(update_p) + update_p.add_argument("key") + update_p.add_argument("value") + update_p.set_defaults(_command=StudioCommand, _studio_leaf=_StudioVariable) + + delete_p = actions.add_parser("delete", help="Delete a plaintext variable.") + _add_studio_id(delete_p) + delete_p.add_argument("key") + delete_p.add_argument("--yes", "-y", action="store_true", help="Skip confirmation.") + delete_p.set_defaults(_command=StudioCommand, _studio_leaf=_StudioVariable) + + def execute(self) -> None: + action = self.args.variable_action + if action == "delete" and not getattr(self.args, "yes", False): + answer = input(f"Delete variable {self.args.key!r} from {self.args.studio_id}? [y/N] ").strip().lower() + if answer not in ("y", "yes"): + info("Aborted.") + return + api = make_api(self.args) + if action == "list": + variables = api.list_variables(self.args.studio_id, RepoType.STUDIO) + if not variables: + info("(no variables)") + return + rows = [ + (item.get("key") or "-", item.get("value") or "") if isinstance(item, dict) else (str(item), "") + for item in variables + ] + info(render_table(rows, headers=["key", "value"])) + return + if action == "add": + api.add_variable(self.args.studio_id, self.args.key, self.args.value, RepoType.STUDIO) + success(f"Variable {self.args.key!r} added.") + return + if action == "update": + api.update_variable(self.args.studio_id, self.args.key, self.args.value, RepoType.STUDIO) + success(f"Variable {self.args.key!r} updated.") + return + if action == "delete": + api.delete_variable(self.args.studio_id, self.args.key, RepoType.STUDIO) + success(f"Variable {self.args.key!r} deleted.") + return + raise ValueError(f"Unknown variable subcommand: {action}") + + +class _StudioHardware(CLICommand): + @staticmethod + def register(subparsers: SubParsers) -> None: + p = subparsers.add_parser("hardware", help="List hardware tiers a Studio can be deployed on.") + p.add_argument("--sdk-type", dest="sdk_type", choices=_STUDIO_SDK_TYPES, default=None) + p.add_argument( + "--studio", + dest="studio_id", + default=None, + help="Scope free tiers to what this 'owner/name' space may use.", + ) + _add_leaf_auth_args(p) + p.set_defaults(_command=StudioCommand, _studio_leaf=_StudioHardware) + + def execute(self) -> None: + api = make_api(self.args) + items = api.list_studio_hardware(sdk_type=self.args.sdk_type, repo_id=self.args.studio_id) + if not items: + info("(no hardware available)") + return + rows = [ + ( + item.get("name") or "-", + item.get("instance_type") or "-", + item.get("resource_type") or "-", + item.get("gpu_type") or "-", + _stock_label(item), + _cost_label(item), + ) + for item in items + ] + info( + render_table( + rows, + headers=["name", "instance_type", "resource_type", "gpu", "stock", "cost"], + ) + ) + info("\nPass a paid tier as --hardware paid/.") + + +class _StudioBaseImages(CLICommand): + @staticmethod + def register(subparsers: SubParsers) -> None: + p = subparsers.add_parser("base-images", help="List base images available to Studio spaces.") + _add_leaf_auth_args(p) + p.set_defaults(_command=StudioCommand, _studio_leaf=_StudioBaseImages) + + def execute(self) -> None: + api = make_api(self.args) + items = api.list_studio_base_images() + if not items: + info("(no base images available)") + return + rows = [(item.get("name") or "-", item.get("tag") or "-") for item in items] + info(render_table(rows, headers=["name", "tag"])) + + +class _StudioSdkVersions(CLICommand): + @staticmethod + def register(subparsers: SubParsers) -> None: + p = subparsers.add_parser("sdk-versions", help="List SDK versions available to Studio spaces.") + p.add_argument( + "--sdk-type", + dest="sdk_type", + choices=_STUDIO_SDK_TYPES, + default="gradio", + help="Only 'gradio' publishes versions (default: gradio).", + ) + _add_leaf_auth_args(p) + p.set_defaults(_command=StudioCommand, _studio_leaf=_StudioSdkVersions) + + def execute(self) -> None: + api = make_api(self.args) + items = api.list_studio_sdk_versions(sdk_type=self.args.sdk_type) + if not items: + info(f"(no SDK versions published for sdk_type={self.args.sdk_type!r})") + return + rows = [(item.get("sdk_type") or "-", item.get("version") or "-", item.get("tag") or "-") for item in items] + info(render_table(rows, headers=["sdk_type", "version", "tag"])) + + def _collect_settings(args) -> dict[str, Any]: settings: dict[str, Any] = parse_kv_pairs(getattr(args, "settings", []) or []) for field in _SETTINGS_FIELDS: @@ -227,6 +476,59 @@ def _collect_settings(args) -> dict[str, Any]: return settings +# Action name -> leaf handler, for callers that dispatch without argparse. +# Declared after the leaf classes because it references them. +_STUDIO_LEAVES: dict[str, type[CLICommand]] = { + "list": _StudioList, + "deploy": _StudioDeploy, + "stop": _StudioStop, + "logs": _StudioLogs, + "settings": _StudioSettings, + "secret": _StudioSecret, + "variable": _StudioVariable, + "hardware": _StudioHardware, + "base-images": _StudioBaseImages, + "sdk-versions": _StudioSdkVersions, +} + + +def _visibility_label(value: object) -> str: + """Render a visibility that may be an enum or the raw ``protected`` string.""" + if value is None: + return "-" + return getattr(value, "label", None) or getattr(value, "name", None) or str(value) + + +def _runtime_status(repo: object) -> str: + runtime = getattr(repo, "runtime", None) + if isinstance(runtime, dict): + return str(runtime.get("status") or "-") + return "-" + + +def _stock_label(item: dict) -> str: + """Summarise availability, distinguishing "none left" from "not reported".""" + if item.get("has_stock") is False: + return "out of stock" + stock = item.get("stock") + if stock is None: + return "-" + return str(stock) + + +def _cost_label(item: dict) -> str: + """Show the discounted price, noting the original when it differs.""" + cost = item.get("cost_after_discount") + original = item.get("original_cost") + if cost is None and original is None: + return "free" + if cost is None: + return str(original) + if original in (None, cost): + return str(cost) + return f"{cost} (was {original})" + + def _print_status(data: object) -> None: if not data: return @@ -253,6 +555,16 @@ def _print_logs(payload: object, *, page_num: int, page_size: int) -> None: info(f"[{ts}] {msg}" if ts else str(msg)) else: info(str(entry)) - total = payload.get("total") - if total is not None: - info(f"-- page {page_num} (size {page_size}), total {total} --") + # The response reports ``total_count`` / ``total_page_num``; the older + # ``total`` spelling this used to read never existed, so the footer was + # always suppressed. + total = payload.get("total_count") + if total is None: + total = payload.get("total") + if total is None: + return + footer = f"-- page {page_num} (size {page_size}), total {total}" + total_pages = payload.get("total_page_num") + if total_pages is not None: + footer += f" across {total_pages} page(s)" + info(f"{footer} --") diff --git a/src/modelscope_hub/compat/hub_api.py b/src/modelscope_hub/compat/hub_api.py index 21b4b6d..8ed9629 100644 --- a/src/modelscope_hub/compat/hub_api.py +++ b/src/modelscope_hub/compat/hub_api.py @@ -36,6 +36,30 @@ META_FILES_FORMAT = {".json", ".csv", ".jsonl", ".tsv", ".py"} +# Transport/credential arguments the legacy surface accepts on almost every +# method. They steer *how* a call is made and are never part of a request body. +_CONTROL_KWARGS: frozenset[str] = frozenset({"token", "endpoint", "cookies", "headers", "timeout", "max_retries"}) + + +def _split_control_kwargs(kwargs: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: + """Split legacy ``**kwargs`` into ``(control, passthrough)``. + + The old ``modelscope.hub.api.HubApi`` signature let callers append + ``token=`` / ``endpoint=`` to any method, so shims declared ``**kwargs`` + and forwarded it wholesale. For the Studio endpoints that turned out to be + actively harmful: ``update_studio_settings`` forwards its kwargs as the + settings payload, so the caller's API token ended up inside the ``PATCH`` + request body (and therefore in server-side request logs), while + ``get_studio_logs`` forwards into a keyword-only signature and raised + ``TypeError: got an unexpected keyword argument 'token'``. + + Splitting them apart keeps the permissive legacy signature while ensuring + only genuine business fields reach the wire. + """ + control = {k: v for k, v in kwargs.items() if k in _CONTROL_KWARGS} + passthrough = {k: v for k, v in kwargs.items() if k not in _CONTROL_KWARGS} + return control, passthrough + class _AigcUploadAdapter: """Expose model-only upload methods expected by legacy ``AigcModel``.""" @@ -529,33 +553,90 @@ def download_model( # ------------------------------------------------------------------ # Studio operations # ------------------------------------------------------------------ + def _api_for(self, control: dict[str, Any]) -> HubApi: + """Resolve the :class:`HubApi` a single legacy call should run against. + + The legacy surface treats ``token`` / ``endpoint`` as per-call + overrides. Honouring them requires a separate client, because the + ambient one is bound to the credential it was constructed with -- which + is why passing ``token=`` used to be silently ignored here. + """ + token = control.get("token") + endpoint = control.get("endpoint") + config = self._api._config + if (token and token != config.token) or (endpoint and endpoint != config.endpoint): + return HubApi(token=token or config.token, endpoint=endpoint or config.endpoint) + return self._api + def deploy_studio(self, studio_id: str, **kwargs: Any) -> dict: - return self._api.deploy_repo( + control, passthrough = _split_control_kwargs(kwargs) + return self._api_for(control).deploy_repo( studio_id, RepoType.STUDIO, - payload=kwargs.get("payload"), + payload=passthrough.get("payload"), ) def stop_studio(self, studio_id: str, **kwargs: Any) -> dict: - return self._api.stop_repo(studio_id, RepoType.STUDIO) + control, _ = _split_control_kwargs(kwargs) + return self._api_for(control).stop_repo(studio_id, RepoType.STUDIO) def get_studio_logs(self, studio_id: str, **kwargs: Any) -> dict: - return self._api.get_repo_logs(studio_id, RepoType.STUDIO, **kwargs) + control, passthrough = _split_control_kwargs(kwargs) + return self._api_for(control).get_repo_logs(studio_id, RepoType.STUDIO, **passthrough) def update_studio_settings(self, studio_id: str, **kwargs: Any) -> dict: - return self._api.update_repo_settings(studio_id, RepoType.STUDIO, **kwargs) + control, settings = _split_control_kwargs(kwargs) + return self._api_for(control).update_repo_settings(studio_id, RepoType.STUDIO, **settings) def list_studio_secrets(self, studio_id: str, **kwargs: Any) -> list: - return self._api.list_secrets(studio_id, RepoType.STUDIO) + control, _ = _split_control_kwargs(kwargs) + return self._api_for(control).list_secrets(studio_id, RepoType.STUDIO) def add_studio_secret(self, studio_id: str, key: str, value: str, **kwargs: Any) -> None: - self._api.add_secret(studio_id, key, value, RepoType.STUDIO) + control, _ = _split_control_kwargs(kwargs) + self._api_for(control).add_secret(studio_id, key, value, RepoType.STUDIO) def update_studio_secret(self, studio_id: str, key: str, value: str, **kwargs: Any) -> None: - self._api.update_secret(studio_id, key, value, RepoType.STUDIO) + control, _ = _split_control_kwargs(kwargs) + self._api_for(control).update_secret(studio_id, key, value, RepoType.STUDIO) def delete_studio_secret(self, studio_id: str, key: str, **kwargs: Any) -> None: - self._api.delete_secret(studio_id, key, RepoType.STUDIO) + control, _ = _split_control_kwargs(kwargs) + self._api_for(control).delete_secret(studio_id, key, RepoType.STUDIO) + + def list_studios(self, **kwargs: Any) -> dict: + """List Studio spaces as a raw paginated dict.""" + control, passthrough = _split_control_kwargs(kwargs) + result = self._api_for(control).list_repos(RepoType.STUDIO, **passthrough) + return result.to_dict() + + def list_studio_variables(self, studio_id: str, **kwargs: Any) -> list: + control, _ = _split_control_kwargs(kwargs) + return self._api_for(control).list_variables(studio_id, RepoType.STUDIO) + + def add_studio_variable(self, studio_id: str, key: str, value: str, **kwargs: Any) -> None: + control, _ = _split_control_kwargs(kwargs) + self._api_for(control).add_variable(studio_id, key, value, RepoType.STUDIO) + + def update_studio_variable(self, studio_id: str, key: str, value: str, **kwargs: Any) -> None: + control, _ = _split_control_kwargs(kwargs) + self._api_for(control).update_variable(studio_id, key, value, RepoType.STUDIO) + + def delete_studio_variable(self, studio_id: str, key: str, **kwargs: Any) -> None: + control, _ = _split_control_kwargs(kwargs) + self._api_for(control).delete_variable(studio_id, key, RepoType.STUDIO) + + def list_studio_hardware(self, **kwargs: Any) -> list: + control, passthrough = _split_control_kwargs(kwargs) + return self._api_for(control).list_studio_hardware(**passthrough) + + def list_studio_base_images(self, **kwargs: Any) -> list: + control, _ = _split_control_kwargs(kwargs) + return self._api_for(control).list_studio_base_images() + + def list_studio_sdk_versions(self, **kwargs: Any) -> list: + control, passthrough = _split_control_kwargs(kwargs) + return self._api_for(control).list_studio_sdk_versions(**passthrough) # ------------------------------------------------------------------ # Revision resolution diff --git a/src/modelscope_hub/constants.py b/src/modelscope_hub/constants.py index 0a244c8..8dd7326 100644 --- a/src/modelscope_hub/constants.py +++ b/src/modelscope_hub/constants.py @@ -105,6 +105,55 @@ def from_label(cls, label: str) -> Visibility: raise ValueError(f"Unknown visibility label: {label!r}") from exc +class StudioVisibility(StrEnum): + """Visibility levels a Studio space can be published under. + + Deliberately separate from :class:`Visibility`: models and datasets encode + visibility as the integers 1/3/5, whereas the Studio endpoints take a string + enum and offer a third state the integer encoding cannot express. + + * ``public`` -- both the code and the running app are public. + * ``protected`` -- the app is public, the code repository is not. + * ``private`` -- neither is public. + """ + + PUBLIC = "public" + PROTECTED = "protected" + PRIVATE = "private" + + @classmethod + def parse(cls, value: object) -> StudioVisibility | None: + """Return the matching member, or ``None`` when *value* is not one. + + Returning ``None`` rather than raising lets callers fall back to the + integer :class:`Visibility` encoding for inputs this enum does not own. + """ + if isinstance(value, cls): + return value + if isinstance(value, str): + try: + return cls(value.strip().lower()) + except ValueError: + return None + return None + + +class TokenScope(StrEnum): + """Permission tiers a ModelScope API token can be issued with. + + The Hub grants tokens one of three levels. The OpenAPI specification does + not model them -- ``securitySchemes`` declares a bare bearer scheme with no + scopes, and ``GET /users/me`` does not report the caller's level -- so the + SDK cannot know a token's tier up front and never pre-validates against it. + These values are used only to annotate what an operation needs, so that a + 403 can name the missing permission instead of leaving the user guessing. + """ + + READ = "read" + WRITE = "write" + ADMIN = "admin" + + class License(StrEnum): """Common open-source licenses used on ModelScope Hub.""" @@ -902,7 +951,9 @@ def get_upload_ignore_file_pattern() -> str | None: "RepoType", "StrEnum", "SESSION_FILE_NAME", + "StudioVisibility", "TEMPORARY_FOLDER_NAME", + "TokenScope", "UPLOAD_ADAPTIVE_BATCHING_ENABLED", "UPLOAD_ADAPTIVE_BATCH_SIZE", "UPLOAD_BATCH_CONSECUTIVE_FAILURE_LIMIT", diff --git a/src/modelscope_hub/errors.py b/src/modelscope_hub/errors.py index 52a685b..13d27b4 100644 --- a/src/modelscope_hub/errors.py +++ b/src/modelscope_hub/errors.py @@ -281,6 +281,22 @@ def __init__( self.retry_after = retry_after +# -- Quota (E3027) ---------------------------------------------------------- +class QuotaExceededError(APIError): + """Raised when the account's quota for a resource is exhausted (E3027). + + The OpenAPI surface reports this as HTTP 403 with ``code=QuotaLimitExceed``, + sharing the status with "insufficient permission". It is deliberately *not* + a :class:`RateLimitError` subclass: rate limiting clears on its own and the + transport retries it with back-off, whereas an exhausted quota will keep + failing, so retrying only delays the error the caller needs to see. + """ + + error_code = "E3027" + retryable = False + suggestion = "Quota exceeded for this resource. Free up existing resources or request a higher quota." + + # -- Server (E1002) --------------------------------------------------------- class ServerError(APIError): """Raised on HTTP 5xx -- upstream service failure.""" @@ -390,6 +406,13 @@ class NotSupportedError(HubError): 403: PermissionDeniedError, 404: NotExistError, 405: InvalidParameter, + # The OpenAPI surface answers 409 (``DuplicateEntity``) when creating a + # Studio, a secret or a plaintext variable that already exists. Without + # this entry the failure degraded to a bare APIError, so callers could not + # catch it as AlreadyExistsError like they can on the legacy surface. + 409: AlreadyExistsError, + # 413 is returned by POST /files/upload when the body exceeds 5 MiB. + 413: InvalidParameter, 429: RateLimitError, } @@ -413,6 +436,41 @@ class NotSupportedError(HubError): } +# --------------------------------------------------------------------------- +# OpenAPI string error-code -> exception mapping +# +# The ``/openapi/v1`` surface publishes ``code`` as a string enum, not the +# numeric business code the legacy surface uses, so it never matched +# _BUSINESS_CODE_MAP above. Two of these codes carry information the HTTP status +# alone destroys: 403 covers both "insufficient permission" and "quota +# exhausted", and 400 covers both bad input and a duplicate entity. +# --------------------------------------------------------------------------- +_OPENAPI_CODE_MAP: dict[str, type[APIError]] = { + "InputParameterError": InvalidParameter, + "InvalidAuthentication": AuthenticationError, + "OperationNotAllowed": PermissionDeniedError, + "ResourceNotFound": NotExistError, + "DuplicateEntity": AlreadyExistsError, + "QuotaLimitExceed": QuotaExceededError, + "RateLimitExceed": RateLimitError, + # ServiceUnavailable / InternalServerError arrive with a 5xx status and are + # deliberately absent: the status-code branch already maps them to the + # retryable ServerError, and reclassifying a 5xx would drop that. +} + + +def _openapi_code(body: Any) -> str | None: + """Return the string error code carried by an OpenAPI response body, if any.""" + if not isinstance(body, dict): + return None + raw = body.get("code") + if raw is None: + raw = body.get("Code") + if isinstance(raw, str) and raw.strip(): + return raw.strip() + return None + + def _business_code(body: Any) -> int | None: """Return the numeric business code carried by a response body, if any.""" if not isinstance(body, dict): @@ -521,6 +579,10 @@ def raise_for_status(response: Response) -> None: if status < 500: code = _business_code(body) business_cls = _BUSINESS_CODE_MAP.get(code) if code is not None else None + if business_cls is None: + # The OpenAPI surface publishes a string code instead of a numeric one. + openapi_code = _openapi_code(body) + business_cls = _OPENAPI_CODE_MAP.get(openapi_code) if openapi_code else None if business_cls is not None: exc_cls = business_cls elif exc_cls is InvalidParameter and isinstance(body, dict): @@ -613,6 +675,7 @@ def is_repo_exists_error(exc: BaseException) -> bool: "InvalidParameter", "AlreadyExistsError", # Rate limiting / Server + "QuotaExceededError", "RateLimitError", "ServerError", # Non-HTTP diff --git a/src/modelscope_hub/types.py b/src/modelscope_hub/types.py index 4efcbeb..6761649 100644 --- a/src/modelscope_hub/types.py +++ b/src/modelscope_hub/types.py @@ -171,6 +171,14 @@ class RepoInfo(_FromDictMixin): private: bool | None = None gated: bool | None = None login_required: bool | None = None + # Studio-native fields. Carried here rather than dropped, because the Studio + # payload's runtime configuration is the whole point of inspecting a space. + sdk_type: str | None = None + sdk_version: str | None = None + base_image: str | None = None + hardware: str | None = None + mcp_support: bool | None = None + runtime: dict[str, Any] | None = None def __post_init__(self) -> None: if isinstance(self.repo_type, str): @@ -193,12 +201,25 @@ def to_dict(self) -> dict: and formats datetimes with Z suffix to match OpenAPI spec. """ _INTERNAL_FIELDS = {"owner", "name", "repo_type", "visibility"} + _OPTIONAL_FIELDS = { + "display_name", + "file_size", + "private", + "gated", + "login_required", + "sdk_type", + "sdk_version", + "base_image", + "hardware", + "mcp_support", + "runtime", + } result = {} for f in fields(self): # type: ignore[arg-type] if f.name in _INTERNAL_FIELDS: continue val = getattr(self, f.name) - if val is None and f.name in ("display_name", "file_size", "private", "gated", "login_required"): + if val is None and f.name in _OPTIONAL_FIELDS: continue # skip None optional OpenAPI fields if isinstance(val, Enum): val = val.value @@ -384,9 +405,10 @@ class CreateStudioPayload(TypedDict, total=False): owner: str display_name: str license: str + visibility: str private: bool description: str - coverImage: str + cover_image: str sdk_type: str sdk_version: str base_image: str @@ -398,9 +420,10 @@ class UpdateStudioSettingsPayload(TypedDict, total=False): display_name: str license: str + visibility: str private: bool description: str - coverImage: str + cover_image: str sdk_type: str sdk_version: str base_image: str diff --git a/src/modelscope_hub/version.py b/src/modelscope_hub/version.py index 88db203..f710337 100644 --- a/src/modelscope_hub/version.py +++ b/src/modelscope_hub/version.py @@ -1,3 +1,3 @@ """Version information for modelscope_hub.""" -__version__ = "0.3.1" +__version__ = "0.4.0" diff --git a/tests/.env.example b/tests/.env.example index 1fb05f8..e4b1f74 100644 --- a/tests/.env.example +++ b/tests/.env.example @@ -5,3 +5,12 @@ MODELSCOPE_TEST_ENDPOINT=https://modelscope.cn # Set to true to run @pytest.mark.remote tests (requires valid token/owner above) MODELSCOPE_RUN_REMOTE_TESTS=true + +# Optional: an existing Studio (owner/repo_name) for the plaintext-variable +# lifecycle tests. Skipped when unset. +MODELSCOPE_TEST_STUDIO=your_username/your_studio + +# Optional: a token issued with *read* permission only. Verifies that read-scoped +# tokens can log in and browse, and that a write attempt names the missing +# permission tier. Skipped when unset. +MODELSCOPE_TEST_READONLY_TOKEN= diff --git a/tests/cli/test_mcp.py b/tests/cli/test_mcp.py index 06be932..80db398 100644 --- a/tests/cli/test_mcp.py +++ b/tests/cli/test_mcp.py @@ -204,6 +204,47 @@ def test_list_with_pagination(self, parser, mock_api, capsys): assert kw["page_number"] == 2 assert kw["page_size"] == 5 + def test_hosted_lists_operational_servers_with_urls(self, parser, mock_api, capsys): + mock_api.list_operational_mcp_servers.return_value = PagedResult( + items=[ + { + "id": "alice/weather", + "name": "Weather", + "operational_urls": [ + {"url": "https://mcp.example/uuid/sse", "transport_type": "sse"}, + {"url": "https://mcp.example/uuid/streamable_http", "transport_type": "streamable_http"}, + ], + } + ], + total_count=1, + ) + args = parser.parse_args(["mcp", "list", "--hosted"]) + with patch("modelscope_hub.cli.mcp.make_api", return_value=mock_api): + _McpList(args).execute() + out = capsys.readouterr().out + assert "alice/weather" in out + assert "https://mcp.example/uuid/sse" in out + assert "streamable_http" in out + mock_api.list_operational_mcp_servers.assert_called_once_with() + mock_api.list_mcp_servers.assert_not_called() + + def test_hosted_empty(self, parser, mock_api, capsys): + mock_api.list_operational_mcp_servers.return_value = PagedResult(items=[], total_count=0) + args = parser.parse_args(["mcp", "list", "--hosted"]) + with patch("modelscope_hub.cli.mcp.make_api", return_value=mock_api): + _McpList(args).execute() + assert "no hosted MCP servers" in capsys.readouterr().out + + def test_hosted_tolerates_missing_urls(self, parser, mock_api, capsys): + mock_api.list_operational_mcp_servers.return_value = PagedResult( + items=[{"id": "alice/weather", "name": "Weather"}], + total_count=1, + ) + args = parser.parse_args(["mcp", "list", "--hosted"]) + with patch("modelscope_hub.cli.mcp.make_api", return_value=mock_api): + _McpList(args).execute() + assert "alice/weather" in capsys.readouterr().out + @pytest.mark.mock_only class TestMcpInfoExecute: diff --git a/tests/cli/test_openapi.py b/tests/cli/test_openapi.py index 43c3798..c66c384 100644 --- a/tests/cli/test_openapi.py +++ b/tests/cli/test_openapi.py @@ -13,7 +13,13 @@ from modelscope_hub._openapi import _RETRYABLE_POST_PATHS, OpenAPIClient from modelscope_hub.api import HubApi from modelscope_hub.config import HubConfig -from modelscope_hub.errors import InvalidParameter, PermissionDeniedError, RateLimitError, ServerError +from modelscope_hub.errors import ( + InvalidParameter, + NotExistError, + PermissionDeniedError, + RateLimitError, + ServerError, +) @pytest.fixture @@ -46,59 +52,98 @@ def _mock_response(status_code=200, json_data=None): # ================================================================== # Item 2: list_mcp_servers filter param +# +# PUT is the only method the specification defines for /mcp/servers, so it must +# be the first thing on the wire. Probing GET first cost every single call a +# wasted round trip to a 404. # ================================================================== class TestListMcpServersFilter: - def test_filter_included_in_get_params(self, client): + def test_put_is_tried_first(self, client): + resp = _mock_response(json_data={"success": True, "data": {"mcp_server_list": [], "total": 0}}) + with patch.object(client._session, "request", return_value=resp) as mock_req: + client.list_mcp_servers(search="weather", page_number=1, page_size=2) + assert mock_req.call_count == 1 + call_kwargs = mock_req.call_args.kwargs + assert call_kwargs["method"] == "PUT" + assert call_kwargs["json"] == {"search": "weather", "page_number": 1, "page_size": 2} + + def test_filter_nested_in_put_body(self, client): resp = _mock_response(json_data={"success": True, "data": {"mcp_server_list": [], "total": 0}}) with patch.object(client._session, "request", return_value=resp) as mock_req: client.list_mcp_servers(filter={"category": "tools", "is_hosted": True}) call_kwargs = mock_req.call_args.kwargs - params = dict(call_kwargs["params"]) - assert call_kwargs["method"] == "GET" - assert call_kwargs["json"] is None - assert params["filter.category"] == "tools" - assert params["filter.is_hosted"] == "true" + assert call_kwargs["method"] == "PUT" + assert call_kwargs["json"]["filter"] == {"category": "tools", "is_hosted": True} - def test_filter_none_not_in_get_params(self, client): + def test_filter_absent_when_not_requested(self, client): resp = _mock_response(json_data={"success": True, "data": {"mcp_server_list": [], "total": 0}}) with patch.object(client._session, "request", return_value=resp) as mock_req: client.list_mcp_servers() - call_kwargs = mock_req.call_args.kwargs - params = dict(call_kwargs["params"]) - assert call_kwargs["method"] == "GET" - assert "filter.category" not in params - assert "filter" not in params + assert "filter" not in mock_req.call_args.kwargs["json"] - def test_get_route_unsupported_falls_back_to_legacy_put(self, client): + def test_put_route_unsupported_falls_back_to_get(self, client): not_found = _mock_response(status_code=404, json_data={"message": "not found"}) success = _mock_response(json_data={"success": True, "data": {"mcp_server_list": [], "total": 0}}) with patch.object(client._session, "request", side_effect=[not_found, success]) as mock_req: client.list_mcp_servers(search="weather", page_number=1, page_size=2) first, second = mock_req.call_args_list - assert first.kwargs["method"] == "GET" - assert dict(first.kwargs["params"])["search"] == "weather" - assert second.kwargs["method"] == "PUT" - assert second.kwargs["json"] == {"search": "weather", "page_number": 1, "page_size": 2} + assert first.kwargs["method"] == "PUT" + assert second.kwargs["method"] == "GET" + assert dict(second.kwargs["params"])["search"] == "weather" + + def test_get_probe_is_not_repeated_when_neither_verb_works(self, client): + """A deployment serving neither verb must not be probed twice per call.""" + not_found = _mock_response(status_code=404, json_data={"message": "not found"}) + with patch.object(client._session, "request", return_value=not_found) as mock_req: + with pytest.raises(NotExistError): + client.list_mcp_servers() + assert [c.kwargs["method"] for c in mock_req.call_args_list] == ["PUT", "GET"] + mock_req.reset_mock() + with pytest.raises(NotExistError): + client.list_mcp_servers() + assert [c.kwargs["method"] for c in mock_req.call_args_list] == ["PUT"] + + def test_put_is_not_retried_once_get_is_known_to_work(self, client): + """A GET-only deployment must not pay a PUT 404 on every single call.""" + not_found = _mock_response(status_code=404, json_data={"message": "not found"}) + success = _mock_response(json_data={"success": True, "data": {"mcp_server_list": [], "total": 0}}) + with patch.object(client._session, "request", side_effect=[not_found, success]) as mock_req: + client.list_mcp_servers() + assert [c.kwargs["method"] for c in mock_req.call_args_list] == ["PUT", "GET"] + with patch.object(client._session, "request", return_value=success) as mock_req: + client.list_mcp_servers() + assert [c.kwargs["method"] for c in mock_req.call_args_list] == ["GET"] - def test_get_permission_denied_does_not_fallback_to_put(self, client): + def test_get_fallback_carries_the_same_query(self, client): + not_found = _mock_response(status_code=404, json_data={"message": "not found"}) + success = _mock_response(json_data={"success": True, "data": {"mcp_server_list": [], "total": 0}}) + with patch.object(client._session, "request", side_effect=[not_found, success]) as mock_req: + client.list_mcp_servers(search="weather", filter={"is_hosted": True}) + params = dict(mock_req.call_args_list[1].kwargs["params"]) + assert params["search"] == "weather" + assert params["filter.is_hosted"] == "true" + + def test_permission_denied_does_not_fall_back_to_get(self, client): + """403 means the route exists; retrying it as GET would be pointless.""" denied = _mock_response(status_code=403, json_data={"message": "denied"}) with patch.object(client._session, "request", return_value=denied) as mock_req: with pytest.raises(PermissionDeniedError): client.list_mcp_servers() - assert mock_req.call_count == 1 - assert mock_req.call_args.kwargs["method"] == "GET" + # One credentialled attempt, then one anonymous retry -- never a GET. + assert [c.kwargs["method"] for c in mock_req.call_args_list] == ["PUT", "PUT"] - def test_put_auth_failure_falls_back_to_anonymous_put(self, client): - not_found = _mock_response(status_code=404, json_data={"message": "not found"}) + def test_put_auth_failure_retries_anonymously(self, client): + """A read-scoped or stale token must not hide the public MCP catalogue.""" denied = _mock_response(status_code=403, json_data={"message": "denied"}) success = _mock_response(json_data={"success": True, "data": {"mcp_server_list": [], "total": 0}}) - with patch.object(client._session, "request", side_effect=[not_found, denied, success]) as mock_req: + with patch.object(client._session, "request", side_effect=[denied, success]) as mock_req: client.list_mcp_servers(page_number=1, page_size=2) - assert [call.kwargs["method"] for call in mock_req.call_args_list] == ["GET", "PUT", "PUT"] - assert "Authorization" in mock_req.call_args_list[1].kwargs["headers"] - assert mock_req.call_args_list[1].kwargs["cookies"] - assert "Authorization" not in mock_req.call_args_list[2].kwargs["headers"] - assert mock_req.call_args_list[2].kwargs["cookies"] == {} + first, second = mock_req.call_args_list + assert [first.kwargs["method"], second.kwargs["method"]] == ["PUT", "PUT"] + assert "Authorization" in first.kwargs["headers"] + assert first.kwargs["cookies"] + assert "Authorization" not in second.kwargs["headers"] + assert second.kwargs["cookies"] == {} # ================================================================== @@ -180,18 +225,24 @@ def test_no_json_body_sent(self, client): # ================================================================== -# Item 6: get_studio requires token +# Item 6: get_studio authentication +# +# The specification marks getStudio's security as optional and states that +# public and experience-public (protected) spaces need no credentials, so +# demanding a token up front locked anonymous callers out of public data. # ================================================================== class TestGetStudioAuth: - def test_requires_token_raises_without_token(self): - from modelscope_hub.errors import AuthenticationError - + def test_anonymous_call_is_attempted_rather_than_refused(self): config = HubConfig(token="placeholder", endpoint="https://modelscope.cn") config.token = None client = OpenAPIClient(config) + resp = _mock_response() with patch.object(HubConfig, "load_token", return_value=None): - with pytest.raises(AuthenticationError, match="Missing API token"): + with patch.object(client._session, "request", return_value=resp) as mock_req: client.get_studio("org", "demo") + call_kwargs = mock_req.call_args.kwargs + assert call_kwargs["method"] == "GET" + assert "Authorization" not in call_kwargs["headers"] def test_sends_auth_header_when_token_present(self, client): resp = _mock_response() @@ -201,6 +252,17 @@ def test_sends_auth_header_when_token_present(self, client): assert "Authorization" in call_kwargs["headers"] assert call_kwargs["headers"]["Authorization"] == "Bearer test-token" + def test_rejected_token_falls_back_to_an_anonymous_read(self, client): + """A read-scoped token must not hide a space anyone else can see.""" + denied = _mock_response(status_code=403, json_data={"message": "denied"}) + success = _mock_response() + with patch.object(client._session, "request", side_effect=[denied, success]) as mock_req: + client.get_studio("org", "demo") + first, second = mock_req.call_args_list + assert "Authorization" in first.kwargs["headers"] + assert "Authorization" not in second.kwargs["headers"] + assert second.kwargs["cookies"] == {} + # ================================================================== # Item 8: page_size defaults @@ -234,11 +296,17 @@ def test_list_skills_default_page_size(self, client): assert param_dict.get("page_size") == "10" def test_list_mcp_servers_default_page_size(self, client): + """The specification's default for this endpoint is 20, not 10.""" resp = _mock_response(json_data={"success": True, "data": {"mcp_server_list": [], "total": 0}}) with patch.object(client._session, "request", return_value=resp) as mock_req: client.list_mcp_servers() - call_kwargs = mock_req.call_args.kwargs - params = dict(call_kwargs["params"]) + assert mock_req.call_args.kwargs["json"]["page_size"] == 20 + + def test_list_studios_default_page_size(self, client): + resp = _mock_response(json_data={"studios": [], "total_count": 0}) + with patch.object(client._session, "request", return_value=resp) as mock_req: + client.list_studios() + params = dict(mock_req.call_args.kwargs["params"]) assert params["page_size"] == "10" diff --git a/tests/cli/test_openapi_studio.py b/tests/cli/test_openapi_studio.py new file mode 100644 index 0000000..5a04e46 --- /dev/null +++ b/tests/cli/test_openapi_studio.py @@ -0,0 +1,360 @@ +"""Wire-level tests for the Studio endpoints added in this cycle. + +Every assertion pins what actually reaches the transport -- method, URL, query +string and body -- because the endpoints silently ignore an unrecognised field or +enum value, so a wrong spelling produces plausible-looking but wrong results +rather than an error. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from modelscope_hub._openapi import OpenAPIClient +from modelscope_hub.api import HubApi +from modelscope_hub.config import HubConfig +from modelscope_hub.constants import RepoType +from modelscope_hub.errors import InvalidParameter, NotSupportedError, PermissionDeniedError + +_BASE = "https://modelscope.cn/openapi/v1" + + +@pytest.fixture +def client() -> OpenAPIClient: + return OpenAPIClient(HubConfig(token="test-token", endpoint="https://modelscope.cn")) + + +def _response(payload=None, status_code=200): + resp = MagicMock(spec=requests.Response) + resp.status_code = status_code + resp.headers = {} + resp.content = b"x" + resp.json.return_value = payload if payload is not None else {"success": True, "data": {}} + resp.request = MagicMock(method="GET", path_url="/x", url=_BASE) + resp.url = _BASE + return resp + + +# --------------------------------------------------------------------------- +# GET /studios +# --------------------------------------------------------------------------- +class TestListStudios: + def test_url_and_defaults(self, client): + resp = _response({"studios": [], "total_count": 0, "page_number": 1, "page_size": 10}) + with patch.object(client._session, "request", return_value=resp) as mock_req: + client.list_studios() + kwargs = mock_req.call_args.kwargs + assert kwargs["method"] == "GET" + assert kwargs["url"] == f"{_BASE}/studios" + assert dict(kwargs["params"]) == {"page_number": "1", "page_size": "10"} + + def test_all_filters_are_serialised(self, client): + resp = _response({"studios": [], "total_count": 0}) + with patch.object(client._session, "request", return_value=resp) as mock_req: + client.list_studios( + search="chat", + owner="alice", + sort="likes", + page_number=2, + page_size=20, + status="all", + mcp_support=True, + hardware_type="xgpu", + ) + params = dict(mock_req.call_args.kwargs["params"]) + assert params == { + "search": "chat", + "owner": "alice", + "sort": "likes", + "page_number": "2", + "page_size": "20", + "status": "all", + "mcp_support": "true", + "hardware_type": "xgpu", + } + + def test_false_flag_is_sent_not_dropped(self, client): + """``False`` is a real filter value; dropping it would silently widen the query.""" + resp = _response({"studios": []}) + with patch.object(client._session, "request", return_value=resp) as mock_req: + client.list_studios(mcp_support=False) + assert dict(mock_req.call_args.kwargs["params"])["mcp_support"] == "false" + + @pytest.mark.parametrize( + ("kwargs", "field"), + [ + ({"sort": "downloads"}, "sort"), + ({"status": "stopped"}, "status"), + ({"hardware_type": "nvidia"}, "hardware_type"), + ], + ) + def test_unknown_enum_value_is_rejected(self, client, kwargs, field): + with pytest.raises(InvalidParameter, match=field): + client.list_studios(**kwargs) + + def test_offset_limit_is_enforced(self, client): + with pytest.raises(InvalidParameter, match="<= 3000"): + client.list_studios(page_number=61, page_size=50) + + def test_anonymous_when_no_token(self): + config = HubConfig(token="placeholder", endpoint="https://modelscope.cn") + config.token = None + client = OpenAPIClient(config) + with patch.object(HubConfig, "load_token", return_value=None): + with patch.object(client._session, "request", return_value=_response({"studios": []})) as mock_req: + client.list_studios() + assert "Authorization" not in mock_req.call_args.kwargs["headers"] + + +# --------------------------------------------------------------------------- +# Resource discovery +# --------------------------------------------------------------------------- +class TestStudioResourceDiscovery: + def test_hardware_url_and_params(self, client): + resp = _response({"success": True, "data": {"hardware": []}}) + with patch.object(client._session, "request", return_value=resp) as mock_req: + client.list_studio_hardware(sdk_type="gradio", studio="alice/demo") + kwargs = mock_req.call_args.kwargs + assert kwargs["method"] == "GET" + assert kwargs["url"] == f"{_BASE}/studios/hardware" + assert dict(kwargs["params"]) == {"sdk_type": "gradio", "studio": "alice/demo"} + + def test_hardware_rejects_unknown_sdk_type(self, client): + with pytest.raises(InvalidParameter, match="sdk_type"): + client.list_studio_hardware(sdk_type="flask") + + def test_base_images_url(self, client): + resp = _response({"success": True, "data": {"base_images": []}}) + with patch.object(client._session, "request", return_value=resp) as mock_req: + client.list_studio_base_images() + kwargs = mock_req.call_args.kwargs + assert kwargs["method"] == "GET" + assert kwargs["url"] == f"{_BASE}/studios/base-images" + + def test_sdk_versions_url_and_params(self, client): + resp = _response({"success": True, "data": {"sdk_versions": []}}) + with patch.object(client._session, "request", return_value=resp) as mock_req: + client.list_studio_sdk_versions(sdk_type="gradio") + kwargs = mock_req.call_args.kwargs + assert kwargs["url"] == f"{_BASE}/studios/sdk-versions" + assert dict(kwargs["params"]) == {"sdk_type": "gradio"} + + @pytest.mark.parametrize( + "method_name", + ["list_studio_hardware", "list_studio_base_images", "list_studio_sdk_versions"], + ) + def test_discovery_works_without_a_token(self, method_name): + config = HubConfig(token="placeholder", endpoint="https://modelscope.cn") + config.token = None + client = OpenAPIClient(config) + with patch.object(HubConfig, "load_token", return_value=None): + with patch.object(client._session, "request", return_value=_response()) as mock_req: + getattr(client, method_name)() + assert "Authorization" not in mock_req.call_args.kwargs["headers"] + + +# --------------------------------------------------------------------------- +# Plaintext variables +# --------------------------------------------------------------------------- +class TestStudioVariables: + def test_list_url(self, client): + resp = _response({"success": True, "data": {"variables": []}}) + with patch.object(client._session, "request", return_value=resp) as mock_req: + client.list_studio_variables("alice", "demo") + kwargs = mock_req.call_args.kwargs + assert kwargs["method"] == "GET" + assert kwargs["url"] == f"{_BASE}/studios/alice/demo/variables" + + @pytest.mark.parametrize( + ("method_name", "verb"), + [("add_studio_variable", "POST"), ("update_studio_variable", "PUT")], + ) + def test_write_body(self, client, method_name, verb): + with patch.object(client._session, "request", return_value=_response()) as mock_req: + getattr(client, method_name)("alice", "demo", "MODEL", "Qwen") + kwargs = mock_req.call_args.kwargs + assert kwargs["method"] == verb + assert kwargs["url"] == f"{_BASE}/studios/alice/demo/variables" + assert kwargs["json"] == {"key": "MODEL", "value": "Qwen"} + + def test_delete_body_carries_key_only(self, client): + with patch.object(client._session, "request", return_value=_response()) as mock_req: + client.delete_studio_variable("alice", "demo", "MODEL") + kwargs = mock_req.call_args.kwargs + assert kwargs["method"] == "DELETE" + assert kwargs["json"] == {"key": "MODEL"} + + def test_variables_never_degrade_to_anonymous(self, client): + """Account-private data: an anonymous retry would report a false 'empty'.""" + denied = _response({"message": "denied"}, status_code=403) + with patch.object(client._session, "request", return_value=denied) as mock_req: + with pytest.raises(PermissionDeniedError): + client.list_studio_variables("alice", "demo") + assert mock_req.call_count == 1 + + def test_variables_mirror_the_secrets_routes(self, client): + """The two blocks must stay symmetrical apart from the path segment.""" + captured = [] + with patch.object(client._session, "request", return_value=_response()) as mock_req: + client.add_studio_variable("a", "d", "K", "V") + captured.append(mock_req.call_args.kwargs) + client.add_studio_secret("a", "d", "K", "V") + captured.append(mock_req.call_args.kwargs) + variable, secret = captured + assert variable["json"] == secret["json"] + assert variable["method"] == secret["method"] + assert variable["url"].replace("/variables", "") == secret["url"].replace("/secrets", "") + + +# --------------------------------------------------------------------------- +# Studio logs +# --------------------------------------------------------------------------- +class TestStudioLogs: + def test_page_size_cap_is_enforced(self, client): + with pytest.raises(InvalidParameter, match="<= 500"): + client.get_studio_logs("alice", "demo", "run", page_size=501) + + def test_page_size_at_boundary_is_accepted(self, client): + with patch.object(client._session, "request", return_value=_response()) as mock_req: + client.get_studio_logs("alice", "demo", "run", page_size=500) + assert dict(mock_req.call_args.kwargs["params"])["page_size"] == "500" + + def test_unknown_log_type_is_rejected(self, client): + with pytest.raises(InvalidParameter, match="log_type"): + client.get_studio_logs("alice", "demo", "trace") + + +# --------------------------------------------------------------------------- +# HubApi facade +# --------------------------------------------------------------------------- +class TestFacade: + @pytest.fixture + def api(self): + api = HubApi(token="test-token") + api._openapi = MagicMock() + return api + + def test_list_repos_studio_is_supported(self, api): + api._openapi.list_studios.return_value = { + "studios": [{"id": "alice/demo", "sdk_type": "gradio", "hardware": "cpu"}], + "total_count": 1, + "page_number": 1, + "page_size": 10, + } + page = api.list_repos(RepoType.STUDIO) + assert page.total_count == 1 + assert page.items[0].repo_id == "alice/demo" + assert page.items[0].sdk_type == "gradio" + assert page.collection_key == "studios" + + def test_list_repos_studio_parses_unenveloped_payload(self, api): + """The endpoint omits the {success, data} envelope other lists use.""" + api._openapi.list_studios.return_value = {"studios": [{"id": "a/b"}], "total_count": 7} + page = api.list_repos("studio") + assert [r.repo_id for r in page.items] == ["a/b"] + assert page.total_count == 7 + + def test_list_repos_studio_forwards_filters(self, api): + api._openapi.list_studios.return_value = {"studios": []} + api.list_repos("studio", owner="alice", status="all", mcp_support=True) + kwargs = api._openapi.list_studios.call_args.kwargs + assert kwargs["owner"] == "alice" + assert kwargs["status"] == "all" + assert kwargs["mcp_support"] is True + + def test_variables_unwrap_the_payload(self, api): + api._openapi.list_studio_variables.return_value = {"variables": [{"key": "K", "value": "V"}]} + assert api.list_variables("alice/demo") == [{"key": "K", "value": "V"}] + + def test_variables_tolerate_a_bare_list(self, api): + api._openapi.list_studio_variables.return_value = [{"key": "K", "value": "V"}] + assert api.list_variables("alice/demo") == [{"key": "K", "value": "V"}] + + def test_variables_tolerate_an_empty_payload(self, api): + api._openapi.list_studio_variables.return_value = None + assert api.list_variables("alice/demo") == [] + + @pytest.mark.parametrize( + ("method_name", "args"), + [ + ("list_variables", ()), + ("add_variable", ("K", "V")), + ("update_variable", ("K", "V")), + ("delete_variable", ("K",)), + ], + ) + def test_variables_are_studio_only(self, api, method_name, args): + with pytest.raises(NotSupportedError): + getattr(api, method_name)("alice/demo", *args, repo_type=RepoType.MODEL) + + def test_hardware_unwraps_the_payload(self, api): + api._openapi.list_studio_hardware.return_value = {"hardware": [{"name": "cpu"}]} + assert api.list_studio_hardware() == [{"name": "cpu"}] + + def test_hardware_forwards_repo_id_as_studio(self, api): + api._openapi.list_studio_hardware.return_value = {"hardware": []} + api.list_studio_hardware(sdk_type="gradio", repo_id="alice/demo") + api._openapi.list_studio_hardware.assert_called_once_with(sdk_type="gradio", studio="alice/demo") + + def test_base_images_unwraps_the_payload(self, api): + api._openapi.list_studio_base_images.return_value = {"base_images": [{"name": "ubuntu"}]} + assert api.list_studio_base_images() == [{"name": "ubuntu"}] + + def test_sdk_versions_unwraps_the_payload(self, api): + api._openapi.list_studio_sdk_versions.return_value = {"sdk_versions": [{"version": "4.44.1"}]} + assert api.list_studio_sdk_versions(sdk_type="gradio") == [{"version": "4.44.1"}] + + def test_operational_mcp_servers_matches_list_mcp_servers_shape(self, api): + api._openapi.list_operational_mcp_servers.return_value = { + "mcp_server_list": [{"id": "alice/weather", "operational_urls": [{"url": "https://x/sse"}]}], + "total_count": 1, + } + page = api.list_operational_mcp_servers() + assert page.total_count == 1 + assert page.items[0]["id"] == "alice/weather" + + +# --------------------------------------------------------------------------- +# Studio field normalisation must not leak onto other repo types +# --------------------------------------------------------------------------- +class TestStudioFieldNormalisation: + @pytest.fixture + def api(self): + api = HubApi(token="test-token") + api._openapi = MagicMock() + return api + + def test_camelcase_cover_image_is_normalised_for_studios(self, api): + api.create_repo("alice/demo", RepoType.STUDIO, coverImage="https://img") + payload = api._openapi.create_studio.call_args[0][0] + assert payload["cover_image"] == "https://img" + assert "coverImage" not in payload + + def test_studio_visibility_expands_to_the_private_companion(self, api): + api.create_repo("alice/demo", RepoType.STUDIO, visibility="protected") + payload = api._openapi.create_studio.call_args[0][0] + assert payload["visibility"] == "protected" + assert payload["private"] is False + + def test_skill_payload_is_left_alone(self, api): + """A Skill has no cover image and no visibility field, so the Studio + normaliser must not invent them on a Skill payload.""" + api.create_repo("alice/my-skill", RepoType.SKILL, coverImage="https://img") + payload = api._openapi.create_skill.call_args[0][0] + assert payload["coverImage"] == "https://img" + assert "cover_image" not in payload + + def test_skill_visibility_does_not_gain_a_studio_field(self, api): + api.create_repo("alice/my-skill", RepoType.SKILL, visibility="private") + payload = api._openapi.create_skill.call_args[0][0] + assert "visibility" not in payload + assert payload["private"] is True + + def test_settings_normalisation_applies_to_studios_only(self, api): + api.update_repo_settings("alice/demo", RepoType.STUDIO, coverImage="https://img") + assert api._openapi.update_studio_settings.call_args[0][2] == {"cover_image": "https://img"} + api.update_repo_settings("alice/my-skill", RepoType.SKILL, logo_url="https://logo") + assert api._openapi.update_skill_settings.call_args[0][2] == {"logo_url": "https://logo"} diff --git a/tests/cli/test_repo.py b/tests/cli/test_repo.py index 3e45300..e4400a1 100644 --- a/tests/cli/test_repo.py +++ b/tests/cli/test_repo.py @@ -193,14 +193,30 @@ def test_subcmd_token_endpoint(self, parser): class TestListParser: """``ms list`` argument parsing.""" - @pytest.mark.parametrize("repo_type", ["model", "dataset", "skill", "mcp"]) + @pytest.mark.parametrize("repo_type", ["model", "dataset", "studio", "skill", "mcp"]) def test_all_repo_types(self, parser, repo_type): args = parser.parse_args(["list", "--repo-type", repo_type]) assert args.repo_type == repo_type def test_invalid_repo_type_rejected(self, parser): with pytest.raises(SystemExit): - parser.parse_args(["list", "--repo-type", "studio"]) + parser.parse_args(["list", "--repo-type", "gallery"]) + + def test_studio_owner_listing_asks_for_every_status(self, parser, mock_api): + """The Studios endpoint keeps filtering to running spaces even with an + owner set, so listing your own would otherwise answer with nothing.""" + mock_api.list_repos.return_value = PagedResult(items=[], total_count=0) + args = parser.parse_args(["list", "--repo-type", "studio", "--owner", "alice"]) + with patch("modelscope_hub.cli.repo.make_api", return_value=mock_api): + ListCommand(args).execute() + assert mock_api.list_repos.call_args.kwargs["status"] == "all" + + def test_non_studio_listing_sends_no_status(self, parser, mock_api): + mock_api.list_repos.return_value = PagedResult(items=[], total_count=0) + args = parser.parse_args(["list", "--repo-type", "model", "--owner", "alice"]) + with patch("modelscope_hub.cli.repo.make_api", return_value=mock_api): + ListCommand(args).execute() + assert "status" not in mock_api.list_repos.call_args.kwargs def test_owner_flag(self, parser): args = parser.parse_args(["list", "--repo-type", "model", "--owner", "my-org"]) diff --git a/tests/cli/test_studio.py b/tests/cli/test_studio.py index 23b47dd..62fea32 100644 --- a/tests/cli/test_studio.py +++ b/tests/cli/test_studio.py @@ -8,6 +8,7 @@ from modelscope_hub.cli.studio import StudioCommand from modelscope_hub.constants import RepoType +from modelscope_hub.types import PagedResult, RepoInfo from .conftest import run_cli @@ -101,6 +102,81 @@ def test_secret_add(self, parser): assert args.key == "API_KEY" assert args.value == "value" + def test_list_filters(self, parser): + args = parser.parse_args( + [ + "studio", + "list", + "--search", + "chat", + "--owner", + "alice", + "--sort", + "likes", + "--status", + "all", + "--mcp-support", + "--hardware-type", + "xgpu", + "--page", + "2", + "--page-size", + "20", + ] + ) + assert args.studio_action == "list" + assert args.search == "chat" + assert args.owner == "alice" + assert args.sort == "likes" + assert args.status == "all" + assert args.mcp_support is True + assert args.hardware_type == "xgpu" + assert args.page_number == 2 + assert args.page_size == 20 + + def test_list_no_mcp_support_flag(self, parser): + args = parser.parse_args(["studio", "list", "--no-mcp-support"]) + assert args.mcp_support is False + + def test_list_rejects_unknown_sort(self, parser): + with pytest.raises(SystemExit): + parser.parse_args(["studio", "list", "--sort", "downloads"]) + + def test_variable_add(self, parser): + args = parser.parse_args(["studio", "variable", "add", "org/demo", "MODEL", "Qwen"]) + assert args.studio_action == "variable" + assert args.variable_action == "add" + assert args.key == "MODEL" + assert args.value == "Qwen" + + def test_variable_delete_has_confirmation_flag(self, parser): + args = parser.parse_args(["studio", "variable", "delete", "org/demo", "MODEL", "--yes"]) + assert args.variable_action == "delete" + assert args.yes is True + + def test_hardware_options(self, parser): + args = parser.parse_args(["studio", "hardware", "--sdk-type", "gradio", "--studio", "org/demo"]) + assert args.studio_action == "hardware" + assert args.sdk_type == "gradio" + assert args.studio_id == "org/demo" + + def test_base_images(self, parser): + args = parser.parse_args(["studio", "base-images"]) + assert args.studio_action == "base-images" + + def test_sdk_versions_defaults_to_gradio(self, parser): + args = parser.parse_args(["studio", "sdk-versions"]) + assert args.studio_action == "sdk-versions" + assert args.sdk_type == "gradio" + + def test_settings_visibility_flag(self, parser): + args = parser.parse_args(["studio", "settings", "org/demo", "--visibility", "protected"]) + assert args.visibility == "protected" + + def test_settings_visibility_and_private_are_mutually_exclusive(self, parser): + with pytest.raises(SystemExit): + parser.parse_args(["studio", "settings", "org/demo", "--visibility", "public", "--private"]) + # =================================================================== # Execution tests @@ -223,3 +299,196 @@ def test_secret_delete(self, mock_api): assert code == 0 assert "deleted" in out.lower() mock_api.delete_secret.assert_called_once_with("org/demo", "API_KEY", RepoType.STUDIO) + + # -- list --------------------------------------------------------- + def test_list_renders_studio_columns(self, mock_api): + mock_api.list_repos.return_value = PagedResult( + items=[ + RepoInfo( + owner="alice", + name="demo", + sdk_type="gradio", + hardware="cpu-basic", + likes=7, + runtime={"status": "Running"}, + ) + ], + total_count=1, + page_number=1, + page_size=10, + ) + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli(["studio", "list"]) + assert code == 0 + for expected in ("alice/demo", "gradio", "cpu-basic", "Running", "7"): + assert expected in out + + def test_list_forwards_every_filter(self, mock_api): + mock_api.list_repos.return_value = PagedResult(items=[], total_count=0) + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli( + ["studio", "list", "--owner", "alice", "--status", "all", "--hardware-type", "xgpu"] + ) + assert code == 0 + assert "(no studios found)" in out + kwargs = mock_api.list_repos.call_args.kwargs + assert kwargs["owner"] == "alice" + assert kwargs["status"] == "all" + assert kwargs["hardware_type"] == "xgpu" + + def test_owner_listing_defaults_to_every_status(self, mock_api): + """The endpoint keeps filtering to running spaces even with an owner set, + so "list my spaces" would otherwise answer with nothing.""" + mock_api.list_repos.return_value = PagedResult(items=[], total_count=0) + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + run_cli(["studio", "list", "--owner", "alice"]) + assert mock_api.list_repos.call_args.kwargs["status"] == "all" + + def test_explicit_status_wins_over_the_owner_default(self, mock_api): + mock_api.list_repos.return_value = PagedResult(items=[], total_count=0) + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + run_cli(["studio", "list", "--owner", "alice", "--status", "running"]) + assert mock_api.list_repos.call_args.kwargs["status"] == "running" + + def test_search_without_owner_leaves_the_status_to_the_server(self, mock_api): + mock_api.list_repos.return_value = PagedResult(items=[], total_count=0) + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + run_cli(["studio", "list", "--search", "chat"]) + assert mock_api.list_repos.call_args.kwargs["status"] is None + + def test_list_shows_protected_visibility_verbatim(self, mock_api): + """``protected`` has no integer equivalent, so it must survive as a string.""" + mock_api.list_repos.return_value = PagedResult( + items=[RepoInfo(owner="alice", name="demo", visibility="protected")], + total_count=1, + ) + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli(["studio", "list"]) + assert code == 0 + assert "protected" in out + + # -- variable ----------------------------------------------------- + def test_variable_list_shows_values(self, mock_api): + """Unlike secrets, plaintext variable values are public and are printed.""" + mock_api.list_variables.return_value = [{"key": "MODEL", "value": "Qwen2.5-7B"}] + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli(["studio", "variable", "list", "org/demo"]) + assert code == 0 + assert "MODEL" in out + assert "Qwen2.5-7B" in out + mock_api.list_variables.assert_called_once_with("org/demo", RepoType.STUDIO) + + def test_variable_list_empty(self, mock_api): + mock_api.list_variables.return_value = [] + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli(["studio", "variable", "list", "org/demo"]) + assert code == 0 + assert "(no variables)" in out + + def test_variable_add(self, mock_api): + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli(["studio", "variable", "add", "org/demo", "MODEL", "Qwen"]) + assert code == 0 + assert "added" in out.lower() + mock_api.add_variable.assert_called_once_with("org/demo", "MODEL", "Qwen", RepoType.STUDIO) + + def test_variable_update(self, mock_api): + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli(["studio", "variable", "update", "org/demo", "MODEL", "Qwen3"]) + assert code == 0 + assert "updated" in out.lower() + mock_api.update_variable.assert_called_once_with("org/demo", "MODEL", "Qwen3", RepoType.STUDIO) + + def test_variable_delete_with_yes(self, mock_api): + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli(["studio", "variable", "delete", "org/demo", "MODEL", "--yes"]) + assert code == 0 + assert "deleted" in out.lower() + mock_api.delete_variable.assert_called_once_with("org/demo", "MODEL", RepoType.STUDIO) + + def test_variable_delete_aborts_without_confirmation(self, mock_api): + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + with patch("builtins.input", return_value="n"): + code, out, err = run_cli(["studio", "variable", "delete", "org/demo", "MODEL"]) + assert code == 0 + assert "Aborted" in out + mock_api.delete_variable.assert_not_called() + + # -- resource discovery ------------------------------------------- + def test_hardware_table(self, mock_api): + mock_api.list_studio_hardware.return_value = [ + { + "name": "CPU basic", + "instance_type": "ecs.c6.large", + "resource_type": "cpu", + "has_stock": True, + "stock": 5, + }, + { + "name": "GPU A10", + "instance_type": "ecs.gn7i", + "resource_type": "gpu", + "gpu_type": "A10", + "has_stock": False, + "cost_after_discount": 8, + "original_cost": 10, + }, + ] + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli(["studio", "hardware", "--sdk-type", "gradio"]) + assert code == 0 + assert "CPU basic" in out + assert "ecs.gn7i" in out + assert "out of stock" in out + assert "8 (was 10)" in out + assert "paid/" in out + mock_api.list_studio_hardware.assert_called_once_with(sdk_type="gradio", repo_id=None) + + def test_hardware_scoped_to_a_studio(self, mock_api): + mock_api.list_studio_hardware.return_value = [] + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli(["studio", "hardware", "--studio", "org/demo"]) + assert code == 0 + mock_api.list_studio_hardware.assert_called_once_with(sdk_type=None, repo_id="org/demo") + + def test_base_images_table(self, mock_api): + mock_api.list_studio_base_images.return_value = [{"name": "ubuntu22.04-py311", "tag": "1.0"}] + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli(["studio", "base-images"]) + assert code == 0 + assert "ubuntu22.04-py311" in out + + def test_sdk_versions_table(self, mock_api): + mock_api.list_studio_sdk_versions.return_value = [{"sdk_type": "gradio", "version": "4.44.1", "tag": "latest"}] + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli(["studio", "sdk-versions"]) + assert code == 0 + assert "4.44.1" in out + mock_api.list_studio_sdk_versions.assert_called_once_with(sdk_type="gradio") + + def test_sdk_versions_empty_names_the_sdk_type(self, mock_api): + mock_api.list_studio_sdk_versions.return_value = [] + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli(["studio", "sdk-versions", "--sdk-type", "docker"]) + assert code == 0 + assert "docker" in out + + # -- settings ----------------------------------------------------- + def test_settings_forwards_visibility(self, mock_api): + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli(["studio", "settings", "org/demo", "--visibility", "protected"]) + assert code == 0 + mock_api.update_repo_settings.assert_called_once_with("org/demo", RepoType.STUDIO, visibility="protected") + + def test_logs_footer_uses_total_count(self, mock_api): + """The payload reports ``total_count``; reading ``total`` printed nothing.""" + mock_api.get_repo_logs.return_value = { + "logs": ["line1"], + "total_count": 42, + "total_page_num": 5, + } + with patch("modelscope_hub.cli.studio.make_api", return_value=mock_api): + code, out, err = run_cli(["studio", "logs", "org/demo"]) + assert code == 0 + assert "total 42" in out + assert "5 page(s)" in out diff --git a/tests/data/README.md b/tests/data/README.md new file mode 100644 index 0000000..2ff2355 --- /dev/null +++ b/tests/data/README.md @@ -0,0 +1,29 @@ +# Vendored ModelScope OpenAPI specification + +`openapi.json` in this directory is a verbatim copy of the live ModelScope +OpenAPI document, kept in-tree so that spec drift becomes a test failure instead +of a silent gap. + +| | | +|---|---| +| Source | `https://modelscope.cn/openapi/v1` (document served alongside it) | +| `info.version` | `1.1.0+master.20260813T030041Z` | +| `openapi` | `3.1.1` | +| Operations | 56 across 11 tags | + +## Why it is here + +`tests/test_openapi_coverage.py` reads this file and asserts that every +`operationId` in the tags the SDK claims to cover is registered in +`modelscope_hub._openapi.OPERATION_REGISTRY`. When the service publishes a new +operation, refreshing this file makes the guard fail by name rather than leaving +the gap to be discovered by hand later. + +Tags not yet implemented are listed in that test's `_DEFERRED_TAGS`; the set +doubles as the remaining to-do list. + +## Refreshing + +Replace the file with the current document and update `info.version` above. Any +new operation in a covered tag will fail the guard until it is either +implemented and registered, or explicitly deferred. diff --git a/tests/data/openapi.json b/tests/data/openapi.json new file mode 100644 index 0000000..aaed4a3 --- /dev/null +++ b/tests/data/openapi.json @@ -0,0 +1,10183 @@ +{ + "openapi": "3.1.1", + "info": { + "title": "ModelScope OpenAPI", + "description": "魔搭社区 OpenAPI 文档", + "version": "1.1.0+master.20260813T030041Z", + "termsOfService": "https://www.modelscope.cn/protocol/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC", + "contact": { + "name": "modelscope", + "url": "https://www.modelscope.cn/protocol/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC", + "email": "contact@modelscope.cn" + }, + "license": { + "name": "Apache 2.0", + "url": "https://www.apache.org/licenses/LICENSE-2.0.html" + } + }, + "servers": [ + { + "url": "https://modelscope.cn/openapi/v1" + } + ], + "security": [ + { + "bearerAuth": [] + } + ], + "tags": [ + { + "name": "User", + "description": "用户相关接口" + }, + { + "name": "Models", + "description": "模型相关接口" + }, + { + "name": "Datasets", + "description": "数据集相关接口" + }, + { + "name": "MCP", + "description": "MCP 服务相关接口" + }, + { + "name": "Studios", + "description": "创空间相关接口" + }, + { + "name": "Skills", + "description": "技能相关接口" + }, + { + "name": "Files", + "description": "文件上传相关接口" + }, + { + "name": "Agent-IDP", + "description": "Agent-IDP身份认证服务（beta版本）" + }, + { + "name": "Magicube", + "description": "魔粒体系相关接口" + }, + { + "name": "Collections", + "description": "合集相关接口" + }, + { + "name": "Galleries", + "description": "Gallery 相关接口" + } + ], + "externalDocs": { + "description": "了解更多关于 ModelScope 的信息", + "url": "https://modelscope.cn" + }, + "paths": { + "/users/me": { + "get": { + "operationId": "getCurrentUser", + "summary": "获取当前用户信息", + "description": "获取当前已认证用户的个人信息。", + "tags": [ + "User" + ], + "parameters": [], + "responses": { + "200": { + "$ref": "#/components/responses/get-current-user-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/models": { + "get": { + "operationId": "listModels", + "summary": "获取模型列表", + "description": "列出模型", + "tags": [ + "Models" + ], + "parameters": [ + { + "name": "search", + "in": "query", + "description": "针对模型名称、作者（包括组织、个人）的子字符串关键词进行搜索", + "required": false, + "schema": { + "type": "string", + "maxLength": 100, + "examples": [ + "Qwen", + "glm" + ] + } + }, + { + "name": "owner", + "in": "query", + "description": "针对模型作者（包括组织、个人）进行搜索", + "required": false, + "schema": { + "type": "string", + "examples": [ + "iic", + "ZhipuAI" + ] + } + }, + { + "name": "sort", + "in": "query", + "description": "排序方式", + "required": false, + "schema": { + "type": "string", + "enum": [ + "default", + "downloads", + "likes", + "last_modified" + ], + "default": "default", + "examples": [ + "default", + "downloads", + "likes", + "last_modified" + ] + } + }, + { + "name": "page_number", + "in": "query", + "description": "页码，限制 page_number * page_size <= 3000", + "required": false, + "schema": { + "type": "integer", + "minimum": 1, + "default": 1, + "examples": [ + 1 + ] + } + }, + { + "name": "page_size", + "in": "query", + "description": "每页大小，限制 page_number * page_size <= 3000", + "required": false, + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 50, + "default": 10, + "examples": [ + 10 + ] + } + }, + { + "name": "filter.task", + "in": "query", + "description": "按任务筛选（示例：text-generation / image-captioning）", + "required": false, + "schema": { + "type": "string", + "examples": [ + "text-generation", + "image-captioning" + ] + } + }, + { + "name": "filter.library", + "in": "query", + "description": "按库/框架筛选（示例：pytorch / safetensors / diffusers / transformer）", + "required": false, + "schema": { + "type": "string", + "examples": [ + "pytorch", + "safetensors", + "diffusers", + "transformer" + ] + } + }, + { + "name": "filter.model_type", + "in": "query", + "description": "按模型类型筛选（示例：qwen3_moe / glm4v）", + "required": false, + "schema": { + "type": "string", + "examples": [ + "qwen3_moe", + "glm4v" + ] + } + }, + { + "name": "filter.custom_tag", + "in": "query", + "description": "按自定义标签筛选（示例：llm）", + "required": false, + "schema": { + "type": "string", + "examples": [ + "common-crawl", + "markdown", + "math", + "html-parsing" + ] + } + }, + { + "name": "filter.license", + "in": "query", + "description": "按许可证筛选（示例：Apache License 2.0 / MIT License）", + "required": false, + "schema": { + "type": "string", + "examples": [ + "Apache License 2.0", + "MIT License" + ] + } + }, + { + "name": "filter.deploy", + "in": "query", + "description": "按部署方式筛选（示例：swingdeploy）", + "required": false, + "schema": { + "type": "string", + "examples": [ + "swingdeploy" + ] + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-models-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/models/{owner}/{repo_name}": { + "get": { + "operationId": "getModel", + "summary": "获取模型详情", + "description": "根据 owner/repo_name 获取模型详情", + "tags": [ + "Models" + ], + "parameters": [ + { + "$ref": "#/components/parameters/owner" + }, + { + "$ref": "#/components/parameters/repo_name" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/get-model-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/datasets": { + "get": { + "operationId": "listDatasets", + "summary": "获取数据集列表", + "description": "列出数据集", + "tags": [ + "Datasets" + ], + "parameters": [ + { + "name": "search", + "in": "query", + "description": "针对数据集中文名称、数据集英文名称、作者（包括组织、个人）的子字符串关键词进行搜索", + "required": false, + "schema": { + "type": "string", + "maxLength": 100, + "examples": [ + "ImageNet", + "中文语料库" + ] + } + }, + { + "name": "owner", + "in": "query", + "description": "针对数据集作者（包括组织、个人）进行搜索", + "required": false, + "schema": { + "type": "string", + "examples": [ + "modelscope", + "AI-ModelScope" + ] + } + }, + { + "name": "sort", + "in": "query", + "description": "排序方式", + "required": false, + "schema": { + "type": "string", + "enum": [ + "default", + "downloads", + "likes", + "last_modified" + ], + "default": "default", + "examples": [ + "default", + "downloads", + "likes", + "last_modified" + ] + } + }, + { + "name": "page_number", + "in": "query", + "description": "页码，限制 page_number * page_size <= 3000", + "required": false, + "schema": { + "type": "integer", + "minimum": 1, + "default": 1, + "examples": [ + 1 + ] + } + }, + { + "name": "page_size", + "in": "query", + "description": "每页大小，限制 page_number * page_size <= 3000", + "required": false, + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 50, + "default": 10, + "examples": [ + 10 + ] + } + }, + { + "name": "filter.task", + "in": "query", + "description": "按任务筛选（示例：text-classification）", + "required": false, + "schema": { + "type": "string", + "examples": [ + "text-classification" + ] + } + }, + { + "name": "filter.license", + "in": "query", + "description": "按许可证筛选（示例：Apache License 2.0 / cc-by-4.0）", + "required": false, + "schema": { + "type": "string", + "examples": [ + "Apache License 2.0", + "cc-by-4.0" + ] + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-datasets-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/datasets/{owner}/{repo_name}": { + "get": { + "operationId": "getDataset", + "summary": "获取数据集详情", + "description": "根据 owner/repo_name 获取数据集详情", + "tags": [ + "Datasets" + ], + "parameters": [ + { + "$ref": "#/components/parameters/owner" + }, + { + "$ref": "#/components/parameters/repo_name" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/get-dataset-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/mcp/servers": { + "put": { + "operationId": "listMcpServers", + "summary": "获取MCP服务列表", + "description": "返回查询的 ModelScope MCP 广场服务列表", + "tags": [ + "MCP" + ], + "requestBody": { + "description": "查询参数", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "search": { + "type": "string", + "description": "支持针对服务中文名称、服务英文名称、作者/所有者用户名进行搜索" + }, + "filter": { + "type": "object", + "description": "筛选条件", + "properties": { + "category": { + "type": "string", + "description": "指定筛选类别标签。与tag,is_hosted同时传入时,取三者的交集", + "examples": [ + "communication" + ] + }, + "is_hosted": { + "type": "boolean", + "description": "指定筛选hosted或未hosted MCP服务。与tag,category同时传入时,取三者的交集", + "examples": [ + true + ] + } + } + }, + "page_number": { + "type": "integer", + "description": "页码，默认为 1，限制 page_number * page_size <= 100", + "default": 1 + }, + "page_size": { + "type": "integer", + "description": "每页大小，默认为20，限制 page_number * page_size <= 100", + "default": 20 + } + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/list-mcp-servers-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/mcp/servers/operational": { + "get": { + "operationId": "listOperationalMcpServers", + "summary": "获取用户托管MCP服务列表", + "description": "获取用户托管MCP服务列表", + "tags": [ + "MCP" + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-operational-mcp-servers-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/mcp/servers/{id}": { + "get": { + "operationId": "getMcpServer", + "summary": "获取指定MCP服务详情", + "description": "返回指定 MCP 服务的详细信息", + "tags": [ + "MCP" + ], + "parameters": [ + { + "$ref": "#/components/parameters/mcp_server_id" + }, + { + "name": "get_operational_url", + "in": "query", + "description": "传入true时返回当前用户在modelscope上托管的MCP服务链接（例如SSE_URL），默认false", + "required": false, + "schema": { + "type": "boolean", + "default": false + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/get-mcp-server-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/mcp/servers/{id}/deploy": { + "post": { + "operationId": "deployMcpServer", + "summary": "部署MCP服务", + "description": "部署MCP服务", + "tags": [ + "MCP" + ], + "parameters": [ + { + "$ref": "#/components/parameters/mcp_server_id" + } + ], + "requestBody": { + "description": "部署参数", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "transport_type": { + "type": "string", + "description": "传输方式: sse/streamable_http", + "enum": [ + "sse", + "streamable_http" + ], + "examples": [ + "streamable_http" + ] + }, + "expiration_minutes": { + "type": "integer", + "description": "有效期（分钟），-1 代表长期有效", + "examples": [ + -1, + 60, + 1440 + ] + }, + "auth_check": { + "type": "boolean", + "description": "远程 url 连接时是否需要提供魔搭访问令牌鉴权", + "default": false + }, + "env_info": { + "type": "object", + "description": "MCP环境变量，具体需要传入的环境变量可在获取指定MCP服务详情接口返回的 env_schema 字段中获取", + "additionalProperties": true + } + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/deploy-mcp-server-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/mcp/servers/{id}/undeploy": { + "delete": { + "operationId": "undeployMcpServer", + "summary": "解除MCP服务部署", + "description": "解除MCP服务部署", + "tags": [ + "MCP" + ], + "parameters": [ + { + "$ref": "#/components/parameters/mcp_server_id" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/undeploy-mcp-server-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/studios": { + "get": { + "operationId": "listStudios", + "summary": "获取创空间列表", + "description": "列出创空间", + "tags": [ + "Studios" + ], + "parameters": [ + { + "name": "search", + "in": "query", + "description": "针对创空间名称、展示名称、作者（包括组织、个人）的子字符串关键词进行搜索", + "required": false, + "schema": { + "type": "string", + "maxLength": 100, + "examples": [ + "chatbot", + "demo" + ] + } + }, + { + "name": "owner", + "in": "query", + "description": "针对创空间作者（包括组织、个人）进行搜索", + "required": false, + "schema": { + "type": "string", + "examples": [ + "iic", + "ZhipuAI" + ] + } + }, + { + "name": "sort", + "in": "query", + "description": "排序方式", + "required": false, + "schema": { + "type": "string", + "enum": [ + "default", + "last_modified", + "view_num", + "likes" + ], + "default": "default", + "examples": [ + "default", + "last_modified", + "view_num", + "likes" + ] + } + }, + { + "name": "page_number", + "in": "query", + "description": "页码，限制 page_number * page_size <= 3000", + "required": false, + "schema": { + "type": "integer", + "minimum": 1, + "default": 1, + "examples": [ + 1 + ] + } + }, + { + "name": "page_size", + "in": "query", + "description": "每页大小（1~50），限制 page_number * page_size <= 3000", + "required": false, + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 50, + "default": 10, + "examples": [ + 10 + ] + } + }, + { + "name": "status", + "in": "query", + "description": "按创空间运行状态筛选。正常搜索仅支持 running；当按 owner 筛选时，默认为 all", + "required": false, + "schema": { + "type": "string", + "enum": [ + "running", + "all" + ], + "default": "running", + "examples": [ + "running", + "all" + ] + } + }, + { + "name": "mcp_support", + "in": "query", + "description": "是否支持通过MCP使用", + "required": false, + "schema": { + "type": "boolean", + "examples": [ + true, + false + ] + } + }, + { + "name": "hardware_type", + "in": "query", + "description": "根据创空间硬件类型筛选，当前仅支持筛选 xGPU", + "required": false, + "schema": { + "type": "string", + "enum": [ + "xgpu", + "amd" + ], + "examples": [ + "xgpu" + ] + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-studios-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "post": { + "operationId": "createStudio", + "summary": "创建创空间", + "description": "创建一个新的 Studio（创空间）", + "tags": [ + "Studios" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateStudioRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/create-studio-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "409": { + "$ref": "#/components/responses/conflict" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/studios/hardware": { + "get": { + "operationId": "listHardware", + "summary": "查询可用硬件配置", + "description": "查询创空间可用的硬件配置列表。\n- 未登录：返回默认免费硬件\n- 已登录：返回用户可用的免费硬件和带价格的付费硬件\n- 已登录且指定创空间：免费硬件以该创空间的可用免费资源为准\n- 付费资源：使用 `paid/` 作为创建或更新 Studio 时的 `hardware` 参数值\n- `sdk_type` 用于按 `supported_sdk_types` 过滤硬件配置\n", + "tags": [ + "Studios" + ], + "parameters": [ + { + "name": "sdk_type", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/StudioSDKType" + }, + "description": "过滤支持该 SDK 类型的硬件" + }, + { + "name": "studio", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "description": "创空间路径（owner/repo_name），指定后返回该创空间可选的硬件" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-hardware-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + }, + {} + ] + } + }, + "/studios/sdk-versions": { + "get": { + "operationId": "listSdkVersions", + "summary": "查询可用 SDK 版本", + "description": "查询创空间可用的 SDK 版本列表。\n仅当 `sdk_type=gradio` 时返回 Gradio 版本列表；其他 SDK 类型或未传 `sdk_type` 时返回空列表。\n", + "tags": [ + "Studios" + ], + "parameters": [ + { + "name": "sdk_type", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/StudioSDKType" + }, + "description": "SDK 类型，仅 `gradio` 返回版本列表", + "example": "gradio" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-sdk-versions-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + }, + {} + ] + } + }, + "/studios/base-images": { + "get": { + "operationId": "listBaseImages", + "summary": "查询可用基础镜像", + "description": "查询创空间可用的基础镜像列表，登录与否返回结果一致", + "tags": [ + "Studios" + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-base-images-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + }, + {} + ] + } + }, + "/studios/{owner}/{repo_name}": { + "get": { + "operationId": "getStudio", + "summary": "获取创空间详情", + "description": "获取指定 Studio 的详细信息。\n公开（public）和公开体验（protected）类型的创空间无需认证即可访问；私有（private）创空间需要认证。\n", + "tags": [ + "Studios" + ], + "parameters": [ + { + "$ref": "#/components/parameters/studio_owner" + }, + { + "$ref": "#/components/parameters/studio_repo_name" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/get-studio-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + }, + {} + ] + } + }, + "/studios/{owner}/{repo_name}/settings": { + "patch": { + "operationId": "updateStudioSettings", + "summary": "更新创空间设置", + "description": "更新指定 Studio 的设置，传入哪个字段修改哪个字段，不传的字段不修改。注意：sdk_type、sdk_version、base_image、hardware 修改后需重新部署才能生效", + "tags": [ + "Studios" + ], + "parameters": [ + { + "$ref": "#/components/parameters/studio_owner" + }, + { + "$ref": "#/components/parameters/studio_repo_name" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateStudioSettingsRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/update-studio-settings-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/studios/{owner}/{repo_name}/secrets": { + "get": { + "operationId": "listStudioSecrets", + "summary": "获取创空间密文变量列表", + "description": "获取指定 Studio 的所有密文变量 key 列表，不返回 value", + "tags": [ + "Studios" + ], + "parameters": [ + { + "$ref": "#/components/parameters/studio_owner" + }, + { + "$ref": "#/components/parameters/studio_repo_name" + } + ], + "responses": { + "200": { + "description": "成功获取密文变量列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListStudioSecretsResponse" + } + } + } + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "post": { + "operationId": "addStudioSecret", + "summary": "添加创空间密文变量", + "description": "为指定的 Studio 添加一个密文变量，value 不会公开展示", + "tags": [ + "Studios" + ], + "parameters": [ + { + "$ref": "#/components/parameters/studio_owner" + }, + { + "$ref": "#/components/parameters/studio_repo_name" + } + ], + "requestBody": { + "description": "添加密文变量请求参数", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AddStudioSecretRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/studio-operation-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "409": { + "$ref": "#/components/responses/conflict" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "put": { + "operationId": "updateStudioSecret", + "summary": "更新创空间密文变量", + "description": "更新指定 Studio 的一个密文变量值", + "tags": [ + "Studios" + ], + "parameters": [ + { + "$ref": "#/components/parameters/studio_owner" + }, + { + "$ref": "#/components/parameters/studio_repo_name" + } + ], + "requestBody": { + "description": "更新密文变量请求参数", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateStudioSecretRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/studio-operation-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "delete": { + "operationId": "deleteStudioSecret", + "summary": "删除创空间密文变量", + "description": "删除指定 Studio 的一个密文变量", + "tags": [ + "Studios" + ], + "parameters": [ + { + "$ref": "#/components/parameters/studio_owner" + }, + { + "$ref": "#/components/parameters/studio_repo_name" + } + ], + "requestBody": { + "description": "删除密文变量请求参数", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteStudioSecretRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/studio-operation-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/studios/{owner}/{repo_name}/variables": { + "get": { + "operationId": "listStudioVariables", + "summary": "获取创空间明文变量列表", + "description": "获取指定 Studio 的所有明文变量的 key 和 value（与密文变量不同，明文变量的值公开可见）", + "tags": [ + "Studios" + ], + "parameters": [ + { + "$ref": "#/components/parameters/studio_owner" + }, + { + "$ref": "#/components/parameters/studio_repo_name" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-studio-variables-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "post": { + "operationId": "addStudioVariable", + "summary": "添加创空间明文变量", + "description": "为指定的 Studio 添加一个明文变量（key 和 value 都公开可见，敏感信息请使用密文变量接口）", + "tags": [ + "Studios" + ], + "parameters": [ + { + "$ref": "#/components/parameters/studio_owner" + }, + { + "$ref": "#/components/parameters/studio_repo_name" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AddStudioVariableRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/studio-operation-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "409": { + "$ref": "#/components/responses/conflict" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "put": { + "operationId": "updateStudioVariable", + "summary": "更新创空间明文变量", + "description": "更新指定 Studio 的一个明文变量值", + "tags": [ + "Studios" + ], + "parameters": [ + { + "$ref": "#/components/parameters/studio_owner" + }, + { + "$ref": "#/components/parameters/studio_repo_name" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateStudioVariableRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/studio-operation-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "delete": { + "operationId": "deleteStudioVariable", + "summary": "删除创空间明文变量", + "description": "删除指定 Studio 的一个明文变量", + "tags": [ + "Studios" + ], + "parameters": [ + { + "$ref": "#/components/parameters/studio_owner" + }, + { + "$ref": "#/components/parameters/studio_repo_name" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteStudioVariableRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/studio-operation-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/studios/{owner}/{repo_name}/deploy": { + "post": { + "operationId": "deployStudio", + "summary": "部署创空间", + "description": "部署指定的 Studio（会重新拉取代码并重建），无论当前状态是停止还是运行中均可调用", + "tags": [ + "Studios" + ], + "parameters": [ + { + "$ref": "#/components/parameters/studio_owner" + }, + { + "$ref": "#/components/parameters/studio_repo_name" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/studio-runtime-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/studios/{owner}/{repo_name}/stop": { + "post": { + "operationId": "stopStudio", + "summary": "停止创空间", + "description": "停止指定的 Studio", + "tags": [ + "Studios" + ], + "parameters": [ + { + "$ref": "#/components/parameters/studio_owner" + }, + { + "$ref": "#/components/parameters/studio_repo_name" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/studio-runtime-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/studios/{owner}/{repo_name}/logs/{log_type}": { + "get": { + "operationId": "getStudioLogs", + "summary": "获取创空间日志", + "description": "获取指定 Studio 的运行日志", + "tags": [ + "Studios" + ], + "parameters": [ + { + "$ref": "#/components/parameters/studio_owner" + }, + { + "$ref": "#/components/parameters/studio_repo_name" + }, + { + "$ref": "#/components/parameters/studio_log_type" + }, + { + "name": "page_num", + "in": "query", + "description": "页码，默认 1", + "required": false, + "schema": { + "type": "integer", + "default": 1 + } + }, + { + "name": "page_size", + "in": "query", + "description": "每页数量，默认 100，最大 500", + "required": false, + "schema": { + "type": "integer", + "default": 100, + "maximum": 500 + } + }, + { + "name": "keyword", + "in": "query", + "description": "关键字过滤", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "start_timestamp", + "in": "query", + "description": "开始时间戳（秒），可不填，自动根据 end_timestamp 计算", + "required": false, + "schema": { + "type": "integer" + } + }, + { + "name": "end_timestamp", + "in": "query", + "description": "结束时间戳（秒），可不填，默认当前时间", + "required": false, + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/get-studio-logs-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/skills": { + "get": { + "operationId": "listSkills", + "summary": "获取技能列表", + "description": "返回 ModelScope 技能广场的技能列表，支持关键词搜索与多维度筛选", + "tags": [ + "Skills" + ], + "parameters": [ + { + "name": "search", + "in": "query", + "description": "关键词搜索，支持技能名称、描述等", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "filter.developer", + "in": "query", + "description": "按开发者筛选", + "required": false, + "schema": { + "type": "string" + }, + "examples": { + "amap-web": { + "value": "AMap-Web", + "summary": "开发者 AMap-Web" + } + } + }, + { + "name": "filter.category", + "in": "query", + "description": "按分类筛选", + "required": false, + "schema": { + "$ref": "#/components/schemas/SkillCategory" + }, + "examples": { + "developer-tools": { + "value": "developer-tools", + "summary": "开发工具" + }, + "ai-media": { + "value": "ai-media", + "summary": "AI 媒体" + } + } + }, + { + "name": "filter.license", + "in": "query", + "description": "按许可证筛选", + "required": false, + "schema": { + "type": "string" + }, + "examples": { + "mit": { + "value": "MIT License", + "summary": "MIT 许可证" + } + } + }, + { + "name": "filter.custom_tag", + "in": "query", + "description": "按自定义标签筛选", + "required": false, + "schema": { + "type": "string" + }, + "examples": { + "api-design": { + "value": "api-design", + "summary": "API 设计" + } + } + }, + { + "name": "filter.owner", + "in": "query", + "description": "按所有者筛选", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "page_number", + "in": "query", + "description": "页码，从 1 开始", + "required": false, + "schema": { + "type": "integer", + "default": 1 + } + }, + { + "name": "page_size", + "in": "query", + "description": "每页条数", + "required": false, + "schema": { + "type": "integer", + "default": 20 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-skills-success" + }, + "400": { + "$ref": "#/components/responses/skill-bad-request" + }, + "401": { + "$ref": "#/components/responses/skill-unauthorized" + }, + "500": { + "$ref": "#/components/responses/skill-internal-server-error" + }, + "503": { + "$ref": "#/components/responses/skill-service-unavailable" + } + } + }, + "post": { + "operationId": "createSkill", + "summary": "创建技能", + "description": "创建一个新的 Skill（技能）。\n\n创建流程：\n1. 先调用 `POST /files/upload` 上传 Skill 项目文件（zip 包，根目录必须且仅可包含 1 个 SKILL.md 文件），获取 `file_id`。\n2. 再调用本接口传入 `skill_file`（上一步返回的 file_id）及其他元信息完成创建。\n\n注意：`skill_name` 仅允许小写字母、数字和连字符，创建后不可修改；`owner` 创建后不可修改。\n", + "tags": [ + "Skills" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateSkillRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/create-skill-success" + }, + "400": { + "$ref": "#/components/responses/skill-bad-request" + }, + "401": { + "$ref": "#/components/responses/skill-unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "409": { + "$ref": "#/components/responses/skill-conflict" + }, + "500": { + "$ref": "#/components/responses/skill-internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/skills/{id}": { + "get": { + "operationId": "getSkill", + "summary": "获取指定技能详情", + "description": "根据技能 ID 返回指定技能的详细信息，ID 格式为 @author/skill_name（例如 @Alipay/alipay-payment-integration），其中 @ 和 / 无需 URL 编码", + "tags": [ + "Skills" + ], + "parameters": [ + { + "name": "id", + "in": "path", + "description": "技能 ID，格式为 @author/skill_name，其中 @ 和 / 无需 URL 编码", + "required": true, + "schema": { + "type": "string" + }, + "examples": { + "alipay-payment": { + "value": "@Alipay/alipay-payment-integration", + "summary": "支付宝支付集成" + }, + "summarize": { + "value": "@steipete/summarize", + "summary": "内容总结" + } + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/get-skill-success" + }, + "401": { + "$ref": "#/components/responses/skill-unauthorized" + }, + "404": { + "$ref": "#/components/responses/skill-not-found" + }, + "500": { + "$ref": "#/components/responses/skill-internal-server-error" + }, + "503": { + "$ref": "#/components/responses/skill-service-unavailable" + } + } + } + }, + "/skills/{owner}/{skill_name}/settings": { + "patch": { + "operationId": "updateSkillSettings", + "summary": "更新技能设置", + "description": "更新指定 Skill 的设置，传入哪个字段修改哪个字段，不传的字段不修改保持原值。\n\n语义说明：\n- `skill_file` 传入时整体覆盖原有项目文件；不传则保留原文件不变。\n- `tags` 传入时整体覆盖原标签列表。\n- `logo_url` 传空字符串时恢复为平台默认图标。\n- `skill_name` 和 `owner` 创建后不可修改，故不在本接口内。\n", + "tags": [ + "Skills" + ], + "parameters": [ + { + "$ref": "#/components/parameters/skill_owner_path" + }, + { + "$ref": "#/components/parameters/skill_name_path" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateSkillSettingsRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/update-skill-settings-success" + }, + "400": { + "$ref": "#/components/responses/skill-bad-request" + }, + "401": { + "$ref": "#/components/responses/skill-unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/skill-not-found" + }, + "500": { + "$ref": "#/components/responses/skill-internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/files/upload": { + "post": { + "operationId": "uploadFile", + "summary": "上传文件", + "description": "通用文件上传接口（最大 5MB），返回文件 ID。该 ID 可作为入参传给其他接口，用于引用已上传的文件。", + "tags": [ + "Files" + ], + "requestBody": { + "required": true, + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/UploadFileRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/upload-file-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "413": { + "$ref": "#/components/responses/bad-request" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/agent_ids": { + "post": { + "operationId": "createAgentIdentity", + "summary": "创建 Agent 身份", + "description": "客户端本地生成 Ed25519 密钥对后，将公钥（JWK 格式）上传到 ModelScope，注册一个新的 Agent 身份。\n\n- 私钥仅由客户端保存，服务端只存储公钥\n- `public_key` 必须包含 `kty=OKP`、`crv=Ed25519`、`x`（base64url 无 padding 编码的公钥）和 `kid`\n- `token_expire_time` 可选 300/600/1800/3600 秒（默认 3600）\n", + "tags": [ + "Agent-IDP" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateAgentIdentityRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/create-agent-identity-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/agent_ids/{agent_id}": { + "get": { + "operationId": "getAgentIdentity", + "summary": "获取 Agent 身份详情", + "description": "根据 agent_id 返回该 Agent 的完整信息（含当前公钥、状态、Token 配置等）", + "tags": [ + "Agent-IDP" + ], + "parameters": [ + { + "name": "agent_id", + "in": "path", + "description": "Agent 身份 ID", + "required": true, + "schema": { + "type": "string" + }, + "examples": { + "sample": { + "value": "agent_id:modelscope:agent_1234567890ab" + } + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/get-agent-identity-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "patch": { + "operationId": "updateAgentIdentity", + "summary": "更新 Agent 身份设置", + "description": "更新 Agent 名称、描述或 Token 过期时间。所有字段均为可选。", + "tags": [ + "Agent-IDP" + ], + "parameters": [ + { + "name": "agent_id", + "in": "path", + "description": "Agent 身份 ID", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateAgentIdentityRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/update-agent-identity-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "delete": { + "operationId": "deleteAgentIdentity", + "summary": "删除 Agent 身份", + "description": "删除指定的 Agent 身份。删除后该 Agent 将无法再签发 Token，已签发的 Token 仍可使用直至过期。", + "tags": [ + "Agent-IDP" + ], + "parameters": [ + { + "name": "agent_id", + "in": "path", + "description": "Agent 身份 ID", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/delete-agent-identity-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/agent_ids/{agent_id}/key_pairs": { + "put": { + "operationId": "resetAgentKeyPair", + "summary": "重置 Agent 密钥对", + "description": "客户端本地生成新的 Ed25519 密钥对后，上传新公钥替换旧公钥。\n\n- 旧 kid 立即失效，新 kid 立即生效\n- 已签发的旧 Token 不受影响（仍可使用直至过期），新签发的 Token 使用新 kid\n", + "tags": [ + "Agent-IDP" + ], + "parameters": [ + { + "name": "agent_id", + "in": "path", + "description": "Agent 身份 ID", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResetKeyPairRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/reset-key-pair-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/agent_ids/{agent_id}/paused": { + "post": { + "operationId": "pauseAgent", + "summary": "启用/停用 Agent", + "description": "设置 Agent 的 paused 状态：\n\n- `paused=true` 停用：该 Agent 无法再签发 Token，但已签发的 Token 仍有效\n- `paused=false` 启用：恢复签发 Token 能力\n", + "tags": [ + "Agent-IDP" + ], + "parameters": [ + { + "name": "agent_id", + "in": "path", + "description": "Agent 身份 ID", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PauseAgentRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/pause-agent-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/agent_ids/{agent_id}/jwt_id_tokens": { + "get": { + "operationId": "listAgentTokenRecords", + "summary": "获取 Agent 的 Token 签发记录", + "description": "按 audience 维度返回该 Agent 最近一次给每个 Hub 应用签发的 Token 记录。\n", + "tags": [ + "Agent-IDP" + ], + "parameters": [ + { + "name": "agent_id", + "in": "path", + "description": "Agent 身份 ID", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "page", + "in": "query", + "description": "页码（默认 1）", + "required": false, + "schema": { + "type": "integer", + "default": 1 + } + }, + { + "name": "page_size", + "in": "query", + "description": "每页数量（默认 20，最大 50）", + "required": false, + "schema": { + "type": "integer", + "default": 20, + "maximum": 50 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-token-records-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/users/{username}/agent_ids": { + "get": { + "operationId": "listUserAgentIdentities", + "summary": "获取指定用户的 Agent 身份列表", + "description": "按用户名维度返回该用户名下所有 Agent 身份摘要列表，支持分页和按状态筛选。", + "tags": [ + "Agent-IDP" + ], + "parameters": [ + { + "name": "username", + "in": "path", + "description": "用户名", + "required": true, + "schema": { + "type": "string" + }, + "examples": { + "sample": { + "value": "alice" + } + } + }, + { + "name": "status", + "in": "query", + "description": "按状态筛选", + "required": false, + "schema": { + "$ref": "#/components/schemas/AgentIdentityStatus" + } + }, + { + "name": "page", + "in": "query", + "description": "页码（默认 1）", + "required": false, + "schema": { + "type": "integer", + "default": 1 + } + }, + { + "name": "page_size", + "in": "query", + "description": "每页数量（默认 20，最大 50）", + "required": false, + "schema": { + "type": "integer", + "default": 20, + "maximum": 50 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-agent-identities-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/agent_id/token": { + "post": { + "operationId": "issueAgentToken", + "summary": "签发 Agent Token", + "description": "Agent 用其私钥对 message=`agent_id|kid|audience|timestamp` 进行 Ed25519 签名，\n服务端用注册时上传的公钥验签通过后，签发一个面向指定 Hub 应用（audience）的 JWT。\n\n本端点为公开端点，不需要 bearerAuth；签名本身即为身份证明。\n", + "tags": [ + "Agent-IDP" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TokenSignRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/issue-token-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [] + } + }, + "/agent_id/.well-known/agentid-configuration": { + "get": { + "operationId": "getAgentIdConfiguration", + "summary": "获取 Agent IDP OIDC 配置", + "description": "OIDC Discovery 端点，返回 Agent IDP 服务的核心元数据：\n`issuer`、`token_endpoint`、`jwks_uri`、`registration_endpoint`、`activity_endpoint`、\n`id_token_signing_alg_values_supported`。\n\n客户端及外部接入方应通过本端点动态发现各端点 URL，避免硬编码。\n本端点为公开端点，不需要 bearerAuth。\n", + "tags": [ + "Agent-IDP" + ], + "responses": { + "200": { + "$ref": "#/components/responses/agentid-configuration-success" + }, + "429": { + "$ref": "#/components/responses/too-many-requests" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [] + } + }, + "/agent_id/.well-known/agentid-jwks": { + "get": { + "operationId": "getAgentIdJWKS", + "summary": "获取 Agent IDP JWKS", + "description": "返回服务端用于签发 Agent JWT 的公钥集合（JSON Web Key Set）。\n\n资源方收到 Agent JWT 后，应通过本端点拉取 JWKS，根据 JWT header 中的 `kid` 选择对应公钥验签。\n支持多 kid 同时存在以实现密钥轮换：旧 kid 在过渡期内继续返回，新签发的 JWT 使用新 kid。\n\n本端点为公开端点，不需要 bearerAuth。\n", + "tags": [ + "Agent-IDP" + ], + "responses": { + "200": { + "$ref": "#/components/responses/agentid-jwks-success" + }, + "429": { + "$ref": "#/components/responses/too-many-requests" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [] + } + }, + "/magicubes/balance": { + "get": { + "operationId": "getBalance", + "summary": "查询余额", + "description": "查询当前用户的魔粒余额信息", + "tags": [ + "Magicube" + ], + "responses": { + "200": { + "$ref": "#/components/responses/get-balance-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/collections": { + "get": { + "operationId": "listCollections", + "summary": "获取 Collection 列表", + "description": "获取 Collection 列表，支持搜索、按所有者过滤与排序、分页。公开列表无需 Token；查询私有 Collection 需 Bearer Token。", + "tags": [ + "Collections" + ], + "parameters": [ + { + "name": "search", + "in": "query", + "description": "针对标题、描述的子字符串搜索", + "required": false, + "schema": { + "type": "string", + "examples": [ + "nlp" + ] + } + }, + { + "name": "owner", + "in": "query", + "description": "所有者过滤。仅允许过滤当前 Token 所属用户自己的 Collection，过滤他人（或匿名使用）返回 403 OperationNotAllowed。", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "sort", + "in": "query", + "description": "排序方式：default（综合）/ last_modified（最近更新）/ likes（喜欢数）", + "required": false, + "schema": { + "type": "string", + "enum": [ + "default", + "last_modified", + "likes" + ], + "default": "default", + "examples": [ + "likes" + ] + } + }, + { + "name": "page_number", + "in": "query", + "description": "页码（≥1）", + "required": false, + "schema": { + "type": "integer", + "minimum": 1, + "default": 1, + "examples": [ + 1 + ] + } + }, + { + "name": "page_size", + "in": "query", + "description": "每页大小（1~50）", + "required": false, + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 50, + "default": 10, + "examples": [ + 10 + ] + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-collections-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + }, + {} + ] + }, + "post": { + "operationId": "createCollection", + "summary": "创建 Collection", + "description": "创建一个新的 Collection。", + "tags": [ + "Collections" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateCollectionRequest" + }, + "examples": { + "create-collection-request": { + "value": { + "title": "最佳 NLP 模型合集", + "owner": "iSolver", + "description": "精选的自然语言处理模型...", + "visibility": "public", + "theme": "Blue" + } + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/create-collection-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/collections/{owner}/{slug}": { + "get": { + "operationId": "getCollection", + "summary": "获取 Collection 详情", + "description": "根据 {owner}/{slug} 获取 Collection 详情。visibility=public 时 Token 可选；visibility=private 时必填 Bearer Token。", + "tags": [ + "Collections" + ], + "parameters": [ + { + "$ref": "#/components/parameters/collection_owner_path" + }, + { + "$ref": "#/components/parameters/collection_slug_path" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/get-collection-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + }, + {} + ] + }, + "patch": { + "operationId": "updateCollection", + "summary": "更新 Collection", + "description": "更新 Collection 元数据。传入哪个字段即更新哪个字段，未传字段保持原值。open/close 语义映射为 visibility（public/private）。", + "tags": [ + "Collections" + ], + "parameters": [ + { + "$ref": "#/components/parameters/collection_owner_path" + }, + { + "$ref": "#/components/parameters/collection_slug_path" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateCollectionRequest" + }, + "examples": { + "update-collection-request": { + "value": { + "title": "最佳 NLP 模型合集（已更名）", + "visibility": "private" + } + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/update-collection-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "delete": { + "operationId": "deleteCollection", + "summary": "删除 Collection", + "description": "删除指定 {owner}/{slug} 的 Collection。需具备 admin 权限的 Bearer Token。", + "tags": [ + "Collections" + ], + "parameters": [ + { + "$ref": "#/components/parameters/collection_owner_path" + }, + { + "$ref": "#/components/parameters/collection_slug_path" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/delete-collection-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/collections/{owner}/{slug}/items": { + "get": { + "operationId": "listCollectionItems", + "summary": "获取 Collection 条目列表", + "description": "分页获取指定 Collection 的条目列表，支持按资源类型过滤。visibility=public 时 Token 可选；private 时必填。", + "tags": [ + "Collections" + ], + "parameters": [ + { + "$ref": "#/components/parameters/collection_owner_path" + }, + { + "$ref": "#/components/parameters/collection_slug_path" + }, + { + "name": "item_type", + "in": "query", + "description": "按资源类型过滤：model / dataset / studio / paper / skill / mcp", + "required": false, + "schema": { + "type": "string", + "enum": [ + "model", + "dataset", + "studio", + "paper", + "skill", + "mcp" + ], + "examples": [ + "model" + ] + } + }, + { + "name": "page_number", + "in": "query", + "description": "页码（≥1）", + "required": false, + "schema": { + "type": "integer", + "minimum": 1, + "default": 1, + "examples": [ + 1 + ] + } + }, + { + "name": "page_size", + "in": "query", + "description": "每页大小（1~50）", + "required": false, + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 50, + "default": 10, + "examples": [ + 10 + ] + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-collection-items-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + }, + {} + ] + }, + "post": { + "operationId": "addCollectionItems", + "summary": "添加 Collection 条目", + "description": "向 Collection 批量添加条目。传入 items 数组。\n\n异常处理：若部分条目不存在、已在 Collection 中或无权限，接口不整体回滚，返回 success=true，并在 data.failed_items 中列出失败项；\n全部失败时仍返回 success=true、added_count=0。仅请求参数格式错误返回 400。部分失败不返回 code 字段。\n", + "tags": [ + "Collections" + ], + "parameters": [ + { + "$ref": "#/components/parameters/collection_owner_path" + }, + { + "$ref": "#/components/parameters/collection_slug_path" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AddCollectionItemsRequest" + }, + "examples": { + "add-collection-items-request": { + "$ref": "#/components/examples/add-collection-items-request" + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/add-collection-items-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "patch": { + "operationId": "updateCollectionItems", + "summary": "批量更新 Collection 条目", + "description": "批量更新 Collection 条目（P2）。通过每个条目的 item_type + item_object_id 定位，传入哪个字段就修改哪个字段。\n\n异常处理：若部分条目不在当前 Collection 或对应资源不存在，接口不整体回滚，返回 success=true，并在 data.failed_items 中列出失败项；\n全部失败时仍返回 success=true、updated_count=0。仅请求参数格式错误返回 400。部分失败不返回 code 字段。\n", + "tags": [ + "Collections" + ], + "parameters": [ + { + "$ref": "#/components/parameters/collection_owner_path" + }, + { + "$ref": "#/components/parameters/collection_slug_path" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateCollectionItemsRequest" + }, + "examples": { + "update-collection-items-request": { + "$ref": "#/components/examples/update-collection-items-request" + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/update-collection-items-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/collections/{owner}/{slug}/items/{item_type}/{item_object_id}": { + "patch": { + "operationId": "updateCollectionItem", + "summary": "更新单个 Collection 条目", + "description": "通过 item_type + item_object_id 定位并更新单个条目。传入哪个字段即更新哪个字段。", + "tags": [ + "Collections" + ], + "parameters": [ + { + "$ref": "#/components/parameters/collection_owner_path" + }, + { + "$ref": "#/components/parameters/collection_slug_path" + }, + { + "$ref": "#/components/parameters/collection_item_type_path" + }, + { + "$ref": "#/components/parameters/collection_item_object_id_path" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateCollectionItemRequest" + }, + "examples": { + "update-collection-item-request": { + "$ref": "#/components/examples/update-collection-item-request" + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/update-collection-item-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "delete": { + "operationId": "removeCollectionItem", + "summary": "移除 Collection 条目", + "description": "通过 item_type + item_object_id 定位并从 Collection 中移除单个条目。", + "tags": [ + "Collections" + ], + "parameters": [ + { + "$ref": "#/components/parameters/collection_owner_path" + }, + { + "$ref": "#/components/parameters/collection_slug_path" + }, + { + "$ref": "#/components/parameters/collection_item_type_path" + }, + { + "$ref": "#/components/parameters/collection_item_object_id_path" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/remove-collection-item-success" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/galleries": { + "post": { + "operationId": "createGallery", + "summary": "创建 Gallery", + "description": "创建一个新的灵感流/Gallery 内容。\n\n**权限要求**\n\n- 基线要求：已注册用户。\n- 若上传到组织名下，用户须为该组织的管理员或写权限成员。\n- 若上传到个人名下，需为和 token 身份指向一致的个人用户。\n\n**字段说明**\n\n- name 名称、labels 标签：需满足 CSI 内容安全审核要求，不合法返回 403。\n- path 自定义 URL 尾缀：系统会校验 owner/path 是否已存在，重复返回 400。\n- entry_file 入口文件：文件后缀须与 category 匹配（后缀不区分大小写，如 .IPYNB 等价于 .ipynb）：notebook→.ipynb，website→.html，pdf→.pdf；file 类别不限后缀，不匹配返回 400。\n- Gallery ID：由服务端生成 UUID v4，不可自定义。\n- 封面颜色：系统随机分配封面颜色模板（purple / pink / blue / turquoise / green）。\n", + "tags": [ + "Galleries" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateGalleryRequest" + }, + "examples": { + "create-gallery-request": { + "$ref": "#/components/examples/create-gallery-request" + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/create-gallery-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + } + }, + "components": { + "schemas": { + "User": { + "allOf": [ + { + "$ref": "#/components/schemas/UserPublic" + }, + { + "$ref": "#/components/schemas/UserPrivate" + } + ], + "description": "用户完整信息(包含私有信息)" + }, + "UserResponse": { + "type": "object", + "description": "用户信息响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功响应时始终为 true", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/User", + "description": "用户信息" + }, + "request_id": { + "type": "string", + "description": "请求ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" + ] + } + }, + "required": [ + "success", + "request_id" + ] + }, + "ErrorResponse": { + "type": "object", + "description": "错误响应格式", + "properties": { + "success": { + "type": "boolean", + "description": "错误响应时始终为 false", + "examples": [ + false + ] + }, + "code": { + "type": "string", + "description": "错误码", + "examples": [ + "InputParameterError" + ] + }, + "message": { + "type": "string", + "description": "错误消息", + "examples": [ + "Unauthorized access" + ] + }, + "request_id": { + "type": "string", + "description": "请求ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" + ] + } + }, + "required": [ + "success", + "code", + "message", + "request_id" + ] + }, + "Model": { + "type": "object", + "description": "模型摘要信息", + "properties": { + "id": { + "type": "string", + "description": "模型仓库名（repo_id，格式为 owner/repo_name）", + "examples": [ + "iic/QwenLong-L1.5-30B-A3B", + "ZhipuAI/AutoGLM-Phone-9B" + ] + }, + "display_name": { + "type": [ + "string", + "null" + ], + "description": "模型展示名称", + "examples": [ + "QwenLong-L1.5-30B-A3B", + "AutoGLM-Phone-9B" + ] + }, + "description": { + "type": [ + "string", + "null" + ], + "description": "模型简介", + "examples": [ + "" + ] + }, + "downloads": { + "type": "integer", + "description": "下载量", + "examples": [ + 1466, + 28525 + ] + }, + "likes": { + "type": "integer", + "description": "喜欢量", + "examples": [ + 25, + 265 + ] + }, + "license": { + "type": [ + "string", + "null" + ], + "description": "许可证", + "examples": [ + "apache-2.0", + "MIT License" + ] + }, + "tasks": { + "type": "array", + "description": "任务列表", + "items": { + "type": "string" + }, + "examples": [ + [ + "text-generation" + ], + [ + "image-captioning" + ] + ] + }, + "created_at": { + "type": "string", + "format": "date-time", + "description": "创建时间（ISO 8601 UTC）", + "examples": [ + "2025-12-14T12:26:15Z" + ] + }, + "last_modified": { + "type": "string", + "format": "date-time", + "description": "Git 文件或设置的最后修改时间（ISO 8601 UTC）", + "examples": [ + "2025-12-16T03:34:26Z" + ] + }, + "file_size": { + "type": "integer", + "description": "文件大小（字节）", + "examples": [ + 61081671123 + ] + }, + "params": { + "type": "integer", + "description": "参数量", + "examples": [ + 30532122624 + ] + }, + "tags": { + "type": "array", + "description": "标签列表", + "items": { + "type": "string" + }, + "examples": [ + [ + "license:apache-2.0", + "model_type:qwen3_moe", + "library:transformer", + "library:safetensors", + "library:pytorch", + "task:text-generation", + "deploy:swingdeploy" + ] + ] + }, + "private": { + "type": "boolean", + "description": "是否非公开模型", + "examples": [ + false + ] + }, + "gated": { + "type": "boolean", + "description": "是否申请制模型，通过申请后可访问", + "examples": [ + false + ] + } + }, + "required": [ + "id", + "downloads", + "likes", + "tasks", + "created_at", + "last_modified", + "file_size", + "params", + "private", + "gated" + ] + }, + "ModelList": { + "type": "object", + "description": "模型列表与分页信息", + "properties": { + "models": { + "type": "array", + "description": "结果模型列表", + "items": { + "$ref": "#/components/schemas/Model" + } + }, + "total_count": { + "type": "integer", + "description": "符合筛选条件的模型总数", + "examples": [ + 43127 + ] + }, + "page_number": { + "type": "integer", + "description": "当前页码", + "examples": [ + 1 + ] + }, + "page_size": { + "type": "integer", + "description": "每页大小", + "examples": [ + 10 + ] + } + }, + "required": [ + "models", + "total_count", + "page_number", + "page_size" + ] + }, + "ModelListResponse": { + "type": "object", + "description": "列出模型的标准响应包裹格式", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/ModelList", + "description": "模型列表数据" + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + } + }, + "required": [ + "success", + "data", + "request_id" + ] + }, + "ModelDetail": { + "allOf": [ + { + "$ref": "#/components/schemas/Model" + }, + { + "type": "object", + "description": "模型详情信息", + "properties": { + "readme": { + "type": [ + "string", + "null" + ], + "description": "README 内容", + "examples": [ + "hello" + ] + } + } + } + ] + }, + "ModelDetailResponse": { + "type": "object", + "description": "获取模型详情的标准响应包裹格式", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/ModelDetail", + "description": "模型详情数据" + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + } + }, + "required": [ + "success", + "data", + "request_id" + ] + }, + "Dataset": { + "type": "object", + "description": "数据集摘要信息", + "properties": { + "id": { + "type": "string", + "description": "数据集仓库名（repo_id，格式为 owner/repo_name）", + "examples": [ + "DAMO_NLP/jd", + "iic/nlp_domain_classification_chinese_testset" + ] + }, + "display_name": { + "type": [ + "string", + "null" + ], + "description": "数据集展示名称", + "examples": [ + "商品评论情感预测", + "中文文本领域分类测试集" + ] + }, + "description": { + "type": [ + "string", + "null" + ], + "description": "数据集简介", + "examples": [ + "", + "..." + ] + }, + "file_size": { + "type": "integer", + "description": "文件大小（字节）", + "examples": [ + 0 + ] + }, + "downloads": { + "type": "integer", + "description": "下载量", + "examples": [ + 53996, + 2401 + ] + }, + "likes": { + "type": "integer", + "description": "喜欢量", + "examples": [ + 102, + 23 + ] + }, + "license": { + "type": [ + "string", + "null" + ], + "description": "许可证", + "examples": [ + "Apache License 2.0", + "cc-by-4.0" + ] + }, + "tasks": { + "type": "array", + "description": "任务列表", + "items": { + "type": "string" + }, + "examples": [ + [ + "text-classification" + ], + [] + ] + }, + "created_at": { + "type": "string", + "format": "date-time", + "description": "创建时间（ISO 8601 UTC）", + "examples": [ + "2022-09-28T06:30:31Z", + "2025-11-25T14:30:22Z" + ] + }, + "last_modified": { + "type": "string", + "format": "date-time", + "description": "Git 文件或设置的最后修改时间（ISO 8601 UTC）", + "examples": [ + "2022-10-30T03:36:39Z", + "2026-01-01T00:31:42Z" + ] + }, + "tags": { + "type": "array", + "description": "标签列表", + "items": { + "type": "string" + }, + "examples": [ + [ + "license:Apache License 2.0", + "task:text-classification" + ], + [ + "license:cc-by-4.0", + "custom_tag:common-crawl" + ] + ] + }, + "private": { + "type": "boolean", + "description": "是否非公开数据集", + "examples": [ + false + ] + }, + "gated": { + "type": "boolean", + "description": "是否申请制数据集，通过申请后可访问", + "examples": [ + false + ] + }, + "login_required": { + "type": "boolean", + "description": "是否登录后才可下载", + "examples": [ + false + ] + } + }, + "required": [ + "id", + "downloads", + "likes", + "private", + "gated", + "login_required", + "created_at", + "last_modified" + ] + }, + "DatasetList": { + "type": "object", + "description": "数据集列表与分页信息", + "properties": { + "datasets": { + "type": "array", + "description": "结果数据集列表", + "items": { + "$ref": "#/components/schemas/Dataset" + } + }, + "total_count": { + "type": "integer", + "description": "符合筛选条件的数据集总数", + "examples": [ + 60 + ] + }, + "page_number": { + "type": "integer", + "description": "当前页码", + "examples": [ + 1 + ] + }, + "page_size": { + "type": "integer", + "description": "每页大小", + "examples": [ + 10 + ] + } + }, + "required": [ + "datasets", + "total_count", + "page_number", + "page_size" + ] + }, + "DatasetListResponse": { + "type": "object", + "description": "列出数据集的标准响应包裹格式", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/DatasetList", + "description": "数据集列表数据" + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + } + }, + "required": [ + "success", + "data", + "request_id" + ] + }, + "DatasetDetail": { + "allOf": [ + { + "$ref": "#/components/schemas/Dataset" + }, + { + "type": "object", + "description": "数据集详情信息", + "properties": { + "readme": { + "type": [ + "string", + "null" + ], + "description": "README 内容", + "examples": [ + "hello" + ] + } + } + } + ] + }, + "DatasetDetailResponse": { + "type": "object", + "description": "获取数据集详情的标准响应包裹格式", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/DatasetDetail", + "description": "数据集详情数据" + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + } + }, + "required": [ + "success", + "data", + "request_id" + ] + }, + "McpServerLocale": { + "type": "object", + "description": "MCP服务在对应语言环境下的内容", + "properties": { + "name": { + "type": "string", + "description": "MCP服务在对应语言环境下的服务名称", + "examples": [ + "Fetch网页内容抓取" + ] + }, + "description": { + "type": "string", + "description": "MCP服务在对应语言环境下的服务简介", + "examples": [ + "该服务器使大型语言模型能够检索和处理网页内容..." + ] + }, + "readme": { + "type": "string", + "description": "MCP服务介绍（仅详情接口返回）", + "examples": [ + "# Fetch MCP Server..." + ] + } + } + }, + "McpServerLocales": { + "type": "object", + "description": "不同语言环境下的内容展示", + "properties": { + "zh": { + "$ref": "#/components/schemas/McpServerLocale", + "description": "中文环境" + }, + "en": { + "$ref": "#/components/schemas/McpServerLocale", + "description": "英文环境" + } + } + }, + "McpOperationalUrl": { + "type": "object", + "description": "MCP服务托管链接信息", + "properties": { + "id": { + "type": "string", + "description": "MCP服务链接命名标识", + "examples": [ + "platform-pool" + ] + }, + "url": { + "type": "string", + "description": "MCP服务远程连接地址", + "examples": [ + "https://mcp.api-inference.modelscope.net/xxxx/mcp" + ] + }, + "transport_type": { + "type": "string", + "description": "MCP服务链接传输方式", + "examples": [ + "streamable_http", + "sse" + ] + }, + "auth_required": { + "type": "boolean", + "description": "远程 url 连接时是否需要提供魔搭访问令牌鉴权", + "examples": [ + false + ] + }, + "expiration": { + "type": "string", + "description": "MCP服务链接有效期", + "examples": [ + "2025-10-01 21:00:00" + ] + }, + "accessible": { + "type": "boolean", + "description": "是否有权限访问该MCP服务详情。当所有者设置为非公开或MCP服务被删除时，无权访问。", + "examples": [ + true + ] + } + } + }, + "McpServerSummary": { + "type": "object", + "description": "MCP服务摘要信息", + "properties": { + "id": { + "type": "string", + "description": "服务ID。由平台后台收录的MCP，ID 默认为@author/server_name，当被社区用户认领后将会变更为user_name/server_name；由社区用户自发提交的MCP，ID 为user_name/server_name。", + "examples": [ + "@modelcontextprotocol/fetch" + ] + }, + "name": { + "type": "string", + "description": "MCP服务在魔搭社区的名称", + "examples": [ + "Fetch网页内容抓取" + ] + }, + "chinese_name": { + "type": "string", + "description": "MCP服务在魔搭社区的中文名称", + "examples": [ + "Fetch网页内容抓取" + ] + }, + "description": { + "type": "string", + "description": "MCP服务简介", + "examples": [ + "该服务器使大型语言模型能够检索和处理网页内容..." + ] + }, + "logo_url": { + "type": "string", + "description": "logo图片URL链接", + "examples": [ + "https://resources.modelscope.cn/..." + ] + }, + "publisher": { + "type": "string", + "description": "发布时原始ID", + "examples": [ + "@modelcontextprotocol/fetch" + ] + }, + "categories": { + "type": "array", + "description": "该MCP服务所属分类", + "items": { + "type": "string" + }, + "examples": [ + [ + "browser-automation" + ] + ] + }, + "tags": { + "type": "array", + "description": "该MCP服务的标签列表", + "items": { + "type": "string" + }, + "examples": [ + [ + "browser-automation" + ] + ] + }, + "view_count": { + "type": "integer", + "description": "该MCP服务的累计访问量", + "examples": [ + 30667 + ] + }, + "locales": { + "$ref": "#/components/schemas/McpServerLocales" + } + }, + "required": [ + "id" + ] + }, + "McpServerDetail": { + "type": "object", + "description": "MCP服务详细信息", + "allOf": [ + { + "$ref": "#/components/schemas/McpServerSummary" + }, + { + "type": "object", + "properties": { + "author": { + "type": "string", + "description": "MCP服务开发者", + "examples": [ + "modelcontextprotocol" + ] + }, + "owner": { + "type": "string", + "description": "MCP服务归属的魔搭用户/组织ID", + "examples": [ + "" + ] + }, + "readme": { + "type": "string", + "description": "MCP服务介绍", + "examples": [ + "# Fetch MCP Server..." + ] + }, + "source_url": { + "type": "string", + "description": "MCP服务来源项目主页地址", + "examples": [ + "https://github.com/modelcontextprotocol/..." + ] + }, + "github_stars": { + "type": "integer", + "description": "该 MCP 服务对应 github 原始仓库的 stars 数量", + "examples": [ + 1234 + ] + }, + "is_hosted": { + "type": "boolean", + "description": "该MCP服务是否支持托管部署", + "examples": [ + true + ] + }, + "is_verified": { + "type": "boolean", + "description": "托管MCP服务是否经过平台验证测试", + "examples": [ + true + ] + }, + "env_schema": { + "type": "object", + "description": "该服务连接部署的环境变量配置", + "additionalProperties": true + }, + "server_config": { + "type": "array", + "description": "MCP服务配置信息", + "items": {} + }, + "operational_urls": { + "type": "array", + "description": "当get_operational_url=True且当前用户已连接该 MCP 服务时则返回，否则为空", + "items": { + "$ref": "#/components/schemas/McpOperationalUrl" + } + } + } + } + ] + }, + "McpServerOperational": { + "type": "object", + "description": "用户托管的MCP服务信息", + "allOf": [ + { + "$ref": "#/components/schemas/McpServerSummary" + }, + { + "type": "object", + "properties": { + "operational_urls": { + "type": "array", + "description": "MCP服务托管链接列表", + "items": { + "$ref": "#/components/schemas/McpOperationalUrl" + } + } + } + } + ] + }, + "McpServerListResponse": { + "type": "object", + "description": "MCP服务列表响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/McpServerList", + "description": "MCP服务列表数据" + }, + "message": { + "type": "string", + "description": "响应消息", + "examples": [ + "success" + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "c55709d2-4e09-4bd8-be84-d8518bc46363" + ] + } + }, + "required": [ + "success", + "request_id" + ] + }, + "McpServerOperationalListResponse": { + "type": "object", + "description": "用户托管MCP服务列表响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/McpServerOperationalList", + "description": "用户托管MCP服务列表数据" + }, + "message": { + "type": "string", + "description": "响应消息", + "examples": [ + "success" + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "c55709d2-4e09-4bd8-be84-d8518bc46363" + ] + } + }, + "required": [ + "success", + "request_id" + ] + }, + "McpServerDetailResponse": { + "type": "object", + "description": "MCP服务详情响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/McpServerDetail", + "description": "MCP服务详情数据" + }, + "message": { + "type": "string", + "description": "响应消息", + "examples": [ + "success" + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "c55709d2-4e09-4bd8-be84-d8518bc46363" + ] + } + }, + "required": [ + "success", + "request_id" + ] + }, + "McpDeployResponse": { + "type": "object", + "description": "MCP服务部署响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/McpOperationalUrl", + "description": "部署后的MCP服务链接信息" + }, + "message": { + "type": "string", + "description": "响应消息", + "examples": [ + "success" + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "c55709d2-4e09-4bd8-be84-d8518bc46363" + ] + } + }, + "required": [ + "success", + "request_id" + ] + }, + "McpUndeployResponse": { + "type": "object", + "description": "MCP服务解除部署响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "message": { + "type": "string", + "description": "响应消息", + "examples": [ + "success" + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "c55709d2-4e09-4bd8-be84-d8518bc46363" + ] + } + }, + "required": [ + "success", + "request_id" + ] + }, + "StudioHardware": { + "type": "string", + "description": "硬件配置，修改后需重新部署才能生效。\n可用值不固定，请通过 `GET /studios/hardware` 接口查询当前可用的硬件配置列表。\n免费资源常见格式为 `platform/...`、`xgpu/...`、`amd/...`；付费资源格式为 `paid/`，例如 `paid/ecs.gn7i-c8g1.2xlarge`。\n", + "examples": [ + "platform/2v-cpu-16g-mem", + "paid/ecs.gn7i-c8g1.2xlarge" + ] + }, + "StudioSDKType": { + "type": "string", + "description": "SDK 类型，修改后需重新部署才能生效", + "enum": [ + "gradio", + "streamlit", + "docker", + "static" + ] + }, + "StudioSDKVersion": { + "type": "string", + "description": "SDK 版本，仅对 Gradio 类型生效，默认最新版，建议选用最新版本，修改后需重新部署才能生效。\n可用版本会随平台更新，请通过 `GET /studios/sdk-versions?sdk_type=gradio` 查询当前可用的 Gradio 版本列表。\n", + "examples": [ + "6.2.0" + ] + }, + "StudioBaseImage": { + "type": "string", + "description": "基础镜像，仅 Docker 类型不支持，建议选用最新版本，修改后需重新部署才能生效。\n可用值不固定，请通过 `GET /studios/base-images` 接口查询当前可用的基础镜像列表。\n", + "examples": [ + "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0" + ] + }, + "CreateStudioRequest": { + "type": "object", + "description": "创建 Studio 请求参数", + "properties": { + "repo_name": { + "type": "string", + "description": "仓库名称", + "maxLength": 64, + "examples": [ + "my-demo-app" + ] + }, + "owner": { + "type": "string", + "description": "所有者（用户名或组织名）", + "examples": [ + "username" + ] + }, + "display_name": { + "type": "string", + "description": "显示名称", + "maxLength": 128, + "examples": [ + "我的演示应用" + ] + }, + "license": { + "type": "string", + "description": "许可证，默认 apache-2.0", + "default": "apache-2.0", + "examples": [ + "apache-2.0" + ] + }, + "visibility": { + "$ref": "#/components/schemas/StudioVisibility" + }, + "private": { + "type": "boolean", + "description": "Deprecated: 请使用 visibility 字段", + "deprecated": true, + "default": false + }, + "description": { + "type": "string", + "description": "描述", + "maxLength": 2000, + "examples": [ + "这是一个演示应用" + ] + }, + "cover_image": { + "type": "string", + "description": "封面图 URL，为空时使用平台默认图" + }, + "sdk_type": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioSDKType" + } + ], + "default": "gradio" + }, + "sdk_version": { + "$ref": "#/components/schemas/StudioSDKVersion" + }, + "base_image": { + "$ref": "#/components/schemas/StudioBaseImage" + }, + "hardware": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioHardware" + } + ], + "default": "platform/2v-cpu-16g-mem", + "description": "硬件配置。未传时使用默认免费硬件配置" + } + }, + "required": [ + "repo_name", + "owner" + ] + }, + "CreateStudioResponse": { + "type": "object", + "description": "创建 Studio 响应", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "data": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Studio ID (owner/repo_name)", + "examples": [ + "username/my-demo-app" + ] + }, + "repo_name": { + "type": "string", + "description": "仓库名称", + "examples": [ + "my-demo-app" + ] + }, + "display_name": { + "type": "string", + "description": "显示名称", + "examples": [ + "我的演示应用" + ] + }, + "owner": { + "type": "string", + "description": "所有者", + "examples": [ + "username" + ] + }, + "url": { + "type": "string", + "description": "Studio URL", + "examples": [ + "https://modelscope.cn/studios/username/my-demo-app" + ] + } + } + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "success", + "request_id" + ] + }, + "StudioConfig": { + "type": "object", + "description": "Studio 运行时配置", + "properties": { + "hardware": { + "$ref": "#/components/schemas/StudioHardware" + }, + "base_image": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioBaseImage" + } + ], + "description": "基础镜像（Docker 类型不返回）" + }, + "sdk_type": { + "$ref": "#/components/schemas/StudioSDKType" + }, + "sdk_version": { + "type": "string", + "description": "SDK 版本（仅 gradio 类型返回）" + } + } + }, + "StudioRuntime": { + "type": "object", + "description": "Studio 运行时信息", + "properties": { + "status": { + "type": "string", + "description": "运行状态", + "enum": [ + "Initialized", + "Building", + "BuildFailed", + "Deploying", + "DeployFailed", + "Running", + "Stopping", + "Stopped", + "Duplicating", + "Sleeping" + ] + }, + "active_config": { + "description": "当前实际运行中的配置", + "$ref": "#/components/schemas/StudioConfig" + }, + "created_at": { + "type": "string", + "description": "部署时间" + }, + "error_message": { + "type": "string", + "description": "失败信息（仅在错误状态时返回）" + } + } + }, + "GetStudioResponse": { + "type": "object", + "description": "获取 Studio 详情响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Studio ID (owner/repo_name)" + }, + "repo_name": { + "type": "string", + "description": "仓库名称" + }, + "display_name": { + "type": "string", + "description": "显示名称" + }, + "owner": { + "type": "string", + "description": "所有者" + }, + "description": { + "type": "string", + "description": "描述" + }, + "cover_image": { + "type": "string", + "description": "封面图 URL" + }, + "likes": { + "type": "integer", + "description": "喜欢数" + }, + "view_count": { + "type": "integer", + "description": "访问量" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "visibility": { + "$ref": "#/components/schemas/StudioVisibility" + }, + "private": { + "type": "boolean", + "description": "Deprecated: 请使用 visibility 字段", + "deprecated": true + }, + "last_modified": { + "type": "string", + "description": "最后修改时间" + }, + "sdk_type": { + "$ref": "#/components/schemas/StudioSDKType" + }, + "sdk_version": { + "$ref": "#/components/schemas/StudioSDKVersion" + }, + "hardware": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioHardware" + } + ], + "description": "期望的硬件配置（数据库中保存的值）" + }, + "base_image": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioBaseImage" + } + ], + "description": "期望的基础镜像（Docker 类型不返回）" + }, + "license": { + "type": "string", + "description": "许可证" + }, + "host": { + "type": "string", + "description": "访问地址" + }, + "mcp_support": { + "type": "boolean", + "description": "是否支持 MCP" + }, + "runtime": { + "$ref": "#/components/schemas/StudioRuntime" + } + } + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "success", + "request_id" + ] + }, + "UpdateStudioSettingsRequest": { + "type": "object", + "description": "更新 Studio 设置请求参数（传入哪个字段修改哪个字段）", + "properties": { + "display_name": { + "type": "string", + "description": "显示名称", + "maxLength": 128 + }, + "license": { + "type": "string", + "description": "许可证" + }, + "visibility": { + "$ref": "#/components/schemas/StudioVisibility" + }, + "private": { + "type": "boolean", + "description": "Deprecated: 请使用 visibility 字段", + "deprecated": true + }, + "description": { + "type": "string", + "description": "描述", + "maxLength": 2000 + }, + "cover_image": { + "type": "string", + "description": "封面图 URL" + }, + "sdk_type": { + "$ref": "#/components/schemas/StudioSDKType" + }, + "sdk_version": { + "$ref": "#/components/schemas/StudioSDKVersion" + }, + "base_image": { + "$ref": "#/components/schemas/StudioBaseImage" + }, + "hardware": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioHardware" + } + ], + "description": "硬件配置，修改后需重新部署才能生效" + } + } + }, + "AddStudioSecretRequest": { + "type": "object", + "description": "添加 Studio 密文变量请求参数", + "properties": { + "key": { + "type": "string", + "description": "密文变量名称", + "maxLength": 128, + "examples": [ + "API_KEY" + ] + }, + "value": { + "type": "string", + "description": "密文变量值（添加后不会公开展示）", + "maxLength": 4096, + "examples": [ + "sk-xxxxxxxxxxxx" + ] + } + }, + "required": [ + "key", + "value" + ] + }, + "UpdateStudioSecretRequest": { + "type": "object", + "description": "更新 Studio 密文变量请求参数", + "properties": { + "key": { + "type": "string", + "description": "密文变量名称", + "maxLength": 128, + "examples": [ + "API_KEY" + ] + }, + "value": { + "type": "string", + "description": "密文变量新值", + "maxLength": 4096, + "examples": [ + "sk-xxxxxxxxxxxx" + ] + } + }, + "required": [ + "key", + "value" + ] + }, + "StudioSecretItem": { + "type": "object", + "description": "密文变量列表项（仅返回 key，不返回 value）", + "properties": { + "key": { + "type": "string", + "description": "密文变量名称" + } + }, + "required": [ + "key" + ] + }, + "ListStudioSecretsResponse": { + "type": "object", + "description": "获取 Studio 密文变量列表响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "secrets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/StudioSecretItem" + } + } + } + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "success", + "data", + "request_id" + ] + }, + "DeleteStudioSecretRequest": { + "type": "object", + "description": "删除 Studio 密文变量请求参数", + "properties": { + "key": { + "type": "string", + "description": "密文变量名称" + } + }, + "required": [ + "key" + ] + }, + "StudioRuntimeResponse": { + "type": "object", + "description": "Studio 运行时操作响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "$ref": "#/components/schemas/StudioRuntime" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "success", + "request_id" + ] + }, + "StudioLogsResponse": { + "type": "object", + "description": "获取 Studio 日志响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "logs": { + "type": "array", + "items": { + "type": "string" + } + }, + "page_num": { + "type": "integer" + }, + "page_size": { + "type": "integer" + }, + "total_count": { + "type": "integer" + }, + "total_page_num": { + "type": "integer" + } + } + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "success", + "request_id" + ] + }, + "SuccessResponse": { + "type": "object", + "description": "通用成功响应", + "properties": { + "success": { + "type": "boolean" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "success", + "request_id" + ] + }, + "StudioVisibility": { + "type": "string", + "description": "创空间可见性：\n- public: 代码和体验都公开\n- protected: 体验公开，代码仓库不可见\n- private: 都不公开\n", + "enum": [ + "public", + "protected", + "private" + ] + }, + "HardwareItem": { + "type": "object", + "description": "硬件配置项。免费资源仅返回基础标识字段；付费资源会返回 ECS 规格、资源明细和价格字段", + "properties": { + "name": { + "type": "string", + "description": "OpenAPI hardware 参数值。付费资源格式为 `paid/`", + "examples": [ + "platform/2v-cpu-16g-mem", + "paid/ecs.gn7i-c8g1.2xlarge" + ] + }, + "resource_type": { + "type": "string", + "description": "资源类型，free 表示平台免费资源，paid 表示使用用户自己的云账号付费部署", + "enum": [ + "free", + "paid" + ] + }, + "instance_type": { + "type": "string", + "description": "ECS 规格名，仅付费资源返回", + "examples": [ + "ecs.gn7i-c8g1.2xlarge" + ] + }, + "cpu": { + "type": "integer", + "description": "CPU 核数，仅付费资源返回" + }, + "gpu": { + "type": "integer", + "description": "GPU 数量，仅付费资源返回" + }, + "memory": { + "type": "integer", + "description": "内存（GB），仅付费资源返回" + }, + "gpu_type": { + "type": "string", + "description": "GPU 类型，仅付费资源返回" + }, + "gpu_memory": { + "type": "integer", + "description": "显存（GB），仅付费资源返回" + }, + "supported_sdk_types": { + "type": "array", + "items": { + "type": "string" + }, + "description": "支持的 SDK 类型" + }, + "has_stock": { + "type": "boolean", + "description": "是否有库存，仅付费资源返回" + }, + "stock": { + "type": "integer", + "description": "库存数量" + }, + "cost_after_discount": { + "type": "number", + "format": "float", + "description": "折后价格，仅付费资源返回" + }, + "original_cost": { + "type": "number", + "format": "float", + "description": "原价，仅付费资源返回" + } + } + }, + "ListHardwareResponse": { + "type": "object", + "description": "查询可用硬件配置响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "hardware": { + "type": "array", + "items": { + "$ref": "#/components/schemas/HardwareItem" + } + } + } + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "success", + "request_id" + ] + }, + "SDKVersionItem": { + "type": "object", + "description": "SDK 版本项", + "properties": { + "sdk_type": { + "type": "string", + "description": "SDK 类型", + "examples": [ + "gradio" + ] + }, + "tag": { + "type": "string", + "description": "版本标签", + "examples": [ + "latest" + ] + }, + "version": { + "type": "string", + "description": "SDK 版本", + "examples": [ + "6.2.0" + ] + } + }, + "required": [ + "sdk_type", + "tag", + "version" + ] + }, + "ListSDKVersionsResponse": { + "type": "object", + "description": "查询可用 SDK 版本响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "sdk_versions": { + "type": "array", + "description": "SDK 版本列表。仅 `sdk_type=gradio` 时返回 Gradio 版本列表；其他 SDK 类型返回空数组", + "items": { + "$ref": "#/components/schemas/SDKVersionItem" + } + }, + "total_count": { + "type": "integer", + "format": "int64", + "description": "SDK 版本总数" + } + } + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "success", + "request_id" + ] + }, + "BaseImageItem": { + "type": "object", + "description": "基础镜像项", + "properties": { + "name": { + "type": "string", + "description": "镜像名称", + "examples": [ + "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0" + ] + }, + "tag": { + "type": "string", + "description": "标签", + "examples": [ + "latest" + ] + } + } + }, + "ListBaseImagesResponse": { + "type": "object", + "description": "查询可用基础镜像响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "base_images": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BaseImageItem" + } + } + } + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "success", + "request_id" + ] + }, + "AddStudioVariableRequest": { + "type": "object", + "description": "添加 Studio 明文变量请求参数（key 和 value 都公开可见，敏感信息请使用密文变量接口）", + "properties": { + "key": { + "type": "string", + "description": "明文变量名称", + "maxLength": 128, + "examples": [ + "GRADIO_TEMP_DIR" + ] + }, + "value": { + "type": "string", + "description": "明文变量值", + "maxLength": 4096, + "examples": [ + "/tmp/gradio" + ] + } + }, + "required": [ + "key", + "value" + ] + }, + "UpdateStudioVariableRequest": { + "type": "object", + "description": "更新 Studio 明文变量请求参数", + "properties": { + "key": { + "type": "string", + "description": "明文变量名称", + "maxLength": 128, + "examples": [ + "GRADIO_TEMP_DIR" + ] + }, + "value": { + "type": "string", + "description": "明文变量新值", + "maxLength": 4096, + "examples": [ + "/new/path" + ] + } + }, + "required": [ + "key", + "value" + ] + }, + "DeleteStudioVariableRequest": { + "type": "object", + "description": "删除 Studio 明文变量请求参数", + "properties": { + "key": { + "type": "string", + "description": "明文变量名称" + } + }, + "required": [ + "key" + ] + }, + "StudioVariableItem": { + "type": "object", + "description": "明文变量列表项（返回 key 和 value）", + "properties": { + "key": { + "type": "string", + "description": "明文变量名称" + }, + "value": { + "type": "string", + "description": "明文变量值" + } + }, + "required": [ + "key", + "value" + ] + }, + "ListStudioVariablesResponse": { + "type": "object", + "description": "获取 Studio 明文变量列表响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "variables": { + "type": "array", + "items": { + "$ref": "#/components/schemas/StudioVariableItem" + } + } + } + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "success", + "data", + "request_id" + ] + }, + "SkillLocale": { + "type": "object", + "description": "技能的单语言本地化信息", + "properties": { + "description": { + "type": "string", + "description": "技能描述的本地化翻译" + }, + "category": { + "type": "string", + "description": "技能分类的本地化翻译" + } + } + }, + "SkillLocales": { + "type": "object", + "description": "技能的多语言本地化信息", + "properties": { + "en": { + "$ref": "#/components/schemas/SkillLocale" + }, + "zh": { + "$ref": "#/components/schemas/SkillLocale" + } + } + }, + "SkillSummary": { + "type": "object", + "description": "技能摘要信息", + "properties": { + "_id": { + "type": "string", + "description": "技能内部唯一标识", + "examples": [ + "62+FGAr3b7f4Bblezwx8hQ==" + ] + }, + "id": { + "type": "string", + "description": "技能 ID，格式为 @author/skill_name 或 owner/skill_name", + "examples": [ + "@AMap-Web/amap-lbs-skill" + ] + }, + "display_name": { + "type": "string", + "description": "技能展示名称", + "examples": [ + "高德地图综合服务Skill" + ] + }, + "description": { + "type": "string", + "description": "技能描述" + }, + "owner": { + "type": "string", + "description": "技能所有者（用户名或组织名）", + "examples": [ + "guoyanlian01" + ] + }, + "license": { + "type": "string", + "description": "许可证", + "examples": [ + "MIT License" + ] + }, + "developer": { + "type": "string", + "description": "开发者", + "examples": [ + "AMap-Web" + ] + }, + "source_url": { + "type": "string", + "description": "源码地址" + }, + "category": { + "type": "string", + "description": "技能分类", + "examples": [ + "developer-tools" + ] + }, + "tags": { + "type": [ + "array", + "null" + ], + "description": "标签列表，格式为 `key:value`，key 包括 category / license / developer / custom_tag", + "items": { + "type": "string" + }, + "examples": [ + [ + "category:developer-tools", + "license:MIT License", + "developer:AMap-Web", + "custom_tag:api-design" + ] + ] + }, + "logo_url": { + "type": "string", + "description": "logo 图片 URL" + }, + "view_count": { + "type": "integer", + "description": "浏览量", + "examples": [ + 13460 + ] + }, + "downloads": { + "type": "integer", + "description": "下载量", + "examples": [ + 1203 + ] + }, + "locales": { + "$ref": "#/components/schemas/SkillLocales" + }, + "private": { + "type": "boolean", + "description": "是否为私有技能", + "examples": [ + false + ] + }, + "custom_tag": { + "type": [ + "array", + "null" + ], + "description": "自定义标签列表（无标签时为 null）", + "items": { + "type": "string" + }, + "examples": [ + [ + "api-design", + "general-tools" + ], + null + ] + } + } + }, + "SkillDetail": { + "type": "object", + "description": "技能详细信息，包含安装命令", + "allOf": [ + { + "$ref": "#/components/schemas/SkillSummary" + }, + { + "type": "object", + "properties": { + "install_command": { + "type": "array", + "description": "安装命令列表", + "items": { + "type": "string" + }, + "examples": [ + [ + "npx skills add https://modelscope.cn/skills/@AMap-Web/amap-lbs-skill", + "curl -fsSL https://modelscope.cn/skills/install.sh | bash -s -- @AMap-Web/amap-lbs-skill" + ] + ] + } + } + } + ] + }, + "SkillListResponse": { + "type": "object", + "description": "技能列表响应", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "examples": [ + "c5bca202-aa4f-478b-b7e2-c568160583cb" + ] + }, + "data": { + "type": "object", + "properties": { + "skills": { + "type": "array", + "description": "技能列表", + "items": { + "$ref": "#/components/schemas/SkillSummary" + } + }, + "total": { + "type": "integer", + "format": "int64", + "description": "总记录数", + "examples": [ + 20 + ] + }, + "page_number": { + "type": "integer", + "description": "当前页码", + "examples": [ + 1 + ] + }, + "page_size": { + "type": "integer", + "description": "每页条数", + "examples": [ + 20 + ] + } + } + } + } + }, + "SkillDetailResponse": { + "type": "object", + "description": "技能详情响应", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "examples": [ + "105c898f-fd6d-437d-aa0b-9006036782da" + ] + }, + "data": { + "$ref": "#/components/schemas/SkillDetail" + } + } + }, + "CreateSkillRequest": { + "type": "object", + "description": "创建 Skill 请求参数", + "properties": { + "skill_name": { + "type": "string", + "description": "英文名称，创建后不可更改，仅允许小写字母、数字和连字符", + "pattern": "^[a-z0-9-]+$", + "maxLength": 64, + "examples": [ + "my-awesome-skill" + ] + }, + "owner": { + "type": "string", + "description": "所有者（用户名或组织名）", + "examples": [ + "iSolver" + ] + }, + "display_name": { + "type": "string", + "description": "展示名称", + "maxLength": 128, + "examples": [ + "我的技能" + ] + }, + "source_url": { + "type": "string", + "format": "uri", + "description": "来源地址", + "examples": [ + "https://github.com/example/my-skill" + ] + }, + "private": { + "type": "boolean", + "description": "是否私有，默认 false（公开）", + "default": false, + "examples": [ + false + ] + }, + "description": { + "type": "string", + "description": "描述", + "maxLength": 2000, + "examples": [ + "这是一个示例技能" + ] + }, + "license": { + "type": "string", + "description": "开源协议证书，不填默认 Apache-2.0", + "default": "Apache-2.0", + "examples": [ + "Apache-2.0" + ] + }, + "category": { + "allOf": [ + { + "$ref": "#/components/schemas/SkillCategory" + } + ], + "description": "Skill 类型/分类", + "examples": [ + "developer-tools" + ] + }, + "tags": { + "type": "array", + "description": "自定义标签", + "items": { + "type": "string" + }, + "examples": [ + [ + "Skill Creation" + ] + ] + }, + "logo_url": { + "type": "string", + "format": "uri", + "description": "Skill 图标 URL", + "examples": [ + "https://example.com/logo.png" + ] + }, + "skill_file": { + "type": "string", + "description": "Skill 项目文件 ID（通过 POST /files/upload 上传 zip 包后获取）。\nzip 包要求：\n- 最大 5MB\n- zip 根目录下必须包含且仅包含 1 个 SKILL.md 文件\n- 可包含子目录及其他辅助文件（如 prompts/、examples/ 等）\n- SKILL.md 必须包含 YAML front-matter（name、version、description）\n", + "examples": [ + "8c378570-8991-431b-a82c-96f3d0b4f0f4" + ] + } + }, + "required": [ + "skill_name", + "owner", + "license", + "category", + "skill_file" + ] + }, + "CreateSkillResponse": { + "type": "object", + "description": "创建 Skill 响应", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "examples": [ + "c55709d2-4e09-4bd8-be84-d8518bc46363" + ] + }, + "data": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Skill ID (owner/skill_name)", + "examples": [ + "iSolver/my-awesome-skill" + ] + }, + "name": { + "type": "string", + "description": "Skill 英文名称", + "examples": [ + "my-awesome-skill" + ] + }, + "display_name": { + "type": "string", + "description": "展示名称", + "examples": [ + "我的技能" + ] + }, + "owner": { + "type": "string", + "description": "所有者", + "examples": [ + "iSolver" + ] + }, + "url": { + "type": "string", + "description": "Skill 页面 URL", + "examples": [ + "https://modelscope.cn/skills/iSolver/my-awesome-skill" + ] + } + } + } + }, + "required": [ + "success", + "request_id" + ] + }, + "UpdateSkillSettingsRequest": { + "type": "object", + "description": "更新 Skill 设置请求参数（传入哪个字段修改哪个字段，不传的字段不修改）", + "properties": { + "display_name": { + "type": "string", + "description": "展示名称", + "maxLength": 128, + "examples": [ + "新的展示名称" + ] + }, + "source_url": { + "type": "string", + "format": "uri", + "description": "来源地址", + "examples": [ + "https://github.com/example/my-skill" + ] + }, + "private": { + "type": "boolean", + "description": "是否私有", + "examples": [ + true + ] + }, + "description": { + "type": "string", + "description": "描述", + "maxLength": 2000, + "examples": [ + "更新后的技能描述" + ] + }, + "license": { + "type": "string", + "description": "开源协议证书", + "examples": [ + "Apache-2.0" + ] + }, + "category": { + "allOf": [ + { + "$ref": "#/components/schemas/SkillCategory" + } + ], + "description": "Skill 类型/分类", + "examples": [ + "developer-tools" + ] + }, + "tags": { + "type": "array", + "description": "自定义标签（传入时整体覆盖原标签列表）", + "items": { + "type": "string" + }, + "examples": [ + [ + "Image/Video Gen", + "Media Processing" + ] + ] + }, + "logo_url": { + "type": "string", + "format": "uri", + "description": "Skill 图标 URL（传空字符串恢复为平台默认图标）", + "examples": [ + "https://example.com/new-logo.png" + ] + }, + "skill_file": { + "type": "string", + "description": "Skill 项目文件 ID（通过 POST /files/upload 上传 zip 包后获取，传入时整体覆盖原文件，不传则保留原文件不变）。\nzip 包要求：\n- 最大 5MB\n- zip 根目录下必须包含且仅包含 1 个 SKILL.md 文件\n- 可包含子目录及其他辅助文件（如 prompts/、examples/ 等）\n- SKILL.md 必须包含 YAML front-matter（name、version、description）\n", + "examples": [ + "new-file-id-xxx" + ] + } + } + }, + "SkillCategory": { + "type": "string", + "description": "Skill 类型/分类枚举", + "enum": [ + "skill-management", + "developer-tools", + "marketing-seo", + "frontend-development", + "ai-media", + "code-quality-testing", + "mobile-development", + "cloud-devops", + "other" + ], + "examples": [ + "developer-tools" + ] + }, + "GetBalanceResponse": { + "type": "object", + "description": "余额查询响应", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "data": { + "type": "object", + "properties": { + "total_balance": { + "type": "number", + "description": "总额度（可用额度 + 预扣额度）", + "examples": [ + 1500 + ] + }, + "available_balance": { + "type": "number", + "description": "可用额度", + "examples": [ + 1300 + ] + }, + "frozen_amount": { + "type": "number", + "description": "预扣额度（进行中任务未返回结果时预扣减）", + "examples": [ + 200 + ] + } + } + }, + "request_id": { + "type": "string", + "examples": [ + "c55709d2-4e09-4bd8-be84-d8518bc46363" + ] + } + }, + "required": [ + "success", + "request_id" + ] + }, + "UploadFileRequest": { + "type": "object", + "description": "上传文件请求参数（multipart/form-data）", + "properties": { + "file": { + "type": "string", + "format": "binary", + "description": "上传的文件，最大 5MB" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "自定义文件名（选填），支持多级目录路径（如 images/sub/logo.png）；不提供则使用上传文件的原始文件名。不允许绝对路径或目录穿越（..）。", + "examples": [ + "images/sub/logo.png" + ] + } + }, + "required": [ + "file" + ] + }, + "UploadFileResponse": { + "type": "object", + "description": "上传文件响应", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "examples": [ + "99ee2083-63e0-46dc-b393-623c22645078" + ] + }, + "data": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "文件 ID，可在其他接口中作为入参引用该文件", + "examples": [ + "8c378570-8991-431b-a82c-96f3d0b4f0f4" + ] + } + }, + "required": [ + "id" + ] + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "JWK": { + "type": "object", + "description": "JSON Web Key（仅用于 Ed25519 公钥）", + "required": [ + "kty", + "crv", + "x", + "kid" + ], + "properties": { + "kty": { + "type": "string", + "description": "密钥类型，固定 OKP", + "enum": [ + "OKP" + ], + "examples": [ + "OKP" + ] + }, + "crv": { + "type": "string", + "description": "椭圆曲线类型，固定 Ed25519", + "enum": [ + "Ed25519" + ], + "examples": [ + "Ed25519" + ] + }, + "x": { + "type": "string", + "description": "公钥 base64url 编码（无 padding）", + "examples": [ + "f7QtzyEJhqFL8Jk5OYOq7010Ry9PipDGQkTzl7XtiEs" + ] + }, + "kid": { + "type": "string", + "description": "密钥 ID", + "examples": [ + "agent-key-001" + ] + }, + "alg": { + "type": "string", + "description": "签名算法", + "examples": [ + "EdDSA" + ] + }, + "use": { + "type": "string", + "description": "用途", + "examples": [ + "sig" + ] + } + } + }, + "Principal": { + "type": "object", + "description": "Agent 归属主体", + "properties": { + "type": { + "type": "string", + "description": "主体类型", + "enum": [ + "user", + "org" + ], + "examples": [ + "user" + ] + }, + "id": { + "type": "string", + "description": "主体 ID", + "examples": [ + "alice" + ] + } + } + }, + "AgentIdentityStatus": { + "type": "string", + "description": "Agent 状态", + "enum": [ + "active", + "paused" + ], + "examples": [ + "active" + ] + }, + "CreateAgentIdentityRequest": { + "type": "object", + "description": "创建 Agent 身份请求", + "required": [ + "agent_name", + "public_key" + ], + "properties": { + "agent_name": { + "type": "string", + "description": "Agent 名称（最长 128 字符）", + "maxLength": 128, + "examples": [ + "my-agent" + ] + }, + "description": { + "type": "string", + "description": "Agent 描述（最长 2000 字符）", + "maxLength": 2000 + }, + "public_key": { + "$ref": "#/components/schemas/JWK" + }, + "key_alg_type": { + "type": "string", + "description": "密钥算法类型，固定 Ed25519", + "enum": [ + "Ed25519" + ], + "examples": [ + "Ed25519" + ] + }, + "token_expire_time": { + "type": "integer", + "description": "Token 过期时间（秒），可选值 300/600/1800/3600", + "enum": [ + 300, + 600, + 1800, + 3600 + ], + "examples": [ + 3600 + ] + } + } + }, + "UpdateAgentIdentityRequest": { + "type": "object", + "description": "更新 Agent 身份设置请求", + "properties": { + "agent_name": { + "type": "string", + "maxLength": 128 + }, + "description": { + "type": "string", + "maxLength": 2000 + }, + "token_expire_time": { + "type": "integer", + "enum": [ + 300, + 600, + 1800, + 3600 + ] + } + } + }, + "ResetKeyPairRequest": { + "type": "object", + "description": "重置密钥对请求（客户端本地生成新密钥后上传公钥）", + "required": [ + "public_key" + ], + "properties": { + "public_key": { + "$ref": "#/components/schemas/JWK" + }, + "key_alg_type": { + "type": "string", + "enum": [ + "Ed25519" + ], + "examples": [ + "Ed25519" + ] + } + } + }, + "PauseAgentRequest": { + "type": "object", + "description": "启用/停用 Agent 请求", + "required": [ + "paused" + ], + "properties": { + "paused": { + "type": "boolean", + "description": "true 表示停用，false 表示启用", + "examples": [ + true + ] + } + } + }, + "AgentIdentity": { + "type": "object", + "description": "Agent 身份详情", + "properties": { + "agent_id": { + "type": "string", + "description": "Agent 唯一 ID", + "examples": [ + "agent_id:modelscope:agent_1234567890ab" + ] + }, + "agent_name": { + "type": "string", + "examples": [ + "my-agent" + ] + }, + "description": { + "type": "string" + }, + "token_expire_time": { + "type": "integer", + "examples": [ + 3600 + ] + }, + "principal": { + "$ref": "#/components/schemas/Principal" + }, + "kid": { + "type": "string", + "description": "当前密钥 ID", + "examples": [ + "agent-key-001" + ] + }, + "public_key": { + "$ref": "#/components/schemas/JWK" + }, + "status": { + "$ref": "#/components/schemas/AgentIdentityStatus" + }, + "create_time": { + "type": "string", + "description": "创建时间 (ISO 8601)", + "examples": [ + "2026-06-03T10:00:00Z" + ] + }, + "update_time": { + "type": "string", + "description": "更新时间 (ISO 8601)", + "examples": [ + "2026-06-03T10:30:00Z" + ] + } + } + }, + "AgentIdentityListItem": { + "type": "object", + "description": "Agent 身份列表项", + "properties": { + "agent_id": { + "type": "string" + }, + "agent_name": { + "type": "string" + }, + "kid": { + "type": "string" + }, + "status": { + "$ref": "#/components/schemas/AgentIdentityStatus" + }, + "token_expire_time": { + "type": "integer" + }, + "create_time": { + "type": "string" + } + } + }, + "ListAgentIdentitiesData": { + "type": "object", + "description": "Agent 身份列表数据", + "properties": { + "agent_identities": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AgentIdentityListItem" + } + }, + "total_count": { + "type": "integer", + "format": "int64" + }, + "page_number": { + "type": "integer" + }, + "page_size": { + "type": "integer" + } + } + }, + "ResetKeyPairData": { + "type": "object", + "description": "重置密钥对返回数据", + "properties": { + "agent_id": { + "type": "string" + }, + "kid": { + "type": "string" + }, + "public_key": { + "$ref": "#/components/schemas/JWK" + } + } + }, + "AgentIdentityResponse": { + "type": "object", + "description": "Agent 身份详情响应", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/AgentIdentity" + }, + "message": { + "type": "string", + "examples": [ + "success" + ] + }, + "request_id": { + "type": "string" + } + } + }, + "ListAgentIdentitiesResponse": { + "type": "object", + "description": "Agent 身份列表响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "$ref": "#/components/schemas/ListAgentIdentitiesData" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + } + }, + "ResetKeyPairResponse": { + "type": "object", + "description": "重置密钥对响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "$ref": "#/components/schemas/ResetKeyPairData" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + } + }, + "DeleteAgentIdentityResponse": { + "type": "object", + "description": "删除 Agent 身份响应", + "properties": { + "success": { + "type": "boolean" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + } + }, + "PauseAgentResponse": { + "type": "object", + "description": "启用/停用 Agent 响应", + "properties": { + "success": { + "type": "boolean" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + } + }, + "TokenSignRequest": { + "type": "object", + "description": "Token 签发请求（Agent 用私钥签名 message=`agent_id|kid|audience|timestamp` 后提交）", + "required": [ + "agent_id", + "kid", + "audience", + "timestamp", + "signature" + ], + "properties": { + "agent_id": { + "type": "string", + "examples": [ + "agent_id:modelscope:agent_1234567890ab" + ] + }, + "kid": { + "type": "string", + "examples": [ + "agent-key-001" + ] + }, + "audience": { + "type": "string", + "description": "目标 Hub 应用的 client_id", + "examples": [ + "hub-app-abcdef" + ] + }, + "timestamp": { + "type": "integer", + "format": "int64", + "description": "签名时间戳（Unix 秒），需在服务器时间 ±5 分钟之内", + "examples": [ + 1717410000 + ] + }, + "signature": { + "type": "string", + "description": "Ed25519 签名 base64url 编码（无 padding）", + "examples": [ + "MEUCIQDxxx..." + ] + } + } + }, + "TokenSignData": { + "type": "object", + "description": "Token 签发返回数据", + "properties": { + "access_token": { + "type": "string", + "description": "签发的 JWT", + "examples": [ + "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCIsImtpZCI6ImlkcC1rZXktMDAxIn0..." + ] + }, + "token_type": { + "type": "string", + "enum": [ + "Bearer" + ], + "examples": [ + "Bearer" + ] + }, + "expire_at": { + "type": "integer", + "format": "int64", + "description": "Token 失效时间（Unix 时间戳，单位秒）", + "examples": [ + 1717413600 + ] + }, + "jti": { + "type": "string", + "description": "JWT ID", + "examples": [ + "jti-1234567890" + ] + } + } + }, + "TokenSignResponse": { + "type": "object", + "description": "Token 签发响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "$ref": "#/components/schemas/TokenSignData" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + } + }, + "TokenRecordItem": { + "type": "object", + "description": "Token 签发记录项", + "properties": { + "token_id": { + "type": "string" + }, + "audience": { + "type": "string", + "description": "目标 Hub 的 client_id" + }, + "issued_at": { + "type": "string", + "description": "签发时间 (ISO 8601)" + }, + "expire_at": { + "type": "string", + "description": "过期时间 (ISO 8601)" + }, + "status": { + "type": "string", + "enum": [ + "active", + "expired", + "revoked" + ] + }, + "jwt": { + "type": "string", + "description": "JWT 字符串" + } + } + }, + "ListTokenRecordsData": { + "type": "object", + "description": "Token 签发记录列表数据", + "properties": { + "token_records": { + "type": "array", + "items": { + "$ref": "#/components/schemas/TokenRecordItem" + } + }, + "total_count": { + "type": "integer", + "format": "int64" + }, + "page_number": { + "type": "integer" + }, + "page_size": { + "type": "integer" + } + } + }, + "ListTokenRecordsResponse": { + "type": "object", + "description": "Token 签发记录列表响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "$ref": "#/components/schemas/ListTokenRecordsData" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + } + }, + "AgentIdConfiguration": { + "type": "object", + "description": "Agent IDP OIDC Discovery 配置", + "properties": { + "issuer": { + "type": "string", + "examples": [ + "https://modelscope.cn" + ] + }, + "token_endpoint": { + "type": "string", + "examples": [ + "https://modelscope.cn/openapi/v1/agent_id/token" + ] + }, + "jwks_uri": { + "type": "string", + "examples": [ + "https://modelscope.cn/openapi/v1/agent_id/.well-known/agentid-jwks" + ] + }, + "registration_endpoint": { + "type": "string", + "examples": [ + "https://modelscope.cn/openapi/v1/agent_ids" + ] + }, + "activity_endpoint": { + "type": "string" + }, + "id_token_signing_alg_values_supported": { + "type": "string", + "examples": [ + "EdDSA" + ] + } + } + }, + "JWKS": { + "type": "object", + "description": "JSON Web Key Set", + "properties": { + "keys": { + "type": "array", + "items": { + "$ref": "#/components/schemas/JWK" + } + } + } + }, + "Collection": { + "type": "object", + "description": "Collection 详情对象", + "properties": { + "slug": { + "type": "string", + "description": "Collection 唯一标识，格式 {owner}/{slug}。新建为 {owner}/{title_slug}；历史为 {owner}/{title_slug}-{short_id}", + "examples": [ + "iSolver/best-nlp-models-6a3b2c" + ] + }, + "title": { + "type": "string", + "description": "Collection 标题", + "examples": [ + "最佳 NLP 模型合集" + ] + }, + "description": { + "type": [ + "string", + "null" + ], + "description": "Collection 描述（Markdown）", + "examples": [ + "精选的自然语言处理模型..." + ] + }, + "owner": { + "type": "string", + "description": "所有者（用户名或组织名）", + "examples": [ + "iSolver" + ] + }, + "visibility": { + "type": "string", + "description": "可见性：public（公开）/ private（私有）", + "enum": [ + "public", + "private" + ], + "examples": [ + "public" + ] + }, + "theme": { + "type": [ + "string", + "null" + ], + "description": "主题标签，枚举值：Blue/Pink/Purple/Cyan", + "examples": [ + "Blue" + ] + }, + "item_count": { + "type": "integer", + "description": "Collection 中条目数量", + "examples": [ + 12 + ] + }, + "likes": { + "type": "integer", + "description": "喜欢数", + "examples": [ + 128 + ] + }, + "view_count": { + "type": "integer", + "description": "浏览量", + "examples": [ + 1024 + ] + }, + "items": { + "type": "array", + "description": "Collection 中的条目列表", + "items": { + "$ref": "#/components/schemas/CollectionItem" + } + }, + "created_at": { + "type": "string", + "format": "date-time", + "description": "创建时间，ISO 8601 UTC", + "examples": [ + "2025-01-15T08:00:00Z" + ] + }, + "updated_at": { + "type": "string", + "format": "date-time", + "description": "更新时间，ISO 8601 UTC", + "examples": [ + "2025-04-20T10:30:00Z" + ] + }, + "url": { + "type": "string", + "description": "Collection 页面 URL", + "examples": [ + "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + ] + }, + "install_command": { + "type": [ + "array", + "null" + ], + "description": "当 items 包含 skill 时，返回下载命令数组", + "items": { + "type": "string" + }, + "examples": [ + [ + "modelscope skill install @iSolver/my-skill" + ] + ] + } + }, + "required": [ + "slug", + "title", + "owner", + "visibility", + "item_count", + "likes", + "view_count", + "items", + "created_at", + "updated_at", + "url" + ] + }, + "CollectionItem": { + "type": "object", + "description": "Collection 条目对象。条目通过 item_type + item_object_id 唯一标识，不维护独立 item_id", + "properties": { + "item_type": { + "type": "string", + "description": "资源类型：model / dataset / studio / paper / skill / mcp", + "enum": [ + "model", + "dataset", + "studio", + "paper", + "skill", + "mcp" + ], + "examples": [ + "model" + ] + }, + "item_object_id": { + "type": "string", + "description": "资源标识，如 damo/nlp_bert_base 或论文 ID", + "examples": [ + "damo/nlp_bert_base" + ] + }, + "note": { + "type": [ + "string", + "null" + ], + "description": "用户对该条目的备注说明（Markdown）", + "examples": [ + "这个模型在中文 NLP 任务上表现优异" + ] + }, + "position": { + "type": "integer", + "description": "排序位置（从 1 开始）", + "examples": [ + 1 + ] + }, + "created_at": { + "type": "string", + "format": "date-time", + "description": "添加时间，ISO 8601 UTC", + "examples": [ + "2025-01-15T08:00:00Z" + ] + }, + "visibility": { + "type": "string", + "description": "可见性：public / protected / private。protected 对应“仅公开体验”语义", + "enum": [ + "public", + "protected", + "private" + ], + "examples": [ + "public" + ] + }, + "gated": { + "type": "boolean", + "description": "visibility 为 public 时，是否需先申请并经同意后才可访问", + "examples": [ + false + ] + }, + "protected_mode": { + "type": "integer", + "description": "protected（仅公开体验）模式标识；用于表达条目/资源的仅公开体验语义，具体取值含义以产品后端实现为准（如 0=无、2=仅公开体验）", + "examples": [ + 0 + ] + } + }, + "required": [ + "item_type", + "item_object_id", + "position", + "created_at", + "visibility", + "gated", + "protected_mode" + ] + }, + "FailedCollectionItem": { + "type": "object", + "description": "批量操作中失败的条目及原因", + "properties": { + "item_type": { + "type": "string", + "description": "资源类型", + "examples": [ + "model" + ] + }, + "item_object_id": { + "type": "string", + "description": "资源标识", + "examples": [ + "not/exist-model" + ] + }, + "reason": { + "type": "string", + "description": "失败原因错误码，如 ResourceNotFound、ItemAlreadyInCollection、ItemNotInCollection、PermissionDenied", + "examples": [ + "ResourceNotFound" + ] + } + }, + "required": [ + "item_type", + "item_object_id", + "reason" + ] + }, + "CreateCollectionRequest": { + "type": "object", + "description": "创建 Collection 请求", + "properties": { + "title": { + "type": "string", + "maxLength": 128, + "description": "Collection 标题（最长 128 字符）", + "examples": [ + "最佳 NLP 模型合集" + ] + }, + "owner": { + "type": "string", + "description": "所有者(用户名或组织名)", + "examples": [ + "iSolver" + ] + }, + "description": { + "type": [ + "string", + "null" + ], + "description": "Collection 描述（Markdown）", + "examples": [ + "精选的自然语言处理模型..." + ] + }, + "visibility": { + "type": "string", + "description": "可见性：public / private，默认 public", + "enum": [ + "public", + "private" + ], + "default": "public", + "examples": [ + "public" + ] + }, + "theme": { + "type": [ + "string", + "null" + ], + "description": "主题标签，枚举值：Blue/Pink/Purple/Cyan", + "examples": [ + "Blue" + ] + } + }, + "required": [ + "title", + "owner" + ] + }, + "UpdateCollectionRequest": { + "type": "object", + "description": "更新 Collection 请求。传入哪个字段即更新哪个字段，未传字段保持原值", + "properties": { + "title": { + "type": [ + "string", + "null" + ], + "maxLength": 128, + "description": "Collection 标题（最长 128 字符）", + "examples": [ + "最佳 NLP 模型合集（已更名）" + ] + }, + "description": { + "type": [ + "string", + "null" + ], + "description": "Collection 描述（Markdown）", + "examples": [ + "更新后的描述..." + ] + }, + "owner": { + "type": [ + "string", + "null" + ], + "description": "所有者", + "examples": [ + "iSolver" + ] + }, + "visibility": { + "type": [ + "string", + "null" + ], + "description": "可见性：public / private", + "enum": [ + "public", + "private", + null + ], + "examples": [ + "private" + ] + }, + "theme": { + "type": [ + "string", + "null" + ], + "description": "主题标签，枚举值：Blue/Pink/Purple/Cyan", + "examples": [ + "Purple" + ] + } + } + }, + "CollectionResponse": { + "type": "object", + "description": "获取 Collection 详情的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "$ref": "#/components/schemas/Collection" + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "CollectionListResponse": { + "type": "object", + "description": "获取 Collection 列表的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "type": "object", + "description": "列表数据", + "properties": { + "collection_list": { + "type": "array", + "description": "Collection 列表，返回完整 Collection 对象", + "items": { + "$ref": "#/components/schemas/Collection" + } + }, + "total_count": { + "type": "integer", + "description": "符合筛选条件的合集总数", + "examples": [ + 128 + ] + }, + "page_number": { + "type": "integer", + "description": "当前页码", + "examples": [ + 1 + ] + }, + "page_size": { + "type": "integer", + "description": "每页大小", + "examples": [ + 10 + ] + } + }, + "required": [ + "collection_list", + "total_count", + "page_number", + "page_size" + ] + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "CollectionItemListResponse": { + "type": "object", + "description": "获取 Collection 条目列表的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "type": "object", + "description": "条目列表数据", + "properties": { + "items": { + "type": "array", + "description": "条目列表", + "items": { + "$ref": "#/components/schemas/CollectionItem" + } + }, + "total_count": { + "type": "integer", + "description": "条目总数", + "examples": [ + 50 + ] + }, + "page_number": { + "type": "integer", + "description": "当前页码", + "examples": [ + 1 + ] + }, + "page_size": { + "type": "integer", + "description": "每页大小", + "examples": [ + 10 + ] + } + }, + "required": [ + "items", + "total_count", + "page_number", + "page_size" + ] + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "CollectionItemRequest": { + "type": "object", + "description": "待添加条目", + "properties": { + "item_type": { + "type": "string", + "description": "资源类型：model / dataset / studio / paper / skill / mcp", + "enum": [ + "model", + "dataset", + "studio", + "paper", + "skill", + "mcp" + ], + "examples": [ + "model" + ] + }, + "item_object_id": { + "type": "string", + "description": "资源标识，如 damo/nlp_bert_base_chinese", + "examples": [ + "damo/nlp_bert_base_chinese" + ] + }, + "note": { + "type": [ + "string", + "null" + ], + "description": "备注（Markdown）", + "examples": [ + "推荐用于中文文本分类" + ] + }, + "position": { + "type": [ + "integer", + "null" + ], + "description": "排序位置（从 1 开始）", + "examples": [ + 1 + ] + } + }, + "required": [ + "item_type", + "item_object_id" + ] + }, + "AddCollectionItemsRequest": { + "type": "object", + "description": "批量添加条目请求", + "properties": { + "items": { + "type": "array", + "description": "待添加条目列表（最少 1 个元素）", + "minItems": 1, + "items": { + "$ref": "#/components/schemas/CollectionItemRequest" + } + } + }, + "required": [ + "items" + ] + }, + "AddCollectionItemsResponse": { + "type": "object", + "description": "批量添加条目的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true（部分失败仍为 true，失败项见 failed_items）", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "type": "object", + "description": "添加结果", + "properties": { + "added_count": { + "type": "integer", + "description": "成功添加的条目数量", + "examples": [ + 2 + ] + }, + "url": { + "type": "string", + "description": "Collection 页面 URL", + "examples": [ + "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + ] + }, + "failed_items": { + "type": "array", + "description": "添加失败的条目及原因", + "items": { + "$ref": "#/components/schemas/FailedCollectionItem" + } + } + }, + "required": [ + "added_count", + "url", + "failed_items" + ] + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "UpdateCollectionItemRequest": { + "type": "object", + "description": "更新单个条目请求。传入哪个字段即更新哪个字段", + "properties": { + "note": { + "type": [ + "string", + "null" + ], + "description": "备注（Markdown）", + "examples": [ + "更新后的备注" + ] + }, + "position": { + "type": [ + "integer", + "null" + ], + "description": "排序位置（从 1 开始）", + "examples": [ + 2 + ] + } + } + }, + "UpdateCollectionItemResponse": { + "type": "object", + "description": "更新单个条目的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "$ref": "#/components/schemas/CollectionItem" + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "CreateGalleryRequest": { + "type": "object", + "description": "创建 Gallery 请求", + "properties": { + "name": { + "type": "string", + "maxLength": 128, + "pattern": "^[0-9a-zA-Z_.-]+$", + "description": "Gallery 名称，最长 128 字符，仅允许字母、数字、下划线、连字符、点", + "examples": [ + "my-gallery" + ] + }, + "owner": { + "type": "string", + "maxLength": 128, + "description": "所属用户或组织名，最长 128 字符", + "examples": [ + "my-org" + ] + }, + "files": { + "type": "array", + "maxItems": 50, + "description": "文件 ID 列表（文件上传接口返回的 ID），最多 50 个，单个元素不能为空", + "items": { + "type": "string", + "minLength": 1 + }, + "examples": [ + [ + "8c378570-8991-431b-a82c-96f3d0b4f0f4" + ] + ] + }, + "entry_file": { + "type": "string", + "minLength": 1, + "description": "入口文件的相对路径，对应 files 中某个已上传文件；后缀须与 category 匹配（后缀不区分大小写）：notebook→.ipynb，website→.html，pdf→.pdf（file 类别不限后缀）", + "examples": [ + "notebook/demo.ipynb" + ] + }, + "private": { + "type": "boolean", + "default": false, + "description": "是否私有，默认 false（公开）", + "examples": [ + false + ] + }, + "category": { + "type": "string", + "description": "类型（必传），可选值：notebook / website / pdf / file；除 file 外须与 entry_file 后缀匹配（后缀不区分大小写）：notebook→.ipynb，website→.html，pdf→.pdf", + "enum": [ + "notebook", + "website", + "pdf", + "file" + ], + "examples": [ + "notebook" + ] + }, + "labels": { + "type": [ + "array", + "null" + ], + "maxItems": 10, + "description": "标签列表，最多 10 个，单个标签最长 20 字符", + "items": { + "type": "string", + "maxLength": 20 + }, + "examples": [ + [ + "demo", + "tutorial" + ] + ] + }, + "path": { + "type": [ + "string", + "null" + ], + "maxLength": 128, + "pattern": "^[0-9a-zA-Z_-]+$", + "description": "自定义 URL 尾缀（选填），仅允许字母、数字、下划线、连字符，最长 128 字符；不填则不生成，填写后灵感流对应 URL：https://modelscope.cn/gallery/{owner}/{path}", + "examples": [ + "my-custom-path" + ] + } + }, + "required": [ + "name", + "owner", + "files", + "entry_file", + "category" + ] + }, + "CreateGalleryResponse": { + "type": "object", + "description": "创建 Gallery 的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "req-xxxxxx" + ] + }, + "data": { + "type": "object", + "description": "创建结果", + "properties": { + "id": { + "type": "string", + "description": "Gallery 唯一标识（UUID）", + "examples": [ + "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + ] + }, + "url": { + "type": "string", + "description": "Gallery 访问链接", + "examples": [ + "https://modelscope.cn/gallery/my-org/a1b2c3d4-e5f6-7890-abcd-ef1234567890" + ] + } + }, + "required": [ + "id", + "url" + ] + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "BatchUpdateCollectionItem": { + "type": "object", + "description": "批量更新条目。通过 item_type + item_object_id 定位，传入哪个字段就修改哪个字段", + "properties": { + "item_type": { + "type": "string", + "description": "资源类型，用于定位", + "enum": [ + "model", + "dataset", + "studio", + "paper", + "skill", + "mcp" + ], + "examples": [ + "model" + ] + }, + "item_object_id": { + "type": "string", + "description": "资源标识，用于定位", + "examples": [ + "damo/nlp_bert_base_chinese" + ] + }, + "note": { + "type": [ + "string", + "null" + ], + "description": "备注（Markdown），不传则不修改", + "examples": [ + "更新后的备注" + ] + }, + "position": { + "type": [ + "integer", + "null" + ], + "description": "排序位置（从 1 开始），不传则不修改", + "examples": [ + 1 + ] + } + }, + "required": [ + "item_type", + "item_object_id" + ] + }, + "UpdateCollectionItemsRequest": { + "type": "object", + "description": "批量更新条目请求", + "properties": { + "items": { + "type": "array", + "description": "待更新条目列表（最少 1 个元素）", + "minItems": 1, + "items": { + "$ref": "#/components/schemas/BatchUpdateCollectionItem" + } + } + }, + "required": [ + "items" + ] + }, + "UpdateCollectionItemsResponse": { + "type": "object", + "description": "批量更新条目的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true（部分失败仍为 true，失败项见 failed_items）", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "type": "object", + "description": "批量更新结果", + "properties": { + "updated_count": { + "type": "integer", + "description": "成功更新的条目数量", + "examples": [ + 2 + ] + }, + "failed_items": { + "type": "array", + "description": "更新失败的条目及原因", + "items": { + "$ref": "#/components/schemas/FailedCollectionItem" + } + } + }, + "required": [ + "updated_count", + "failed_items" + ] + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "UserPublic": { + "type": "object", + "description": "用户公开信息", + "properties": { + "username": { + "type": "string", + "description": "用户名", + "examples": [ + "tester" + ] + }, + "nickname": { + "type": "string", + "description": "昵称", + "examples": [ + "测试用户" + ] + }, + "description": { + "type": "string", + "description": "描述", + "examples": [ + "这是一个测试用户" + ] + }, + "avatar_url": { + "type": "string", + "format": "uri", + "description": "头像图片URL地址", + "examples": [ + "https://resouces.modelscope.cn/avatar/516aa51f-5fb7-49eb-930c-e90500ae13ca.png" + ] + } + }, + "required": [ + "username" + ] + }, + "UserPrivate": { + "type": "object", + "description": "用户私有信息(仅自己可见)", + "properties": { + "email": { + "type": "string", + "format": "email", + "description": "邮箱地址", + "examples": [ + "test@modelscope.cn" + ] + } + } + }, + "McpServerList": { + "type": "object", + "description": "MCP服务列表与分页信息", + "properties": { + "mcp_server_list": { + "type": "array", + "description": "MCP服务列表", + "items": { + "$ref": "#/components/schemas/McpServerSummary" + } + }, + "total_count": { + "type": "integer", + "description": "搜索符合条件的mcp server总个数", + "examples": [ + 100 + ] + } + }, + "required": [ + "mcp_server_list", + "total_count" + ] + }, + "McpServerOperationalList": { + "type": "object", + "description": "用户托管MCP服务列表", + "properties": { + "mcp_server_list": { + "type": "array", + "description": "用户托管MCP服务列表", + "items": { + "$ref": "#/components/schemas/McpServerOperational" + } + }, + "total_count": { + "type": "integer", + "description": "服务链接总数", + "examples": [ + 5 + ] + } + }, + "required": [ + "mcp_server_list", + "total_count" + ] + }, + "data": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Studio ID (owner/repo_name)" + }, + "repo_name": { + "type": "string", + "description": "仓库名称" + }, + "display_name": { + "type": "string", + "description": "显示名称" + }, + "owner": { + "type": "string", + "description": "所有者" + }, + "description": { + "type": "string", + "description": "描述" + }, + "cover_image": { + "type": "string", + "description": "封面图 URL" + }, + "likes": { + "type": "integer", + "description": "喜欢数" + }, + "view_count": { + "type": "integer", + "description": "访问量" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "visibility": { + "$ref": "#/components/schemas/StudioVisibility" + }, + "private": { + "type": "boolean", + "description": "Deprecated: 请使用 visibility 字段", + "deprecated": true + }, + "last_modified": { + "type": "string", + "description": "最后修改时间" + }, + "sdk_type": { + "$ref": "#/components/schemas/StudioSDKType" + }, + "sdk_version": { + "$ref": "#/components/schemas/StudioSDKVersion" + }, + "hardware": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioHardware" + } + ], + "description": "期望的硬件配置（数据库中保存的值）" + }, + "base_image": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioBaseImage" + } + ], + "description": "期望的基础镜像（Docker 类型不返回）" + }, + "license": { + "type": "string", + "description": "许可证" + }, + "host": { + "type": "string", + "description": "访问地址" + }, + "mcp_support": { + "type": "boolean", + "description": "是否支持 MCP" + }, + "runtime": { + "$ref": "#/components/schemas/StudioRuntime" + } + } + }, + "ListStudiosResponse": { + "type": "object", + "description": "获取创空间列表响应", + "properties": { + "studios": { + "type": "array", + "description": "创空间列表", + "items": { + "$ref": "#/components/schemas/data" + } + }, + "total_count": { + "type": "integer", + "description": "符合筛选条件的创空间总数", + "examples": [ + 120 + ] + }, + "page_number": { + "type": "integer", + "description": "当前页码", + "examples": [ + 1 + ] + }, + "page_size": { + "type": "integer", + "description": "每页大小", + "examples": [ + 10 + ] + } + }, + "required": [ + "studios", + "total_count", + "page_number", + "page_size" + ] + }, + "CollectionMutationResponse": { + "type": "object", + "description": "创建/更新 Collection 的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "type": "object", + "description": "变更结果", + "properties": { + "slug": { + "type": "string", + "description": "Collection slug（重命名时可能变化）", + "examples": [ + "iSolver/best-nlp-models-6a3b2c" + ] + }, + "title": { + "type": "string", + "description": "Collection 标题", + "examples": [ + "最佳 NLP 模型合集" + ] + }, + "url": { + "type": "string", + "description": "Collection 页面 URL", + "examples": [ + "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + ] + } + }, + "required": [ + "slug", + "title", + "url" + ] + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "EmptyDataResponse": { + "type": "object", + "description": "删除/移除类操作的标准响应，结果以 success 字段为准，data 为空对象", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "type": "object", + "description": "空对象" + } + }, + "required": [ + "success", + "request_id", + "data" + ] + } + }, + "responses": { + "bad-request": { + "description": "请求参数错误", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "input-parameter-error": { + "$ref": "#/components/examples/input-parameter-error" + } + } + } + } + }, + "unauthorized": { + "description": "认证失败（token 无效、过期或未提供）", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "invalid-authentication": { + "$ref": "#/components/examples/invalid-authentication" + } + } + } + } + }, + "forbidden": { + "description": "权限不足（token 有效但无权执行该操作，或配额超限）", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "operation-not-allowed": { + "$ref": "#/components/examples/operation-not-allowed" + }, + "quota-limit-exceed": { + "$ref": "#/components/examples/quota-limit-exceed" + } + } + } + } + }, + "not-found": { + "description": "资源未找到", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "resource-not-found": { + "$ref": "#/components/examples/resource-not-found" + } + } + } + } + }, + "internal-server-error": { + "description": "内部服务错误", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "internal-server-error": { + "$ref": "#/components/examples/internal-server-error" + } + } + } + } + }, + "service-unavailable": { + "description": "服务不可用", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "service-unavailable": { + "$ref": "#/components/examples/service-unavailable" + } + } + } + } + }, + "get-current-user-success": { + "description": "成功获取当前用户信息", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UserResponse" + }, + "examples": { + "user-all-response": { + "$ref": "#/components/examples/user-all-response" + } + } + } + } + }, + "list-models-success": { + "description": "成功获取模型列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ModelListResponse" + }, + "examples": { + "list-models-response": { + "$ref": "#/components/examples/list-models-response" + } + } + } + } + }, + "get-model-success": { + "description": "成功获取模型详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ModelDetailResponse" + }, + "examples": { + "get-model-response": { + "$ref": "#/components/examples/get-model-response" + } + } + } + } + }, + "list-datasets-success": { + "description": "成功获取数据集列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DatasetListResponse" + }, + "examples": { + "list-datasets-response": { + "$ref": "#/components/examples/list-datasets-response" + } + } + } + } + }, + "get-dataset-success": { + "description": "成功获取数据集详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DatasetDetailResponse" + }, + "examples": { + "get-dataset-response": { + "$ref": "#/components/examples/get-dataset-response" + } + } + } + } + }, + "list-mcp-servers-success": { + "description": "成功获取MCP服务列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpServerListResponse" + }, + "examples": { + "list-mcp-servers-response": { + "$ref": "#/components/examples/list-mcp-servers-response" + } + } + } + } + }, + "list-operational-mcp-servers-success": { + "description": "成功获取用户托管MCP服务列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpServerOperationalListResponse" + }, + "examples": { + "list-operational-mcp-servers-response": { + "$ref": "#/components/examples/list-operational-mcp-servers-response" + } + } + } + } + }, + "get-mcp-server-success": { + "description": "成功获取MCP服务详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpServerDetailResponse" + }, + "examples": { + "get-mcp-server-response": { + "$ref": "#/components/examples/get-mcp-server-response" + } + } + } + } + }, + "deploy-mcp-server-success": { + "description": "成功部署MCP服务", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpDeployResponse" + }, + "examples": { + "deploy-mcp-server-response": { + "$ref": "#/components/examples/deploy-mcp-server-response" + } + } + } + } + }, + "undeploy-mcp-server-success": { + "description": "成功解除MCP服务部署", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpUndeployResponse" + }, + "examples": { + "undeploy-mcp-server-response": { + "$ref": "#/components/examples/undeploy-mcp-server-response" + } + } + } + } + }, + "create-studio-success": { + "description": "成功创建 Studio", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateStudioResponse" + }, + "examples": { + "create-studio-response": { + "$ref": "#/components/examples/create-studio-response" + } + } + } + } + }, + "get-studio-success": { + "description": "成功获取 Studio 详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetStudioResponse" + }, + "examples": { + "get-studio-response": { + "$ref": "#/components/examples/get-studio-response" + } + } + } + } + }, + "update-studio-settings-success": { + "description": "成功更新 Studio 设置", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetStudioResponse" + }, + "examples": { + "update-studio-settings-response": { + "$ref": "#/components/examples/get-studio-response" + } + } + } + } + }, + "studio-operation-success": { + "description": "Studio 操作成功", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessResponse" + } + } + } + }, + "studio-runtime-success": { + "description": "Studio 运行时操作成功", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StudioRuntimeResponse" + }, + "examples": { + "studio-runtime-response": { + "$ref": "#/components/examples/studio-runtime-response" + } + } + } + } + }, + "get-studio-logs-success": { + "description": "成功获取 Studio 日志", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StudioLogsResponse" + }, + "examples": { + "get-studio-logs-response": { + "$ref": "#/components/examples/get-studio-logs-response" + } + } + } + } + }, + "list-hardware-success": { + "description": "成功获取可用硬件配置列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListHardwareResponse" + } + } + } + }, + "list-sdk-versions-success": { + "description": "成功获取可用 SDK 版本列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListSDKVersionsResponse" + }, + "examples": { + "list-sdk-versions-response": { + "$ref": "#/components/examples/list-sdk-versions-response" + } + } + } + } + }, + "list-base-images-success": { + "description": "成功获取可用基础镜像列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListBaseImagesResponse" + } + } + } + }, + "list-studio-variables-success": { + "description": "成功获取明文变量列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListStudioVariablesResponse" + } + } + } + }, + "list-skills-success": { + "description": "成功获取技能列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillListResponse" + }, + "examples": { + "list-skills-response": { + "$ref": "#/components/examples/list-skills-response" + } + } + } + } + }, + "get-skill-success": { + "description": "成功获取技能详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillDetailResponse" + }, + "examples": { + "get-skill-response": { + "$ref": "#/components/examples/get-skill-response" + } + } + } + } + }, + "create-skill-success": { + "description": "成功创建技能", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateSkillResponse" + }, + "examples": { + "create-skill-response": { + "$ref": "#/components/examples/create-skill-response" + } + } + } + } + }, + "update-skill-settings-success": { + "description": "成功更新技能设置", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessResponse" + }, + "examples": { + "update-skill-settings-response": { + "$ref": "#/components/examples/update-skill-settings-response" + } + } + } + } + }, + "skill-bad-request": { + "description": "请求参数错误", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "skill-input-parameter-error": { + "$ref": "#/components/examples/skill-input-parameter-error" + } + } + } + } + }, + "skill-unauthorized": { + "description": "未通过认证", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "skill-invalid-authentication": { + "$ref": "#/components/examples/skill-invalid-authentication" + } + } + } + } + }, + "skill-not-found": { + "description": "技能未找到", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "skill-not-found": { + "$ref": "#/components/examples/skill-not-found" + } + } + } + } + }, + "skill-conflict": { + "description": "资源冲突（如 Skill 已存在）", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "skill-duplicate-entity": { + "$ref": "#/components/examples/skill-duplicate-entity" + } + } + } + } + }, + "skill-internal-server-error": { + "description": "内部服务错误", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "skill-internal-server-error": { + "$ref": "#/components/examples/skill-internal-server-error" + } + } + } + } + }, + "skill-service-unavailable": { + "description": "服务不可用", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "skill-service-unavailable": { + "$ref": "#/components/examples/skill-service-unavailable" + } + } + } + } + }, + "upload-file-success": { + "description": "成功上传文件", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UploadFileResponse" + }, + "examples": { + "upload-file-response": { + "$ref": "#/components/examples/upload-file-response" + } + } + } + } + }, + "create-agent-identity-success": { + "description": "成功创建 Agent 身份", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentIdentityResponse" + }, + "examples": { + "create-agent-identity-response": { + "$ref": "#/components/examples/create-agent-identity-response" + } + } + } + } + }, + "get-agent-identity-success": { + "description": "成功获取 Agent 身份详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentIdentityResponse" + }, + "examples": { + "get-agent-identity-response": { + "$ref": "#/components/examples/get-agent-identity-response" + } + } + } + } + }, + "update-agent-identity-success": { + "description": "成功更新 Agent 身份设置", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentIdentityResponse" + }, + "examples": { + "update-agent-identity-response": { + "$ref": "#/components/examples/update-agent-identity-response" + } + } + } + } + }, + "list-agent-identities-success": { + "description": "成功获取 Agent 身份列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListAgentIdentitiesResponse" + }, + "examples": { + "list-agent-identities-response": { + "$ref": "#/components/examples/list-agent-identities-response" + } + } + } + } + }, + "reset-key-pair-success": { + "description": "成功重置 Agent 密钥对", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResetKeyPairResponse" + }, + "examples": { + "reset-key-pair-response": { + "$ref": "#/components/examples/reset-key-pair-response" + } + } + } + } + }, + "pause-agent-success": { + "description": "成功启用/停用 Agent", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PauseAgentResponse" + }, + "examples": { + "pause-agent-response": { + "$ref": "#/components/examples/pause-agent-response" + } + } + } + } + }, + "delete-agent-identity-success": { + "description": "成功删除 Agent 身份", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteAgentIdentityResponse" + }, + "examples": { + "delete-agent-identity-response": { + "$ref": "#/components/examples/delete-agent-identity-response" + } + } + } + } + }, + "issue-token-success": { + "description": "成功签发 Token", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TokenSignResponse" + }, + "examples": { + "issue-token-response": { + "$ref": "#/components/examples/issue-token-response" + } + } + } + } + }, + "list-token-records-success": { + "description": "成功获取 Token 签发记录列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListTokenRecordsResponse" + }, + "examples": { + "list-token-records-response": { + "$ref": "#/components/examples/list-token-records-response" + } + } + } + } + }, + "agentid-configuration-success": { + "description": "成功获取 Agent IDP OIDC 配置", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentIdConfiguration" + }, + "examples": { + "agentid-configuration-response": { + "$ref": "#/components/examples/agentid-configuration-response" + } + } + } + } + }, + "agentid-jwks-success": { + "description": "成功获取 JWKS", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JWKS" + }, + "examples": { + "agentid-jwks-response": { + "$ref": "#/components/examples/agentid-jwks-response" + } + } + } + } + }, + "get-collection-success": { + "description": "成功获取合集详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CollectionResponse" + }, + "examples": { + "get-collection-response": { + "$ref": "#/components/examples/get-collection-response" + } + } + } + } + }, + "create-collection-success": { + "description": "成功创建合集", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CollectionMutationResponse" + }, + "examples": { + "create-collection-response": { + "$ref": "#/components/examples/create-collection-response" + } + } + } + } + }, + "update-collection-success": { + "description": "成功更新合集", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CollectionMutationResponse" + }, + "examples": { + "update-collection-response": { + "$ref": "#/components/examples/update-collection-response" + } + } + } + } + }, + "delete-collection-success": { + "description": "成功删除合集", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EmptyDataResponse" + }, + "examples": { + "delete-collection-response": { + "$ref": "#/components/examples/delete-collection-response" + } + } + } + } + }, + "list-collections-success": { + "description": "成功获取合集列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CollectionListResponse" + }, + "examples": { + "list-collections-response": { + "$ref": "#/components/examples/list-collections-response" + } + } + } + } + }, + "list-collection-items-success": { + "description": "成功获取合集条目列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CollectionItemListResponse" + }, + "examples": { + "list-collection-items-response": { + "$ref": "#/components/examples/list-collection-items-response" + } + } + } + } + }, + "add-collection-items-success": { + "description": "成功添加合集条目（部分失败仍返回 success=true，见 failed_items）", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AddCollectionItemsResponse" + }, + "examples": { + "add-collection-items-response": { + "$ref": "#/components/examples/add-collection-items-response" + } + } + } + } + }, + "update-collection-item-success": { + "description": "成功更新单个合集条目", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateCollectionItemResponse" + }, + "examples": { + "update-collection-item-response": { + "$ref": "#/components/examples/update-collection-item-response" + } + } + } + } + }, + "remove-collection-item-success": { + "description": "成功移除合集条目", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EmptyDataResponse" + }, + "examples": { + "remove-collection-item-response": { + "$ref": "#/components/examples/remove-collection-item-response" + } + } + } + } + }, + "update-collection-items-success": { + "description": "成功批量更新合集条目（部分失败仍返回 success=true，见 failed_items）", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateCollectionItemsResponse" + }, + "examples": { + "update-collection-items-response": { + "$ref": "#/components/examples/update-collection-items-response" + } + } + } + } + }, + "create-gallery-success": { + "description": "成功创建 Gallery", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateGalleryResponse" + }, + "examples": { + "create-gallery-response": { + "$ref": "#/components/examples/create-gallery-response" + } + } + } + } + }, + "list-studios-success": { + "description": "成功获取创空间列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListStudiosResponse" + } + } + } + }, + "conflict": { + "description": "资源冲突", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "duplicate-entity": { + "$ref": "#/components/examples/duplicate-entity" + } + } + } + } + }, + "too-many-requests": { + "description": "请求过于频繁", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "rate-limit-exceed": { + "$ref": "#/components/examples/rate-limit-exceed" + } + } + } + } + }, + "get-balance-success": { + "description": "余额查询成功", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetBalanceResponse" + }, + "examples": { + "get-balance-response": { + "$ref": "#/components/examples/get-balance-response" + } + } + } + } + } + }, + "securitySchemes": { + "bearerAuth": { + "type": "http", + "scheme": "bearer", + "description": "ModelScope API Token" + } + }, + "parameters": { + "owner": { + "name": "owner", + "in": "path", + "description": "仓库 owner（组织或个人）", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "examples": [ + "OpenDataLab", + "Qwen", + "iic" + ] + } + }, + "repo_name": { + "name": "repo_name", + "in": "path", + "description": "仓库 repo_name 名称", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "examples": [ + "AICC", + "Qwen-Image-Edit-2511" + ] + } + }, + "mcp_server_id": { + "name": "id", + "in": "path", + "description": "服务ID。由平台后台收录的MCP，ID 默认为@author/server_name，当被社区用户认领后将会变更为user_name/server_name；由社区用户自发提交的MCP，ID 为user_name/server_name。", + "required": true, + "schema": { + "type": "string", + "examples": [ + "@modelcontextprotocol/fetch", + "username/server-name" + ] + } + }, + "studio_owner": { + "name": "owner", + "in": "path", + "description": "所有者（用户名或组织名）", + "required": true, + "schema": { + "type": "string", + "examples": [ + "username", + "organization" + ] + } + }, + "studio_repo_name": { + "name": "repo_name", + "in": "path", + "description": "仓库名称", + "required": true, + "schema": { + "type": "string", + "examples": [ + "my-demo-app" + ] + } + }, + "studio_log_type": { + "name": "log_type", + "in": "path", + "description": "日志类型：build（构建日志）或 run（运行日志）", + "required": true, + "schema": { + "type": "string", + "enum": [ + "build", + "run" + ], + "examples": [ + "run" + ] + } + }, + "skill_owner_path": { + "name": "owner", + "in": "path", + "description": "技能作者/所有者，支持 `@author` 格式（官方认证）或 `owner`（用户名或组织名），其中 `@` 无需 URL 编码", + "required": true, + "schema": { + "type": "string", + "minLength": 1 + }, + "examples": { + "author": { + "value": "@iSolver", + "summary": "官方认证作者" + }, + "user": { + "value": "iSolver", + "summary": "用户名或组织名" + } + } + }, + "skill_name_path": { + "name": "skill_name", + "in": "path", + "description": "技能英文名称，仅允许小写字母、数字和连字符", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9-]+$", + "examples": [ + "my-awesome-skill" + ] + } + }, + "collection_owner_path": { + "name": "owner", + "in": "path", + "description": "Collection 所有者（用户名或组织名）。具体长度与格式约束以当前产品页面实现为准。", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "examples": [ + "iSolver" + ] + } + }, + "collection_slug_path": { + "name": "slug", + "in": "path", + "description": "Collection slug 尾段，由 owner 和 title 字段按一定规则自动生成；创建接口会返回该字段。", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "examples": [ + "best-nlp-models-6a3b2c" + ] + } + }, + "collection_item_type_path": { + "name": "item_type", + "in": "path", + "description": "资源类型：model / dataset / studio / paper / skill / mcp", + "required": true, + "schema": { + "type": "string", + "enum": [ + "model", + "dataset", + "studio", + "paper", + "skill", + "mcp" + ], + "examples": [ + "model" + ] + } + }, + "collection_item_object_id_path": { + "name": "item_object_id", + "in": "path", + "description": "资源标识，如 damo/nlp_bert_base。可包含 `/`（作为 items/{item_type}/ 之后的整段剩余路径），无需 URL 编码。", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "examples": [ + "damo/nlp_bert_base" + ] + } + } + }, + "examples": { + "user-all-response": { + "value": { + "success": true, + "data": { + "username": "tester", + "nickname": "测试用户", + "description": "这是一个测试用户", + "email": "test@modelscope.cn", + "avatar_url": "https://resouces.modelscope.cn/avatar/516aa51f-5fb7-49eb-930c-e90500ae13ca.png" + }, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" + } + }, + "list-models-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": { + "models": [ + { + "id": "iic/QwenLong-L1.5-30B-A3B", + "display_name": "QwenLong-L1.5-30B-A3B", + "description": "", + "downloads": 1466, + "likes": 25, + "license": "apache-2.0", + "tasks": [ + "text-generation" + ], + "created_at": "2025-12-14T12:26:15Z", + "last_modified": "2025-12-16T03:34:26Z", + "file_size": 61081671123, + "params": 30532122624, + "tags": [ + "license:apache-2.0", + "model_type:qwen3_moe", + "library:transformer", + "library:safetensors", + "library:pytorch", + "task:text-generation", + "deploy:swingdeploy" + ], + "private": false, + "gated": false + }, + { + "id": "ZhipuAI/AutoGLM-Phone-9B", + "display_name": "AutoGLM-Phone-9B", + "description": "", + "downloads": 28525, + "likes": 265, + "license": "MIT License", + "tasks": [ + "image-captioning" + ], + "created_at": "2025-12-08T08:00:28Z", + "last_modified": "2025-12-09T15:22:57Z", + "file_size": 20605693975, + "params": 5368163840, + "tags": [ + "license:MIT License", + "model_type:glm4v", + "library:safetensors", + "library:pytorch", + "task:image-captioning", + "deploy:swingdeploy" + ], + "private": false, + "gated": false + } + ], + "total_count": 43127, + "page_number": 1, + "page_size": 2 + } + } + }, + "get-model-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": { + "id": "Qwen/Qwen-Image-Edit-2511", + "display_name": "Qwen-Image-Edit-2511", + "description": "", + "downloads": 20175, + "likes": 149, + "license": "apache-2.0", + "tasks": [ + "image-to-image" + ], + "created_at": "2025-12-18T01:32:15Z", + "last_modified": "2025-12-23T14:09:52Z", + "file_size": 57720465119, + "params": 0, + "tags": [ + "license:apache-2.0", + "library:pytorch", + "library:safetensors", + "library:diffusers", + "task:image-to-image" + ], + "private": false, + "gated": false, + "readme": "hello" + } + } + }, + "list-datasets-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": { + "datasets": [ + { + "id": "DAMO_NLP/jd", + "display_name": "商品评论情感预测", + "description": "该数据集包含从2011年1月1日到2014年3月31日（3年多）某电商网站的消费者购买行为，用户评分，评论和产品元数据，涵盖15个一级产品类别，987个二级产品类别，近2个百万用户，超过10万种产品和超过6,000万条评论。该数据集中的每个文本评论都包含三个子评论：正面评论，负面评论和整体评论。", + "file_size": 0, + "downloads": 53996, + "likes": 102, + "license": "Apache License 2.0", + "tasks": [ + "text-classification" + ], + "created_at": "2022-09-28T06:30:31Z", + "last_modified": "2022-10-30T03:36:39Z", + "tags": [ + "license:Apache License 2.0", + "task:text-classification" + ], + "private": false, + "gated": false, + "login_required": false + }, + { + "id": "iic/nlp_domain_classification_chinese_testset", + "display_name": "中文文本领域分类测试集", + "description": "中文文本领域分类测试集，涉及国民经济行业标准的18个行业，部分收集自公开数据", + "file_size": 0, + "downloads": 2401, + "likes": 23, + "license": "Apache License 2.0", + "tasks": [ + "text-classification" + ], + "created_at": "2023-02-06T09:21:21Z", + "last_modified": "2024-09-02T15:16:35Z", + "tags": [ + "license:Apache License 2.0", + "task:text-classification", + "custom_tag:文本领域", + "custom_tag:领域", + "custom_tag:文本分类" + ], + "private": false, + "gated": false, + "login_required": false + } + ], + "total_count": 60, + "page_number": 1, + "page_size": 2 + } + } + }, + "get-dataset-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": { + "id": "OpenDataLab/AICC", + "display_name": "AICC", + "description": "", + "file_size": 0, + "downloads": 12119, + "likes": 7, + "license": "cc-by-4.0", + "tasks": [], + "created_at": "2025-11-25T14:30:22Z", + "last_modified": "2026-01-01T00:31:42Z", + "tags": [ + "license:cc-by-4.0", + "custom_tag:common-crawl", + "custom_tag:code", + "custom_tag:markdown", + "custom_tag:math", + "custom_tag:html-parsing" + ], + "private": false, + "gated": false, + "login_required": false, + "readme": "hello" + } + } + }, + "list-mcp-servers-response": { + "value": { + "success": true, + "data": { + "mcp_server_list": [ + { + "id": "@modelcontextprotocol/fetch", + "name": "Fetch网页内容抓取", + "chinese_name": "Fetch网页内容抓取", + "description": "该服务器使大型语言模型能够检索和处理网页内容，支持多种格式的内容提取和转换。", + "logo_url": "https://resources.modelscope.cn/mcp-cover-img/fetch.png", + "publisher": "@modelcontextprotocol/fetch", + "categories": [ + "browser-automation" + ], + "tags": [ + "browser-automation", + "web-scraping" + ], + "view_count": 30667, + "locales": { + "zh": { + "name": "Fetch网页内容抓取", + "description": "该服务器使大型语言模型能够检索和处理网页内容..." + }, + "en": { + "name": "Fetch", + "description": "A server that enables LLMs to retrieve and process web content..." + } + } + }, + { + "id": "@anthropic/claude-mcp", + "name": "Claude MCP", + "chinese_name": "Claude MCP服务", + "description": "Anthropic 官方 MCP 服务，提供 Claude 模型的标准化访问接口。", + "logo_url": "https://resources.modelscope.cn/mcp-cover-img/claude.png", + "publisher": "@anthropic/claude-mcp", + "categories": [ + "ai-assistant" + ], + "tags": [ + "ai-assistant", + "claude" + ], + "view_count": 15234, + "locales": { + "zh": { + "name": "Claude MCP服务", + "description": "Anthropic 官方 MCP 服务..." + }, + "en": { + "name": "Claude MCP", + "description": "Official Anthropic MCP service..." + } + } + } + ], + "total_count": 100 + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "list-operational-mcp-servers-response": { + "value": { + "success": true, + "data": { + "mcp_server_list": [ + { + "id": "@modelcontextprotocol/fetch", + "name": "Fetch网页内容抓取", + "chinese_name": "Fetch网页内容抓取", + "description": "该服务器使大型语言模型能够检索和处理网页内容...", + "logo_url": "https://resources.modelscope.cn/mcp-cover-img/fetch.png", + "publisher": "@modelcontextprotocol/fetch", + "categories": [ + "browser-automation" + ], + "tags": [ + "browser-automation" + ], + "view_count": 30667, + "operational_urls": [ + { + "id": "platform-pool", + "url": "https://mcp.api-inference.modelscope.net/abc123/mcp", + "transport_type": "streamable_http", + "auth_required": false, + "expiration": "2025-10-01 21:00:00", + "accessible": true + } + ] + } + ], + "total_count": 1 + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "get-mcp-server-response": { + "value": { + "success": true, + "data": { + "id": "@modelcontextprotocol/fetch", + "name": "Fetch网页内容抓取", + "chinese_name": "Fetch网页内容抓取", + "description": "该服务器使大型语言模型能够检索和处理网页内容，支持多种格式的内容提取和转换。", + "logo_url": "https://resources.modelscope.cn/mcp-cover-img/fetch.png", + "publisher": "@modelcontextprotocol/fetch", + "author": "modelcontextprotocol", + "owner": "", + "readme": "# Fetch MCP Server\n\nA Model Context Protocol server that provides web content fetching capabilities.\n\n## Features\n\n- Fetch web pages and extract content\n- Support for multiple output formats\n- Automatic content cleaning and formatting", + "source_url": "https://github.com/modelcontextprotocol/servers/tree/main/src/fetch", + "github_stars": 1234, + "is_hosted": true, + "is_verified": true, + "categories": [ + "browser-automation" + ], + "tags": [ + "browser-automation", + "web-scraping" + ], + "view_count": 30667, + "env_schema": {}, + "server_config": [], + "operational_urls": [], + "locales": { + "zh": { + "name": "Fetch网页内容抓取", + "description": "该服务器使大型语言模型能够检索和处理网页内容...", + "readme": "# Fetch MCP Server\n\n提供网页内容抓取能力的 MCP 服务。" + }, + "en": { + "name": "Fetch", + "description": "A server that enables LLMs to retrieve and process web content...", + "readme": "# Fetch MCP Server\n\nA Model Context Protocol server that provides web content fetching capabilities." + } + } + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "deploy-mcp-server-response": { + "value": { + "success": true, + "data": { + "id": "platform-pool", + "url": "https://mcp.api-inference.modelscope.net/abc123def456/mcp", + "transport_type": "streamable_http", + "auth_required": false, + "expiration": "2025-10-01 21:00:00" + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "undeploy-mcp-server-response": { + "value": { + "success": true, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "create-studio-response": { + "value": { + "success": true, + "data": { + "id": "username/my-demo-app", + "repo_name": "my-demo-app", + "display_name": "我的演示应用", + "owner": "username", + "url": "https://modelscope.cn/studios/username/my-demo-app" + }, + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "get-studio-response": { + "value": { + "success": true, + "data": { + "id": "username/my-demo-app", + "repo_name": "my-demo-app", + "display_name": "我的演示应用", + "owner": "username", + "description": "一个演示创空间应用", + "likes": 100, + "view_count": 5000, + "tags": [ + "nlp", + "chatbot" + ], + "visibility": "public", + "private": false, + "last_modified": "2025-01-15T10:30:00Z", + "sdk_type": "gradio", + "sdk_version": "6.2.0", + "hardware": "platform/2v-cpu-16g-mem", + "base_image": "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0", + "license": "apache-2.0", + "host": "https://username-my-demo-app.ms.show", + "mcp_support": false, + "runtime": { + "status": "Running", + "active_config": { + "hardware": "platform/2v-cpu-16g-mem", + "base_image": "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0", + "sdk_type": "gradio", + "sdk_version": "6.2.0" + }, + "created_at": "2025-01-15T10:30:00Z" + } + }, + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "studio-runtime-response": { + "value": { + "success": true, + "data": { + "status": "Running", + "active_config": { + "hardware": "platform/2v-cpu-16g-mem", + "base_image": "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0", + "sdk_type": "gradio", + "sdk_version": "6.2.0" + }, + "created_at": "2025-01-15T10:30:00Z" + }, + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "get-studio-logs-response": { + "value": { + "success": true, + "data": { + "logs": [ + "2025-01-15 10:30:00 INFO Starting application...", + "2025-01-15 10:30:01 INFO Application started successfully" + ], + "page_num": 1, + "page_size": 100, + "total_count": 500, + "total_page_num": 5 + }, + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "list-sdk-versions-response": { + "value": { + "success": true, + "data": { + "sdk_versions": [ + { + "sdk_type": "gradio", + "tag": "latest", + "version": "6.2.0" + }, + { + "sdk_type": "gradio", + "tag": "stable", + "version": "5.49.1" + } + ], + "total_count": 2 + }, + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "list-skills-response": { + "value": { + "success": true, + "request_id": "c5bca202-aa4f-478b-b7e2-c568160583cb", + "data": { + "skills": [ + { + "_id": "CIK3H+DYKlSAMlHnX60AUg==", + "id": "@AMap-Web/amap-lbs-skill", + "display_name": "高德地图综合服务Skill", + "description": "高德地图综合服务，支持POI搜索、路径规划、旅游规划、周边搜索和热力图数据可视化", + "owner": "", + "license": "MIT License", + "developer": "AMap-Web", + "source_url": "https://github.com/AMap-Web/amap-lbs-skill", + "category": "developer-tools", + "tags": [ + "category:developer-tools", + "license:MIT License", + "developer:AMap-Web", + "custom_tag:api-design", + "custom_tag:general-tools" + ], + "logo_url": "", + "view_count": 6144, + "downloads": 791, + "locales": { + "en": { + "description": "Amap comprehensive services, supporting POI search, route planning, travel planning, nearby search, and heatmap data visualization", + "category": "DevTools" + }, + "zh": { + "description": "高德地图综合服务，支持POI搜索、路径规划、旅游规划、周边搜索和热力图数据可视化", + "category": "开发工具" + } + }, + "private": false, + "custom_tag": [ + "api-design", + "general-tools" + ] + }, + { + "_id": "0ry9hAiAiJ+2pklbvQjepg==", + "id": "@steipete/summarize", + "display_name": "summarize", + "description": "使用summarize CLI（适用于网页、PDF、图片、音频、YouTube）来总结URL或文件。", + "owner": "steipete", + "license": "", + "developer": "steipete", + "source_url": "https://clawhub.ai/steipete/summarize", + "category": "ai-media", + "tags": [ + "category:ai-media", + "developer:steipete", + "custom_tag:image-video-gen", + "custom_tag:media-processing" + ], + "logo_url": "", + "view_count": 4696, + "downloads": 690, + "locales": { + "en": { + "description": "Summarize URLs or files with the summarize CLI (web, PDFs, images, audio, YouTube).", + "category": "MediaAI" + }, + "zh": { + "description": "使用summarize CLI（适用于网页、PDF、图片、音频、YouTube）来总结URL或文件。", + "category": "媒体处理" + } + }, + "private": false, + "custom_tag": [ + "image-video-gen", + "media-processing" + ] + } + ], + "total": 20, + "page_number": 1, + "page_size": 20 + } + } + }, + "get-skill-response": { + "value": { + "success": true, + "request_id": "105c898f-fd6d-437d-aa0b-9006036782da", + "data": { + "_id": "CIK3H+DYKlSAMlHnX60AUg==", + "id": "@AMap-Web/amap-lbs-skill", + "display_name": "高德地图综合服务Skill", + "description": "高德地图综合服务，支持POI搜索、路径规划、旅游规划、周边搜索和热力图数据可视化", + "owner": "", + "license": "MIT License", + "developer": "AMap-Web", + "source_url": "https://github.com/AMap-Web/amap-lbs-skill", + "category": "developer-tools", + "tags": [ + "category:developer-tools", + "license:MIT License", + "developer:AMap-Web", + "custom_tag:api-design", + "custom_tag:general-tools" + ], + "logo_url": "", + "view_count": 6144, + "downloads": 791, + "locales": { + "en": { + "description": "Amap comprehensive services, supporting POI search, route planning, travel planning, nearby search, and heatmap data visualization", + "category": "DevTools" + }, + "zh": { + "description": "高德地图综合服务，支持POI搜索、路径规划、旅游规划、周边搜索和热力图数据可视化", + "category": "开发工具" + } + }, + "private": false, + "custom_tag": [ + "api-design", + "general-tools" + ], + "install_command": [ + "npx skills add https://modelscope.cn/skills/@AMap-Web/amap-lbs-skill", + "curl -fsSL https://modelscope.cn/skills/install.sh | bash -s -- @AMap-Web/amap-lbs-skill" + ] + } + } + }, + "create-skill-response": { + "value": { + "success": true, + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363", + "data": { + "id": "iSolver/my-awesome-skill", + "name": "my-awesome-skill", + "display_name": "我的技能", + "owner": "iSolver", + "url": "https://modelscope.cn/skills/iSolver/my-awesome-skill" + } + } + }, + "update-skill-settings-response": { + "value": { + "success": true, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "upload-file-response": { + "value": { + "success": true, + "request_id": "99ee2083-63e0-46dc-b393-623c22645078", + "data": { + "id": "8c378570-8991-431b-a82c-96f3d0b4f0f4" + } + } + }, + "create-agent-identity-response": { + "value": { + "success": true, + "data": { + "agent_id": "agent_id:modelscope:agent_1234567890ab", + "agent_name": "my-agent", + "description": "用于演示的测试 Agent", + "token_expire_time": 3600, + "principal": { + "type": "user", + "id": "alice" + }, + "kid": "agent-key-001", + "public_key": { + "kty": "OKP", + "crv": "Ed25519", + "x": "f7QtzyEJhqFL8Jk5OYOq7010Ry9PipDGQkTzl7XtiEs", + "kid": "agent-key-001", + "alg": "EdDSA", + "use": "sig" + }, + "status": "active", + "create_time": "2026-06-08T10:00:00Z" + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "get-agent-identity-response": { + "value": { + "success": true, + "data": { + "agent_id": "agent_id:modelscope:agent_1234567890ab", + "agent_name": "my-agent", + "description": "用于演示的测试 Agent", + "token_expire_time": 3600, + "principal": { + "type": "user", + "id": "alice" + }, + "kid": "agent-key-001", + "public_key": { + "kty": "OKP", + "crv": "Ed25519", + "x": "f7QtzyEJhqFL8Jk5OYOq7010Ry9PipDGQkTzl7XtiEs", + "kid": "agent-key-001", + "alg": "EdDSA", + "use": "sig" + }, + "status": "active", + "create_time": "2026-06-08T10:00:00Z", + "update_time": "2026-06-08T10:30:00Z" + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "update-agent-identity-response": { + "value": { + "success": true, + "data": { + "agent_id": "agent_id:modelscope:agent_1234567890ab", + "agent_name": "my-agent-renamed", + "description": "更新后的描述", + "token_expire_time": 1800, + "principal": { + "type": "user", + "id": "alice" + }, + "kid": "agent-key-001", + "public_key": { + "kty": "OKP", + "crv": "Ed25519", + "x": "f7QtzyEJhqFL8Jk5OYOq7010Ry9PipDGQkTzl7XtiEs", + "kid": "agent-key-001", + "alg": "EdDSA", + "use": "sig" + }, + "status": "active", + "create_time": "2026-06-08T10:00:00Z", + "update_time": "2026-06-08T11:00:00Z" + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "list-agent-identities-response": { + "value": { + "success": true, + "data": { + "agent_identities": [ + { + "agent_id": "agent_id:modelscope:agent_1234567890ab", + "agent_name": "my-agent", + "kid": "agent-key-001", + "status": "active", + "token_expire_time": 3600, + "create_time": "2026-06-08T10:00:00Z" + }, + { + "agent_id": "agent-cdef01234567", + "agent_name": "build-bot", + "kid": "agent-key-002", + "status": "paused", + "token_expire_time": 1800, + "create_time": "2026-06-07T15:20:00Z" + } + ], + "total_count": 2, + "page_number": 1, + "page_size": 20 + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "reset-key-pair-response": { + "value": { + "success": true, + "data": { + "agent_id": "agent_id:modelscope:agent_1234567890ab", + "kid": "agent-key-002", + "public_key": { + "kty": "OKP", + "crv": "Ed25519", + "x": "UnZsPxBV8ZyK5zhQlpnT1nf5GjxJQbpHQMwoS0jZeNc", + "kid": "agent-key-002", + "alg": "EdDSA", + "use": "sig" + } + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "pause-agent-response": { + "value": { + "success": true, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "delete-agent-identity-response": { + "value": { + "success": true, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "issue-token-response": { + "value": { + "success": true, + "data": { + "access_token": "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCIsImtpZCI6ImlkcC1rZXktMDAxIn0.eyJpc3MiOiJodHRwczovL21vZGVsc2NvcGUuY24iLCJzdWIiOiJhZ2VudC0xMjM0NTY3ODkwYWIiLCJhdWQiOiJodWItYXBwLWFiY2RlZjEyMzQ1NiIsImV4cCI6MTcxNzQxMzYwMCwiaWF0IjoxNzE3NDEwMDAwLCJqdGkiOiJqdGktMTIzNDU2Nzg5MCJ9.dummy_signature_base64url", + "token_type": "Bearer", + "expire_at": 1717413600, + "jti": "jti-1234567890" + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "list-token-records-response": { + "value": { + "success": true, + "data": { + "token_records": [ + { + "token_id": "jti-1234567890", + "audience": "hub-app-abcdef123456", + "issued_at": "2026-06-08T11:30:00Z", + "expire_at": "2026-06-08T12:30:00Z", + "status": "active", + "jwt": "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCIsImtpZCI6ImlkcC1rZXktMDAxIn0.eyJpc3MiOiJodHRwczovL21vZGVsc2NvcGUuY24iLCJzdWIiOiJhZ2VudC0xMjM0NTY3ODkwYWIiLCJhdWQiOiJodWItYXBwLWFiY2RlZjEyMzQ1NiIsImV4cCI6MTcxNzQxMzYwMCwiaWF0IjoxNzE3NDEwMDAwLCJqdGkiOiJqdGktMTIzNDU2Nzg5MCJ9.dummy_signature_base64url" + }, + { + "token_id": "jti-2345678901", + "audience": "hub-app-fedcba654321", + "issued_at": "2026-06-07T09:15:00Z", + "expire_at": "2026-06-07T10:15:00Z", + "status": "expired", + "jwt": "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCIsImtpZCI6ImlkcC1rZXktMDAxIn0.eyJpc3MiOiJodHRwczovL21vZGVsc2NvcGUuY24iLCJzdWIiOiJhZ2VudC0xMjM0NTY3ODkwYWIiLCJhdWQiOiJodWItYXBwLWZlZGNiYTY1NDMyMSIsImV4cCI6MTcxNzMyMjUwMCwiaWF0IjoxNzE3MzE4OTAwLCJqdGkiOiJqdGktMjM0NTY3ODkwMSJ9.dummy_signature_base64url" + } + ], + "total_count": 2, + "page_number": 1, + "page_size": 20 + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "agentid-configuration-response": { + "value": { + "issuer": "https://modelscope.cn", + "token_endpoint": "https://modelscope.cn/openapi/v1/agent_id/token", + "jwks_uri": "https://modelscope.cn/openapi/v1/agent_id/.well-known/agentid-jwks", + "registration_endpoint": "https://modelscope.cn/openapi/v1/agent_ids", + "activity_endpoint": "https://modelscope.cn/openapi/v1/agent_ids/{agent_id}/jwt_id_tokens", + "id_token_signing_alg_values_supported": "EdDSA" + } + }, + "agentid-jwks-response": { + "value": { + "keys": [ + { + "kty": "OKP", + "crv": "Ed25519", + "x": "UnZsPxBV8ZyK5zhQlpnT1nf5GjxJQbpHQMwoS0jZeNc", + "kid": "idp-key-001", + "alg": "EdDSA", + "use": "sig" + }, + { + "kty": "OKP", + "crv": "Ed25519", + "x": "WoxPa7_oZOn1HtdPNn-3Btu7UKmtMVRUyQMlYMM0SnQ", + "kid": "idp-key-002", + "alg": "EdDSA", + "use": "sig" + } + ] + } + }, + "get-collection-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": { + "slug": "iSolver/best-nlp-models-6a3b2c", + "title": "最佳 NLP 模型合集", + "description": "精选的自然语言处理模型...", + "owner": "iSolver", + "visibility": "public", + "theme": "Blue", + "item_count": 2, + "likes": 128, + "view_count": 1024, + "items": [ + { + "item_type": "model", + "item_object_id": "damo/nlp_bert_base", + "note": "这个模型在中文 NLP 任务上表现优异", + "position": 1, + "created_at": "2025-01-15T08:00:00Z", + "visibility": "public", + "gated": false, + "protected_mode": 2 + }, + { + "item_type": "dataset", + "item_object_id": "damo/nlp_corpus", + "note": null, + "position": 2, + "created_at": "2025-01-16T09:00:00Z", + "visibility": "public", + "gated": false, + "protected_mode": 2 + } + ], + "created_at": "2025-01-15T08:00:00Z", + "updated_at": "2025-04-20T10:30:00Z", + "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + } + } + }, + "create-collection-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": { + "slug": "iSolver/best-nlp-models-6a3b2c", + "title": "最佳 NLP 模型合集", + "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + } + } + }, + "update-collection-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": { + "slug": "iSolver/best-nlp-models-renamed-6a3b2c", + "title": "最佳 NLP 模型合集（已更名）", + "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-renamed-6a3b2c" + } + } + }, + "delete-collection-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": {} + } + }, + "list-collections-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": { + "collection_list": [ + { + "slug": "iSolver/best-nlp-models-6a3b2c", + "title": "最佳 NLP 模型合集", + "description": "精选的自然语言处理模型...", + "owner": "iSolver", + "visibility": "public", + "theme": "Blue", + "item_count": 12, + "likes": 128, + "view_count": 1024, + "items": [], + "created_at": "2025-01-15T08:00:00Z", + "updated_at": "2025-04-20T10:30:00Z", + "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + } + ], + "total_count": 128, + "page_number": 1, + "page_size": 10 + } + } + }, + "list-collection-items-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": { + "items": [ + { + "item_type": "model", + "item_object_id": "damo/nlp_bert_base", + "note": "这个模型在中文 NLP 任务上表现优异", + "position": 1, + "created_at": "2025-01-15T08:00:00Z", + "visibility": "public", + "gated": false, + "protected_mode": 2 + } + ], + "total_count": 50, + "page_number": 1, + "page_size": 10 + } + } + }, + "add-collection-items-request": { + "value": { + "items": [ + { + "item_type": "model", + "item_object_id": "damo/nlp_bert_base_chinese", + "note": "推荐用于中文文本分类", + "position": 1 + }, + { + "item_type": "dataset", + "item_object_id": "damo/nlp_corpus" + } + ] + } + }, + "add-collection-items-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": { + "added_count": 2, + "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c", + "failed_items": [ + { + "item_type": "model", + "item_object_id": "not/exist-model", + "reason": "ResourceNotFound" + } + ] + } + } + }, + "update-collection-item-request": { + "value": { + "note": "更新后的备注", + "position": 2 + } + }, + "update-collection-item-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": { + "item_type": "model", + "item_object_id": "damo/nlp_bert_base", + "note": "更新后的备注", + "position": 2, + "created_at": "2025-01-15T08:00:00Z", + "visibility": "public", + "gated": false, + "protected_mode": 2 + } + } + }, + "remove-collection-item-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": {} + } + }, + "update-collection-items-request": { + "value": { + "items": [ + { + "item_type": "model", + "item_object_id": "damo/nlp_bert_base", + "position": 1 + }, + { + "item_type": "dataset", + "item_object_id": "damo/nlp_corpus", + "note": "更新备注", + "position": 2 + } + ] + } + }, + "update-collection-items-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": { + "updated_count": 2, + "failed_items": [ + { + "item_type": "model", + "item_object_id": "not/exist-model", + "reason": "ItemNotInCollection" + } + ] + } + } + }, + "create-gallery-request": { + "value": { + "name": "my-gallery", + "owner": "my-org", + "files": [ + "8c378570-8991-431b-a82c-96f3d0b4f0f4", + "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + ], + "entry_file": "notebook/demo.ipynb", + "private": false, + "category": "notebook", + "labels": [ + "demo", + "tutorial" + ], + "path": "my-custom-path" + } + }, + "create-gallery-response": { + "value": { + "success": true, + "request_id": "req-xxxxxx", + "data": { + "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "url": "https://modelscope.cn/gallery/my-org/a1b2c3d4-e5f6-7890-abcd-ef1234567890" + } + } + }, + "invalid-authentication": { + "value": { + "success": false, + "code": "InvalidAuthentication", + "message": "Invalid authentication, please check your Authorization header", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + } + }, + "internal-server-error": { + "value": { + "success": false, + "code": "InternalServerError", + "message": "Internal server error: database connection failed", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd8" + } + }, + "service-unavailable": { + "value": { + "success": false, + "code": "ServiceUnavailable", + "message": "Service unavailable: model inference service is down", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd9" + } + }, + "resource-not-found": { + "value": { + "success": false, + "code": "ResourceNotFound", + "message": "Resource not found: model Qwen/Qwen3-72B", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd5" + } + }, + "input-parameter-error": { + "value": { + "success": false, + "code": "InputParameterError", + "message": "Input parameter 'model_id' is invalid: cannot be empty", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" + } + }, + "operation-not-allowed": { + "value": { + "success": false, + "code": "OperationNotAllowed", + "message": "Operation not allowed: you don't have permission to access this studio", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375be0" + } + }, + "quota-limit-exceed": { + "value": { + "success": false, + "code": "QuotaLimitExceed", + "message": "Quota limit exceeded: API calls per minute", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd4" + } + }, + "duplicate-entity": { + "value": { + "success": false, + "code": "DuplicateEntity", + "message": "Duplicate entity: model already exists", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd6" + } + }, + "skill-input-parameter-error": { + "value": { + "success": false, + "code": "InputParameterError", + "message": "Input parameter 'page_size' is invalid: must be between 1 and 100", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375be1" + } + }, + "skill-invalid-authentication": { + "value": { + "success": false, + "code": "InvalidAuthentication", + "message": "Invalid authentication, please check your Authorization header", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375be5" + } + }, + "skill-internal-server-error": { + "value": { + "success": false, + "code": "InternalServerError", + "message": "Internal server error: failed to query skill marketplace", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375be3" + } + }, + "skill-service-unavailable": { + "value": { + "success": false, + "code": "ServiceUnavailable", + "message": "Service unavailable: skill marketplace is temporarily unavailable", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375be4" + } + }, + "skill-duplicate-entity": { + "value": { + "success": false, + "code": "DuplicateEntity", + "message": "Duplicate entity: skill iSolver/my-awesome-skill already exists", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375be6" + } + }, + "skill-not-found": { + "value": { + "success": false, + "code": "ResourceNotFound", + "message": "Resource not found: skill @Alipay/alipay-payment-integration", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375be2" + } + }, + "rate-limit-exceed": { + "value": { + "success": false, + "code": "RateLimitExceed", + "message": "Rate limit exceeded: 100 requests per minute", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd7" + } + }, + "get-balance-response": { + "value": { + "success": true, + "data": { + "total_balance": 1000, + "available_balance": 998.5, + "frozen_amount": 1.5 + }, + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + } + } + } +} diff --git a/tests/integration/test_openapi.py b/tests/integration/test_openapi.py index a4c0c5c..b4cbca8 100644 --- a/tests/integration/test_openapi.py +++ b/tests/integration/test_openapi.py @@ -9,10 +9,21 @@ from __future__ import annotations +import contextlib +import os +import uuid + import pytest from modelscope_hub._openapi import OpenAPIClient +from modelscope_hub.api import HubApi from modelscope_hub.config import HubConfig +from modelscope_hub.errors import ( + AlreadyExistsError, + AuthenticationError, + InvalidParameter, + PermissionDeniedError, +) @pytest.fixture @@ -138,6 +149,169 @@ def test_get_studio_public(self, openapi): except Exception: pytest.skip("Public studio not available or requires auth") + def test_list_studios(self, openapi): + result = openapi.list_studios(page_size=5) + assert isinstance(result, dict) + studios = result.get("studios") or [] + assert isinstance(studios, list) + assert len(studios) <= 5 + + def test_list_studios_with_search(self, openapi): + result = openapi.list_studios(search="chat", page_size=3) + assert isinstance(result, dict) + + def test_list_studios_by_owner(self, openapi, test_owner): + result = openapi.list_studios(owner=test_owner, page_size=5) + for studio in result.get("studios") or []: + assert (studio.get("owner") or "").lower() == test_owner.lower() + + def test_list_studios_rejects_unknown_sort(self, openapi): + with pytest.raises(InvalidParameter): + openapi.list_studios(sort="downloads") + + +@pytest.mark.remote +class TestOpenAPIStudioResources: + """Resource-discovery endpoints backing --hardware / --base-image / --sdk-version.""" + + def test_list_hardware(self, openapi): + result = openapi.list_studio_hardware() + hardware = (result or {}).get("hardware") + assert isinstance(hardware, list) + if hardware: + assert "name" in hardware[0] + + def test_list_hardware_filtered_by_sdk_type(self, openapi): + result = openapi.list_studio_hardware(sdk_type="gradio") + assert isinstance((result or {}).get("hardware"), list) + + def test_list_base_images(self, openapi): + result = openapi.list_studio_base_images() + images = (result or {}).get("base_images") + assert isinstance(images, list) + if images: + assert "name" in images[0] + + def test_gradio_publishes_sdk_versions(self, openapi): + result = openapi.list_studio_sdk_versions(sdk_type="gradio") + versions = (result or {}).get("sdk_versions") + assert isinstance(versions, list) + + def test_non_gradio_sdk_has_no_versions(self, openapi): + """The specification states only gradio publishes a version list.""" + result = openapi.list_studio_sdk_versions(sdk_type="docker") + assert not ((result or {}).get("sdk_versions") or []) + + +@pytest.mark.remote +class TestOpenAPIStudioVariables: + """Full add -> list -> update -> delete cycle against a real Studio. + + Requires MODELSCOPE_TEST_STUDIO (``owner/repo_name``) in tests/.env, because + the endpoints operate on an existing space. The key is randomised so parallel + runs cannot collide, and cleanup runs even when an assertion fails. + """ + + @pytest.fixture + def studio_id(self) -> str: + studio = os.environ.get("MODELSCOPE_TEST_STUDIO") + if not studio: + pytest.skip("Set MODELSCOPE_TEST_STUDIO=owner/repo_name to exercise Studio variables") + return studio + + @pytest.fixture + def variable_key(self, openapi, studio_id): + owner, repo_name = studio_id.split("/", 1) + key = f"MSHUB_TEST_{uuid.uuid4().hex[:8].upper()}" + yield key + with contextlib.suppress(Exception): + openapi.delete_studio_variable(owner, repo_name, key) + + def test_variable_lifecycle(self, openapi, studio_id, variable_key): + owner, repo_name = studio_id.split("/", 1) + + openapi.add_studio_variable(owner, repo_name, variable_key, "first") + listed = openapi.list_studio_variables(owner, repo_name) + entries = {v["key"]: v.get("value") for v in (listed or {}).get("variables") or []} + assert entries.get(variable_key) == "first" + + openapi.update_studio_variable(owner, repo_name, variable_key, "second") + listed = openapi.list_studio_variables(owner, repo_name) + entries = {v["key"]: v.get("value") for v in (listed or {}).get("variables") or []} + assert entries.get(variable_key) == "second" + + openapi.delete_studio_variable(owner, repo_name, variable_key) + listed = openapi.list_studio_variables(owner, repo_name) + remaining = {v["key"] for v in (listed or {}).get("variables") or []} + assert variable_key not in remaining + + def test_adding_a_duplicate_reports_already_exists(self, openapi, studio_id, variable_key): + """The endpoint answers 409, which must surface as AlreadyExistsError.""" + owner, repo_name = studio_id.split("/", 1) + openapi.add_studio_variable(owner, repo_name, variable_key, "first") + with pytest.raises(AlreadyExistsError): + openapi.add_studio_variable(owner, repo_name, variable_key, "again") + + +@pytest.mark.remote +class TestOpenAPIMcp: + """MCP discovery and the caller's own hosted servers.""" + + def test_list_mcp_servers(self, openapi): + result = openapi.list_mcp_servers(page_size=5) + assert isinstance(result, dict) + assert isinstance(result.get("mcp_server_list"), list) + + def test_list_mcp_servers_rejects_excessive_offset(self, openapi): + with pytest.raises(InvalidParameter): + openapi.list_mcp_servers(page_number=11, page_size=10) + + def test_list_operational_mcp_servers(self, openapi): + result = openapi.list_operational_mcp_servers() + assert isinstance(result, dict) + assert isinstance(result.get("mcp_server_list"), list) + + +@pytest.mark.remote +class TestOpenAPIReadOnlyToken: + """Behaviour under a read-scoped token. + + Set MODELSCOPE_TEST_READONLY_TOKEN in tests/.env to a token issued with read + permission only. This is the only way to verify the permission-tier handling + end to end: the spec models no scopes, so it can only be observed at runtime. + """ + + @pytest.fixture + def readonly_config(self, test_endpoint): + token = os.environ.get("MODELSCOPE_TEST_READONLY_TOKEN") + if not token: + pytest.skip("Set MODELSCOPE_TEST_READONLY_TOKEN to exercise read-scoped behaviour") + return HubConfig(token=token, endpoint=test_endpoint) + + def test_read_operations_succeed(self, readonly_config): + client = OpenAPIClient(readonly_config) + try: + result = client.list_studios(page_size=3) + assert isinstance(result, dict) + finally: + client.close() + + def test_login_succeeds_with_reduced_capability(self, readonly_config, tmp_path, monkeypatch): + monkeypatch.setenv("MODELSCOPE_HOME", str(tmp_path)) + api = HubApi(config=HubConfig(config_dir=tmp_path, endpoint=readonly_config.endpoint)) + user = api.login(readonly_config.token) + assert user.username + + def test_write_operation_names_the_missing_permission(self, readonly_config, test_owner): + client = OpenAPIClient(readonly_config) + try: + with pytest.raises((PermissionDeniedError, AuthenticationError)) as excinfo: + client.create_studio({"owner": test_owner, "repo_name": f"mshub-perm-{uuid.uuid4().hex[:8]}"}) + finally: + client.close() + if isinstance(excinfo.value, PermissionDeniedError): + assert "write" in excinfo.value.suggestion + @pytest.mark.remote class TestOpenAPIPagination: diff --git a/tests/test_compat_studio.py b/tests/test_compat_studio.py new file mode 100644 index 0000000..5a0d2a2 --- /dev/null +++ b/tests/test_compat_studio.py @@ -0,0 +1,300 @@ +"""Regression tests for the Studio methods on the legacy compat surface. + +The old ``modelscope.hub.api.HubApi`` signature let callers append ``token=`` / +``endpoint=`` to any method, and the umbrella SDK's Studio CLI does exactly +that. Forwarding those kwargs wholesale caused two concrete defects that these +tests pin down: + +* ``update_studio_settings`` forwards its kwargs as the settings payload, so the + caller's API token was serialised into the ``PATCH`` request body. +* ``get_studio_logs`` forwards into a keyword-only signature, so passing + ``token=`` raised ``TypeError`` and the call could never succeed. + +Network-free: the OpenAPI client is mocked, so assertions run against the exact +arguments that would have gone on the wire. +""" + +from __future__ import annotations + +from unittest import mock + +import pytest + +from modelscope_hub.api import HubApi +from modelscope_hub.compat import LegacyHubApi +from modelscope_hub.compat.hub_api import _split_control_kwargs + + +def _compat(token: str = "ms-ambient-token") -> tuple[LegacyHubApi, mock.MagicMock]: + """Build a compat wrapper whose OpenAPI transport is mocked out.""" + api = HubApi(token=token) + openapi = mock.MagicMock() + api._openapi = openapi + legacy = LegacyHubApi.__new__(LegacyHubApi) + legacy._api = api + legacy._endpoint = None + return legacy, openapi + + +# --------------------------------------------------------------------------- +# The helper itself +# --------------------------------------------------------------------------- +class TestSplitControlKwargs: + def test_control_args_are_separated(self): + control, passthrough = _split_control_kwargs({"token": "ms-x", "endpoint": "https://e", "display_name": "X"}) + assert control == {"token": "ms-x", "endpoint": "https://e"} + assert passthrough == {"display_name": "X"} + + def test_all_control_keys_recognised(self): + control, passthrough = _split_control_kwargs( + { + "token": "t", + "endpoint": "e", + "cookies": "c", + "headers": {}, + "timeout": 1, + "max_retries": 2, + } + ) + assert passthrough == {} + assert set(control) == {"token", "endpoint", "cookies", "headers", "timeout", "max_retries"} + + def test_business_only_kwargs_pass_through_untouched(self): + control, passthrough = _split_control_kwargs({"key": "K", "value": "V"}) + assert control == {} + assert passthrough == {"key": "K", "value": "V"} + + +# --------------------------------------------------------------------------- +# Defect 1: the token must never reach a request body +# --------------------------------------------------------------------------- +class TestSettingsDoesNotLeakCredentials: + def test_token_and_endpoint_absent_from_patch_body(self): + legacy, openapi = _compat() + legacy.update_studio_settings( + "owner/demo", + token="ms-ambient-token", + endpoint="https://modelscope.cn", + display_name="Demo", + ) + settings = openapi.update_studio_settings.call_args[0][2] + assert "token" not in settings + assert "endpoint" not in settings + assert settings == {"display_name": "Demo"} + + def test_every_control_key_is_stripped(self): + legacy, openapi = _compat() + legacy.update_studio_settings( + "owner/demo", + token="ms-ambient-token", + endpoint="https://modelscope.cn", + cookies="session=1", + headers={"X": "1"}, + timeout=5, + max_retries=1, + description="d", + ) + assert openapi.update_studio_settings.call_args[0][2] == {"description": "d"} + + def test_business_fields_still_reach_the_wire(self): + legacy, openapi = _compat() + legacy.update_studio_settings( + "owner/demo", + token="ms-ambient-token", + display_name="Demo", + description="d", + license="apache-2.0", + private=True, + ) + assert openapi.update_studio_settings.call_args[0][2] == { + "display_name": "Demo", + "description": "d", + "license": "apache-2.0", + "private": True, + } + + +# --------------------------------------------------------------------------- +# Defect 2: ``studio logs`` used to raise TypeError outright +# --------------------------------------------------------------------------- +class TestLogsAcceptsControlKwargs: + def test_token_kwarg_does_not_raise_type_error(self): + legacy, openapi = _compat() + legacy.get_studio_logs( + "owner/demo", + token="ms-ambient-token", + endpoint="https://modelscope.cn", + log_type="run", + page_size=5, + ) + args, kwargs = openapi.get_studio_logs.call_args + assert args == ("owner", "demo", "run") + assert kwargs["page_size"] == 5 + + def test_log_options_are_forwarded(self): + legacy, openapi = _compat() + legacy.get_studio_logs( + "owner/demo", + token="ms-ambient-token", + log_type="build", + page_num=3, + page_size=50, + keyword="ERROR", + start_timestamp=1, + end_timestamp=2, + ) + args, kwargs = openapi.get_studio_logs.call_args + assert args == ("owner", "demo", "build") + assert kwargs == { + "page_num": 3, + "page_size": 50, + "keyword": "ERROR", + "start_timestamp": 1, + "end_timestamp": 2, + } + + +# --------------------------------------------------------------------------- +# Defect 3: a per-call token used to be silently ignored +# --------------------------------------------------------------------------- +class TestPerCallTokenOverride: + @pytest.mark.parametrize( + ("method", "args"), + [ + ("deploy_studio", ("owner/demo",)), + ("stop_studio", ("owner/demo",)), + ("list_studio_secrets", ("owner/demo",)), + ("add_studio_secret", ("owner/demo", "K", "V")), + ("update_studio_secret", ("owner/demo", "K", "V")), + ("delete_studio_secret", ("owner/demo", "K")), + ("update_studio_settings", ("owner/demo",)), + ], + ) + def test_distinct_token_builds_a_dedicated_client(self, method, args): + legacy, _ = _compat(token="ms-ambient-token") + with mock.patch("modelscope_hub.compat.hub_api.HubApi") as hub_cls: + getattr(legacy, method)(*args, token="ms-per-call") + hub_cls.assert_called_once() + assert hub_cls.call_args.kwargs["token"] == "ms-per-call" + + def test_matching_token_reuses_the_ambient_client(self): + legacy, openapi = _compat(token="ms-ambient-token") + with mock.patch("modelscope_hub.compat.hub_api.HubApi") as hub_cls: + legacy.stop_studio("owner/demo", token="ms-ambient-token") + hub_cls.assert_not_called() + openapi.stop_studio.assert_called_once_with("owner", "demo") + + def test_no_token_reuses_the_ambient_client(self): + legacy, openapi = _compat() + with mock.patch("modelscope_hub.compat.hub_api.HubApi") as hub_cls: + legacy.deploy_studio("owner/demo") + hub_cls.assert_not_called() + openapi.deploy_studio.assert_called_once() + + def test_distinct_endpoint_builds_a_dedicated_client(self): + legacy, _ = _compat() + with mock.patch("modelscope_hub.compat.hub_api.HubApi") as hub_cls: + legacy.stop_studio("owner/demo", endpoint="https://modelscope.ai") + hub_cls.assert_called_once() + assert hub_cls.call_args.kwargs["endpoint"] == "https://modelscope.ai" + + +# --------------------------------------------------------------------------- +# Untouched delegation behaviour +# --------------------------------------------------------------------------- +class TestStudioDelegation: + def test_deploy_forwards_payload(self): + legacy, openapi = _compat() + legacy.deploy_studio("owner/demo", payload={"a": 1}, token="ms-ambient-token") + assert openapi.deploy_studio.call_args[0] == ("owner", "demo", {"a": 1}) + + def test_secret_writes_return_none(self): + legacy, _ = _compat() + assert legacy.add_studio_secret("owner/demo", "K", "V") is None + assert legacy.update_studio_secret("owner/demo", "K", "V") is None + assert legacy.delete_studio_secret("owner/demo", "K") is None + + +# --------------------------------------------------------------------------- +# Newly delegated Studio operations +# --------------------------------------------------------------------------- +class TestNewStudioDelegation: + def test_list_studios_returns_a_paginated_dict(self): + legacy, openapi = _compat() + openapi.list_studios.return_value = { + "studios": [{"id": "owner/demo"}], + "total_count": 1, + "page_number": 1, + "page_size": 10, + } + result = legacy.list_studios(token="ms-ambient-token") + assert result["total_count"] == 1 + assert result["studios"][0]["id"] == "owner/demo" + + def test_list_studios_forwards_filters(self): + legacy, openapi = _compat() + openapi.list_studios.return_value = {"studios": []} + legacy.list_studios(owner="alice", page_size=20, token="ms-ambient-token") + kwargs = openapi.list_studios.call_args.kwargs + assert kwargs["owner"] == "alice" + assert kwargs["page_size"] == 20 + + def test_list_variables_unwraps_the_payload(self): + legacy, openapi = _compat() + openapi.list_studio_variables.return_value = {"variables": [{"key": "K", "value": "V"}]} + assert legacy.list_studio_variables("owner/demo") == [{"key": "K", "value": "V"}] + assert openapi.list_studio_variables.call_args[0] == ("owner", "demo") + + @pytest.mark.parametrize( + ("method", "args", "target"), + [ + ("add_studio_variable", ("owner/demo", "K", "V"), "add_studio_variable"), + ("update_studio_variable", ("owner/demo", "K", "V"), "update_studio_variable"), + ("delete_studio_variable", ("owner/demo", "K"), "delete_studio_variable"), + ], + ) + def test_variable_writes_return_none_and_reach_the_endpoint(self, method, args, target): + legacy, openapi = _compat() + assert getattr(legacy, method)(*args) is None + assert getattr(openapi, target).call_args[0][:2] == ("owner", "demo") + + def test_hardware_unwraps_and_forwards_options(self): + legacy, openapi = _compat() + openapi.list_studio_hardware.return_value = {"hardware": [{"name": "cpu"}]} + assert legacy.list_studio_hardware(sdk_type="gradio") == [{"name": "cpu"}] + assert openapi.list_studio_hardware.call_args.kwargs["sdk_type"] == "gradio" + + def test_base_images_unwraps_the_payload(self): + legacy, openapi = _compat() + openapi.list_studio_base_images.return_value = {"base_images": [{"name": "ubuntu"}]} + assert legacy.list_studio_base_images() == [{"name": "ubuntu"}] + + def test_sdk_versions_unwraps_and_forwards_options(self): + legacy, openapi = _compat() + openapi.list_studio_sdk_versions.return_value = {"sdk_versions": [{"version": "4.44.1"}]} + assert legacy.list_studio_sdk_versions(sdk_type="gradio") == [{"version": "4.44.1"}] + assert openapi.list_studio_sdk_versions.call_args.kwargs["sdk_type"] == "gradio" + + @pytest.mark.parametrize( + ("method", "args"), + [ + ("list_studios", ()), + ("list_studio_variables", ("owner/demo",)), + ("add_studio_variable", ("owner/demo", "K", "V")), + ("update_studio_variable", ("owner/demo", "K", "V")), + ("delete_studio_variable", ("owner/demo", "K")), + ("list_studio_hardware", ()), + ("list_studio_base_images", ()), + ("list_studio_sdk_versions", ()), + ], + ) + def test_control_kwargs_never_become_business_arguments(self, method, args): + """Every new shim must strip token/endpoint like the existing ones do.""" + legacy, openapi = _compat() + openapi.list_studios.return_value = {"studios": []} + with mock.patch("modelscope_hub.compat.hub_api.HubApi") as hub_cls: + getattr(legacy, method)(*args, token="ms-per-call", endpoint="https://modelscope.ai") + hub_cls.assert_called_once() + for call in hub_cls.return_value.mock_calls: + assert "token" not in call.kwargs + assert "endpoint" not in call.kwargs diff --git a/tests/test_errors_openapi_codes.py b/tests/test_errors_openapi_codes.py new file mode 100644 index 0000000..fdfd14e --- /dev/null +++ b/tests/test_errors_openapi_codes.py @@ -0,0 +1,165 @@ +"""Classification tests for the OpenAPI error envelope. + +The ``/openapi/v1`` surface reports failures as ``{"success": false, "code": +"", "message": ..., "request_id": ...}``. The SDK only understood the +legacy *numeric* business code, so every one of these string codes fell through +to the bare HTTP status -- collapsing distinctions the status cannot express: +403 covers both "insufficient token permission" and "quota exhausted", and a 409 +duplicate had no mapping at all. +""" + +from __future__ import annotations + +import json + +import pytest +import requests + +from modelscope_hub._openapi import _RETRYABLE_EXC +from modelscope_hub.errors import ( + AlreadyExistsError, + APIError, + AuthenticationError, + InvalidParameter, + NotExistError, + PermissionDeniedError, + QuotaExceededError, + RateLimitError, + ServerError, + raise_for_status, +) + +_URL = "https://modelscope.cn/openapi/v1/studios" + + +def _openapi_error(status: int, code: str, message: str = "failed") -> requests.Response: + """Build the exact envelope the OpenAPI surface returns on failure.""" + resp = requests.Response() + resp.status_code = status + resp._content = json.dumps( + { + "success": False, + "code": code, + "message": message, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd2", + } + ).encode() + resp.headers["Content-Type"] = "application/json" + resp.url = _URL + return resp + + +def _plain_error(status: int) -> requests.Response: + resp = requests.Response() + resp.status_code = status + resp._content = b'{"success": false, "message": "no code"}' + resp.headers["Content-Type"] = "application/json" + resp.url = _URL + return resp + + +# --------------------------------------------------------------------------- +# String code -> exception +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + ("status", "code", "expected"), + [ + (400, "InputParameterError", InvalidParameter), + (401, "InvalidAuthentication", AuthenticationError), + (403, "OperationNotAllowed", PermissionDeniedError), + (404, "ResourceNotFound", NotExistError), + (409, "DuplicateEntity", AlreadyExistsError), + (403, "QuotaLimitExceed", QuotaExceededError), + (429, "RateLimitExceed", RateLimitError), + ], +) +def test_string_code_selects_the_exception(status, code, expected): + with pytest.raises(expected) as excinfo: + raise_for_status(_openapi_error(status, code)) + # AlreadyExistsError subclasses InvalidParameter, so assert the exact type. + assert type(excinfo.value) is expected + + +def test_quota_and_permission_share_403_but_not_the_exception(): + """403 is overloaded upstream; the code is the only thing that separates them.""" + with pytest.raises(PermissionDeniedError): + raise_for_status(_openapi_error(403, "OperationNotAllowed")) + with pytest.raises(QuotaExceededError): + raise_for_status(_openapi_error(403, "QuotaLimitExceed")) + + +def test_message_request_id_and_status_survive_classification(): + with pytest.raises(QuotaExceededError) as excinfo: + raise_for_status(_openapi_error(403, "QuotaLimitExceed", "magicube balance exhausted")) + exc = excinfo.value + assert exc.status_code == 403 + assert exc.request_id == "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" + assert "magicube balance exhausted" in exc.message + + +@pytest.mark.parametrize("code", ["ServiceUnavailable", "InternalServerError"]) +@pytest.mark.parametrize("status", [500, 503]) +def test_server_side_codes_stay_retryable(status, code): + """A 5xx must never be reclassified: doing so would drop ``retryable``.""" + with pytest.raises(ServerError) as excinfo: + raise_for_status(_openapi_error(status, code)) + assert type(excinfo.value) is ServerError + assert excinfo.value.retryable is True + + +def test_unknown_string_code_falls_back_to_the_status(): + with pytest.raises(NotExistError): + raise_for_status(_openapi_error(404, "SomeCodeWeHaveNeverSeen")) + + +# --------------------------------------------------------------------------- +# Status-code table gaps this work closed +# --------------------------------------------------------------------------- +def test_409_maps_to_already_exists_even_without_a_code(): + """createStudio / addStudioSecret / addStudioVariable all answer bare 409.""" + with pytest.raises(AlreadyExistsError) as excinfo: + raise_for_status(_plain_error(409)) + assert type(excinfo.value) is AlreadyExistsError + + +def test_413_maps_to_invalid_parameter(): + """POST /files/upload answers 413 when the body exceeds 5 MiB.""" + with pytest.raises(InvalidParameter): + raise_for_status(_plain_error(413)) + + +# --------------------------------------------------------------------------- +# Retry policy +# --------------------------------------------------------------------------- +def test_quota_exceeded_is_not_retryable(): + """Reusing RateLimitError here would make the transport retry a hard failure.""" + assert QuotaExceededError.retryable is False + assert not issubclass(QuotaExceededError, RateLimitError) + assert QuotaExceededError not in _RETRYABLE_EXC + assert not issubclass(QuotaExceededError, _RETRYABLE_EXC) + + +def test_rate_limit_exceeded_stays_retryable(): + assert RateLimitError.retryable is True + assert issubclass(RateLimitError, _RETRYABLE_EXC) + + +def test_quota_error_is_an_api_error(): + assert issubclass(QuotaExceededError, APIError) + assert QuotaExceededError.error_code == "E3027" + + +def test_quota_error_is_exported_from_the_package_root(): + import modelscope_hub + + assert modelscope_hub.QuotaExceededError is QuotaExceededError + assert "QuotaExceededError" in modelscope_hub.__all__ + + +def test_retry_after_still_honoured_for_a_string_rate_limit_code(): + """RateLimitError carries retry_after regardless of how it was selected.""" + resp = _openapi_error(429, "RateLimitExceed") + resp.headers["Retry-After"] = "7" + with pytest.raises(RateLimitError) as excinfo: + raise_for_status(resp) + assert excinfo.value.retry_after == 7 diff --git a/tests/test_login_scoped_token.py b/tests/test_login_scoped_token.py new file mode 100644 index 0000000..e4db1f5 --- /dev/null +++ b/tests/test_login_scoped_token.py @@ -0,0 +1,203 @@ +"""Login behaviour for permission-tiered API tokens. + +Tokens are issued as read / write / admin. The legacy ``POST /api/v1/login`` +endpoint exists to mint *git* credentials, which a read-only token is not +entitled to -- yet such a token authenticates fine and is all a caller needs to +browse and download. Reporting that refusal as "invalid token" sends the user +after the wrong remedy (re-issuing a token that was never the problem). + +The transport is stubbed with ``responses`` so the real config, HubApi, +LegacyClient and error-translation layers all take part. +""" + +from __future__ import annotations + +from unittest import mock + +import pytest +import requests +import responses + +from modelscope_hub.api import HubApi +from modelscope_hub.config import HubConfig +from modelscope_hub.constants import DEFAULT_ENDPOINT, DEFAULT_INTL_ENDPOINT, USER_INFO_FILE_NAME +from modelscope_hub.errors import AuthenticationError, NetworkError + +CN_LOGIN = f"{DEFAULT_ENDPOINT}/api/v1/login" +AI_LOGIN = f"{DEFAULT_INTL_ENDPOINT}/api/v1/login" +CN_USERS_ME = f"{DEFAULT_ENDPOINT}/openapi/v1/users/me" + +READONLY_TOKEN = "ms-readonly-token" + +# What the legacy login endpoint answers for a token it will not mint git +# credentials for. +LOGIN_REFUSED_BODY = { + "Code": 10010103009, + "Message": "登录失败，AccessToken错误，请从用户中心获取AccessToken或刷新", + "RequestId": "8a039827-3f7c-4378-9b7c-3f8341b73649", + "Success": False, +} + +USERS_ME_BODY = { + "success": True, + "data": {"username": "alice", "email": "alice@example.com", "nickname": "Alice"}, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd2", +} + + +@pytest.fixture(autouse=True) +def isolated_home(tmp_path, monkeypatch): + """Redirect credential storage and drop ambient endpoint/token overrides.""" + for name in ("MODELSCOPE_ENDPOINT", "MODELSCOPE_API_TOKEN", "MODELSCOPE_DOMAIN"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("MODELSCOPE_HOME", str(tmp_path)) + return tmp_path + + +def _api(home) -> HubApi: + return HubApi(config=HubConfig(config_dir=home), endpoint=DEFAULT_ENDPOINT) + + +# --------------------------------------------------------------------------- +# The token is valid, only its tier is limited +# --------------------------------------------------------------------------- +@responses.activate +def test_scoped_token_logs_in_successfully(isolated_home): + responses.add(responses.POST, CN_LOGIN, json=LOGIN_REFUSED_BODY, status=400) + responses.add(responses.GET, CN_USERS_ME, json=USERS_ME_BODY, status=200) + + user = _api(isolated_home).login(READONLY_TOKEN) + + assert user.username == "alice" + assert user.email == "alice@example.com" + + +@responses.activate +def test_scoped_login_persists_the_token(isolated_home): + responses.add(responses.POST, CN_LOGIN, json=LOGIN_REFUSED_BODY, status=400) + responses.add(responses.GET, CN_USERS_ME, json=USERS_ME_BODY, status=200) + + api = _api(isolated_home) + api.login(READONLY_TOKEN) + + assert api._config.token == READONLY_TOKEN + assert HubConfig(config_dir=isolated_home).load_token() == READONLY_TOKEN + + +@responses.activate +def test_scoped_login_records_the_user_but_no_git_token(isolated_home): + """Cookies and the git token are unavailable at this tier, so they stay unset.""" + responses.add(responses.POST, CN_LOGIN, json=LOGIN_REFUSED_BODY, status=400) + responses.add(responses.GET, CN_USERS_ME, json=USERS_ME_BODY, status=200) + + _api(isolated_home).login(READONLY_TOKEN) + + config = HubConfig(config_dir=isolated_home) + user_file = config.credentials_dir / USER_INFO_FILE_NAME + assert user_file.read_text(encoding="utf-8") == "alice:alice@example.com" + assert not config.load_git_token() + + +@responses.activate +def test_scoped_login_warns_about_write_operations(isolated_home): + responses.add(responses.POST, CN_LOGIN, json=LOGIN_REFUSED_BODY, status=400) + responses.add(responses.GET, CN_USERS_ME, json=USERS_ME_BODY, status=200) + + # The SDK logger deliberately does not propagate, so caplog cannot see it. + with mock.patch("modelscope_hub.api.logger.warning") as warn: + _api(isolated_home).login(READONLY_TOKEN) + + warn.assert_called_once() + rendered = warn.call_args[0][0] % warn.call_args[0][1:] + assert "reduced permissions" in rendered + assert "write" in rendered + assert "alice" in rendered + + +# --------------------------------------------------------------------------- +# A genuinely bad token must still fail, exactly as before +# --------------------------------------------------------------------------- +@responses.activate +def test_invalid_token_still_raises(isolated_home): + responses.add(responses.POST, CN_LOGIN, json=LOGIN_REFUSED_BODY, status=400) + responses.add( + responses.GET, + CN_USERS_ME, + json={"success": False, "code": "InvalidAuthentication", "message": "bad token"}, + status=401, + ) + + with pytest.raises(AuthenticationError) as excinfo: + _api(isolated_home).login("ms-bad-token") + + assert LOGIN_REFUSED_BODY["Message"] in excinfo.value.message + assert excinfo.value.request_id == LOGIN_REFUSED_BODY["RequestId"] + + +@responses.activate +def test_invalid_token_leaves_the_stored_credential_alone(isolated_home): + config = HubConfig(config_dir=isolated_home) + config.save_token("ms-previously-working") + + responses.add(responses.POST, CN_LOGIN, json=LOGIN_REFUSED_BODY, status=400) + responses.add(responses.GET, CN_USERS_ME, json={"success": False}, status=401) + + api = _api(isolated_home) + with pytest.raises(AuthenticationError): + api.login("ms-bad-token") + + assert HubConfig(config_dir=isolated_home).load_token() == "ms-previously-working" + assert api._config.token == "ms-previously-working" + + +@responses.activate +def test_probe_returning_no_username_is_not_treated_as_success(isolated_home): + """A 200 with an empty profile cannot confirm the token; the refusal stands.""" + responses.add(responses.POST, CN_LOGIN, json=LOGIN_REFUSED_BODY, status=400) + responses.add(responses.GET, CN_USERS_ME, json={"success": True, "data": {}}, status=200) + + with pytest.raises(AuthenticationError): + _api(isolated_home).login("ms-odd-token") + + +@responses.activate +def test_probe_transport_failure_does_not_mask_the_refusal(isolated_home): + """The probe is advisory: if it cannot run, the original error is surfaced.""" + responses.add(responses.POST, CN_LOGIN, json=LOGIN_REFUSED_BODY, status=400) + responses.add(responses.GET, CN_USERS_ME, body=requests.ConnectionError("reset")) + + with pytest.raises(AuthenticationError): + _api(isolated_home).login("ms-bad-token") + + +# --------------------------------------------------------------------------- +# Non-authentication failures are untouched by this path +# --------------------------------------------------------------------------- +@responses.activate +def test_network_failure_never_reaches_the_probe(isolated_home): + """A transport error is not a credential verdict, so no probe is attempted.""" + responses.add(responses.POST, CN_LOGIN, body=requests.ConnectionError("connection reset")) + + with pytest.raises(NetworkError): + _api(isolated_home).login("ms-any-token") + + assert all(CN_USERS_ME not in call.request.url for call in responses.calls) + + +@responses.activate +def test_successful_login_never_reaches_the_probe(isolated_home): + responses.add( + responses.POST, + CN_LOGIN, + json={ + "Code": 200, + "Data": {"AccessToken": "git-token", "Email": "a@b.c", "Username": "alice"}, + "Success": True, + }, + status=200, + ) + responses.add(responses.GET, CN_USERS_ME, json=USERS_ME_BODY, status=200) + + api = _api(isolated_home) + assert api.login("ms-write-token").username == "alice" + assert HubConfig(config_dir=isolated_home).load_git_token() == "git-token" diff --git a/tests/test_openapi_coverage.py b/tests/test_openapi_coverage.py new file mode 100644 index 0000000..ce2cead --- /dev/null +++ b/tests/test_openapi_coverage.py @@ -0,0 +1,128 @@ +"""Guard against the OpenAPI specification drifting away from the client. + +Every gap this work closed had the same cause: the service published new +operations and nothing in the suite noticed. `tests/data/openapi.json` is a +vendored copy of the live document, and these tests assert that the tags the SDK +claims to cover are covered *completely*. + +When the spec is refreshed and a covered tag gained an operation, the first test +fails and names it. Tags still to be implemented are listed in `_DEFERRED_TAGS`, +which doubles as the remaining to-do list. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from modelscope_hub._openapi import OPERATION_REGISTRY, OpenAPIClient +from modelscope_hub.constants import TokenScope + +_SPEC_PATH = Path(__file__).parent / "data" / "openapi.json" +_HTTP_METHODS = frozenset({"get", "post", "put", "delete", "patch", "head", "options"}) + +# Tags the SDK implements end to end. +_COVERED_TAGS = frozenset({"MCP", "Studios"}) + +# Tags not implemented yet. Remove an entry as its tag lands; until then the +# guard would otherwise fail on operations nobody has promised. +_DEFERRED_TAGS = frozenset( + { + "User", + "Models", + "Datasets", + "Skills", + "Files", + "Collections", + "Agent-IDP", + "Galleries", + "Magicube", + } +) + + +@pytest.fixture(scope="module") +def spec() -> dict: + return json.loads(_SPEC_PATH.read_text(encoding="utf-8")) + + +def _operations(spec: dict) -> list[tuple[str, str, str, str]]: + """Flatten the spec into ``(tag, operation_id, method, path)`` rows.""" + rows: list[tuple[str, str, str, str]] = [] + for path, item in spec["paths"].items(): + for method, operation in item.items(): + if method not in _HTTP_METHODS: + continue + tags = operation.get("tags") or [""] + rows.append((tags[0], operation["operationId"], method.upper(), path)) + return rows + + +# --------------------------------------------------------------------------- +# Coverage +# --------------------------------------------------------------------------- +def test_every_covered_operation_is_registered(spec): + expected = {op_id for tag, op_id, _, _ in _operations(spec) if tag in _COVERED_TAGS} + missing = sorted(expected - set(OPERATION_REGISTRY)) + assert not missing, ( + f"{len(missing)} operation(s) in the covered tags {sorted(_COVERED_TAGS)} are not in " + f"OPERATION_REGISTRY: {missing}. Implement them on OpenAPIClient and register them, " + f"or move the tag to _DEFERRED_TAGS." + ) + + +def test_registry_has_no_unknown_operations(spec): + known = {op_id for _, op_id, _, _ in _operations(spec)} + unknown = sorted(set(OPERATION_REGISTRY) - known) + assert not unknown, f"OPERATION_REGISTRY references operationIds absent from the spec: {unknown}" + + +def test_registry_does_not_claim_deferred_tags(spec): + deferred = {op_id for tag, op_id, _, _ in _operations(spec) if tag in _DEFERRED_TAGS} + overlap = sorted(deferred & set(OPERATION_REGISTRY)) + assert not overlap, ( + f"These operations are registered but their tag is still listed as deferred: {overlap}. " + f"Remove the tag from _DEFERRED_TAGS once it is fully covered." + ) + + +def test_every_tag_is_accounted_for(spec): + """A brand-new tag must be an explicit decision, not an oversight.""" + tags = {tag for tag, _, _, _ in _operations(spec)} + unclassified = sorted(tags - _COVERED_TAGS - _DEFERRED_TAGS) + assert not unclassified, f"Unclassified tag(s) {unclassified}: add each to _COVERED_TAGS or _DEFERRED_TAGS." + + +# --------------------------------------------------------------------------- +# Registry integrity +# --------------------------------------------------------------------------- +@pytest.mark.parametrize(("operation_id", "entry"), sorted(OPERATION_REGISTRY.items())) +def test_registered_method_exists_and_is_callable(operation_id, entry): + method_name, scope = entry + method = getattr(OpenAPIClient, method_name, None) + assert method is not None, f"{operation_id} maps to OpenAPIClient.{method_name}, which does not exist" + assert callable(method) + assert isinstance(scope, TokenScope) + + +def test_registry_maps_each_operation_to_a_distinct_method(): + methods = [method for method, _ in OPERATION_REGISTRY.values()] + duplicates = sorted({m for m in methods if methods.count(m) > 1}) + assert not duplicates, f"One method serves several operations: {duplicates}" + + +# --------------------------------------------------------------------------- +# The vendored spec itself +# --------------------------------------------------------------------------- +def test_spec_is_the_expected_document(spec): + assert spec["openapi"].startswith("3.1") + assert spec["info"]["title"] == "ModelScope OpenAPI" + assert spec["servers"][0]["url"].endswith("/openapi/v1") + + +def test_covered_tags_account_for_every_registered_entry(spec): + """Registry size must equal the covered operation count -- no silent drift.""" + covered = [op_id for tag, op_id, _, _ in _operations(spec) if tag in _COVERED_TAGS] + assert len(OPERATION_REGISTRY) == len(covered) From 49af7449e46ce07dfe50f044d409d1795dab7345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Tue, 1 Sep 2026 20:33:16 +0800 Subject: [PATCH 02/22] [Docs] Add the missing 0.3.x release notes and condense the News section The News list had no entry for 0.3.0 or 0.3.1, so it jumped straight from 0.4.0 to 0.2.0 and left the release that took over all four console scripts undocumented. - Adds v0.3.0 (unified console scripts, repo/files truncation mitigation, get_current_username, agent visibility) and v0.3.1 (upload env var rename with deprecation shims, logout wiring, download lock paths, UserInfo and legacy compat field mapping), both reconstructed from the commits in each range. - Condenses v0.4.0 from eleven bullets to nine by merging entries that share a root cause: the three compat kwargs defects are one leak, and the two MCP/ Studio request-shape improvements are one behaviour change. - Folds v0.2.0 and everything older into the "Older releases" block, so only the three current releases are visible by default. --- README.md | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 103c228..1c1fa39 100644 --- a/README.md +++ b/README.md @@ -34,17 +34,28 @@ The official Python SDK & CLI for [ModelScope Hub](https://modelscope.cn) — do ## News **v0.4.0** (2026-09-01) -- **Feature**: full OpenAPI coverage for MCP and Studios — `ms-hub studio list`, `ms-hub list --repo-type studio`, `ms-hub studio variable` (plaintext environment variables), and `ms-hub studio hardware` / `base-images` / `sdk-versions` so valid `--hardware` / `--base-image` / `--sdk-version` values no longer have to be guessed; `ms-hub mcp list --hosted` and `HubApi.list_operational_mcp_servers()` -- **Feature**: Studio `--visibility protected` (app public, code repository hidden), previously inexpressible; `RepoInfo` now carries the Studio runtime fields (`sdk_type`, `sdk_version`, `base_image`, `hardware`, `mcp_support`, `runtime`) instead of discarding them -- **Feature**: permission-tiered token support — a read-only token can now `login` (with a warning that write operations need a higher tier), and a rejected write reports which tier it requires -- **Fix**: the compat layer no longer forwards `token` / `endpoint` into request bodies — `HubApi().update_studio_settings(..., token=...)` was serialising your **API token into the `PATCH` body** -- **Fix**: `HubApi().get_studio_logs(..., token=...)` raised `TypeError` and could never succeed; a per-call `token=` on the Studio compat methods was silently ignored -- **Fix**: `cover_image` was renamed to `coverImage` on the way out and silently dropped by the server, so Studio cover images never applied (`ms-hub studio settings --cover-image`, `create_repo(cover_image=...)`) -- **Fix**: HTTP 403 now distinguishes insufficient permission (`PermissionDeniedError`) from an exhausted quota (new `QuotaExceededError`, never retried); 409 maps to `AlreadyExistsError`; the OpenAPI string error codes are recognised at last -- **Fix**: `ms-hub info --repo-type studio` works anonymously for public spaces; `ms-hub studio logs` prints its pagination footer again (it read a field the response never had) -- **Enhance**: MCP discovery sends `PUT` first — the only verb the spec defines — instead of spending a wasted round trip on a `GET` 404 every call; whichever verb a deployment serves is remembered, so the loser is attempted at most once -- **Enhance**: `ms-hub studio list --owner` / `ms-hub list --repo-type studio --owner` now ask for every status, because the endpoint keeps filtering to running spaces even with an owner set and "list my spaces" therefore answered with nothing -- **Quality**: the OpenAPI specification is vendored under `tests/data/` and checked against an operation registry, so a newly published MCP/Studios endpoint now fails the suite by name +- **Feature**: complete OpenAPI coverage for MCP and Studios — `ms-hub studio list` / `ms-hub list --repo-type studio`, `ms-hub studio variable` (plaintext env vars), `ms-hub studio hardware` / `base-images` / `sdk-versions` so valid `--hardware` / `--base-image` / `--sdk-version` values no longer have to be guessed, and `ms-hub mcp list --hosted` +- **Feature**: Studio `--visibility protected` (app public, code hidden), previously inexpressible; `RepoInfo` now carries the Studio runtime fields (`sdk_type`, `hardware`, `runtime`, …) instead of discarding them +- **Feature**: permission-tiered tokens — a read-only token can `login` (warning that writes need a higher tier), and a rejected write names the tier it requires +- **Fix**: the compat layer forwarded `token` / `endpoint` as business fields, so `update_studio_settings(..., token=...)` serialised your **API token into the `PATCH` body**, `get_studio_logs(..., token=...)` raised `TypeError`, and a per-call `token=` was silently ignored +- **Fix**: `cover_image` was renamed to `coverImage` on the way out and dropped by the server, so Studio cover images never applied +- **Fix**: HTTP 403 now separates insufficient permission (`PermissionDeniedError`) from an exhausted quota (new `QuotaExceededError`, never retried); 409 maps to `AlreadyExistsError`; the OpenAPI string error codes are recognised at last +- **Fix**: `ms-hub info --repo-type studio` works anonymously for public spaces; `ms-hub studio logs` prints its pagination footer again +- **Enhance**: MCP discovery sends `PUT` first (the only verb the spec defines) and remembers which verb a deployment serves; listing a Studio owner asks for every status, since the endpoint otherwise answers "list my spaces" with nothing +- **Quality**: the OpenAPI spec is vendored under `tests/data/` and checked against an operation registry, so a newly published MCP/Studios endpoint fails the suite by name + +**v0.3.1** (2026-09-01) +- **Refactor**: upload environment variables renamed to carry their units and semantics (`UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS`, `UPLOAD_BLOB_MAX_ATTEMPTS`, `UPLOAD_BLOB_PROGRESS_THRESHOLD_BYTES`, …); the previous names keep working and emit a deprecation warning +- **Fix**: `ms-hub logout` is wired up (the API supported it but the CLI never exposed it); download lock filenames hash the remote path, so files sharing a basename no longer contend for one lock +- **Fix**: `whoami` and `UserInfo` map the pre-production field names to their canonical ones; `get_model_files` and other legacy compat paths align with the current server responses + +**v0.3.0** (2026-08-19) +- **Feature**: this package now installs all four console scripts — `modelscope`, `ms`, `modelscope-hub`, `ms-hub` — so exactly one distribution owns them and neither package can strand the other's CLI on upgrade; the umbrella SDK contributes its commands as plugins instead +- **Fix**: the server truncates `repo/files` at a fixed entry count with no marker and ignores pagination, silently hiding files in large repos; listings that land on the cap are now re-enumerated per directory +- **Feature**: `HubApi.get_current_username()`; agent visibility reads the API's new boolean `private` field while still accepting the legacy `visibility` string + +
+Older releases **v0.2.0** (2026-08-01) - **Breaking**: `HubApi.login()` now raises `NetworkError` / `ServerError` / `RequestTimeoutError` for non-authentication failures instead of always raising `AuthenticationError` — widen `except AuthenticationError` to `HubError` if you catch *any* login failure @@ -62,9 +73,6 @@ The official Python SDK & CLI for [ModelScope Hub](https://modelscope.cn) — do - **Fix**: forward `progress_callbacks` through `HubApi.download_repo` so custom download-progress callbacks work end-to-end; harden legacy (pre-1.38) cache auto-detection (reuse existing `{cache}/models/...` and default `{cache}/hub/models/...` layouts); normal (non-LFS) file upload - **Packaging**: rename console scripts to `modelscope-hub` / `ms-hub` to avoid a file conflict with the `modelscope` package (e.g. FreeBSD pkg) -
-Older releases - **v0.1.7** (2026-07-07) - **Feature**: intra-/inter-region cloud download acceleration, with a source marker in the progress bar - **Fix**: align `snapshot_download` cache path with the CLI; add legacy cache fallback From db79f8b97f05ececcc1d1cae2ff40b3cb625a106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Tue, 1 Sep 2026 21:05:01 +0800 Subject: [PATCH 03/22] [Docs] Merge like-category entries and tighten the News bullets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - v0.4.0: fold the second Feature bullet into the OpenAPI-coverage one, join the cover_image leak into the compat-kwarg fix, and merge the two small studio fixes into the error-handling bullet — nine bullets become six, one per category except Feature. - v0.3.0/v0.3.1: trim wordy phrasing and group Feature items first. --- README.md | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 1c1fa39..69351fe 100644 --- a/README.md +++ b/README.md @@ -34,25 +34,22 @@ The official Python SDK & CLI for [ModelScope Hub](https://modelscope.cn) — do ## News **v0.4.0** (2026-09-01) -- **Feature**: complete OpenAPI coverage for MCP and Studios — `ms-hub studio list` / `ms-hub list --repo-type studio`, `ms-hub studio variable` (plaintext env vars), `ms-hub studio hardware` / `base-images` / `sdk-versions` so valid `--hardware` / `--base-image` / `--sdk-version` values no longer have to be guessed, and `ms-hub mcp list --hosted` -- **Feature**: Studio `--visibility protected` (app public, code hidden), previously inexpressible; `RepoInfo` now carries the Studio runtime fields (`sdk_type`, `hardware`, `runtime`, …) instead of discarding them +- **Feature**: complete OpenAPI coverage for MCP and Studios — `studio list` / `list --repo-type studio`, `studio variable` (plaintext env vars), `studio hardware` / `base-images` / `sdk-versions` so valid option values no longer have to be guessed, `mcp list --hosted`; `--visibility protected` (app public, code hidden); `RepoInfo` carries the Studio runtime fields instead of discarding them - **Feature**: permission-tiered tokens — a read-only token can `login` (warning that writes need a higher tier), and a rejected write names the tier it requires -- **Fix**: the compat layer forwarded `token` / `endpoint` as business fields, so `update_studio_settings(..., token=...)` serialised your **API token into the `PATCH` body**, `get_studio_logs(..., token=...)` raised `TypeError`, and a per-call `token=` was silently ignored -- **Fix**: `cover_image` was renamed to `coverImage` on the way out and dropped by the server, so Studio cover images never applied -- **Fix**: HTTP 403 now separates insufficient permission (`PermissionDeniedError`) from an exhausted quota (new `QuotaExceededError`, never retried); 409 maps to `AlreadyExistsError`; the OpenAPI string error codes are recognised at last -- **Fix**: `ms-hub info --repo-type studio` works anonymously for public spaces; `ms-hub studio logs` prints its pagination footer again -- **Enhance**: MCP discovery sends `PUT` first (the only verb the spec defines) and remembers which verb a deployment serves; listing a Studio owner asks for every status, since the endpoint otherwise answers "list my spaces" with nothing +- **Fix**: the compat layer forwarded `token` / `endpoint` as business fields (your **API token was serialised into the `PATCH` body**, and per-call `token=` raised or was silently ignored); `cover_image` reached the server as `coverImage` and was dropped, so cover images never applied +- **Fix**: HTTP 403 now separates insufficient permission (`PermissionDeniedError`) from an exhausted quota (`QuotaExceededError`, never retried); 409 maps to `AlreadyExistsError`; the OpenAPI string error codes are recognised; `info --repo-type studio` works anonymously; `studio logs` prints its pagination footer again +- **Enhance**: MCP discovery sends `PUT` first and remembers which verb a deployment serves; listing a Studio owner asks for every status instead of answering with nothing - **Quality**: the OpenAPI spec is vendored under `tests/data/` and checked against an operation registry, so a newly published MCP/Studios endpoint fails the suite by name **v0.3.1** (2026-09-01) -- **Refactor**: upload environment variables renamed to carry their units and semantics (`UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS`, `UPLOAD_BLOB_MAX_ATTEMPTS`, `UPLOAD_BLOB_PROGRESS_THRESHOLD_BYTES`, …); the previous names keep working and emit a deprecation warning +- **Refactor**: upload environment variables renamed to carry their units and semantics (`UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS`, `UPLOAD_BLOB_MAX_ATTEMPTS`, …); the previous names keep working with a deprecation warning - **Fix**: `ms-hub logout` is wired up (the API supported it but the CLI never exposed it); download lock filenames hash the remote path, so files sharing a basename no longer contend for one lock -- **Fix**: `whoami` and `UserInfo` map the pre-production field names to their canonical ones; `get_model_files` and other legacy compat paths align with the current server responses +- **Fix**: `whoami` and legacy compat paths (`get_model_files`, …) map the pre-production field names to their canonical ones and align with the current server responses **v0.3.0** (2026-08-19) -- **Feature**: this package now installs all four console scripts — `modelscope`, `ms`, `modelscope-hub`, `ms-hub` — so exactly one distribution owns them and neither package can strand the other's CLI on upgrade; the umbrella SDK contributes its commands as plugins instead -- **Fix**: the server truncates `repo/files` at a fixed entry count with no marker and ignores pagination, silently hiding files in large repos; listings that land on the cap are now re-enumerated per directory +- **Feature**: this package now installs all four console scripts — `modelscope`, `ms`, `modelscope-hub`, `ms-hub` — so one distribution owns them and neither package can strand the other's CLI on upgrade; the umbrella SDK contributes its commands as plugins - **Feature**: `HubApi.get_current_username()`; agent visibility reads the API's new boolean `private` field while still accepting the legacy `visibility` string +- **Fix**: the server truncates `repo/files` at a fixed entry count with no marker and ignores pagination, silently hiding files in large repos; listings that land on the cap are now re-enumerated per directory
Older releases From 82b458cbc936dae0a8af857a3939c8a9abb0ab9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Tue, 1 Sep 2026 21:37:13 +0800 Subject: [PATCH 04/22] [Docs] Consolidate News entries by category --- README.md | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 69351fe..d91c85d 100644 --- a/README.md +++ b/README.md @@ -34,22 +34,18 @@ The official Python SDK & CLI for [ModelScope Hub](https://modelscope.cn) — do ## News **v0.4.0** (2026-09-01) -- **Feature**: complete OpenAPI coverage for MCP and Studios — `studio list` / `list --repo-type studio`, `studio variable` (plaintext env vars), `studio hardware` / `base-images` / `sdk-versions` so valid option values no longer have to be guessed, `mcp list --hosted`; `--visibility protected` (app public, code hidden); `RepoInfo` carries the Studio runtime fields instead of discarding them -- **Feature**: permission-tiered tokens — a read-only token can `login` (warning that writes need a higher tier), and a rejected write names the tier it requires -- **Fix**: the compat layer forwarded `token` / `endpoint` as business fields (your **API token was serialised into the `PATCH` body**, and per-call `token=` raised or was silently ignored); `cover_image` reached the server as `coverImage` and was dropped, so cover images never applied -- **Fix**: HTTP 403 now separates insufficient permission (`PermissionDeniedError`) from an exhausted quota (`QuotaExceededError`, never retried); 409 maps to `AlreadyExistsError`; the OpenAPI string error codes are recognised; `info --repo-type studio` works anonymously; `studio logs` prints its pagination footer again -- **Enhance**: MCP discovery sends `PUT` first and remembers which verb a deployment serves; listing a Studio owner asks for every status instead of answering with nothing -- **Quality**: the OpenAPI spec is vendored under `tests/data/` and checked against an operation registry, so a newly published MCP/Studios endpoint fails the suite by name +- **Feature**: full MCP/Studios OpenAPI coverage: Studio lists, variables and configuration options; hosted MCP discovery; protected visibility and runtime metadata; read-only tokens can log in and rejected writes name the required tier +- **Fix**: Studio compat calls no longer leak connection options or API tokens, or drop cover images; errors distinguish permission, quota and conflicts; anonymous Studio info and log pagination work +- **Enhance**: MCP verb negotiation and Studio-owner listings adapt to endpoint behaviour +- **Quality**: the vendored OpenAPI spec and operation registry flag unimplemented published MCP/Studios endpoints **v0.3.1** (2026-09-01) -- **Refactor**: upload environment variables renamed to carry their units and semantics (`UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS`, `UPLOAD_BLOB_MAX_ATTEMPTS`, …); the previous names keep working with a deprecation warning -- **Fix**: `ms-hub logout` is wired up (the API supported it but the CLI never exposed it); download lock filenames hash the remote path, so files sharing a basename no longer contend for one lock -- **Fix**: `whoami` and legacy compat paths (`get_model_files`, …) map the pre-production field names to their canonical ones and align with the current server responses +- **Refactor**: upload environment variables now state their units and semantics; deprecated names remain supported with warnings +- **Fix**: logout works; download locks avoid same-basename contention; `whoami` and legacy responses normalise pre-production field names **v0.3.0** (2026-08-19) -- **Feature**: this package now installs all four console scripts — `modelscope`, `ms`, `modelscope-hub`, `ms-hub` — so one distribution owns them and neither package can strand the other's CLI on upgrade; the umbrella SDK contributes its commands as plugins -- **Feature**: `HubApi.get_current_username()`; agent visibility reads the API's new boolean `private` field while still accepting the legacy `visibility` string -- **Fix**: the server truncates `repo/files` at a fixed entry count with no marker and ignores pagination, silently hiding files in large repos; listings that land on the cap are now re-enumerated per directory +- **Feature**: this package owns all four console scripts (`modelscope`, `ms`, `modelscope-hub`, `ms-hub`), adds `HubApi.get_current_username()`, and supports both agent visibility wire formats +- **Fix**: large `repo/files` listings are re-enumerated when the server silently truncates results
Older releases From 60ece18a6102d02d9b34c3905fde7a081a0ef67f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Wed, 2 Sep 2026 02:19:30 +0800 Subject: [PATCH 05/22] [Feature] Add OpenAPI-first Agent-IDP support --- README.md | 36 ++++ pyproject.toml | 1 + src/modelscope_hub/__init__.py | 25 +++ src/modelscope_hub/_openapi.py | 218 +++++++++++++++++++++++ src/modelscope_hub/agent_idp.py | 206 ++++++++++++++++++++++ src/modelscope_hub/api.py | 120 ++++++++++++- src/modelscope_hub/cli/agent.py | 7 +- src/modelscope_hub/cli/agent_idp.py | 261 ++++++++++++++++++++++++++++ src/modelscope_hub/cli/main.py | 2 + src/modelscope_hub/errors.py | 5 + src/modelscope_hub/types.py | 178 +++++++++++++++++++ src/modelscope_hub/version.py | 2 +- tests/cli/test_agent_idp.py | 81 +++++++++ tests/test_agent_idp.py | 147 ++++++++++++++++ tests/test_agent_idp_keys.py | 75 ++++++++ tests/test_openapi_coverage.py | 3 +- 16 files changed, 1360 insertions(+), 7 deletions(-) create mode 100644 src/modelscope_hub/agent_idp.py create mode 100644 src/modelscope_hub/cli/agent_idp.py create mode 100644 tests/cli/test_agent_idp.py create mode 100644 tests/test_agent_idp.py create mode 100644 tests/test_agent_idp_keys.py diff --git a/README.md b/README.md index d91c85d..719e6d0 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ The official Python SDK & CLI for [ModelScope Hub](https://modelscope.cn) — do ## News +**v0.5.0** (2026-09-02) +- **Feature**: complete Agent-IDP OpenAPI support for Ed25519 identity management, OIDC discovery/JWKS, and signed JWT issuance through `HubApi` and `ms-hub agent-idp` +- **Security**: private JWKs are never stored automatically; `keygen` writes only an explicitly requested owner-only file, while token issuance prints only the credential + **v0.4.0** (2026-09-01) - **Feature**: full MCP/Studios OpenAPI coverage: Studio lists, variables and configuration options; hosted MCP discovery; protected visibility and runtime metadata; read-only tokens can log in and rejected writes name the required tier - **Fix**: Studio compat calls no longer leak connection options or API tokens, or drop cover images; errors distinguish permission, quota and conflicts; anonymous Studio info and log pagination work @@ -685,6 +689,32 @@ ms-hub agent upload -r user/my-agent --local-dir ./my-agent --dry-run
+### `ms-hub agent-idp` + +Agent-IDP manages an Agent's **identity and signing key**, not its repository files. Use `ms-hub agent` for raw Agent repository transfer; use `ms-hub agent-idp` to register an Ed25519 public key, inspect OIDC metadata, and issue a short-lived JWT. + +```bash +# Private JWK storage is always explicit; the command prints only its public JWK. +ms-hub agent-idp keygen --private-key-out ./agent.jwk +ms-hub agent-idp create --agent-name my-agent --private-key-file ./agent.jwk +ms-hub agent-idp issue-token --agent-id agent_id:modelscope:agent_xxx --audience my-hub --private-key-file ./agent.jwk + +# These discovery endpoints are public and do not require login. +ms-hub agent-idp configuration +ms-hub agent-idp jwks +``` + +Protect the private JWK file: it is created with mode `0600` on POSIX, never copied into the SDK configuration or cache, and must not be committed. External key stores can pass a public JWK with `--public-jwk-file` for registration or rotation. + +```python +from modelscope_hub import HubApi, generate_agent_key_pair + +api = HubApi(token="ms-write-token") +private_jwk, public_jwk = generate_agent_key_pair() +identity = api.create_agent_identity({"agent_name": "my-agent", "public_key": public_jwk.to_dict()}) +token = api.issue_agent_token_with_private_key(private_jwk, agent_id=identity.agent_id, audience="my-hub") +``` + --- ## SDK API Overview @@ -742,6 +772,12 @@ api = HubApi(token="...", endpoint="https://modelscope.ai") | | `get_mcp_server(server_id)` | Get server details | | | `deploy_mcp_server(server_id)` | Deploy an MCP server | | | `undeploy_mcp_server(server_id)` | Undeploy an MCP server | +| **Agent-IDP** | `create_agent_identity(payload)` | Register an Agent Ed25519 public key | +| | `get_agent_identity(agent_id)` / `update_agent_identity(...)` / `delete_agent_identity(agent_id)` | Manage identity metadata | +| | `reset_agent_key_pair(agent_id, payload)` / `pause_agent(agent_id, paused=...)` | Rotate a key or control token issuance | +| | `list_user_agent_identities(...)` / `list_agent_token_records(...)` | List identities and non-sensitive issuance records | +| | `issue_agent_token_with_private_key(...)` | Locally sign and exchange a short-lived JWT | +| | `get_agent_id_configuration()` / `get_agent_id_jwks()` | Anonymous OIDC discovery and JWT verification keys | | **Cache** | `scan_cache(cache_dir)` | Inspect local cache | | | `clear_cache(cache_dir, ...)` | Free disk space | diff --git a/pyproject.toml b/pyproject.toml index 4ebd36d..afb9ff3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "tqdm>=4.64.0", "filelock>=3.9", "urllib3>=1.26", + "cryptography>=41", ] [project.optional-dependencies] diff --git a/src/modelscope_hub/__init__.py b/src/modelscope_hub/__init__.py index 342d70d..46f2d95 100644 --- a/src/modelscope_hub/__init__.py +++ b/src/modelscope_hub/__init__.py @@ -10,6 +10,13 @@ from __future__ import annotations from ._download import ProgressCallback, TqdmCallback +from .agent_idp import ( + generate_agent_key_pair, + load_private_jwk, + public_jwk_from_private, + sign_agent_token_request, + write_private_jwk, +) from .api import HubApi from .config import HubConfig, get_default_config, set_default_config from .constants import License, RepoType, StudioVisibility, TokenScope, Visibility @@ -36,6 +43,12 @@ ValidationError, ) from .types import ( + AgentIdConfiguration, + AgentIdentity, + AgentIdentitySummary, + AgentJWK, + AgentToken, + AgentTokenRecord, CachedRepoInfo, CacheInfo, CacheVerification, @@ -65,7 +78,19 @@ # Progress callbacks "ProgressCallback", "TqdmCallback", + # Agent-IDP local key helpers + "generate_agent_key_pair", + "load_private_jwk", + "public_jwk_from_private", + "sign_agent_token_request", + "write_private_jwk", # Data classes + "AgentIdentity", + "AgentIdentitySummary", + "AgentIdConfiguration", + "AgentJWK", + "AgentToken", + "AgentTokenRecord", "CacheInfo", "CacheVerification", "CachedRepoInfo", diff --git a/src/modelscope_hub/_openapi.py b/src/modelscope_hub/_openapi.py index 30592f7..24ee5c8 100644 --- a/src/modelscope_hub/_openapi.py +++ b/src/modelscope_hub/_openapi.py @@ -45,9 +45,14 @@ raise_for_status, ) from .types import ( + CreateAgentIdentityPayload, CreateSkillPayload, CreateStudioPayload, DeployMcpServerPayload, + PauseAgentPayload, + ResetAgentKeyPairPayload, + TokenSignPayload, + UpdateAgentIdentityPayload, UpdateSkillSettingsPayload, UpdateStudioSettingsPayload, ) @@ -84,6 +89,10 @@ # ``GET /studios/{owner}/{repo}/logs/{type}`` caps page_size at 500. _STUDIO_LOG_MAX_PAGE_SIZE = 500 +# Agent-IDP declares these closed sets in its OpenAPI request schemas. +_AGENT_IDP_TOKEN_EXPIRIES: frozenset[int] = frozenset({300, 600, 1800, 3600}) +_AGENT_IDENTITY_STATUSES: frozenset[str] = frozenset({"active", "paused"}) + JSON = dict[str, Any] QueryParams = list[tuple[str, str]] Filters = Mapping[str, str | int | float | bool] | None @@ -123,6 +132,18 @@ def _as_wire_bool(value: bool | None) -> str | None: # deferred in that test. # --------------------------------------------------------------------------- OPERATION_REGISTRY: dict[str, tuple[str, TokenScope]] = { + # -- Agent-IDP ---------------------------------------------------------- + "createAgentIdentity": ("create_agent_identity", TokenScope.WRITE), + "getAgentIdentity": ("get_agent_identity", TokenScope.READ), + "updateAgentIdentity": ("update_agent_identity", TokenScope.WRITE), + "deleteAgentIdentity": ("delete_agent_identity", TokenScope.WRITE), + "resetAgentKeyPair": ("reset_agent_key_pair", TokenScope.WRITE), + "pauseAgent": ("pause_agent", TokenScope.WRITE), + "listAgentTokenRecords": ("list_agent_token_records", TokenScope.READ), + "listUserAgentIdentities": ("list_user_agent_identities", TokenScope.READ), + "issueAgentToken": ("issue_agent_token", TokenScope.READ), + "getAgentIdConfiguration": ("get_agent_id_configuration", TokenScope.READ), + "getAgentIdJWKS": ("get_agent_id_jwks", TokenScope.READ), # -- MCP ---------------------------------------------------------------- "listMcpServers": ("list_mcp_servers", TokenScope.READ), "listOperationalMcpServers": ("list_operational_mcp_servers", TokenScope.READ), @@ -1067,6 +1088,203 @@ def _flatten_mcp_list_params(body: Mapping[str, Any]) -> QueryParams: def _is_method_or_route_unsupported(exc: APIError) -> bool: return exc.status_code in (404, 405, 501) + # ================================================================== + # Agent-IDP + # ================================================================== + @staticmethod + def _validate_agent_idp_page(page: int, page_size: int) -> None: + """Validate the common Agent-IDP pagination contract (1-based, max 50).""" + if isinstance(page, bool) or not isinstance(page, int) or page < 1: + raise InvalidParameter("page must be an integer >= 1.") + if isinstance(page_size, bool) or not isinstance(page_size, int) or not 1 <= page_size <= 50: + raise InvalidParameter("page_size must be an integer between 1 and 50.") + + @staticmethod + def _validate_agent_public_jwk(value: Any) -> dict[str, Any]: + """Validate the public-only Ed25519 JWK accepted by registration routes.""" + if not isinstance(value, Mapping): + raise InvalidParameter("public_key must be an Ed25519 JWK object.") + key = dict(value) + if "d" in key: + raise InvalidParameter("public_key must not contain private JWK material ('d').") + required = {"kty", "crv", "x", "kid"} + missing = sorted(name for name in required if not isinstance(key.get(name), str) or not key[name]) + if missing: + raise InvalidParameter(f"public_key is missing required JWK field(s): {', '.join(missing)}.") + if key["kty"] != "OKP" or key["crv"] != "Ed25519": + raise InvalidParameter("public_key must use kty='OKP' and crv='Ed25519'.") + for field in ("alg", "use"): + if field in key and key[field] is not None and not isinstance(key[field], str): + raise InvalidParameter(f"public_key.{field} must be a string.") + return {field: value for field, value in key.items() if value is not None} + + @staticmethod + def _validate_agent_token_expiry(value: Any) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value not in _AGENT_IDP_TOKEN_EXPIRIES: + allowed = ", ".join(str(item) for item in sorted(_AGENT_IDP_TOKEN_EXPIRIES)) + raise InvalidParameter(f"token_expire_time must be one of {allowed} seconds.") + return value + + def create_agent_identity(self, payload: CreateAgentIdentityPayload | Mapping[str, Any]) -> JSON: + """``POST /agent_ids`` — register an Agent-IDP public Ed25519 identity.""" + body = {key: value for key, value in dict(payload).items() if value is not None} + allowed = {"agent_name", "description", "public_key", "key_alg_type", "token_expire_time"} + unknown = sorted(set(body) - allowed) + if unknown: + raise InvalidParameter(f"Unsupported create Agent-IDP field(s): {', '.join(unknown)}.") + if not isinstance(body.get("agent_name"), str) or not body["agent_name"].strip(): + raise InvalidParameter("agent_name must be a non-empty string.") + body["public_key"] = self._validate_agent_public_jwk(body.get("public_key")) + if "key_alg_type" in body and body["key_alg_type"] != "Ed25519": + raise InvalidParameter("key_alg_type must be 'Ed25519'.") + if "token_expire_time" in body: + body["token_expire_time"] = self._validate_agent_token_expiry(body["token_expire_time"]) + return self._request("POST", "/agent_ids", json_body=body, required_scope=TokenScope.WRITE) + + def get_agent_identity(self, agent_id: str) -> JSON: + """``GET /agent_ids/{agent_id}`` — fetch one Agent-IDP identity.""" + if not agent_id: + raise InvalidParameter("agent_id must not be empty.") + return self._request("GET", f"/agent_ids/{agent_id}", required_scope=TokenScope.READ) + + def update_agent_identity( + self, + agent_id: str, + payload: UpdateAgentIdentityPayload | Mapping[str, Any], + ) -> JSON: + """``PATCH /agent_ids/{agent_id}`` — update mutable Agent-IDP metadata.""" + if not agent_id: + raise InvalidParameter("agent_id must not be empty.") + body = {key: value for key, value in dict(payload).items() if value is not None} + allowed = {"agent_name", "description", "token_expire_time"} + unknown = sorted(set(body) - allowed) + if unknown: + raise InvalidParameter(f"Unsupported Agent-IDP update field(s): {', '.join(unknown)}.") + if not body: + raise InvalidParameter("update_agent_identity requires at least one field.") + if "agent_name" in body and (not isinstance(body["agent_name"], str) or not body["agent_name"].strip()): + raise InvalidParameter("agent_name must be a non-empty string.") + if "description" in body and not isinstance(body["description"], str): + raise InvalidParameter("description must be a string.") + if "token_expire_time" in body: + body["token_expire_time"] = self._validate_agent_token_expiry(body["token_expire_time"]) + return self._request("PATCH", f"/agent_ids/{agent_id}", json_body=body, required_scope=TokenScope.WRITE) + + def delete_agent_identity(self, agent_id: str) -> JSON: + """``DELETE /agent_ids/{agent_id}`` — delete an Agent-IDP identity.""" + if not agent_id: + raise InvalidParameter("agent_id must not be empty.") + return self._request("DELETE", f"/agent_ids/{agent_id}", required_scope=TokenScope.WRITE) + + def reset_agent_key_pair( + self, + agent_id: str, + payload: ResetAgentKeyPairPayload | Mapping[str, Any], + ) -> JSON: + """``PUT /agent_ids/{agent_id}/key_pairs`` — replace an identity public key.""" + if not agent_id: + raise InvalidParameter("agent_id must not be empty.") + body = {key: value for key, value in dict(payload).items() if value is not None} + allowed = {"public_key", "key_alg_type"} + unknown = sorted(set(body) - allowed) + if unknown: + raise InvalidParameter(f"Unsupported Agent-IDP key-reset field(s): {', '.join(unknown)}.") + body["public_key"] = self._validate_agent_public_jwk(body.get("public_key")) + if "key_alg_type" in body and body["key_alg_type"] != "Ed25519": + raise InvalidParameter("key_alg_type must be 'Ed25519'.") + return self._request( + "PUT", + f"/agent_ids/{agent_id}/key_pairs", + json_body=body, + required_scope=TokenScope.WRITE, + ) + + def pause_agent(self, agent_id: str, payload: PauseAgentPayload | Mapping[str, Any]) -> JSON: + """``POST /agent_ids/{agent_id}/paused`` — enable or pause token issuance.""" + if not agent_id: + raise InvalidParameter("agent_id must not be empty.") + body = dict(payload) + if set(body) != {"paused"} or not isinstance(body.get("paused"), bool): + raise InvalidParameter("pause_agent requires exactly a boolean 'paused' field.") + return self._request( + "POST", + f"/agent_ids/{agent_id}/paused", + json_body=body, + required_scope=TokenScope.WRITE, + ) + + def list_agent_token_records(self, agent_id: str, *, page: int = 1, page_size: int = 20) -> JSON: + """``GET /agent_ids/{agent_id}/jwt_id_tokens`` — list issued-token records.""" + if not agent_id: + raise InvalidParameter("agent_id must not be empty.") + self._validate_agent_idp_page(page, page_size) + return self._request( + "GET", + f"/agent_ids/{agent_id}/jwt_id_tokens", + params=self._merge_params({"page": page, "page_size": page_size}), + required_scope=TokenScope.READ, + ) + + def list_user_agent_identities( + self, + username: str, + *, + status: str | None = None, + page: int = 1, + page_size: int = 20, + ) -> JSON: + """``GET /users/{username}/agent_ids`` — list a user's Agent-IDP identities.""" + if not username: + raise InvalidParameter("username must not be empty.") + if status is not None and status not in _AGENT_IDENTITY_STATUSES: + allowed = ", ".join(sorted(_AGENT_IDENTITY_STATUSES)) + raise InvalidParameter(f"status must be one of {allowed}.") + self._validate_agent_idp_page(page, page_size) + return self._request( + "GET", + f"/users/{username}/agent_ids", + params=self._merge_params({"status": status, "page": page, "page_size": page_size}), + required_scope=TokenScope.READ, + ) + + def issue_agent_token(self, payload: TokenSignPayload | Mapping[str, Any]) -> JSON: + """``POST /agent_id/token`` — exchange a locally signed request for a JWT. + + The signature itself authenticates this public endpoint. Explicit + anonymous transport ensures an ambient Hub token is never attached. + """ + body = dict(payload) + required = {"agent_id", "kid", "audience", "timestamp", "signature"} + if set(body) != required: + raise InvalidParameter("issue_agent_token requires agent_id, kid, audience, timestamp and signature.") + for field in ("agent_id", "kid", "audience", "signature"): + if not isinstance(body[field], str) or not body[field]: + raise InvalidParameter(f"{field} must be a non-empty string.") + if isinstance(body["timestamp"], bool) or not isinstance(body["timestamp"], int) or body["timestamp"] <= 0: + raise InvalidParameter("timestamp must be a positive Unix timestamp in seconds.") + return self._request("POST", "/agent_id/token", json_body=body, require_token=False, anonymous=True) + + def get_agent_id_configuration(self) -> JSON: + """Fetch anonymous Agent-IDP OIDC discovery metadata.""" + return self._request( + "GET", + "/agent_id/.well-known/agentid-configuration", + require_token=False, + anonymous=True, + ) + + def get_agent_id_jwks(self) -> JSON: + """Fetch the anonymous Agent-IDP JWT validation key set.""" + return self._request( + "GET", + "/agent_id/.well-known/agentid-jwks", + require_token=False, + anonymous=True, + ) + + # ================================================================== + # MCP + # ================================================================== def list_mcp_servers( self, *, diff --git a/src/modelscope_hub/agent_idp.py b/src/modelscope_hub/agent_idp.py new file mode 100644 index 0000000..e2ad5da --- /dev/null +++ b/src/modelscope_hub/agent_idp.py @@ -0,0 +1,206 @@ +"""Local Ed25519/JWK helpers for the Agent-IDP OpenAPI surface. + +This module deliberately owns no HTTP transport and never persists a key unless +an application explicitly calls :func:`write_private_jwk`. Agent repository +transfer lives in :mod:`modelscope_hub.agent` and is unrelated to Agent-IDP +identities, signing keys, OIDC discovery, or JWT issuance. +""" + +from __future__ import annotations + +import base64 +import json +import os +import secrets +import stat +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from .errors import InvalidParameter +from .types import AgentJWK, TokenSignPayload + +__all__ = [ + "generate_agent_key_pair", + "load_private_jwk", + "public_jwk_from_private", + "sign_agent_token_request", + "write_private_jwk", +] + + +_JWK_KEY_TYPE = "OKP" +_JWK_CURVE = "Ed25519" + + +def _encode_base64url(value: bytes) -> str: + """Encode bytes as unpadded base64url, the JWK wire representation.""" + return base64.urlsafe_b64encode(value).rstrip(b"=").decode("ascii") + + +def _decode_base64url(value: object, field: str) -> bytes: + """Decode an unpadded base64url JWK member without accepting aliases.""" + if not isinstance(value, str) or not value: + raise InvalidParameter(f"Agent private key field {field!r} must be a non-empty base64url string.") + try: + decoded = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + except (ValueError, UnicodeEncodeError) as exc: + raise InvalidParameter(f"Agent private key field {field!r} is not valid base64url.") from exc + if _encode_base64url(decoded) != value: + raise InvalidParameter(f"Agent private key field {field!r} must be unpadded base64url.") + return decoded + + +def _normalise_private_jwk(value: AgentJWK | Mapping[str, Any]) -> AgentJWK: + """Validate an Ed25519 private JWK and return a safe structured copy.""" + if isinstance(value, AgentJWK): + key = value + elif isinstance(value, Mapping): + key = AgentJWK.from_dict(value) + else: + raise InvalidParameter("Agent private key must be an Ed25519 JWK mapping.") + + if key.kty != _JWK_KEY_TYPE or key.crv != _JWK_CURVE: + raise InvalidParameter("Agent private key must use kty='OKP' and crv='Ed25519'.") + if not key.kid: + raise InvalidParameter("Agent private key must contain a non-empty 'kid'.") + public_bytes = _decode_base64url(key.x, "x") + private_bytes = _decode_base64url(key.d, "d") + if len(public_bytes) != 32 or len(private_bytes) != 32: + raise InvalidParameter("Agent Ed25519 JWK public and private values must each be 32 bytes.") + derived_public = ( + Ed25519PrivateKey.from_private_bytes(private_bytes) + .public_key() + .public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + ) + if derived_public != public_bytes: + raise InvalidParameter("Agent private JWK public and private values do not form an Ed25519 key pair.") + return AgentJWK( + kty=_JWK_KEY_TYPE, + crv=_JWK_CURVE, + x=key.x, + kid=key.kid, + alg=key.alg, + use=key.use, + d=key.d, + ) + + +def generate_agent_key_pair(kid: str | None = None) -> tuple[AgentJWK, AgentJWK]: + """Generate a local Ed25519 key pair as ``(private_jwk, public_jwk)``. + + The caller chooses whether and where to persist the private JWK. This helper + never writes files or returns the private material in a public JWK. + """ + if kid is not None and not kid.strip(): + raise InvalidParameter("Agent key id 'kid' must not be empty.") + private_key = Ed25519PrivateKey.generate() + private_bytes = private_key.private_bytes( + serialization.Encoding.Raw, + serialization.PrivateFormat.Raw, + serialization.NoEncryption(), + ) + public_bytes = private_key.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + resolved_kid = kid.strip() if kid is not None else secrets.token_urlsafe(16) + private_jwk = AgentJWK( + kty=_JWK_KEY_TYPE, + crv=_JWK_CURVE, + x=_encode_base64url(public_bytes), + kid=resolved_kid, + alg="EdDSA", + use="sig", + d=_encode_base64url(private_bytes), + ) + return private_jwk, public_jwk_from_private(private_jwk) + + +def public_jwk_from_private(private_jwk: AgentJWK | Mapping[str, Any]) -> AgentJWK: + """Return the upload-safe public JWK corresponding to a private JWK.""" + key = _normalise_private_jwk(private_jwk) + return AgentJWK(kty=key.kty, crv=key.crv, x=key.x, kid=key.kid, alg=key.alg, use=key.use) + + +def write_private_jwk(path: str | Path, private_jwk: AgentJWK | Mapping[str, Any], *, overwrite: bool = False) -> Path: + """Write a validated private JWK with owner-only (``0600``) permissions. + + Existing files are never replaced unless *overwrite* is explicit. Symlinks + are rejected to keep a CLI invocation from overwriting an unexpected file. + """ + key = _normalise_private_jwk(private_jwk) + target = Path(path).expanduser() + target.parent.mkdir(parents=True, exist_ok=True) + if target.is_symlink(): + raise InvalidParameter("Refusing to write an Agent private key through a symbolic link.") + flags = os.O_WRONLY | os.O_CREAT | (os.O_TRUNC if overwrite else os.O_EXCL) + try: + descriptor = os.open(target, flags, 0o600) + except FileExistsError as exc: + raise InvalidParameter(f"Agent private key file already exists: {target}. Pass --force to replace it.") from exc + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + json.dump(key.to_dict(include_private=True), output, sort_keys=True) + output.write("\n") + os.chmod(target, 0o600) + except BaseException: + try: + target.unlink(missing_ok=True) + except OSError: + pass + raise + return target + + +def load_private_jwk(path: str | Path) -> AgentJWK: + """Load and strictly validate an owner-only Ed25519 private JWK file.""" + target = Path(path).expanduser() + try: + mode = target.stat().st_mode + except OSError as exc: + raise InvalidParameter(f"Unable to read Agent private key file: {target}.") from exc + if target.is_symlink() or not stat.S_ISREG(mode): + raise InvalidParameter("Agent private key path must be a regular, non-symbolic-link file.") + if os.name == "posix" and mode & 0o077: + raise InvalidParameter("Agent private key file must not be readable by group or other users (mode 0600).") + try: + raw = json.loads(target.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise InvalidParameter(f"Agent private key file is not valid JSON: {target}.") from exc + if not isinstance(raw, Mapping): + raise InvalidParameter("Agent private key file must contain a JSON object.") + return _normalise_private_jwk(raw) + + +def sign_agent_token_request( + private_jwk: AgentJWK | Mapping[str, Any], + *, + agent_id: str, + audience: str, + timestamp: int, +) -> TokenSignPayload: + """Build the signed body required by anonymous ``POST /agent_id/token``.""" + if not isinstance(agent_id, str) or not agent_id: + raise InvalidParameter("agent_id must be a non-empty string.") + if not isinstance(audience, str) or not audience: + raise InvalidParameter("audience must be a non-empty string.") + if isinstance(timestamp, bool) or not isinstance(timestamp, int) or timestamp <= 0: + raise InvalidParameter("timestamp must be a positive Unix timestamp in seconds.") + key = _normalise_private_jwk(private_jwk) + message = f"{agent_id}|{key.kid}|{audience}|{timestamp}".encode("ascii") + private_bytes = _decode_base64url(key.d, "d") + signature = Ed25519PrivateKey.from_private_bytes(private_bytes).sign(message) + return { + "agent_id": agent_id, + "kid": key.kid, + "audience": audience, + "timestamp": timestamp, + "signature": _encode_base64url(signature), + } diff --git a/src/modelscope_hub/api.py b/src/modelscope_hub/api.py index fd22be0..5fb1996 100644 --- a/src/modelscope_hub/api.py +++ b/src/modelscope_hub/api.py @@ -23,6 +23,7 @@ from __future__ import annotations +import time from collections.abc import Iterable, Mapping from pathlib import Path from typing import Any, BinaryIO, TypeAlias @@ -38,6 +39,7 @@ from ._legacy_api import LegacyClient from ._openapi import OpenAPIClient from ._upload import UploadManager +from .agent_idp import sign_agent_token_request from .config import HubConfig, get_default_config from .constants import DEFAULT_ENDPOINT, RepoType, StudioVisibility, Visibility from .errors import ( @@ -49,7 +51,20 @@ NotSupportedError, PermissionDeniedError, ) -from .types import CacheInfo, CacheVerification, FileInfo, PagedResult, RepoInfo, UserInfo +from .types import ( + AgentIdConfiguration, + AgentIdentity, + AgentIdentitySummary, + AgentJWK, + AgentToken, + AgentTokenRecord, + CacheInfo, + CacheVerification, + FileInfo, + PagedResult, + RepoInfo, + UserInfo, +) from .utils.logger import get_logger __all__ = ["HubApi"] @@ -2266,6 +2281,109 @@ def undeploy_mcp_server(self, server_id: str) -> dict: """ return self.openapi.undeploy_mcp_server(server_id) + # ================================================================== + # Agent-IDP + # ================================================================== + def create_agent_identity(self, payload: Mapping[str, Any]) -> AgentIdentity: + """Register an Agent-IDP identity with an Ed25519 public JWK.""" + return AgentIdentity.from_dict(self.openapi.create_agent_identity(payload)) + + def get_agent_identity(self, agent_id: str) -> AgentIdentity: + """Return an Agent-IDP identity and its current public key.""" + return AgentIdentity.from_dict(self.openapi.get_agent_identity(agent_id)) + + def update_agent_identity(self, agent_id: str, payload: Mapping[str, Any]) -> AgentIdentity: + """Update an Agent-IDP identity's mutable metadata.""" + return AgentIdentity.from_dict(self.openapi.update_agent_identity(agent_id, payload)) + + def delete_agent_identity(self, agent_id: str) -> dict: + """Delete an Agent-IDP identity and prevent future token issuance.""" + result = self.openapi.delete_agent_identity(agent_id) + return result if isinstance(result, dict) else {} + + def reset_agent_key_pair(self, agent_id: str, payload: Mapping[str, Any]) -> AgentIdentity: + """Replace an Agent-IDP identity's registered Ed25519 public key.""" + return AgentIdentity.from_dict(self.openapi.reset_agent_key_pair(agent_id, payload)) + + def pause_agent(self, agent_id: str, *, paused: bool) -> dict: + """Pause or resume JWT issuance for an Agent-IDP identity.""" + result = self.openapi.pause_agent(agent_id, {"paused": paused}) + return result if isinstance(result, dict) else {} + + def list_agent_token_records( + self, + agent_id: str, + *, + page: int = 1, + page_size: int = 20, + ) -> PagedResult[AgentTokenRecord]: + """List one Agent-IDP identity's issued token records.""" + payload = self.openapi.list_agent_token_records(agent_id, page=page, page_size=page_size) + data = payload if isinstance(payload, Mapping) else {} + raw_records = data.get("token_records") + records: list[Any] = raw_records if isinstance(raw_records, list) else [] + return PagedResult( + items=[AgentTokenRecord.from_dict(item) for item in records if isinstance(item, Mapping)], + total_count=int(data.get("total_count") or 0), + page_number=int(data.get("page_number") or page), + page_size=int(data.get("page_size") or page_size), + collection_key="token_records", + ) + + def list_user_agent_identities( + self, + username: str, + *, + status: str | None = None, + page: int = 1, + page_size: int = 20, + ) -> PagedResult[AgentIdentitySummary]: + """List a user's Agent-IDP identities via the OpenAPI surface.""" + payload = self.openapi.list_user_agent_identities(username, status=status, page=page, page_size=page_size) + data = payload if isinstance(payload, Mapping) else {} + raw_identities = data.get("agent_identities") + identities: list[Any] = raw_identities if isinstance(raw_identities, list) else [] + return PagedResult( + items=[AgentIdentitySummary.from_dict(item) for item in identities if isinstance(item, Mapping)], + total_count=int(data.get("total_count") or 0), + page_number=int(data.get("page_number") or page), + page_size=int(data.get("page_size") or page_size), + collection_key="agent_identities", + ) + + def issue_agent_token(self, payload: Mapping[str, Any]) -> AgentToken: + """Exchange a locally signed Agent-IDP request for a short-lived JWT.""" + return AgentToken.from_dict(self.openapi.issue_agent_token(payload)) + + def issue_agent_token_with_private_key( + self, + private_jwk: AgentJWK | Mapping[str, Any], + *, + agent_id: str, + audience: str, + timestamp: int | None = None, + ) -> AgentToken: + """Locally sign and exchange an Agent-IDP token request without persisting a key.""" + signed = sign_agent_token_request( + private_jwk, + agent_id=agent_id, + audience=audience, + timestamp=int(time.time()) if timestamp is None else timestamp, + ) + return self.issue_agent_token(signed) + + def get_agent_id_configuration(self) -> AgentIdConfiguration: + """Fetch anonymous Agent-IDP OIDC discovery metadata.""" + return AgentIdConfiguration.from_dict(self.openapi.get_agent_id_configuration()) + + def get_agent_id_jwks(self) -> list[AgentJWK]: + """Fetch anonymous Agent-IDP JWT verification keys.""" + payload = self.openapi.get_agent_id_jwks() + keys = payload.get("keys") if isinstance(payload, Mapping) else None + if not isinstance(keys, list): + return [] + return [AgentJWK.from_dict(item) for item in keys if isinstance(item, Mapping)] + # ================================================================== # Cache # ================================================================== diff --git a/src/modelscope_hub/cli/agent.py b/src/modelscope_hub/cli/agent.py index 3f8fe33..e70748c 100644 --- a/src/modelscope_hub/cli/agent.py +++ b/src/modelscope_hub/cli/agent.py @@ -2,9 +2,10 @@ """``ms agent`` command -- low-level raw file transfer for agent repositories. This is the *slim* Hub CLI. It supports only ``download``/``upload``/``list`` -for raw file transfer to and from remote agent repositories, with no framework -awareness. Framework-aware operations (convert, watch/sync, status, backups, -restore, stop) live in **modelscope-agent** -- use ``ms-agent agent ...``. +for raw file transfer to and from remote agent repositories; Agent-IDP identity, +Ed25519-key, and token operations live in ``ms agent-idp``. Framework-aware +operations (convert, watch/sync, status, backups, restore, stop) live in +**modelscope-agent** -- use ``ms-agent agent ...``. """ from __future__ import annotations diff --git a/src/modelscope_hub/cli/agent_idp.py b/src/modelscope_hub/cli/agent_idp.py new file mode 100644 index 0000000..9ee112e --- /dev/null +++ b/src/modelscope_hub/cli/agent_idp.py @@ -0,0 +1,261 @@ +"""``ms agent-idp`` — manage Agent-IDP identities and signed token issuance. + +This command is intentionally separate from ``ms agent``: the latter transfers +raw files to Agent repositories, while this command manages Ed25519 identities, +OIDC discovery, and short-lived JWT issuance. +""" + +from __future__ import annotations + +import json +from argparse import Namespace +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from ..agent_idp import ( + generate_agent_key_pair, + load_private_jwk, + public_jwk_from_private, + write_private_jwk, +) +from ..errors import InvalidParameter +from ..types import AgentIdConfiguration, AgentIdentity +from .base import CLICommand, SubParsers, info, make_api, render_table + +__all__ = ["AgentIdpCommand"] + + +_EXPIRY_CHOICES = (300, 600, 1800, 3600) +_STATUS_CHOICES = ("active", "paused") + + +def _read_public_jwk(path: str) -> dict[str, Any]: + """Read an upload-safe public JWK without accepting a private ``d`` member.""" + target = Path(path).expanduser() + try: + payload = json.loads(target.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise InvalidParameter(f"Public JWK file is not valid JSON: {target}.") from exc + if not isinstance(payload, Mapping) or "d" in payload: + raise InvalidParameter("Public JWK file must be an object without private key material ('d').") + # The low-level client performs the complete protocol validation. Constructing + # the dataclass here keeps formatting and optional JWK fields consistent. + return {key: value for key, value in payload.items() if value is not None} + + +def _public_jwk_from_args(args: Namespace) -> dict[str, Any]: + if getattr(args, "private_key_file", None): + return public_jwk_from_private(load_private_jwk(args.private_key_file)).to_dict() + return _read_public_jwk(args.public_jwk_file) + + +def _identity_json(identity: AgentIdentity) -> dict[str, Any]: + """Render identity data while never serialising local private material.""" + result: dict[str, Any] = { + "agent_id": identity.agent_id, + "agent_name": identity.agent_name, + "description": identity.description, + "token_expire_time": identity.token_expire_time, + "principal": identity.principal, + "kid": identity.kid, + "status": identity.status, + "create_time": identity.create_time, + "update_time": identity.update_time, + } + if identity.public_key is not None: + result["public_key"] = identity.public_key.to_dict() + return {key: value for key, value in result.items() if value is not None and value != ""} + + +def _configuration_json(configuration: AgentIdConfiguration) -> dict[str, Any]: + return { + key: value + for key, value in { + "issuer": configuration.issuer, + "token_endpoint": configuration.token_endpoint, + "jwks_uri": configuration.jwks_uri, + "registration_endpoint": configuration.registration_endpoint, + "activity_endpoint": configuration.activity_endpoint, + "id_token_signing_alg_values_supported": configuration.id_token_signing_alg_values_supported, + }.items() + if value is not None + } + + +class AgentIdpCommand(CLICommand): + """Manage Agent-IDP identities, keys, OIDC discovery, and JWT issuance.""" + + name = "agent-idp" + + @staticmethod + def register(subparsers: SubParsers) -> None: + parser = subparsers.add_parser( + AgentIdpCommand.name, + help="Manage Agent-IDP identities, signing keys, and OIDC metadata.", + ) + parser.set_defaults(_command=AgentIdpCommand) + actions = parser.add_subparsers(dest="agent_idp_action", metavar="ACTION") + actions.required = True + + keygen = actions.add_parser("keygen", help="Generate an Ed25519 private JWK file (0600).") + keygen.add_argument("--private-key-out", required=True) + keygen.add_argument("--kid", default=None) + keygen.add_argument("--force", action="store_true", help="Replace an existing private-key file.") + + create = actions.add_parser("create", help="Register an Agent-IDP identity.") + create.add_argument("--agent-name", required=True) + create.add_argument("--description", default=None) + create.add_argument("--token-expire-time", type=int, choices=_EXPIRY_CHOICES, default=None) + AgentIdpCommand._add_key_source(create) + + get = actions.add_parser("get", help="Show one Agent-IDP identity.") + get.add_argument("agent_id") + + update = actions.add_parser("update", help="Update Agent-IDP identity metadata.") + update.add_argument("agent_id") + update.add_argument("--agent-name", default=None) + update.add_argument("--description", default=None) + update.add_argument("--token-expire-time", type=int, choices=_EXPIRY_CHOICES, default=None) + + delete = actions.add_parser("delete", help="Delete an Agent-IDP identity.") + delete.add_argument("agent_id") + delete.add_argument("--yes", action="store_true", help="Confirm permanent deletion.") + + reset = actions.add_parser("reset-key", help="Replace an Agent-IDP identity public key.") + reset.add_argument("agent_id") + AgentIdpCommand._add_key_source(reset) + + pause = actions.add_parser("pause", help="Pause or resume Agent-IDP token issuance.") + pause.add_argument("agent_id") + state = pause.add_mutually_exclusive_group(required=True) + state.add_argument("--paused", action="store_true", help="Pause token issuance.") + state.add_argument("--active", action="store_true", help="Resume token issuance.") + + listed = actions.add_parser("list", help="List a user's Agent-IDP identities.") + listed.add_argument("username") + listed.add_argument("--status", choices=_STATUS_CHOICES, default=None) + AgentIdpCommand._add_paging(listed) + + records = actions.add_parser("list-tokens", help="List non-sensitive issued-token records.") + records.add_argument("agent_id") + AgentIdpCommand._add_paging(records) + + issue = actions.add_parser("issue-token", help="Sign locally and exchange an Agent-IDP JWT request.") + issue.add_argument("--agent-id", required=True) + issue.add_argument("--audience", required=True) + issue.add_argument("--private-key-file", required=True) + issue.add_argument("--timestamp", type=int, default=None) + + actions.add_parser("configuration", help="Show anonymous Agent-IDP OIDC discovery metadata.") + actions.add_parser("jwks", help="Show anonymous Agent-IDP JWT verification keys.") + + @staticmethod + def _add_key_source(parser: Any) -> None: + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--private-key-file", default=None, help="Owner-only Ed25519 private JWK file.") + source.add_argument("--public-jwk-file", default=None, help="Public Ed25519 JWK file (for HSM-managed keys).") + + @staticmethod + def _add_paging(parser: Any) -> None: + parser.add_argument("--page", type=int, default=1) + parser.add_argument("--page-size", type=int, choices=range(1, 51), default=20) + + def execute(self) -> None: + action = self.args.agent_idp_action + if action == "keygen": + private_jwk, public_jwk = generate_agent_key_pair(self.args.kid) + write_private_jwk(self.args.private_key_out, private_jwk, overwrite=self.args.force) + info(json.dumps(public_jwk.to_dict(), ensure_ascii=False, sort_keys=True)) + return + + api = make_api(self.args) + if action == "create": + payload = { + "agent_name": self.args.agent_name, + "public_key": _public_jwk_from_args(self.args), + "description": self.args.description, + "token_expire_time": self.args.token_expire_time, + } + info(json.dumps(_identity_json(api.create_agent_identity(payload)), ensure_ascii=False, indent=2)) + elif action == "get": + info(json.dumps(_identity_json(api.get_agent_identity(self.args.agent_id)), ensure_ascii=False, indent=2)) + elif action == "update": + payload = { + key: value + for key, value in { + "agent_name": self.args.agent_name, + "description": self.args.description, + "token_expire_time": self.args.token_expire_time, + }.items() + if value is not None + } + identity = api.update_agent_identity(self.args.agent_id, payload) + info(json.dumps(_identity_json(identity), ensure_ascii=False, indent=2)) + elif action == "delete": + if not self.args.yes: + raise InvalidParameter("Deletion requires --yes.") + api.delete_agent_identity(self.args.agent_id) + info(f"Deleted Agent-IDP identity {self.args.agent_id}.") + elif action == "reset-key": + identity = api.reset_agent_key_pair( + self.args.agent_id, + {"public_key": _public_jwk_from_args(self.args)}, + ) + info(json.dumps(_identity_json(identity), ensure_ascii=False, indent=2)) + elif action == "pause": + api.pause_agent(self.args.agent_id, paused=self.args.paused) + info(f"Agent-IDP identity {self.args.agent_id} is now {'paused' if self.args.paused else 'active'}.") + elif action == "list": + identity_page = api.list_user_agent_identities( + self.args.username, + status=self.args.status, + page=self.args.page, + page_size=self.args.page_size, + ) + identity_rows = [ + ( + item.agent_id, + item.agent_name, + item.kid or "-", + item.status or "-", + item.token_expire_time or "-", + item.create_time or "-", + ) + for item in identity_page.items + ] + info(render_table(identity_rows, headers=["agent_id", "name", "kid", "status", "expiry", "created"])) + info( + f"page {identity_page.page_number} / total {identity_page.total_count} " + f"(page_size={identity_page.page_size})" + ) + elif action == "list-tokens": + token_page = api.list_agent_token_records( + self.args.agent_id, + page=self.args.page, + page_size=self.args.page_size, + ) + token_rows = [ + (item.token_id, item.audience, item.issued_at or "-", item.expire_at or "-", item.status or "-") + for item in token_page.items + ] + info(render_table(token_rows, headers=["token_id", "audience", "issued", "expires", "status"])) + info(f"page {token_page.page_number} / total {token_page.total_count} (page_size={token_page.page_size})") + elif action == "issue-token": + key = load_private_jwk(self.args.private_key_file) + token = api.issue_agent_token_with_private_key( + key, + agent_id=self.args.agent_id, + audience=self.args.audience, + timestamp=self.args.timestamp, + ) + # Deliberately the only stdout content: callers may pipe this exact + # credential to another process without parsing a status message. + print(token.access_token) + elif action == "configuration": + info(json.dumps(_configuration_json(api.get_agent_id_configuration()), ensure_ascii=False, indent=2)) + elif action == "jwks": + keys = [key.to_dict() for key in api.get_agent_id_jwks()] + info(json.dumps({"keys": keys}, ensure_ascii=False, indent=2)) + else: # pragma: no cover - argparse makes this defensive only + raise InvalidParameter(f"Unknown Agent-IDP action: {action}") diff --git a/src/modelscope_hub/cli/main.py b/src/modelscope_hub/cli/main.py index 88a6933..ce6528e 100644 --- a/src/modelscope_hub/cli/main.py +++ b/src/modelscope_hub/cli/main.py @@ -34,6 +34,7 @@ from ..constants import MODELSCOPE_ASCII from ..errors import HubError, InvalidParameter, NotSupportedError from .agent import AgentCommand +from .agent_idp import AgentIdpCommand from .base import CLICommand, error, info from .cache import CacheCommand, _CacheClear, _CacheScan from .deploy import DeployCommand, LogsCommand, SettingsCommand, StopCommand @@ -66,6 +67,7 @@ McpCommand, CacheCommand, AgentCommand, + AgentIdpCommand, ] # Plugin entry-point group name diff --git a/src/modelscope_hub/errors.py b/src/modelscope_hub/errors.py index 13d27b4..d3a3799 100644 --- a/src/modelscope_hub/errors.py +++ b/src/modelscope_hub/errors.py @@ -37,6 +37,9 @@ "session", "api_key", "apikey", + "signature", + "jwt", + "private_key", ) _SENSITIVE_QUERY_KEYS: frozenset[str] = frozenset( { @@ -53,6 +56,8 @@ "key", "authorization", "credentials", + "signature", + "jwt", } ) _SENSITIVE_BODY_KEYS: re.Pattern[str] = re.compile( diff --git a/src/modelscope_hub/types.py b/src/modelscope_hub/types.py index 6761649..4cf8316 100644 --- a/src/modelscope_hub/types.py +++ b/src/modelscope_hub/types.py @@ -147,6 +147,119 @@ def _as_str_or_none(value: Any | None) -> str | None: return str(value) +# --------------------------------------------------------------------------- +# Agent-IDP +# --------------------------------------------------------------------------- +@dataclass(slots=True) +class AgentJWK(_FromDictMixin): + """An Ed25519 JSON Web Key used by the Agent-IDP protocol. + + ``d`` exists only in local private-key material. Server responses and public + request payloads contain the other fields only; :meth:`to_dict` therefore + excludes it unless explicitly requested. + """ + + kty: str = "OKP" + crv: str = "Ed25519" + x: str = "" + kid: str = "" + alg: str | None = None + use: str | None = None + d: str | None = field(default=None, repr=False) + + def to_dict(self, *, include_private: bool = False) -> dict[str, str]: + """Return the public JWK, optionally including the private ``d`` value.""" + result = {"kty": self.kty, "crv": self.crv, "x": self.x, "kid": self.kid} + if self.alg is not None: + result["alg"] = self.alg + if self.use is not None: + result["use"] = self.use + if include_private and self.d is not None: + result["d"] = self.d + return result + + +@dataclass(slots=True) +class AgentIdentity(_FromDictMixin): + """A registered Agent-IDP identity returned by the OpenAPI service.""" + + agent_id: str = "" + agent_name: str = "" + description: str | None = None + token_expire_time: int | None = None + principal: dict[str, Any] | None = None + kid: str | None = None + public_key: AgentJWK | None = None + status: str | None = None + create_time: str | None = None + update_time: str | None = None + + @classmethod + def from_dict(cls, data: Mapping[str, Any] | None) -> AgentIdentity: + if not isinstance(data, Mapping): + return cls() + raw_key = data.get("public_key") + principal = data.get("principal") + return cls( + agent_id=str(data.get("agent_id") or ""), + agent_name=str(data.get("agent_name") or ""), + description=data.get("description"), + token_expire_time=data.get("token_expire_time"), + principal=dict(principal) if isinstance(principal, Mapping) else None, + kid=data.get("kid"), + public_key=AgentJWK.from_dict(raw_key) if isinstance(raw_key, Mapping) else None, + status=data.get("status"), + create_time=data.get("create_time"), + update_time=data.get("update_time"), + ) + + +@dataclass(slots=True) +class AgentIdentitySummary(_FromDictMixin): + """The paginated, non-sensitive projection of an Agent-IDP identity.""" + + agent_id: str = "" + agent_name: str = "" + kid: str | None = None + status: str | None = None + token_expire_time: int | None = None + create_time: str | None = None + + +@dataclass(slots=True) +class AgentTokenRecord(_FromDictMixin): + """One issued Agent JWT record returned by the service.""" + + token_id: str = "" + audience: str = "" + issued_at: str | None = None + expire_at: str | None = None + status: str | None = None + jwt: str | None = field(default=None, repr=False) + + +@dataclass(slots=True) +class AgentToken(_FromDictMixin): + """A short-lived JWT issued by ``POST /agent_id/token``.""" + + access_token: str = field(default="", repr=False) + token_type: str = "Bearer" + expire_at: int | None = None + jti: str | None = None + + +@dataclass(slots=True) +class AgentIdConfiguration(_FromDictMixin): + """OIDC discovery metadata served by Agent-IDP.""" + + issuer: str | None = None + token_endpoint: str | None = None + jwks_uri: str | None = None + registration_endpoint: str | None = None + activity_endpoint: str | None = None + id_token_signing_alg_values_supported: str | None = None + + # --------------------------------------------------------------------------- # Repository # --------------------------------------------------------------------------- @@ -439,17 +552,82 @@ class DeployMcpServerPayload(TypedDict, total=False): env_info: dict[str, str] +class JWKPayload(TypedDict, total=False): + """Public or private Ed25519 JSON Web Key wire shape.""" + + kty: str + crv: str + x: str + kid: str + alg: str + use: str + d: str + + +class CreateAgentIdentityPayload(TypedDict, total=False): + """Payload for POST /agent_ids.""" + + agent_name: str + description: str + public_key: JWKPayload + key_alg_type: str + token_expire_time: int + + +class UpdateAgentIdentityPayload(TypedDict, total=False): + """Payload for PATCH /agent_ids/{agent_id}.""" + + agent_name: str + description: str + token_expire_time: int + + +class ResetAgentKeyPairPayload(TypedDict, total=False): + """Payload for PUT /agent_ids/{agent_id}/key_pairs.""" + + public_key: JWKPayload + key_alg_type: str + + +class PauseAgentPayload(TypedDict): + """Payload for POST /agent_ids/{agent_id}/paused.""" + + paused: bool + + +class TokenSignPayload(TypedDict): + """Signed request body for anonymous POST /agent_id/token.""" + + agent_id: str + kid: str + audience: str + timestamp: int + signature: str + + __all__ = [ + "AgentIdentity", + "AgentIdentitySummary", + "AgentIdConfiguration", + "AgentJWK", + "AgentToken", + "AgentTokenRecord", "CacheVerification", "CacheInfo", "CachedRepoInfo", "CommitInfo", + "CreateAgentIdentityPayload", "CreateSkillPayload", "CreateStudioPayload", "DeployMcpServerPayload", "FileInfo", + "JWKPayload", "PagedResult", + "PauseAgentPayload", "RepoInfo", + "ResetAgentKeyPairPayload", + "TokenSignPayload", + "UpdateAgentIdentityPayload", "UpdateSkillSettingsPayload", "UpdateStudioSettingsPayload", "UserInfo", diff --git a/src/modelscope_hub/version.py b/src/modelscope_hub/version.py index f710337..cfc4219 100644 --- a/src/modelscope_hub/version.py +++ b/src/modelscope_hub/version.py @@ -1,3 +1,3 @@ """Version information for modelscope_hub.""" -__version__ = "0.4.0" +__version__ = "0.5.0" diff --git a/tests/cli/test_agent_idp.py b/tests/cli/test_agent_idp.py new file mode 100644 index 0000000..36311be --- /dev/null +++ b/tests/cli/test_agent_idp.py @@ -0,0 +1,81 @@ +"""CLI tests for the Agent-IDP command, with no network access.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from modelscope_hub.agent_idp import generate_agent_key_pair, write_private_jwk +from modelscope_hub.cli.agent_idp import AgentIdpCommand +from modelscope_hub.errors import InvalidParameter +from modelscope_hub.types import AgentIdentity, AgentToken + + +def _execute(parser, arguments: list[str]) -> None: + args = parser.parse_args(["agent-idp", *arguments]) + assert args._command is AgentIdpCommand + AgentIdpCommand(args).execute() + + +def test_parser_registers_agent_idp(parser): + args = parser.parse_args(["agent-idp", "configuration"]) + assert args.agent_idp_action == "configuration" + assert args._command is AgentIdpCommand + + +def test_keygen_prints_only_public_jwk_and_writes_private_file(parser, tmp_path, capsys): + path = tmp_path / "agent.jwk" + _execute(parser, ["keygen", "--private-key-out", str(path), "--kid", "key-1"]) + public_jwk = json.loads(capsys.readouterr().out) + private_jwk = json.loads(path.read_text()) + assert public_jwk["kid"] == "key-1" + assert "d" not in public_jwk + assert private_jwk["d"] + + +def test_create_reads_public_key_without_private_material(parser, tmp_path): + public_path = tmp_path / "public.jwk" + public_path.write_text(json.dumps({"kty": "OKP", "crv": "Ed25519", "x": "x", "kid": "key-1"})) + api = MagicMock() + api.create_agent_identity.return_value = AgentIdentity(agent_id="agent-1", agent_name="builder") + with patch("modelscope_hub.cli.agent_idp.make_api", return_value=api): + _execute(parser, ["create", "--agent-name", "builder", "--public-jwk-file", str(public_path)]) + payload = api.create_agent_identity.call_args.args[0] + assert payload["public_key"] == {"kty": "OKP", "crv": "Ed25519", "x": "x", "kid": "key-1"} + + +def test_delete_requires_explicit_confirmation(parser): + with pytest.raises(InvalidParameter, match="--yes"): + _execute(parser, ["delete", "agent-1"]) + + +def test_issue_token_stdout_is_only_the_credential(parser, tmp_path, capsys): + private_jwk, _ = generate_agent_key_pair() + path = tmp_path / "agent.jwk" + write_private_jwk(path, private_jwk) + api = MagicMock() + api.issue_agent_token_with_private_key.return_value = AgentToken(access_token="issued.jwt") + with patch("modelscope_hub.cli.agent_idp.make_api", return_value=api): + _execute( + parser, + ["issue-token", "--agent-id", "agent-1", "--audience", "hub", "--private-key-file", str(path)], + ) + assert capsys.readouterr().out == "issued.jwt\n" + + +def test_public_oidc_commands_can_run_without_token(parser, capsys): + api = MagicMock() + api.get_agent_id_configuration.return_value = MagicMock( + issuer="https://issuer", + token_endpoint="https://issuer/token", + jwks_uri=None, + registration_endpoint=None, + activity_endpoint=None, + id_token_signing_alg_values_supported=None, + ) + with patch("modelscope_hub.cli.agent_idp.make_api", return_value=api): + _execute(parser, ["configuration"]) + assert "https://issuer" in capsys.readouterr().out + api.get_agent_id_configuration.assert_called_once_with() diff --git a/tests/test_agent_idp.py b/tests/test_agent_idp.py new file mode 100644 index 0000000..5a34b94 --- /dev/null +++ b/tests/test_agent_idp.py @@ -0,0 +1,147 @@ +"""Mock-based contract tests for the Agent-IDP OpenAPI client and facade.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from modelscope_hub._openapi import OpenAPIClient +from modelscope_hub.api import HubApi +from modelscope_hub.config import HubConfig +from modelscope_hub.errors import InvalidParameter +from modelscope_hub.types import AgentIdentity, AgentToken + + +def _response(data: object) -> MagicMock: + response = MagicMock(spec=requests.Response) + response.status_code = 200 + response.content = b"x" + response.headers = {} + response.request = MagicMock() + response.request.method = "GET" + response.request.path_url = "/test" + response.request.url = "https://modelscope.cn/openapi/v1/test" + response.url = response.request.url + response.json.return_value = {"success": True, "data": data} + return response + + +@pytest.fixture +def client() -> OpenAPIClient: + return OpenAPIClient(HubConfig(token="test-token", endpoint="https://modelscope.cn")) + + +@pytest.fixture +def public_jwk() -> dict[str, str]: + return {"kty": "OKP", "crv": "Ed25519", "x": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "kid": "key-1"} + + +class TestAgentIdpOpenApi: + def test_create_identity_uses_openapi_payload(self, client, public_jwk): + with patch.object(client._session, "request", return_value=_response({"agent_id": "agent-1"})) as request: + client.create_agent_identity({"agent_name": "builder", "public_key": public_jwk, "token_expire_time": 600}) + call = request.call_args.kwargs + assert call["method"] == "POST" + assert call["url"].endswith("/openapi/v1/agent_ids") + assert call["json"]["public_key"] == public_jwk + assert call["headers"]["Authorization"] == "Bearer test-token" + + def test_update_and_key_reset_use_spec_methods(self, client, public_jwk): + with patch.object(client._session, "request", side_effect=[_response({}), _response({})]) as request: + client.update_agent_identity("agent-1", {"description": "updated"}) + client.reset_agent_key_pair("agent-1", {"public_key": public_jwk}) + update, reset = request.call_args_list + assert update.kwargs["method"] == "PATCH" + assert update.kwargs["url"].endswith("/agent_ids/agent-1") + assert reset.kwargs["method"] == "PUT" + assert reset.kwargs["url"].endswith("/agent_ids/agent-1/key_pairs") + + def test_delete_and_pause_use_spec_routes(self, client): + with patch.object(client._session, "request", side_effect=[_response({}), _response({})]) as request: + client.delete_agent_identity("agent-1") + client.pause_agent("agent-1", {"paused": True}) + delete, pause = request.call_args_list + assert delete.kwargs["method"] == "DELETE" + assert delete.kwargs["url"].endswith("/agent_ids/agent-1") + assert pause.kwargs["method"] == "POST" + assert pause.kwargs["url"].endswith("/agent_ids/agent-1/paused") + assert pause.kwargs["json"] == {"paused": True} + + def test_list_routes_use_spec_page_fields(self, client): + with patch.object(client._session, "request", side_effect=[_response({}), _response({})]) as request: + client.list_agent_token_records("agent-1", page=2, page_size=10) + client.list_user_agent_identities("alice", status="paused", page=3, page_size=20) + tokens, identities = request.call_args_list + assert tokens.kwargs["url"].endswith("/agent_ids/agent-1/jwt_id_tokens") + assert dict(tokens.kwargs["params"]) == {"page": "2", "page_size": "10"} + assert identities.kwargs["url"].endswith("/users/alice/agent_ids") + assert dict(identities.kwargs["params"]) == {"status": "paused", "page": "3", "page_size": "20"} + + @pytest.mark.parametrize(("page", "page_size"), [(0, 20), (1, 0), (1, 51)]) + def test_paging_is_validated(self, client, page, page_size): + with pytest.raises(InvalidParameter): + client.list_agent_token_records("agent-1", page=page, page_size=page_size) + + def test_update_and_public_jwk_are_strict(self, client, public_jwk): + with pytest.raises(InvalidParameter, match="at least one"): + client.update_agent_identity("agent-1", {}) + with pytest.raises(InvalidParameter, match="private"): + client.create_agent_identity({"agent_name": "builder", "public_key": {**public_jwk, "d": "not-allowed"}}) + with pytest.raises(InvalidParameter, match="status"): + client.list_user_agent_identities("alice", status="retired") + + def test_public_operations_never_attach_ambient_credentials(self, client): + with patch.object( + client._session, + "request", + side_effect=[_response({"access_token": "jwt"}), _response({}), _response({"keys": []})], + ) as request: + client.issue_agent_token( + { + "agent_id": "agent-1", + "kid": "key-1", + "audience": "hub", + "timestamp": 1, + "signature": "signature", + } + ) + client.get_agent_id_configuration() + client.get_agent_id_jwks() + for call in request.call_args_list: + assert "Authorization" not in call.kwargs["headers"] + assert call.kwargs["cookies"] == {} + assert request.call_args_list[0].kwargs["method"] == "POST" + assert request.call_args_list[0].kwargs["url"].endswith("/agent_id/token") + assert request.call_args_list[1].kwargs["url"].endswith("/agent_id/.well-known/agentid-configuration") + assert request.call_args_list[2].kwargs["url"].endswith("/agent_id/.well-known/agentid-jwks") + + +class TestAgentIdpFacade: + def test_facade_converts_identity_and_pagination(self): + api = HubApi(token="test-token") + api._openapi = MagicMock() + api._openapi.create_agent_identity.return_value = {"agent_id": "agent-1", "agent_name": "builder"} + api._openapi.list_user_agent_identities.return_value = { + "agent_identities": [{"agent_id": "agent-1", "agent_name": "builder"}], + "total_count": 1, + "page_number": 1, + "page_size": 20, + } + created = api.create_agent_identity({"agent_name": "builder", "public_key": {}}) + page = api.list_user_agent_identities("alice") + assert isinstance(created, AgentIdentity) + assert created.agent_id == "agent-1" + assert page.total_count == 1 + assert page.items[0].agent_name == "builder" + + def test_facade_converts_issued_token(self): + api = HubApi(token="test-token") + api._openapi = MagicMock() + api._openapi.issue_agent_token.return_value = {"access_token": "jwt", "token_type": "Bearer", "expire_at": 10} + token = api.issue_agent_token( + {"agent_id": "agent-1", "kid": "key-1", "audience": "hub", "timestamp": 1, "signature": "sig"} + ) + assert isinstance(token, AgentToken) + assert token.access_token == "jwt" diff --git a/tests/test_agent_idp_keys.py b/tests/test_agent_idp_keys.py new file mode 100644 index 0000000..ec5bd2f --- /dev/null +++ b/tests/test_agent_idp_keys.py @@ -0,0 +1,75 @@ +"""Security-sensitive local Ed25519/JWK tests for Agent-IDP.""" + +from __future__ import annotations + +import base64 +import os +import stat + +import pytest +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + +from modelscope_hub.agent_idp import ( + generate_agent_key_pair, + load_private_jwk, + public_jwk_from_private, + sign_agent_token_request, + write_private_jwk, +) +from modelscope_hub.errors import InvalidParameter + + +def _decode(value: str) -> bytes: + return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + + +def test_generated_jwks_round_trip_and_sign(): + private_jwk, public_jwk = generate_agent_key_pair("key-1") + assert private_jwk.d + assert "d" not in public_jwk.to_dict() + assert private_jwk.x == public_jwk.x + assert "=" not in private_jwk.x + + signed = sign_agent_token_request(private_jwk, agent_id="agent-1", audience="hub", timestamp=100) + message = b"agent-1|key-1|hub|100" + Ed25519PublicKey.from_public_bytes(_decode(public_jwk.x)).verify(_decode(signed["signature"]), message) + with pytest.raises(InvalidSignature): + Ed25519PublicKey.from_public_bytes(_decode(public_jwk.x)).verify(_decode(signed["signature"]), b"wrong") + + +def test_private_file_is_owner_only_and_not_overwritten(tmp_path): + private_jwk, _ = generate_agent_key_pair() + destination = tmp_path / "agent.jwk" + write_private_jwk(destination, private_jwk) + assert stat.S_IMODE(destination.stat().st_mode) == 0o600 + loaded = load_private_jwk(destination) + assert loaded.to_dict(include_private=True) == private_jwk.to_dict(include_private=True) + with pytest.raises(InvalidParameter, match="already exists"): + write_private_jwk(destination, private_jwk) + + +def test_private_file_rejects_insecure_permissions_on_posix(tmp_path): + if os.name != "posix": + pytest.skip("POSIX permission enforcement only") + private_jwk, _ = generate_agent_key_pair() + destination = tmp_path / "agent.jwk" + write_private_jwk(destination, private_jwk) + destination.chmod(0o644) + with pytest.raises(InvalidParameter, match="0600"): + load_private_jwk(destination) + + +def test_mismatched_key_material_is_rejected_without_leaking_private_value(): + private_jwk, _ = generate_agent_key_pair() + invalid = private_jwk.to_dict(include_private=True) + invalid["x"] = "A" * len(invalid["x"]) + with pytest.raises(InvalidParameter) as raised: + public_jwk_from_private(invalid) + assert private_jwk.d not in str(raised.value) + + +def test_signing_rejects_invalid_timestamp(): + private_jwk, _ = generate_agent_key_pair() + with pytest.raises(InvalidParameter, match="timestamp"): + sign_agent_token_request(private_jwk, agent_id="agent-1", audience="hub", timestamp=0) diff --git a/tests/test_openapi_coverage.py b/tests/test_openapi_coverage.py index ce2cead..344f87b 100644 --- a/tests/test_openapi_coverage.py +++ b/tests/test_openapi_coverage.py @@ -24,7 +24,7 @@ _HTTP_METHODS = frozenset({"get", "post", "put", "delete", "patch", "head", "options"}) # Tags the SDK implements end to end. -_COVERED_TAGS = frozenset({"MCP", "Studios"}) +_COVERED_TAGS = frozenset({"Agent-IDP", "MCP", "Studios"}) # Tags not implemented yet. Remove an entry as its tag lands; until then the # guard would otherwise fail on operations nobody has promised. @@ -36,7 +36,6 @@ "Skills", "Files", "Collections", - "Agent-IDP", "Galleries", "Magicube", } From e8f79d2591ef8af0053b3714bb086d62ac769ecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Wed, 2 Sep 2026 03:00:12 +0800 Subject: [PATCH 06/22] update readme --- README.md | 8 ++------ src/modelscope_hub/version.py | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 719e6d0..2c5f2ea 100644 --- a/README.md +++ b/README.md @@ -33,15 +33,11 @@ The official Python SDK & CLI for [ModelScope Hub](https://modelscope.cn) — do ## News -**v0.5.0** (2026-09-02) -- **Feature**: complete Agent-IDP OpenAPI support for Ed25519 identity management, OIDC discovery/JWKS, and signed JWT issuance through `HubApi` and `ms-hub agent-idp` -- **Security**: private JWKs are never stored automatically; `keygen` writes only an explicitly requested owner-only file, while token issuance prints only the credential - **v0.4.0** (2026-09-01) -- **Feature**: full MCP/Studios OpenAPI coverage: Studio lists, variables and configuration options; hosted MCP discovery; protected visibility and runtime metadata; read-only tokens can log in and rejected writes name the required tier +- **Feature**: complete OpenAPI coverage for Agent-IDP, MCP, and Studios — Agent Ed25519 identities, OIDC discovery/JWKS and signed JWT issuance (`HubApi`, `ms-hub agent-idp`); Studio lists, variables and configuration options; hosted MCP discovery; protected visibility and runtime metadata; read-only tokens can log in and rejected writes name the required tier. Agent private JWKs are only written to an explicitly requested owner-only file. - **Fix**: Studio compat calls no longer leak connection options or API tokens, or drop cover images; errors distinguish permission, quota and conflicts; anonymous Studio info and log pagination work - **Enhance**: MCP verb negotiation and Studio-owner listings adapt to endpoint behaviour -- **Quality**: the vendored OpenAPI spec and operation registry flag unimplemented published MCP/Studios endpoints +- **Quality**: the vendored OpenAPI spec and operation registry flag unimplemented published Agent-IDP, MCP, and Studios endpoints **v0.3.1** (2026-09-01) - **Refactor**: upload environment variables now state their units and semantics; deprecated names remain supported with warnings diff --git a/src/modelscope_hub/version.py b/src/modelscope_hub/version.py index cfc4219..f710337 100644 --- a/src/modelscope_hub/version.py +++ b/src/modelscope_hub/version.py @@ -1,3 +1,3 @@ """Version information for modelscope_hub.""" -__version__ = "0.5.0" +__version__ = "0.4.0" From cb7602aa9997b10f81878b11b4ce09e01e00d628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Wed, 9 Sep 2026 15:00:55 +0800 Subject: [PATCH 07/22] bump version --- src/modelscope_hub/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modelscope_hub/version.py b/src/modelscope_hub/version.py index f710337..95f9844 100644 --- a/src/modelscope_hub/version.py +++ b/src/modelscope_hub/version.py @@ -1,3 +1,3 @@ """Version information for modelscope_hub.""" -__version__ = "0.4.0" +__version__ = "0.4.1+main" From f357a12a4fcadb0ac315b4dd68323ff1eadaaaa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Wed, 9 Sep 2026 16:37:50 +0800 Subject: [PATCH 08/22] add del files in upload manager --- src/modelscope_hub/_upload.py | 36 ++++++++++ src/modelscope_hub/api.py | 54 ++++---------- tests/integration/test_remote_file_ops.py | 4 -- tests/integration/test_sdk_api.py | 1 - tests/test_upload_lfs_gate.py | 87 ++++++++++++++++++++++- 5 files changed, 135 insertions(+), 47 deletions(-) diff --git a/src/modelscope_hub/_upload.py b/src/modelscope_hub/_upload.py index 43d49f8..cb6cd18 100644 --- a/src/modelscope_hub/_upload.py +++ b/src/modelscope_hub/_upload.py @@ -635,6 +635,42 @@ def upload_file( revision=revision, ) + # ------------------------------------------------------------------ + # Public: delete_files + # ------------------------------------------------------------------ + def delete_files( + self, + repo_id: str, + repo_type: str, + file_paths: list[str], + *, + commit_message: str = "Delete files", + revision: str = "master", + ) -> dict: + """Delete repository files through a commit operation. + + The direct repository DELETE endpoints reject API-token authentication. + Commit ``delete`` actions use the same supported write path as uploads + and are applied atomically in a single commit. + """ + paths = list(dict.fromkeys(path for path in file_paths if path)) + if not paths: + raise InvalidParameter( + "file_paths must contain at least one non-empty path.") + + self._commit_with_retry( + repo_id=repo_id, + repo_type=repo_type, + operations=self._build_delete_operations(paths), + commit_message=commit_message, + revision=revision, + ) + return { + "deleted_files": paths, + "failed_files": [], + "total_files": len(paths), + } + # ------------------------------------------------------------------ # Public: upload_folder # ------------------------------------------------------------------ diff --git a/src/modelscope_hub/api.py b/src/modelscope_hub/api.py index 5fb1996..ac22516 100644 --- a/src/modelscope_hub/api.py +++ b/src/modelscope_hub/api.py @@ -46,7 +46,6 @@ AuthenticationError, HubError, InvalidParameter, - NetworkError, NotExistError, NotSupportedError, PermissionDeniedError, @@ -1675,12 +1674,11 @@ def delete_files( commit_message: str | None = None, revision: str | None = None, ) -> dict: - """Delete one or more files from a repository. + """Delete one or more files from a repository in a single commit. - .. note:: - File deletion is restricted by the server to cookie-based session - auth (interactive login). API tokens (``ms-...``) may receive a 401 - "token no longer supports deletion operations" error. + The direct repository DELETE endpoints reject API-token authentication. + This method therefore sends commit ``delete`` actions through the same + supported write path as :meth:`upload_file` and :meth:`upload_folder`. Parameters ---------- @@ -1691,7 +1689,7 @@ def delete_files( file_paths : iterable of str Paths of files to remove. Empty entries are ignored. commit_message : str, optional - Unused (kept for API compatibility). + Message for the delete commit. Defaults to ``"Delete files"``. revision : str, optional Branch to delete from. Defaults to ``"master"``. @@ -1699,42 +1697,16 @@ def delete_files( ------- dict Summary with ``deleted_files`` and ``failed_files`` lists. - - Raises - ------ - InvalidParameter - When ``file_paths`` resolves to an empty list. - - Examples - -------- - >>> api.delete_files( - ... "alice/llama-7b", - ... "model", - ... ["old_weights.bin", "deprecated/config.json"], - ... ) """ rt = self._normalize_repo_type(repo_type) - paths = [p for p in file_paths if p] - if not paths: - raise InvalidParameter("file_paths must contain at least one non-empty path.") - - deleted, failed = [], [] - for p in paths: - try: - self.legacy.delete_file( - repo_id=repo_id, - repo_type=str(rt), - file_path=p, - revision=revision or "master", - ) - deleted.append(p) - except (AuthenticationError, NetworkError): - failed.append(p) - raise - except Exception: - failed.append(p) - - return {"deleted_files": deleted, "failed_files": failed, "total_files": len(paths)} + paths = [file_paths] if isinstance(file_paths, str) else file_paths + return self.uploader.delete_files( + repo_id=repo_id, + repo_type=str(rt), + file_paths=[path for path in paths if path], + commit_message=commit_message or "Delete files", + revision=revision or "master", + ) # ================================================================== # Versioning diff --git a/tests/integration/test_remote_file_ops.py b/tests/integration/test_remote_file_ops.py index 282bd5e..9b434ea 100644 --- a/tests/integration/test_remote_file_ops.py +++ b/tests/integration/test_remote_file_ops.py @@ -63,10 +63,6 @@ def test_download_file(self, tmp_path): assert local_path.exists() assert content == "hello modelscope" - @pytest.mark.xfail( - reason="Server restricts file deletion to cookie-based session auth; " - "API tokens get 401 'token no longer supports deletion operations'" - ) def test_delete_files(self): """delete_files removes the file from the repo.""" print(f"\n** repo_id: {self.repo_id}") diff --git a/tests/integration/test_sdk_api.py b/tests/integration/test_sdk_api.py index 769daef..223b9f1 100644 --- a/tests/integration/test_sdk_api.py +++ b/tests/integration/test_sdk_api.py @@ -211,7 +211,6 @@ def test_list_repo_files(self): paths = [f.path for f in files] assert "list_test.txt" in paths - @pytest.mark.xfail(reason="Server restricts file deletion to cookie-based session auth") def test_delete_files(self): self.api.upload_file( self.repo_id, diff --git a/tests/test_upload_lfs_gate.py b/tests/test_upload_lfs_gate.py index 7728cef..1b64ba2 100644 --- a/tests/test_upload_lfs_gate.py +++ b/tests/test_upload_lfs_gate.py @@ -6,9 +6,12 @@ from pathlib import Path from unittest.mock import MagicMock +import pytest + import modelscope_hub._upload as upload_module +from modelscope_hub import HubApi from modelscope_hub._upload import UploadManager -from modelscope_hub.errors import NetworkError +from modelscope_hub.errors import InvalidParameter, NetworkError def _make_manager() -> tuple[UploadManager, MagicMock]: @@ -27,6 +30,88 @@ def upload_blob(*, upload_url: str, data, size: int) -> None: return UploadManager(client, MagicMock()), client +@pytest.mark.parametrize("repo_type", ["model", "dataset"]) +def test_delete_files_commits_atomic_delete_actions(repo_type: str) -> None: + manager, client = _make_manager() + + result = manager.delete_files( + repo_id="owner/repo", + repo_type=repo_type, + file_paths=["old.bin", "nested/deprecated.json", "old.bin"], + commit_message="Remove obsolete files", + revision="main", + ) + + client.create_commit.assert_called_once_with( + repo_id="owner/repo", + repo_type=repo_type, + operations=[ + { + "action": "delete", + "path": "old.bin", + "type": "normal", + "size": 0, + "sha256": "", + "content": "", + "encoding": "", + }, + { + "action": "delete", + "path": "nested/deprecated.json", + "type": "normal", + "size": 0, + "sha256": "", + "content": "", + "encoding": "", + }, + ], + commit_message="Remove obsolete files", + revision="main", + ) + assert result == { + "deleted_files": ["old.bin", "nested/deprecated.json"], + "failed_files": [], + "total_files": 2, + } + + +def test_delete_files_rejects_empty_paths() -> None: + manager, client = _make_manager() + + with pytest.raises(InvalidParameter, match="at least one"): + manager.delete_files( + repo_id="owner/repo", repo_type="model", file_paths=["", ""]) + + client.create_commit.assert_not_called() + + +@pytest.mark.parametrize("repo_type", ["model", "dataset"]) +@pytest.mark.parametrize("file_paths", [["", "old.bin"], "old.bin"]) +def test_hub_api_delete_files_delegates_to_upload_manager( + repo_type: str, file_paths: list[str] | str +) -> None: + api = HubApi(token="test-token") + api._uploader = MagicMock() + api._uploader.delete_files.return_value = {"deleted_files": ["old.bin"]} + + result = api.delete_files( + "owner/repo", + repo_type, + file_paths, + commit_message="Remove obsolete files", + revision="main", + ) + + api._uploader.delete_files.assert_called_once_with( + repo_id="owner/repo", + repo_type=repo_type, + file_paths=["old.bin"], + commit_message="Remove obsolete files", + revision="main", + ) + assert result == {"deleted_files": ["old.bin"]} + + def test_upload_file_normal_commits_inline_without_blob_api() -> None: manager, client = _make_manager() From f52fd4a09137a853d8a2da5f069737b96482959b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Thu, 10 Sep 2026 10:51:31 +0800 Subject: [PATCH 09/22] fix delete files --- src/modelscope_hub/api.py | 76 ++++++++++++++++++++++++---- src/modelscope_hub/compat/hub_api.py | 30 +++++++++++ src/modelscope_hub/version.py | 2 +- tests/test_compat_delete_files.py | 54 ++++++++++++++++++++ tests/test_upload_lfs_gate.py | 55 ++++++++++++++++++++ 5 files changed, 206 insertions(+), 11 deletions(-) create mode 100644 tests/test_compat_delete_files.py diff --git a/src/modelscope_hub/api.py b/src/modelscope_hub/api.py index ac22516..00ae696 100644 --- a/src/modelscope_hub/api.py +++ b/src/modelscope_hub/api.py @@ -23,6 +23,7 @@ from __future__ import annotations +import fnmatch import time from collections.abc import Iterable, Mapping from pathlib import Path @@ -1669,16 +1670,18 @@ def delete_files( self, repo_id: str, repo_type: RepoTypeLike, - file_paths: Iterable[str], + file_paths: Iterable[str] | str | None = None, *, + delete_patterns: Iterable[str] | str | None = None, commit_message: str | None = None, revision: str | None = None, ) -> dict: - """Delete one or more files from a repository in a single commit. + """Delete repository files selected by paths or glob patterns. The direct repository DELETE endpoints reject API-token authentication. - This method therefore sends commit ``delete`` actions through the same - supported write path as :meth:`upload_file` and :meth:`upload_folder`. + This method resolves optional glob patterns against the remote file list, + then sends commit ``delete`` actions through the same supported write + path as :meth:`upload_file` and :meth:`upload_folder`. Parameters ---------- @@ -1686,8 +1689,11 @@ def delete_files( Canonical ``owner/name`` identifier. repo_type : str or RepoType Repository type. - file_paths : iterable of str - Paths of files to remove. Empty entries are ignored. + file_paths : iterable of str or str, optional + Explicit repository-relative paths to remove. + delete_patterns : iterable of str or str, optional + Glob patterns matched against remote file paths. For example, + ``"*.json"`` matches JSON files at any repository depth. commit_message : str, optional Message for the delete commit. Defaults to ``"Delete files"``. revision : str, optional @@ -1696,18 +1702,68 @@ def delete_files( Returns ------- dict - Summary with ``deleted_files`` and ``failed_files`` lists. + Summary with ``deleted_files``, ``failed_files``, and ``total_files``. + A pattern with no remote matches returns an empty successful summary. """ rt = self._normalize_repo_type(repo_type) - paths = [file_paths] if isinstance(file_paths, str) else file_paths + paths = self._normalize_delete_values(file_paths, "file_paths") + patterns = self._normalize_delete_values( + delete_patterns, "delete_patterns") + if not paths and not patterns: + raise InvalidParameter( + "Provide at least one file path or delete pattern.") + + resolved_revision = revision or "master" + if patterns: + remote_paths = [ + file.path + for file in self.list_repo_files( + repo_id, + rt, + revision=resolved_revision, + recursive=True, + ) + if file.path and file.type != "tree" + ] + paths.extend( + path for path in remote_paths + if any(fnmatch.fnmatchcase(path, pattern) + for pattern in patterns)) + + paths = list(dict.fromkeys(paths)) + if not paths: + return { + "deleted_files": [], + "failed_files": [], + "total_files": 0, + } + return self.uploader.delete_files( repo_id=repo_id, repo_type=str(rt), - file_paths=[path for path in paths if path], + file_paths=paths, commit_message=commit_message or "Delete files", - revision=revision or "master", + revision=resolved_revision, ) + @staticmethod + def _normalize_delete_values( + values: Iterable[str] | str | None, + parameter_name: str, + ) -> list[str]: + """Normalize a delete path or glob-pattern argument.""" + if values is None: + return [] + raw_values = [values] if isinstance(values, str) else list(values) + normalized: list[str] = [] + for value in raw_values: + if not isinstance(value, str): + raise InvalidParameter( + f"{parameter_name} must contain only strings.") + if value: + normalized.append(value) + return normalized + # ================================================================== # Versioning # ================================================================== diff --git a/src/modelscope_hub/compat/hub_api.py b/src/modelscope_hub/compat/hub_api.py index 8ed9629..f011f91 100644 --- a/src/modelscope_hub/compat/hub_api.py +++ b/src/modelscope_hub/compat/hub_api.py @@ -195,6 +195,36 @@ def get_model_files( result = [f for f in result if f["Path"] == prefix or str(f["Path"]).startswith(prefix + "/")] return result + def delete_files( + self, + repo_id: str, + repo_type: str | RepoType = RepoType.MODEL, + delete_patterns: str | list[str] | None = None, + *, + file_paths: str | list[str] | None = None, + revision: str | None = DEFAULT_DATASET_REVISION, + commit_message: str | None = None, + endpoint: str | None = None, + token: str | None = None, + ) -> dict: + """Delete files selected by legacy glob patterns or explicit paths. + + ``delete_patterns`` preserves the historical ``modelscope.hub.api`` + contract. Patterns are resolved against the remote repository by the + modelscope-hub facade before it commits atomic delete actions. + """ + api = self._api + if token or endpoint: + api = HubApi(endpoint=endpoint or self._endpoint, token=token) + return api.delete_files( + repo_id, + repo_type, + file_paths=file_paths, + delete_patterns=delete_patterns, + commit_message=commit_message, + revision=revision, + ) + def create_repo( self, repo_id: str, diff --git a/src/modelscope_hub/version.py b/src/modelscope_hub/version.py index 95f9844..e82eace 100644 --- a/src/modelscope_hub/version.py +++ b/src/modelscope_hub/version.py @@ -1,3 +1,3 @@ """Version information for modelscope_hub.""" -__version__ = "0.4.1+main" +__version__ = "0.4.2+main" diff --git a/tests/test_compat_delete_files.py b/tests/test_compat_delete_files.py new file mode 100644 index 0000000..dfa2104 --- /dev/null +++ b/tests/test_compat_delete_files.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from unittest import mock + +from modelscope_hub.compat import LegacyHubApi +from modelscope_hub.constants import RepoType + + +def test_legacy_hub_api_preserves_delete_patterns_contract() -> None: + api = LegacyHubApi(endpoint="https://modelscope.cn", token="test-token") + expected = { + "deleted_files": ["config.json"], + "failed_files": [], + "total_files": 1, + } + + with mock.patch.object(api._api, "delete_files", return_value=expected) as delete_files: + result = api.delete_files( + repo_id="owner/repo", + repo_type=RepoType.MODEL, + delete_patterns="*.json", + revision="main", + commit_message="Delete JSON files", + ) + + delete_files.assert_called_once_with( + "owner/repo", + RepoType.MODEL, + file_paths=None, + delete_patterns="*.json", + commit_message="Delete JSON files", + revision="main", + ) + assert result == expected + + +def test_legacy_hub_api_delete_files_supports_explicit_paths() -> None: + api = LegacyHubApi(token="test-token") + + with mock.patch.object(api._api, "delete_files", return_value={}) as delete_files: + api.delete_files( + "owner/repo", + repo_type="dataset", + file_paths=["data/old.json"], + ) + + delete_files.assert_called_once_with( + "owner/repo", + "dataset", + file_paths=["data/old.json"], + delete_patterns=None, + commit_message=None, + revision="master", + ) diff --git a/tests/test_upload_lfs_gate.py b/tests/test_upload_lfs_gate.py index 1b64ba2..6532b8c 100644 --- a/tests/test_upload_lfs_gate.py +++ b/tests/test_upload_lfs_gate.py @@ -4,6 +4,7 @@ import hashlib import json from pathlib import Path +from types import SimpleNamespace from unittest.mock import MagicMock import pytest @@ -112,6 +113,60 @@ def test_hub_api_delete_files_delegates_to_upload_manager( assert result == {"deleted_files": ["old.bin"]} +@pytest.mark.parametrize("repo_type", ["model", "dataset"]) +def test_hub_api_delete_patterns_resolve_remote_paths(repo_type: str) -> None: + api = HubApi(token="test-token") + api._uploader = MagicMock() + api._uploader.delete_files.return_value = { + "deleted_files": ["config.json", "nested/metadata.json"], + "failed_files": [], + "total_files": 2, + } + api.list_repo_files = MagicMock( + return_value=[ + SimpleNamespace(path="config.json", type="blob"), + SimpleNamespace(path="nested/metadata.json", type="blob"), + SimpleNamespace(path="weights.bin", type="blob"), + SimpleNamespace(path="nested", type="tree"), + ]) + + result = api.delete_files( + "owner/repo", + repo_type, + delete_patterns="*.json", + commit_message="Remove JSON files", + revision="main", + ) + + api.list_repo_files.assert_called_once_with( + "owner/repo", repo_type, revision="main", recursive=True) + api._uploader.delete_files.assert_called_once_with( + repo_id="owner/repo", + repo_type=repo_type, + file_paths=["config.json", "nested/metadata.json"], + commit_message="Remove JSON files", + revision="main", + ) + assert result["total_files"] == 2 + + +def test_hub_api_delete_patterns_with_no_match_is_noop() -> None: + api = HubApi(token="test-token") + api._uploader = MagicMock() + api.list_repo_files = MagicMock( + return_value=[SimpleNamespace(path="weights.bin", type="blob")]) + + result = api.delete_files( + "owner/repo", "model", delete_patterns="*.json") + + api._uploader.delete_files.assert_not_called() + assert result == { + "deleted_files": [], + "failed_files": [], + "total_files": 0, + } + + def test_upload_file_normal_commits_inline_without_blob_api() -> None: manager, client = _make_manager() From c413dbd81117aa072f64301876f9daf2b1eb99aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Thu, 17 Sep 2026 22:31:50 +0800 Subject: [PATCH 10/22] feat(upload): route small files to LFS, size commits by bytes, honor server limits Uploading a large dataset of small files was dominated by commit count and commit body size, not by bandwidth. Non-LFS files travel base64-inlined inside the commit request, so a 60k-file corpus put gigabytes through the API servers and forced tiny batches, multiplying commits against the per-repo hourly commit budget. Upload path: - LFS threshold becomes byte-level (MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD, accepts 32KiB/512K/1MiB/0); the MB-only name stays as a deprecated alias. Default is unchanged at 1MiB. - UPLOAD_INLINE_METADATA_PATHS keeps README.md, .gitattributes and the configuration files inline whatever the threshold, so lowering it can never turn a file the Hub parses into an LFS pointer. - Commit batches are planned on two axes: operation count and inlined content volume (UPLOAD_COMMIT_MAX_INLINE_BYTES, 8MiB). Adaptive sizing no longer silently overrides an explicit cap, and both are clamped to the server's COMMIT_MAX_ACTIONS_PER_REQUEST ceiling, which is rejected rather than trimmed. - delete_files splits at the same ceiling instead of sending every path at once. - All LFS candidates are hashed up front and pre-signed in parallel groups, replacing one round trip per file. A group that fails degrades to per-file negotiation; an oid absent from a *failed* group is never read as "already exists", which would skip the transfer and commit a dangling pointer. Transport and resilience: - Both HTTP sessions size their pool from API_CONNECTION_POOL_MAXSIZE (32). urllib3's default of 10 discarded connections under the upload worker count and paid for a fresh TLS handshake on each. - Commit retries honor a server Retry-After, budgeted apart from the transient-error allowance, plus an opt-in per-hour commit governor shared by every commit path. upload_file now uses that path too. - A tree listing denied with 403 after the credential has been proven by an earlier successful read is retried; a large listing spans hundreds of requests, so a per-request denial rate otherwise guarantees whole-listing failure. Observability: - upload_folder takes progress_callback and emits per-batch commit events plus throttled wire-level upload_progress events carrying byte counts, so a consumer can build a rate and an ETA instead of watching a silent run. - upload_folder takes tracker_path, so resume state can live outside a staging tree that gets discarded between runs. - Invalid environment values now warn and name the fallback instead of reverting silently. Verified end to end against the Hub: 25000 files / 5.00 GiB and 10000 files / 2.00 GiB, both 0 failures, sha256 spot-checks identical, commit count 254 -> 49 and inlined commit volume 1.96GiB -> 55MiB on the 5 GiB corpus. --- src/modelscope_hub/_legacy_api.py | 92 +++- src/modelscope_hub/_openapi.py | 20 +- src/modelscope_hub/_upload.py | 655 ++++++++++++++++++++---- src/modelscope_hub/agent/_api.py | 2 +- src/modelscope_hub/api.py | 18 +- src/modelscope_hub/constants.py | 317 +++++++++++- src/modelscope_hub/version.py | 2 +- tests/test_commit_action_limit.py | 110 ++++ tests/test_repo_tree_page_retry.py | 143 ++++++ tests/test_upload_batching.py | 785 +++++++++++++++++++++++++++++ tests/test_upload_config.py | 140 +++++ tests/test_upload_lfs_gate.py | 19 +- 12 files changed, 2185 insertions(+), 118 deletions(-) create mode 100644 tests/test_commit_action_limit.py create mode 100644 tests/test_repo_tree_page_retry.py create mode 100644 tests/test_upload_batching.py diff --git a/src/modelscope_hub/_legacy_api.py b/src/modelscope_hub/_legacy_api.py index 2e70d52..aa3d217 100644 --- a/src/modelscope_hub/_legacy_api.py +++ b/src/modelscope_hub/_legacy_api.py @@ -14,6 +14,7 @@ from __future__ import annotations +import time import uuid from concurrent.futures import ThreadPoolExecutor, as_completed from typing import IO, Any, BinaryIO @@ -24,18 +25,28 @@ from .constants import ( API_CONNECT_TIMEOUT, + API_CONNECTION_POOL_MAXSIZE, API_MAX_RETRIES, API_TIMEOUT, LEGACY_API_PREFIX, REPO_FILES_TRUNCATION_LIMIT, REPO_TREE_MAX_REQUESTS, + REPO_TREE_PAGE_MAX_ATTEMPTS, + REPO_TREE_PAGE_RETRY_MAX_DELAY_SECONDS, REPO_TREE_WALK_WORKERS, UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS, UPLOAD_BLOB_READ_TIMEOUT_SECONDS, UPLOAD_HTTP_RETRY_ALLOWED_METHODS, RepoType, ) -from .errors import InvalidParameter, NetworkError, RequestTimeoutError, ServerError, raise_for_status +from .errors import ( + InvalidParameter, + NetworkError, + PermissionDeniedError, + RequestTimeoutError, + ServerError, + raise_for_status, +) from .utils.logger import get_logger logger = get_logger("legacy_api") @@ -100,6 +111,10 @@ def __init__( self._endpoint = endpoint.rstrip("/") self._timeout: int | tuple[int, int] = (API_CONNECT_TIMEOUT, timeout) self._session_authenticated = False + # Set once any file-tree read succeeds. From then on a 403 on a tree + # request cannot be an authorization result, so it is retried instead of + # aborting an enumeration that spans hundreds of requests. + self._tree_reads_ok = False self._session = requests.Session() if user_agent: @@ -110,7 +125,11 @@ def __init__( status_forcelist=[429, 500, 502, 503, 504], allowed_methods=UPLOAD_HTTP_RETRY_ALLOWED_METHODS, ) - adapter = HTTPAdapter(max_retries=retry) + adapter = HTTPAdapter( + max_retries=retry, + pool_connections=API_CONNECTION_POOL_MAXSIZE, + pool_maxsize=API_CONNECTION_POOL_MAXSIZE, + ) self._session.mount("https://", adapter) self._session.mount("http://", adapter) @@ -394,7 +413,12 @@ def _list_files_page( params["Root"] = root suffix = "repo/tree" if _is_dataset(repo_type) else "repo/files" - resp = self._request("GET", f"{segment}/{repo_id}/{suffix}", params=params) + resp = self._request_repo_tree( + f"{segment}/{repo_id}/{suffix}", + params, + repo_id=repo_id, + authorized=self._tree_reads_ok, + ) data = self._json_data(resp) if isinstance(data, list): return data @@ -567,6 +591,13 @@ def list_dataset_files_paginated( Datasets can have millions of files, so this method pages through ``GET /api/v1/datasets/{repo_id}/repo/tree`` with ``PageNumber``/``PageSize`` params. + + A page occasionally answers ``403 无权访问该数据集`` on a repository the + caller demonstrably can read. Once any page has succeeded the credential + is proven, so a later-page denial is a server-side hiccup rather than an + authorization result, and it is retried instead of discarding every page + collected so far -- a large listing spans hundreds of pages, which makes + hitting it near-certain. """ all_files: list[dict] = [] page_number = 1 @@ -579,10 +610,11 @@ def list_dataset_files_paginated( } if root_path and root_path != "/": params["Root"] = root_path - resp = self._request( - "GET", + resp = self._request_repo_tree( f"datasets/{repo_id}/repo/tree", - params=params, + params, + repo_id=repo_id, + authorized=self._tree_reads_ok, ) data = self._json_data(resp) if isinstance(data, list): @@ -598,6 +630,54 @@ def list_dataset_files_paginated( page_number += 1 return all_files + def _request_repo_tree( + self, + path: str, + params: dict[str, Any], + *, + repo_id: str, + authorized: bool, + ) -> Any: + """Fetch one file-tree listing, retrying a spurious denial. + + The server intermittently answers a tree request with ``403 无权访问该数据 + 集`` on a repository the caller has just read successfully -- observed on + both ``PageNumber``-paginated and ``Root``-scoped listings. Enumerating a + large repository takes hundreds of such requests, so at that scale a + single-request failure rate is effectively a guaranteed whole-listing + failure, and it discards every entry gathered so far. + + ``authorized`` means some tree read already succeeded on this client, so + the credential is proven and a denial cannot be an authorization result. + Until then a 403 is taken at face value, keeping a real permission error + fast and honest. + """ + last_error: PermissionDeniedError | None = None + for attempt in range(REPO_TREE_PAGE_MAX_ATTEMPTS): + try: + resp = self._request("GET", path, params=params) + except PermissionDeniedError as error: + if not authorized: + raise + last_error = error + if attempt < REPO_TREE_PAGE_MAX_ATTEMPTS - 1: + wait = min(2**attempt, REPO_TREE_PAGE_RETRY_MAX_DELAY_SECONDS) + logger.warning( + "Repo %s: tree listing (%s) denied on an already-authorized repo, retrying in %ds ...", + repo_id, + params.get("Root") or params.get("PageNumber") or "/", + wait, + ) + time.sleep(wait) + continue + self._tree_reads_ok = True + return resp + raise NetworkError( + f"Repo {repo_id}: tree listing ({params.get('Root') or params.get('PageNumber') or '/'}) kept " + f"returning a denial after {REPO_TREE_PAGE_MAX_ATTEMPTS} attempts on an already-authorized " + f"repo: {last_error}" + ) from last_error + # ------------------------------------------------------------------ # Revisions # ------------------------------------------------------------------ diff --git a/src/modelscope_hub/_openapi.py b/src/modelscope_hub/_openapi.py index 93a71b1..ebdd71f 100644 --- a/src/modelscope_hub/_openapi.py +++ b/src/modelscope_hub/_openapi.py @@ -30,9 +30,17 @@ from urllib.parse import urljoin, urlsplit import requests +from requests.adapters import HTTPAdapter from .config import HubConfig, get_default_config -from .constants import API_CONNECT_TIMEOUT, API_MAX_RETRIES, API_TIMEOUT, OPENAPI_PREFIX, TokenScope +from .constants import ( + API_CONNECT_TIMEOUT, + API_CONNECTION_POOL_MAXSIZE, + API_MAX_RETRIES, + API_TIMEOUT, + OPENAPI_PREFIX, + TokenScope, +) from .errors import ( APIError, AuthenticationError, @@ -200,6 +208,16 @@ def __init__( ) -> None: self._config = config or get_default_config() self._session = session or requests.Session() + if session is None: + # A bare Session caps the pool at urllib3's default of 10, which is + # below the concurrency bulk transfers use; the excess connections + # are discarded and pay for a new TLS handshake on next use. + adapter = HTTPAdapter( + pool_connections=API_CONNECTION_POOL_MAXSIZE, + pool_maxsize=API_CONNECTION_POOL_MAXSIZE, + ) + self._session.mount("https://", adapter) + self._session.mount("http://", adapter) self._timeout: float | tuple[float, float] = ( float(timeout) if timeout is not None else (float(API_CONNECT_TIMEOUT), float(API_TIMEOUT)) ) diff --git a/src/modelscope_hub/_upload.py b/src/modelscope_hub/_upload.py index cb6cd18..19749a9 100644 --- a/src/modelscope_hub/_upload.py +++ b/src/modelscope_hub/_upload.py @@ -30,6 +30,8 @@ from tqdm.auto import tqdm from .constants import ( + API_CONNECTION_POOL_MAXSIZE, + COMMIT_MAX_ACTIONS_PER_REQUEST, DATASET_LFS_SUFFIX, DEFAULT_IGNORE_PATTERNS, MODEL_LFS_SUFFIX, @@ -44,8 +46,12 @@ UPLOAD_COMMIT_BATCH_MAX_OPERATIONS, UPLOAD_COMMIT_MAX_ATTEMPTS, UPLOAD_COMMIT_MAX_CONSECUTIVE_FAILED_BATCHES, + UPLOAD_COMMIT_MAX_INLINE_BYTES, + UPLOAD_COMMIT_MAX_PER_HOUR, + UPLOAD_COMMIT_MAX_RETRY_AFTER_SECONDS, UPLOAD_COMMIT_RETRY_TOTAL_WAIT_SECONDS, UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS, + UPLOAD_INLINE_METADATA_PATHS, UPLOAD_LEGACY_PROGRESS_FILE, UPLOAD_LFS_FORCE_THRESHOLD_BYTES, UPLOAD_MAX_CONCURRENT_WORKERS, @@ -53,6 +59,7 @@ UPLOAD_MAX_FILE_SIZE_BYTES, UPLOAD_MAX_FILES_PER_DIRECTORY, UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES, + UPLOAD_PROGRESS_MIN_INTERVAL_SECONDS, UPLOAD_RECOVERY_BACKOFF_MAX_EXPONENT, UPLOAD_RECOVERY_ENABLED, UPLOAD_RECOVERY_MAX_DELAY_SECONDS, @@ -64,6 +71,7 @@ HubError, InvalidParameter, NetworkError, + RateLimitError, StorageError, ) from .utils.file_utils import compute_hash @@ -119,8 +127,21 @@ def verify_complete(self) -> None: ) +def _is_inline_metadata(path: str | Path) -> bool: + """Return whether *path* must stay inline in the commit rather than go to LFS. + + Checked before the size and suffix rules so that lowering the LFS threshold + can never turn a repository file the Hub parses server-side into a pointer. + """ + if not isinstance(path, (str, Path)): + return False + return Path(path).name.upper() in UPLOAD_INLINE_METADATA_PATHS + + def _is_lfs(path: str | Path, size: int, repo_type: str) -> bool: """Determine if a file should use LFS upload mode (suffix + size threshold).""" + if _is_inline_metadata(path): + return False if size > UPLOAD_LFS_FORCE_THRESHOLD_BYTES: return True suffix = Path(path).suffix.lower() if isinstance(path, (str, Path)) else "" @@ -136,15 +157,68 @@ def _upload_mode(path: str | Path, size: int, repo_type: str) -> str: return "lfs" if _is_lfs(path, size, repo_type) else "normal" -def _calculate_adaptive_batch_size(total_files: int) -> int: - """Calculate optimal commit batch size based on total file count.""" +def _calculate_adaptive_batch_size(total_files: int, max_operations: int) -> int: + """Commit batch size from the file count alone, capped by *max_operations*. + + Fewer, fuller commits are strictly better: the Hub throttles commits per + repository, so commit count -- not commit size -- is the scarce resource. + The only reason to stop growing a batch is the operation cap, or the inlined + content limit that :func:`_plan_commit_batches` applies on top of this. + + The cap is itself clamped to :data:`COMMIT_MAX_ACTIONS_PER_REQUEST`, which + the server rejects outright rather than truncating. + """ if total_files <= 0: return 1 - if total_files <= 100: - return total_files - if total_files <= 10_000: - return max(64, min(256, total_files // 80)) - return 512 + ceiling = max(1, COMMIT_MAX_ACTIONS_PER_REQUEST) + cap = max_operations if max_operations > 0 else total_files + return max(1, min(cap, ceiling, total_files)) + + +def _plan_commit_batches( + files: list[tuple[str, str]], + repo_type: str, + *, + max_operations: int, + max_inline_bytes: int, + sizes: dict[str, int] | None = None, +) -> list[int]: + """Split *files* into commit batches, returning each batch's file count. + + Two limits close a batch, whichever is reached first: ``max_operations`` + files, or ``max_inline_bytes`` of content that will ride *inside* the commit + request. Only non-LFS files contribute to the byte total -- an LFS file adds + a fixed-size pointer -- and base64 expansion is accounted for, because the + request carries the encoded form. A batch always holds at least one file, so + a single oversized inline file still makes progress instead of deadlocking. + """ + if not files: + return [] + cap = max_operations if max_operations > 0 else len(files) + cap = max(1, min(cap, max(1, COMMIT_MAX_ACTIONS_PER_REQUEST))) + + batches: list[int] = [] + count = 0 + inline_bytes = 0 + for path_in_repo, file_path in files: + size = sizes.get(file_path, 0) if sizes is not None else _safe_size(file_path) + encoded = 0 if _is_lfs(path_in_repo, size, repo_type) else (size + 2) // 3 * 4 + if count > 0 and (count >= cap or (max_inline_bytes > 0 and inline_bytes + encoded > max_inline_bytes)): + batches.append(count) + count = 0 + inline_bytes = 0 + count += 1 + inline_bytes += encoded + if count: + batches.append(count) + return batches + + +def _safe_size(file_path: str) -> int: + try: + return os.stat(file_path).st_size + except OSError: + return 0 def _compute_file_hash( @@ -482,18 +556,38 @@ def clear(self) -> None: class BatchTracker: - """Thread-safe tracker for pre-assigned upload batches.""" + """Thread-safe tracker for pre-assigned upload batches. + + Batch sizes are supplied as a plan rather than a single number so that a + batch can be closed on inlined-content volume as well as on file count. + """ + + def __init__(self, total_files: int, batch_sizes: list[int] | int) -> None: + if isinstance(batch_sizes, int): + step = max(1, batch_sizes) + sizes = [min(step, total_files - start) for start in range(0, total_files, step)] + else: + sizes = [size for size in batch_sizes if size > 0] + assigned = sum(sizes) + if assigned < total_files: + # Never drop files: a short plan gets the remainder as a final batch. + sizes.append(total_files - assigned) + self._batch_sizes = sizes + self._num_batches = len(sizes) + + # file index -> batch index, so a completed upload can find its batch + # without assuming batches are uniform. + self._owner: list[int] = [] + self._batch_start: list[int] = [] + offset = 0 + for batch_idx, size in enumerate(sizes): + self._batch_start.append(offset) + self._owner.extend([batch_idx] * size) + offset += size - def __init__(self, total_files: int, batch_size: int) -> None: - self._batch_size = batch_size - self._num_batches = (total_files - 1) // batch_size + 1 if total_files > 0 else 0 self._batch_results: list[list[dict]] = [[] for _ in range(self._num_batches)] self._batch_failures: list[list[tuple]] = [[] for _ in range(self._num_batches)] - self._batch_expected: list[int] = [] - for i in range(self._num_batches): - start = i * batch_size - end = min(start + batch_size, total_files) - self._batch_expected.append(end - start) + self._batch_expected: list[int] = list(sizes) self._batch_events: list[threading.Event] = [threading.Event() for _ in range(self._num_batches)] self._lock = threading.Lock() @@ -501,8 +595,13 @@ def __init__(self, total_files: int, batch_size: int) -> None: def num_batches(self) -> int: return self._num_batches + def batch_range(self, batch_idx: int) -> tuple[int, int]: + """Return the ``[start, end)`` file-index range owned by *batch_idx*.""" + start = self._batch_start[batch_idx] + return start, start + self._batch_sizes[batch_idx] + def batch_index(self, file_index: int) -> int: - return file_index // self._batch_size + return self._owner[file_index] def record_success(self, file_index: int, result: dict) -> None: idx = self.batch_index(file_index) @@ -538,6 +637,49 @@ def _is_batch_complete(self, batch_idx: int) -> bool: return count >= self._batch_expected[batch_idx] +class _CommitRateGovernor: + """Sliding-window limiter that keeps commits under a per-hour budget. + + Reacting to a throttle costs a wasted round trip, and when the server holds + the connection open instead of answering, a full read timeout. Pacing ahead + of the limit avoids both. Disabled when the budget is not positive, so it is + inert for interactive uploads that never approach the ceiling. + """ + + _WINDOW_SECONDS = 3600.0 + + def __init__(self, max_per_hour: int) -> None: + self._max_per_hour = max_per_hour + self._timestamps: list[float] = [] + self._lock = threading.Lock() + + @property + def enabled(self) -> bool: + return self._max_per_hour > 0 + + def acquire(self) -> float: + """Block until a commit slot is free; return the seconds spent waiting.""" + if not self.enabled: + return 0.0 + waited = 0.0 + while True: + with self._lock: + now = time.monotonic() + cutoff = now - self._WINDOW_SECONDS + self._timestamps = [ts for ts in self._timestamps if ts > cutoff] + if len(self._timestamps) < self._max_per_hour: + self._timestamps.append(now) + return waited + sleep_for = self._timestamps[0] - cutoff + logger.info( + "Commit budget reached (%d/hour), pausing %.0fs before the next commit ...", + self._max_per_hour, + sleep_for, + ) + time.sleep(max(sleep_for, 0.1)) + waited += max(sleep_for, 0.1) + + # ==================================================================== # Upload Manager # ==================================================================== @@ -558,6 +700,12 @@ def __init__( self._config = config self._openapi = openapi_client self._create_repo_fn = create_repo_fn + # The commit budget is a server-side property of the repository, not of + # one call, so the governor is shared by every commit this manager makes + # -- batch commits, recovery rounds, sync deletes and single files alike. + # Pacing only the happy path would leave recovery free to hammer a + # server that is already throttling. + self._commit_governor = _CommitRateGovernor(UPLOAD_COMMIT_MAX_PER_HOUR) # ------------------------------------------------------------------ # Public: upload_file @@ -627,7 +775,10 @@ def upload_file( ) print(f"Committing file to {repo_id} ...", flush=True) - return self._client.create_commit( + # Same commit path as folder uploads: a single file gets the transient + # retry and the Retry-After handling too. Committing directly meant a + # throttled or briefly unavailable server failed the call outright. + return self._commit_with_retry( repo_id=repo_id, repo_type=repo_type, operations=[operation], @@ -650,25 +801,55 @@ def delete_files( """Delete repository files through a commit operation. The direct repository DELETE endpoints reject API-token authentication. - Commit ``delete`` actions use the same supported write path as uploads - and are applied atomically in a single commit. + Commit ``delete`` actions use the same supported write path as uploads. + + A single commit is capped by the server at + :data:`COMMIT_MAX_ACTIONS_PER_REQUEST` actions, so a larger request is + split across sequential commits. Deletions within one commit are atomic; + across a split they are not, and a failure part-way leaves the earlier + commits applied -- which is reported rather than hidden, so the caller can + retry with the remaining paths. """ paths = list(dict.fromkeys(path for path in file_paths if path)) if not paths: - raise InvalidParameter( - "file_paths must contain at least one non-empty path.") + raise InvalidParameter("file_paths must contain at least one non-empty path.") + + chunk_size = max(1, COMMIT_MAX_ACTIONS_PER_REQUEST) + chunks = [paths[i : i + chunk_size] for i in range(0, len(paths), chunk_size)] + if len(chunks) > 1: + logger.info( + "Deleting %d file(s) in %d commit(s) (server caps one commit at %d actions).", + len(paths), + len(chunks), + chunk_size, + ) + + deleted: list[str] = [] + for index, chunk in enumerate(chunks): + message = commit_message if len(chunks) == 1 else f"{commit_message} ({index + 1}/{len(chunks)})" + try: + self._commit_with_retry( + repo_id=repo_id, + repo_type=repo_type, + operations=self._build_delete_operations(chunk), + commit_message=message, + revision=revision, + ) + except Exception: + if deleted: + logger.error( + "Delete commit %d/%d failed after %d file(s) were already removed.", + index + 1, + len(chunks), + len(deleted), + ) + raise + deleted.extend(chunk) - self._commit_with_retry( - repo_id=repo_id, - repo_type=repo_type, - operations=self._build_delete_operations(paths), - commit_message=commit_message, - revision=revision, - ) return { - "deleted_files": paths, + "deleted_files": deleted, "failed_files": [], - "total_files": len(paths), + "total_files": len(deleted), } # ------------------------------------------------------------------ @@ -690,6 +871,8 @@ def upload_folder( use_cache: bool | None = None, disable_tqdm: bool = False, sync_remote_repo: bool = False, + tracker_path: str | Path | None = None, + progress_callback: Any = None, ) -> dict | list[dict] | None: """Upload a folder with resumable support, adaptive batching, and retry.""" start_time = time.time() @@ -727,12 +910,14 @@ def upload_folder( # Collect files logger.info("Preparing files to upload ...") + file_sizes: dict[str, int] = {} sorted_files = self._prepare_upload_folder( folder_path=folder_path, path_in_repo=path_in_repo, repo_type=repo_type, allow_patterns=allow_patterns, ignore_patterns=ignore_patterns, + sizes_out=file_sizes, ) # For sync mode: collect ALL local files (unfiltered) to avoid @@ -760,27 +945,40 @@ def upload_folder( # Sort for deterministic batch assignment sorted_files = sorted(sorted_files, key=lambda x: x[0]) - # Calculate batch size - if UPLOAD_ADAPTIVE_BATCHING_ENABLED: - commit_batch_size = _calculate_adaptive_batch_size(len(sorted_files)) - logger.info( - "Adaptive batch size: %d (for %d files)", - commit_batch_size, - len(sorted_files), - ) - else: - commit_batch_size = ( - UPLOAD_COMMIT_BATCH_MAX_OPERATIONS if UPLOAD_COMMIT_BATCH_MAX_OPERATIONS > 0 else len(sorted_files) - ) + # Plan commit batches. The operation cap bounds the file count; the + # inline-content cap bounds how many bytes a commit body carries, which + # only non-LFS files add to. + max_operations = ( + _calculate_adaptive_batch_size(len(sorted_files), UPLOAD_COMMIT_BATCH_MAX_OPERATIONS) + if UPLOAD_ADAPTIVE_BATCHING_ENABLED + else (UPLOAD_COMMIT_BATCH_MAX_OPERATIONS if UPLOAD_COMMIT_BATCH_MAX_OPERATIONS > 0 else len(sorted_files)) + ) + batch_plan = _plan_commit_batches( + sorted_files, + repo_type, + max_operations=max_operations, + max_inline_bytes=UPLOAD_COMMIT_MAX_INLINE_BYTES, + sizes=file_sizes, + ) + logger.info( + "Commit plan: %d batch(es) for %d file(s) (max %d ops, max %d inline bytes per commit).", + len(batch_plan), + len(sorted_files), + max_operations, + UPLOAD_COMMIT_MAX_INLINE_BYTES, + ) - # Initialize tracker + # Initialize tracker. The cache normally lives in the uploaded folder, + # but a caller that stages files into a throwaway tree (a link tree, a + # per-chunk directory) must be able to keep it outside, or every run + # rediscovers hashes and re-commits what was already committed. folder_path_resolved = Path(folder_path).resolve() if use_cache: - cache_path = folder_path_resolved / UPLOAD_CACHE_FILE + cache_path = Path(tracker_path).expanduser() if tracker_path else folder_path_resolved / UPLOAD_CACHE_FILE tracker: UploadTracker | NullTracker = UploadTracker(cache_path, repo_id=repo_id) else: tracker = NullTracker() - batch_tracker = BatchTracker(len(sorted_files), commit_batch_size) + batch_tracker = BatchTracker(len(sorted_files), batch_plan) # Skip individually committed files files_to_upload: list[tuple[int, tuple[str, str]]] = [] @@ -800,38 +998,63 @@ def upload_folder( ) files_to_upload.append((file_idx, (file_path_in_repo, file_path))) - # Batch pre-validation for LFS files with cached hashes + # Batch pre-validation for every LFS candidate. + # + # Without a cached hash the per-file upload path would ask the git-lfs + # batch endpoint for its own pre-signed URL, one round trip per file. + # Hashing up front lets all candidates be pre-signed in groups instead, + # which is the difference between one request per file and one per group + # once small files are routed to LFS. pre_validated_map: dict[str, str | None] = {} - lfs_hash_info_map: dict[int, tuple[dict, os.stat_result]] = {} + lfs_hash_info_map: dict[int, dict] = {} for file_idx, (file_path_in_repo, file_path) in files_to_upload: + size = file_sizes.get(file_path, 0) + if _upload_mode(file_path_in_repo, size, repo_type) != "lfs": + continue try: st = os.stat(file_path) - cached = tracker.get_hash(file_path_in_repo, st.st_mtime, st.st_size) - if cached is not None: - if ( - _upload_mode( - file_path_in_repo, - cached["file_size"], - repo_type, - ) - == "lfs" - ): - lfs_hash_info_map[file_idx] = (cached, st) - continue except OSError: - pass + continue + cached = tracker.get_hash(file_path_in_repo, st.st_mtime, st.st_size) + if cached is None: + continue + if _upload_mode(file_path_in_repo, cached["file_size"], repo_type) == "lfs": + lfs_hash_info_map[file_idx] = cached + + uncached_lfs = [ + (file_idx, file_info) + for file_idx, file_info in files_to_upload + if file_idx not in lfs_hash_info_map + and _upload_mode(file_info[0], file_sizes.get(file_info[1], 0), repo_type) == "lfs" + ] + if uncached_lfs: + lfs_hash_info_map.update(self._hash_files_parallel(uncached_lfs, tracker, max_workers)) if lfs_hash_info_map: - objects = [{"oid": info["file_hash"], "size": info["file_size"]} for info, _ in lfs_hash_info_map.values()] - validated = self._validate_blobs_batch(repo_id=repo_id, repo_type=repo_type, objects=objects) + objects = [{"oid": info["file_hash"], "size": info["file_size"]} for info in lfs_hash_info_map.values()] + # Identical content shares one oid, so the request set is keyed by + # digest rather than by file. + unique_objects = list({obj["oid"]: obj for obj in objects}.values()) + validated = self._validate_blobs_batch( + repo_id=repo_id, + repo_type=repo_type, + objects=unique_objects, + max_workers=max_workers, + ) pre_validated_map = validated reused = sum(1 for v in validated.values() if v is None) + unresolved = len(unique_objects) - len(validated) logger.info( - "Pre-validated %d cached LFS hash(es): %d globally existing, %d need upload.", + "Pre-validated %d/%d distinct blob(s) for %d file(s) in %d request(s): " + "%d already stored, %d to upload%s.", + len(validated), + len(unique_objects), len(objects), + -(-len(unique_objects) // max(1, UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS)), reused, - len(objects) - reused, + len(validated) - reused, + f", {unresolved} unresolved (will negotiate per file)" if unresolved else "", ) skipped_count = len(skipped_indices) @@ -846,10 +1069,9 @@ def upload_folder( ) logger.info( - "Uploading %d file(s) in %d batch(es) of size %d (pipeline mode).", + "Uploading %d file(s) in %d batch(es) (pipeline mode).", len(files_to_upload), batch_tracker.num_batches, - commit_batch_size, ) # Pipeline: upload workers @@ -864,10 +1086,12 @@ def _upload_worker(file_idx: int, file_info: tuple, pre_validated: Any = None) - repo_type=repo_type, tracker=tracker, pre_validated=pre_validated, + hash_info=lfs_hash_info_map.get(file_idx), disable_tqdm=disable_tqdm, ) logger.debug("Uploaded: %s", path_in_repo_w) batch_tracker.record_success(file_idx, result) + _report_wire(result) except Exception as e: logger.error("Upload failed: %s - %s", path_in_repo_w, e) batch_tracker.record_failure(file_idx, file_info, e) @@ -877,16 +1101,146 @@ def _upload_worker(file_idx: int, file_info: tuple, pre_validated: Any = None) - all_results: list[dict] = [] total_failed_files: list[tuple] = [] num_batches = batch_tracker.num_batches + committed_files = 0 + committed_bytes = 0 + total_bytes = sum(file_sizes.values()) + # Wire-level counters, advanced as each file's transfer finishes rather + # than when its commit lands. + wire_lock = threading.Lock() + wire_state = {"bytes": 0, "files": 0, "reported_bytes": 0, "last_emit": 0.0} + + def _emit(payload: dict) -> None: + if progress_callback is None: + return + try: + progress_callback(payload) + except Exception as cb_error: # noqa: BLE001 - a reporter must not fail the upload + logger.warning("Progress callback raised %s, continuing upload.", cb_error) + + def _report( + event: str, + batch_idx: int, + files: int, + num_bytes: int, + inline_bytes: int = 0, + error: str | None = None, + ) -> None: + """Emit a batch-level progress event. + + A folder upload is otherwise silent between batches, so a long run is + indistinguishable from a hung one. Byte counts are included because a + consumer that only learns file counts cannot compute a throughput rate + or an ETA, which is most of what progress is for. ``inline_bytes`` is + the part of the batch that travels inside this commit rather than + having already gone to object storage, so a consumer can attribute + wire traffic to the right moment without double counting. + """ + with wire_lock: + wire_bytes, wire_files = wire_state["bytes"], wire_state["files"] + _emit( + { + "event": event, + "repo_id": repo_id, + "batch_index": batch_idx, + "num_batches": num_batches, + "batch_files": files, + "batch_bytes": num_bytes, + "batch_inline_bytes": inline_bytes, + "committed_files": committed_files, + "committed_bytes": committed_bytes, + "uploaded_bytes": wire_bytes, + "uploaded_files": wire_files, + "total_files": len(sorted_files), + "total_bytes": total_bytes, + "skipped_files": skipped_count, + "elapsed": time.time() - start_time, + "error": error, + } + ) + + def _report_wire(result: dict) -> None: + """Account one finished file transfer, emitting at a throttled rate. + + Commits land in lumps tens of seconds apart, so a consumer fed only by + commit events sees a rate that alternates between a spike and zero and + cannot tell a slow batch from a hung one. Blob uploads finish + continuously, which is the signal a rate should be built from. Only + bytes that really went to object storage count: a deduplicated blob + transfers nothing. + """ + if progress_callback is None: + return + moved = result["file_size_on_disk"] if result.get("is_blob_uploaded") else 0 + now = time.monotonic() + with wire_lock: + wire_state["bytes"] += moved + wire_state["files"] += 1 + due = now - wire_state["last_emit"] >= UPLOAD_PROGRESS_MIN_INTERVAL_SECONDS + if not due: + return + wire_state["last_emit"] = now + delta = wire_state["bytes"] - wire_state["reported_bytes"] + wire_state["reported_bytes"] = wire_state["bytes"] + snapshot = (wire_state["bytes"], wire_state["files"]) + _emit( + { + "event": "upload_progress", + "repo_id": repo_id, + "uploaded_bytes": snapshot[0], + "uploaded_bytes_delta": delta, + "uploaded_files": snapshot[1], + "committed_files": committed_files, + "committed_bytes": committed_bytes, + "total_files": len(sorted_files), + "total_bytes": total_bytes, + "skipped_files": skipped_count, + "elapsed": time.time() - start_time, + "error": None, + } + ) + + def _flush_wire() -> None: + """Emit whatever wire bytes the throttle has not reported yet.""" + if progress_callback is None: + return + with wire_lock: + delta = wire_state["bytes"] - wire_state["reported_bytes"] + if delta <= 0: + return + wire_state["reported_bytes"] = wire_state["bytes"] + snapshot = (wire_state["bytes"], wire_state["files"]) + _emit( + { + "event": "upload_progress", + "repo_id": repo_id, + "uploaded_bytes": snapshot[0], + "uploaded_bytes_delta": delta, + "uploaded_files": snapshot[1], + "committed_files": committed_files, + "committed_bytes": committed_bytes, + "total_files": len(sorted_files), + "total_bytes": total_bytes, + "skipped_files": skipped_count, + "elapsed": time.time() - start_time, + "error": None, + } + ) try: with ThreadPoolExecutor(max_workers=max_workers) as executor: for file_idx, file_info in files_to_upload: pv: str | bool | None = None if file_idx in lfs_hash_info_map: - cached_hash = lfs_hash_info_map[file_idx][0]["file_hash"] - pv = pre_validated_map.get(cached_hash) - if pv is None: - pv = True + cached_hash = lfs_hash_info_map[file_idx]["file_hash"] + # "Answered, and the server did not ask for an upload" + # means the blob already exists and is reused. "Never + # answered" means the group failed, and the file has to + # negotiate its own URL -- treating that as reuse would + # skip the transfer and commit a pointer to a blob that + # was never stored. + if cached_hash in pre_validated_map: + url = pre_validated_map[cached_hash] + pv = True if url is None else url executor.submit(_upload_worker, file_idx, file_info, pv) consecutive_failures = 0 @@ -896,8 +1250,7 @@ def _upload_worker(file_idx: int, file_info: tuple, pre_validated: Any = None) - total=num_batches, disable=disable_tqdm, ): - batch_start = batch_idx * commit_batch_size - batch_end = min(batch_start + commit_batch_size, len(sorted_files)) + batch_start, batch_end = batch_tracker.batch_range(batch_idx) if all(i in skipped_indices for i in range(batch_start, batch_end)): logger.info( "Batch %d/%d fully committed, skipping.", @@ -907,6 +1260,10 @@ def _upload_worker(file_idx: int, file_info: tuple, pre_validated: Any = None) - continue results, failures = batch_tracker.wait_for_batch(batch_idx) + # Every file of this batch has finished its transfer by now, + # so publish the bytes the throttle may still be holding + # before the commit event reports the batch as done. + _flush_wire() if failures: total_failed_files.extend(failures) @@ -922,6 +1279,7 @@ def _upload_worker(file_idx: int, file_info: tuple, pre_validated: Any = None) - batch_idx + 1, num_batches, ) + _report("batch_failed", batch_idx, len(failures), 0, error="all files failed to upload") continue batch_commit_message = f"{commit_message} (batch {batch_idx + 1}/{num_batches})" @@ -943,6 +1301,13 @@ def _upload_worker(file_idx: int, file_info: tuple, pre_validated: Any = None) - ) self._track_committed_batch(tracker, results) consecutive_failures = 0 + batch_bytes = sum(r["file_size_on_disk"] for r in results) + batch_inline_bytes = sum( + r["file_size_on_disk"] for r in results if r.get("upload_mode") != "lfs" + ) + committed_files += len(results) + committed_bytes += batch_bytes + _report("batch_committed", batch_idx, len(results), batch_bytes, batch_inline_bytes) except Exception as e: logger.error( "Batch %d/%d commit failed: %s", @@ -950,6 +1315,13 @@ def _upload_worker(file_idx: int, file_info: tuple, pre_validated: Any = None) - num_batches, e, ) + _report( + "batch_failed", + batch_idx, + len(results), + sum(r["file_size_on_disk"] for r in results), + error=str(e), + ) category = classify_error(e) if not _ErrorCategory.is_retryable(category): for r in results: @@ -1170,6 +1542,7 @@ def _upload_single_file( repo_type: str, tracker: UploadTracker | NullTracker | None = None, pre_validated: Any = None, + hash_info: dict | None = None, disable_tqdm: bool = False, ) -> dict: if tracker is None: @@ -1178,7 +1551,13 @@ def _upload_single_file( file_stat = None is_real_path = isinstance(file_path, (str, os.PathLike)) - if is_real_path: + if hash_info is not None: + # Already computed during batch pre-validation; re-reading the file + # to hash it again would double the disk cost of every LFS file. + hash_info_d = dict(hash_info) + hash_info_d["file_path_or_obj"] = file_path + + if hash_info_d is None and is_real_path: try: file_stat = os.stat(file_path) cached = tracker.get_hash(file_path_in_repo, file_stat.st_mtime, file_stat.st_size) @@ -1366,24 +1745,97 @@ def _upload_blob( # ------------------------------------------------------------------ # Internal: batch blob validation # ------------------------------------------------------------------ + def _hash_files_parallel( + self, + files: list[tuple[int, tuple[str, str]]], + tracker: UploadTracker | NullTracker, + max_workers: int, + ) -> dict[int, dict]: + """Hash *files* concurrently and record the results in *tracker*. + + Hashing ahead of the upload pipeline is what makes group pre-signing + possible: the git-lfs batch endpoint is keyed by ``sha256``, so without + the digests up front each file has to negotiate its own upload URL. + """ + hashed: dict[int, dict] = {} + + def _hash_one(entry: tuple[int, tuple[str, str]]) -> tuple[int, dict] | None: + file_idx, (path_in_repo, file_path) = entry + try: + st = os.stat(file_path) + info = _compute_file_hash(file_path_or_obj=file_path) + except OSError as error: + # Leave it to the upload worker, which reports per-file failures. + logger.debug("Cannot pre-hash %s: %s", path_in_repo, error) + return None + tracker.put_hash(path_in_repo, st.st_mtime, st.st_size, info) + return file_idx, info + + with ThreadPoolExecutor(max_workers=max(1, max_workers)) as executor: + for outcome in executor.map(_hash_one, files): + if outcome is not None: + hashed[outcome[0]] = outcome[1] + + logger.info("Hashed %d LFS candidate(s) for batch pre-validation.", len(hashed)) + return hashed + def _validate_blobs_batch( self, repo_id: str, repo_type: str, objects: list[dict], + max_workers: int = 1, ) -> dict[str, str | None]: - result: dict[str, str | None] = {} - batch_size = UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS + """Pre-sign every object, in parallel groups, tolerating group failures. + + The groups are independent requests, so running them serially made the + pre-sign phase scale linearly with the file count -- measurably so: 150 + groups took 25s of pure round-trip latency before a single byte moved. + + The batch endpoint answers only about objects that *need* uploading; an + object it does not mention already exists server-side. That is normalised + here into an explicit ``oid -> None`` entry, so the returned map covers + every object of every group that answered. + + A group that fails is not fatal: its oids are simply absent from the map + and the per-file upload path negotiates their URL itself. Keeping + "answered: already exists" and "never answered" distinguishable is what + makes that safe -- conflating them would skip the transfer and commit a + pointer to a blob that was never stored. + """ + batch_size = max(1, UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS) + chunks = [objects[i : i + batch_size] for i in range(0, len(objects), batch_size)] + if not chunks: + return {} - for i in range(0, len(objects), batch_size): - chunk = objects[i : i + batch_size] - validated = self._client.validate_blobs( - repo_id=repo_id, - repo_type=repo_type, - objects=chunk, - ) - result.update(validated) + def validate(chunk: list[dict]) -> dict[str, str | None]: + try: + validated = self._client.validate_blobs( + repo_id=repo_id, + repo_type=repo_type, + objects=chunk, + ) + except Exception as error: # noqa: BLE001 - degrades to per-file negotiation + logger.warning( + "Blob pre-validation failed for %d object(s) (%s); those files will negotiate " + "their upload URL individually.", + len(chunk), + error, + ) + return {} + answered: dict[str, str | None] = {obj["oid"]: None for obj in chunk} + answered.update(validated) + return answered + result: dict[str, str | None] = {} + if len(chunks) == 1: + return validate(chunks[0]) + # Bounded by the HTTP pool: more in-flight requests than pooled + # connections just trades latency for TLS handshakes. + workers = max(1, min(max_workers, API_CONNECTION_POOL_MAXSIZE, len(chunks))) + with ThreadPoolExecutor(max_workers=workers) as executor: + for validated in executor.map(validate, chunks): + result.update(validated) return result # ------------------------------------------------------------------ @@ -1401,7 +1853,9 @@ def _commit_with_retry( ) -> dict: last_error: Exception | None = None start_time = time.monotonic() + throttled_wait = 0.0 for attempt in range(max_attempts): + self._commit_governor.acquire() try: return self._client.create_commit( repo_id=repo_id, @@ -1422,8 +1876,32 @@ def _commit_with_retry( except Exception as e: last_error = e + # A throttled commit carries the wait the server wants; honoring it + # beats guessing, and it is budgeted apart from the transient-error + # allowance because its duration is known and can legitimately + # exceed it. + retry_after = getattr(last_error, "retry_after", None) if isinstance(last_error, RateLimitError) else None + if retry_after is not None: + wait = float(retry_after) + if wait > UPLOAD_COMMIT_MAX_RETRY_AFTER_SECONDS: + logger.error( + "Commit throttled with Retry-After=%.0fs, above the %ds ceiling; aborting retries.", + wait, + UPLOAD_COMMIT_MAX_RETRY_AFTER_SECONDS, + ) + break + throttled_wait += wait + logger.warning( + "Commit attempt %d/%d throttled, honoring Retry-After=%.0fs ...", + attempt + 1, + max_attempts, + wait, + ) + time.sleep(wait) + continue + wait = min(2**attempt, 60) - elapsed = time.monotonic() - start_time + elapsed = time.monotonic() - start_time - throttled_wait if elapsed + wait > UPLOAD_COMMIT_RETRY_TOTAL_WAIT_SECONDS: logger.error( "Commit total wait time would exceed %ds (already %.1fs elapsed), aborting retries.", @@ -1536,6 +2014,7 @@ def _prepare_upload_folder( repo_type: str = "model", allow_patterns: list[str] | None = None, ignore_patterns: list[str] | None = None, + sizes_out: dict[str, int] | None = None, ) -> list[tuple[str, str]]: folder = Path(folder_path).expanduser().resolve() if not folder.is_dir(): @@ -1558,7 +2037,9 @@ def _prepare_upload_folder( f"max allowed per directory: {UPLOAD_MAX_FILES_PER_DIRECTORY}" ) - # File size checks + # File size checks. Sizes are handed back through ``sizes_out`` because + # batch planning needs them next; re-stating a large tree costs one + # syscall per file for no new information. total_size = 0 normal_size = 0 for path in all_files: @@ -1571,6 +2052,8 @@ def _prepare_upload_folder( total_size += fsize if not _is_lfs(str(path), fsize, repo_type): normal_size += fsize + if sizes_out is not None: + sizes_out[str(path)] = fsize if normal_size > UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES: logger.warning( diff --git a/src/modelscope_hub/agent/_api.py b/src/modelscope_hub/agent/_api.py index 531bf41..86c2d0f 100644 --- a/src/modelscope_hub/agent/_api.py +++ b/src/modelscope_hub/agent/_api.py @@ -241,7 +241,7 @@ def _decode_dolphin_list_response(response: object, *, list_url: str) -> object: from an owner with no repositories. """ try: - payload = response.json() # type: ignore[union-attr] + payload = response.json() # type: ignore[attr-defined] except (AttributeError, ValueError) as exc: raise APIError( "Agent repository list endpoint returned a non-JSON response.", diff --git a/src/modelscope_hub/api.py b/src/modelscope_hub/api.py index 0244ea5..2718b29 100644 --- a/src/modelscope_hub/api.py +++ b/src/modelscope_hub/api.py @@ -25,7 +25,7 @@ import fnmatch import time -from collections.abc import Iterable, Mapping +from collections.abc import Callable, Iterable, Mapping from pathlib import Path from typing import Any, BinaryIO, TypeAlias from urllib.parse import urlparse @@ -1368,6 +1368,8 @@ def upload_folder( use_cache: bool | None = None, disable_tqdm: bool = False, sync_remote_repo: bool = False, + tracker_path: str | Path | None = None, + progress_callback: Callable[[dict], None] | None = None, ) -> dict | list[dict] | None: """Upload an entire folder to a repository with resumable support. @@ -1406,6 +1408,18 @@ def upload_folder( sync_remote_repo : bool, optional If True, delete remote files that are not present locally after a successful upload (sync semantics). Default False. + tracker_path : str or Path, optional + Where to keep the resumable-upload cache. Defaults to + ``.ms_upload_cache`` inside ``folder_path``. Point this outside the + uploaded tree when ``folder_path`` is a staging directory that gets + discarded between runs, so resume state survives. + progress_callback : callable, optional + Called with one dict per batch outcome, carrying ``event`` + (``"batch_committed"`` or ``"batch_failed"``), ``batch_index``, + ``num_batches``, ``batch_files``, ``batch_bytes``, + ``committed_files``, ``committed_bytes``, ``total_files``, + ``total_bytes``, ``skipped_files``, ``elapsed`` and ``error``. + Exceptions raised by the callback are logged and swallowed. Returns ------- @@ -1441,6 +1455,8 @@ def upload_folder( use_cache=use_cache, disable_tqdm=disable_tqdm, sync_remote_repo=sync_remote_repo, + tracker_path=tracker_path, + progress_callback=progress_callback, ) def download_file( diff --git a/src/modelscope_hub/constants.py b/src/modelscope_hub/constants.py index 1a9f970..4d53fb6 100644 --- a/src/modelscope_hub/constants.py +++ b/src/modelscope_hub/constants.py @@ -227,6 +227,7 @@ def _warn_deprecated_env( name: str, *, expects_mb: bool = False, + expects_bytes: bool = False, stacklevel: int = 3, ) -> None: """Warn that a legacy environment variable remains temporarily supported.""" @@ -235,9 +236,19 @@ def _warn_deprecated_env( ) if expects_mb: message += f" {name!r} expects a value in MB." + if expects_bytes: + message += f" {name!r} expects bytes, or a value with a unit suffix such as '32KiB'." warnings.warn(message, FutureWarning, stacklevel=stacklevel) +def _format_bytes(num_bytes: int) -> str: + """Render a byte count using the largest binary unit that stays exact.""" + for unit, scale in (("GiB", 1024**3), ("MiB", 1024**2), ("KiB", 1024)): + if num_bytes and num_bytes % scale == 0: + return f"{num_bytes // scale}{unit}" + return f"{num_bytes}" + + def _env(name: str, *deprecated_names: str) -> str | None: """Read an env var, falling back to deprecated names with a warning.""" value = os.environ.get(name) @@ -251,12 +262,71 @@ def _env(name: str, *deprecated_names: str) -> str | None: return None +def _warn_invalid_env(name: str, raw: str, reason: str, fallback: object) -> None: + """Warn that an environment value was rejected, naming the value used instead. + + Silently falling back to the default made misconfiguration invisible: a + pipeline could export a tuning value, observe none of its effect, and have + no signal to look at. Every rejected value now says so. + """ + warnings.warn( + f"Environment variable {name}={raw!r} is invalid ({reason}); using {fallback!r} instead.", + UserWarning, + stacklevel=3, + ) + + +_BYTE_UNITS: dict[str, int] = { + "": 1, + "B": 1, + "K": 1024, + "KB": 1000, + "KIB": 1024, + "M": 1024**2, + "MB": 1000**2, + "MIB": 1024**2, + "G": 1024**3, + "GB": 1000**3, + "GIB": 1024**3, +} + + +def _parse_byte_size(raw: str, *, bare_unit: int = 1) -> int: + """Parse a size string into bytes, accepting an optional unit suffix. + + ``bare_unit`` scales a value given without a suffix, which is what lets a + deprecated ``*_MB`` alias keep its megabyte meaning while the canonical + name treats a bare number as bytes. Raises :class:`ValueError` so callers + decide between warning and propagating. + """ + text = raw.strip() + if not text: + raise ValueError("empty value") + digits = text + suffix = "" + while digits and not (digits[-1].isdigit() or digits[-1] == "."): + suffix = digits[-1] + suffix + digits = digits[:-1] + digits = digits.strip() + suffix = suffix.strip().upper() + if not digits: + raise ValueError("no numeric part") + if suffix not in _BYTE_UNITS: + raise ValueError(f"unknown size unit {suffix!r}") + number = float(digits) + if number != int(number): + raise ValueError("fractional byte counts are not supported") + unit = bare_unit if suffix == "" else _BYTE_UNITS[suffix] + return int(number) * unit + + def _env_int( name: str, default: int, description: str = "", category: str = "", *deprecated_names: str, + allow_zero: bool = False, ) -> int: """Read a positive integer from the environment and register it.""" all_deprecated = deprecated_names or _DEPRECATED_LOOKUP.get(name, ()) @@ -268,8 +338,13 @@ def _env_int( try: value = int(raw) except ValueError: + _warn_invalid_env(name, raw, "not an integer", default) + return default + if value < 0 or (value == 0 and not allow_zero): + reason = "must not be negative" if allow_zero else "must be a positive integer" + _warn_invalid_env(name, raw, reason, default) return default - return value if value > 0 else default + return value def _env_int_mb( @@ -294,8 +369,12 @@ def _env_int_mb( try: value = int(raw) except ValueError: + _warn_invalid_env(name, raw, "not an integer", f"{default_mb} MB") + return default_mb * 1024 * 1024 + if value <= 0: + _warn_invalid_env(name, raw, "must be a positive integer", f"{default_mb} MB") return default_mb * 1024 * 1024 - return value * 1024 * 1024 if value > 0 else default_mb * 1024 * 1024 + return value * 1024 * 1024 # Fall back to deprecated names (value already in bytes) for old in all_deprecated: raw = os.environ.get(old) @@ -304,8 +383,12 @@ def _env_int_mb( try: value = int(raw) except ValueError: + _warn_invalid_env(old, raw, "not an integer", f"{default_mb} MB") return default_mb * 1024 * 1024 - return value if value > 0 else default_mb * 1024 * 1024 + if value <= 0: + _warn_invalid_env(old, raw, "must be a positive integer", f"{default_mb} MB") + return default_mb * 1024 * 1024 + return value return default_mb * 1024 * 1024 @@ -328,8 +411,12 @@ def _env_int_mb_with_deprecated_units( try: value = int(raw) except ValueError: + _warn_invalid_env(name, raw, "not an integer", f"{default_mb} MB") + return default_bytes + if value <= 0: + _warn_invalid_env(name, raw, "must be a positive integer", f"{default_mb} MB") return default_bytes - return value * 1024 * 1024 if value > 0 else default_bytes + return value * 1024 * 1024 for old in deprecated_mb_names: raw = os.environ.get(old) @@ -338,8 +425,12 @@ def _env_int_mb_with_deprecated_units( try: value = int(raw) except ValueError: + _warn_invalid_env(old, raw, "not an integer", f"{default_mb} MB") return default_bytes - return value * 1024 * 1024 if value > 0 else default_bytes + if value <= 0: + _warn_invalid_env(old, raw, "must be a positive integer", f"{default_mb} MB") + return default_bytes + return value * 1024 * 1024 for old in deprecated_byte_names: raw = os.environ.get(old) @@ -348,8 +439,68 @@ def _env_int_mb_with_deprecated_units( try: value = int(raw) except ValueError: + _warn_invalid_env(old, raw, "not an integer", f"{default_mb} MB") + return default_bytes + if value <= 0: + _warn_invalid_env(old, raw, "must be a positive integer", f"{default_mb} MB") return default_bytes - return value if value > 0 else default_bytes + return value + + return default_bytes + + +def _env_bytes( + name: str, + default_bytes: int, + description: str, + category: str, + *, + deprecated_mb_names: tuple[str, ...] = (), + deprecated_byte_names: tuple[str, ...] = (), + allow_zero: bool = False, +) -> int: + """Read a byte-size setting whose canonical name accepts a unit suffix. + + The canonical name treats a bare number as **bytes** and understands the + suffixes ``B``, ``K``/``KiB``, ``KB``, ``M``/``MiB``, ``MB``, ``G``/``GiB`` + and ``GB`` (binary for the ``iB``/bare-letter forms, decimal for ``KB``/ + ``MB``/``GB``). A megabyte-only knob cannot express thresholds below 1 MB, + which is exactly the range that matters when deciding whether a small file + rides inline in a commit or goes to object storage. + + Names in ``deprecated_mb_names`` keep their megabyte meaning for a bare + number; names in ``deprecated_byte_names`` keep their byte meaning. + """ + deprecated_names = deprecated_mb_names + deprecated_byte_names + _env_register(name, _format_bytes(default_bytes), description, category, deprecated_names=deprecated_names) + + def _accept(source: str, raw: str, bare_unit: int) -> int: + try: + value = _parse_byte_size(raw, bare_unit=bare_unit) + except ValueError as exc: + _warn_invalid_env(source, raw, str(exc), _format_bytes(default_bytes)) + return default_bytes + if value < 0 or (value == 0 and not allow_zero): + reason = "must not be negative" if allow_zero else "must be a positive size" + _warn_invalid_env(source, raw, reason, _format_bytes(default_bytes)) + return default_bytes + return value + + raw = os.environ.get(name) + if raw is not None and raw.strip(): + return _accept(name, raw, 1) + + for old in deprecated_mb_names: + raw = os.environ.get(old) + if raw is not None and raw.strip(): + _warn_deprecated_env(old, name, expects_bytes=True, stacklevel=2) + return _accept(old, raw, 1024 * 1024) + + for old in deprecated_byte_names: + raw = os.environ.get(old) + if raw is not None and raw.strip(): + _warn_deprecated_env(old, name, expects_bytes=True, stacklevel=2) + return _accept(old, raw, 1) return default_bytes @@ -439,6 +590,42 @@ def _env_register( "API_MAX_RETRIES", ) +API_CONNECTION_POOL_MAXSIZE: int = _env_int( + "MODELSCOPE_API_CONNECTION_POOL_MAXSIZE", + 32, + "Per-host HTTP connection pool size", + "Network", +) +"""Connections kept alive per host, and the concurrency the pool can serve. + +urllib3 defaults this to 10. A folder upload runs ``max_workers`` requests at +once -- commonly 16 or more for bulk transfers -- so the default silently +discards the excess connections ("Connection pool is full, discarding +connection") and every discarded one costs a fresh TLS handshake on its next +use. This must be at least as large as the worker count to avoid that churn. +""" + +REPO_TREE_PAGE_MAX_ATTEMPTS: int = _env_int( + "MODELSCOPE_REPO_TREE_PAGE_MAX_ATTEMPTS", + 4, + "Attempts for one repo-tree page denied on an already-authorized listing", + "Network", +) +"""Retries for a spurious ``403`` on a single page of a paginated tree listing. + +A large dataset listing spans hundreds of pages and an occasional page answers +``403 无权访问该数据集`` on a repository the caller has just read successfully. +Treating that as an authorization result throws away every page already +collected, so it is retried once the credential has been proven by an earlier +page. +""" +REPO_TREE_PAGE_RETRY_MAX_DELAY_SECONDS: int = _env_int( + "MODELSCOPE_REPO_TREE_PAGE_RETRY_MAX_DELAY_SECONDS", + 8, + "Maximum backoff between repo-tree page retries (seconds)", + "Network", +) + REPO_FILES_TRUNCATION_LIMIT: int = 3000 """Server-side hard cap on a single ``repo/files`` listing. @@ -649,6 +836,78 @@ def _env_register( "Upload", "UPLOAD_ADAPTIVE_BATCH_SIZE", ) +UPLOAD_COMMIT_MAX_INLINE_BYTES: int = _env_bytes( + "MODELSCOPE_UPLOAD_COMMIT_MAX_INLINE_BYTES", + 8 * 1024 * 1024, + "Maximum inlined (non-LFS) content carried by one commit request", + "Upload", +) +"""Byte ceiling on the base64 content a single commit may carry. + +Non-LFS files travel *inside* the commit request body, so a batch sized purely +by file count can produce a request tens of megabytes large -- big enough for +the server to time out mid-read, which surfaces on the client as an unrelated +write timeout. Batching therefore closes a batch on whichever limit is reached +first, this one or :data:`UPLOAD_COMMIT_BATCH_MAX_OPERATIONS`. LFS files +contribute only a pointer, so they do not count against it. +""" +UPLOAD_COMMIT_MAX_PER_HOUR: int = _env_int( + "MODELSCOPE_UPLOAD_COMMIT_MAX_PER_HOUR", + 0, + "Client-side commit rate ceiling per hour (0 disables the governor)", + "Upload", + allow_zero=True, +) +"""Opt-in client-side commit budget, disabled by default. + +The Hub throttles commits per repository. Reacting to a throttle costs a failed +round trip and, when the server holds the connection instead of answering, a +full read timeout. A bulk pipeline that knows its budget can set this to spread +commits out and never trip the limit; interactive uploads stay unthrottled. +""" +UPLOAD_COMMIT_MAX_RETRY_AFTER_SECONDS: int = _env_int( + "MODELSCOPE_UPLOAD_COMMIT_MAX_RETRY_AFTER_SECONDS", + 1800, + "Longest server-provided Retry-After a commit will honor (seconds)", + "Upload", +) +"""Upper bound on an honored ``Retry-After`` for a throttled commit. + +A rate limit is a bounded wait the server declares, unlike a transient failure +of unknown duration, so it is budgeted separately from +:data:`UPLOAD_COMMIT_RETRY_TOTAL_WAIT_SECONDS` rather than exhausting it. +""" +COMMIT_MAX_ACTIONS_PER_REQUEST: int = _env_int( + "MODELSCOPE_COMMIT_MAX_ACTIONS_PER_REQUEST", + 2000, + "Server-enforced maximum actions in one commit request", + "Upload", +) +"""Hard ceiling the server puts on a single commit request. + +Exceeding it is rejected outright with ``HTTP 422``:: + + commit request exceeds actions limit: 3300 > 2000; split the commit into + smaller batches + +This is a server contract, not a tuning preference, so every commit path clamps +to it -- uploads, and deletes, which otherwise put every path in one request. +""" + +UPLOAD_PROGRESS_MIN_INTERVAL_SECONDS: int = _env_int( + "MODELSCOPE_UPLOAD_PROGRESS_MIN_INTERVAL_SECONDS", + 1, + "Minimum gap between wire-level upload progress events (seconds)", + "Upload", +) +"""Throttle for per-file upload progress events. + +Commits land in lumps tens of seconds apart, so a rate built only from commit +events alternates between a spike and zero and cannot distinguish a slow batch +from a hung one. Blob uploads finish continuously and are the honest source for +a rate -- but there is one per file, so the events are coalesced to this interval +rather than fanning out tens of thousands of callback invocations. +""" UPLOAD_COMMIT_MAX_ATTEMPTS: int = _env_int( "MODELSCOPE_UPLOAD_COMMIT_MAX_ATTEMPTS", 5, @@ -751,13 +1010,42 @@ def get_upload_ignore_file_pattern() -> str | None: UPLOAD_LEGACY_PROGRESS_FILE: str = ".ms_upload_progress" # Upload: limits -UPLOAD_LFS_FORCE_THRESHOLD_BYTES: int = _env_int_mb_with_deprecated_units( - "MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD_MB", - 1, - "File-size threshold that forces LFS mode (MB)", +UPLOAD_LFS_FORCE_THRESHOLD_BYTES: int = _env_bytes( + "MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD", + 1024 * 1024, + "File size above which LFS mode is forced (bytes; accepts a unit suffix, 0 forces LFS for every file)", "Upload", + deprecated_mb_names=("MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD_MB",), deprecated_byte_names=("UPLOAD_LFS_ENFORCE_THRESHOLD", "UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFS"), + allow_zero=True, ) +"""Size above which a file is uploaded as LFS regardless of its suffix. + +Files at or below the threshold are committed inline as base64, which puts their +bytes in the commit request body and caps how many of them one commit can carry. +Lowering the threshold moves that content onto the pre-signed object-storage +path instead, leaving the commit with only ``sha256`` plus ``size``. + +``0`` forces LFS for every non-empty file. It stays safe because +:data:`UPLOAD_INLINE_METADATA_PATHS` is consulted first, so the repository files +the Hub itself parses are never turned into LFS pointers. +""" +UPLOAD_INLINE_METADATA_PATHS: frozenset[str] = _env_csv_frozenset( + "MODELSCOPE_UPLOAD_INLINE_METADATA_PATHS", + "README.md,.gitattributes,.gitignore,configuration.json,configuration.yaml,configuration.yml," + "dataset_infos.json,config.json,.msc,.mdl", + "Repository file names always committed inline, never as LFS", + "Upload", +) +"""Repository-relative file names that must stay inline in the commit. + +The Hub parses these server-side -- the dataset/model card front matter, the +configuration files, the git attribute rules. Stored as an LFS pointer, the +server would read the 130-byte pointer text instead of the real content and the +card or configuration would silently render empty. Matching is on the file name +(case-insensitive), so the rule holds at any depth in the tree, and it is +checked before the size and suffix rules. +""" UPLOAD_MAX_FILE_SIZE_BYTES: int = _env_int_mb_with_deprecated_units( "MODELSCOPE_UPLOAD_MAX_FILE_SIZE_MB", 100 * 1024, @@ -955,9 +1243,11 @@ def get_upload_ignore_file_pattern() -> str | None: __all__ = [ "API_CONNECT_TIMEOUT", + "API_CONNECTION_POOL_MAXSIZE", "API_MAX_RETRIES", "API_TIMEOUT", "CATEGORY_ORDER", + "COMMIT_MAX_ACTIONS_PER_REQUEST", "CONFIG_DIR_NAME", "DATASET_LFS_SUFFIX", "DEFAULT_CACHE_DIR_NAME", @@ -996,6 +1286,8 @@ def get_upload_ignore_file_pattern() -> str | None: "MODEL_ID_SEPARATOR", "MODEL_LFS_SUFFIX", "OPENAPI_PREFIX", + "REPO_TREE_PAGE_MAX_ATTEMPTS", + "REPO_TREE_PAGE_RETRY_MAX_DELAY_SECONDS", "REPO_TYPE_DATASET", "REPO_TYPE_MODEL", "REPO_TYPE_STUDIO", @@ -1029,12 +1321,16 @@ def get_upload_ignore_file_pattern() -> str | None: "UPLOAD_COMMIT_BATCH_SIZE", "UPLOAD_COMMIT_MAX_ATTEMPTS", "UPLOAD_COMMIT_MAX_CONSECUTIVE_FAILED_BATCHES", + "UPLOAD_COMMIT_MAX_INLINE_BYTES", + "UPLOAD_COMMIT_MAX_PER_HOUR", "UPLOAD_COMMIT_MAX_RETRIES", + "UPLOAD_COMMIT_MAX_RETRY_AFTER_SECONDS", "UPLOAD_COMMIT_MAX_TOTAL_WAIT", "UPLOAD_COMMIT_RETRY_TOTAL_WAIT_SECONDS", "UPLOAD_FAILED_FILE_MAX_RETRIES", "UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS", "UPLOAD_HTTP_RETRY_ALLOWED_METHODS", + "UPLOAD_INLINE_METADATA_PATHS", "UPLOAD_LEGACY_PROGRESS_FILE", "UPLOAD_LFS_ENFORCE_THRESHOLD", "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", @@ -1047,6 +1343,7 @@ def get_upload_ignore_file_pattern() -> str | None: "UPLOAD_MAX_FILES_PER_DIRECTORY", "UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT", "UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES", + "UPLOAD_PROGRESS_MIN_INTERVAL_SECONDS", "UPLOAD_REACT_BACKOFF_MAX_EXPONENT", "UPLOAD_REACT_ENABLED", "UPLOAD_REACT_MAX_DELAY", diff --git a/src/modelscope_hub/version.py b/src/modelscope_hub/version.py index 06d4093..a59471b 100644 --- a/src/modelscope_hub/version.py +++ b/src/modelscope_hub/version.py @@ -1,3 +1,3 @@ """Version information for modelscope_hub.""" -__version__ = "0.4.3+main" +__version__ = "0.4.6+main" diff --git a/tests/test_commit_action_limit.py b/tests/test_commit_action_limit.py new file mode 100644 index 0000000..9bbbf94 --- /dev/null +++ b/tests/test_commit_action_limit.py @@ -0,0 +1,110 @@ +"""Commit requests must respect the server's hard action ceiling. + +The server rejects an oversized commit outright rather than truncating it:: + + HTTP 422 commit request exceeds actions limit: 3300 > 2000; + split the commit into smaller batches + +Observed while removing 3300 verification files from a test dataset. Uploads were +accidentally safe because the default operation cap is well below the ceiling, +but an explicitly configured larger cap, or any delete of more than 2000 paths, +would hit it. +""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +import modelscope_hub._upload as upload_module +from modelscope_hub._upload import ( + UploadManager, + _calculate_adaptive_batch_size, + _plan_commit_batches, +) +from modelscope_hub.errors import APIError + + +def _make_manager() -> tuple[UploadManager, MagicMock]: + client = MagicMock() + client.create_commit.return_value = {"ok": True} + client.validate_blobs.side_effect = lambda **kw: {o["oid"]: f"https://upload/{o['oid']}" for o in kw["objects"]} + client.upload_blob.side_effect = lambda **kw: None + return UploadManager(client, MagicMock()), client + + +def test_batch_size_is_clamped_to_the_server_ceiling(monkeypatch) -> None: + monkeypatch.setattr(upload_module, "COMMIT_MAX_ACTIONS_PER_REQUEST", 2000) + + # An explicit cap above the ceiling must not be honored: the server would + # reject the request instead of trimming it. + assert _calculate_adaptive_batch_size(50_000, 4096) == 2000 + assert _calculate_adaptive_batch_size(50_000, 512) == 512 + # "No cap" must not mean "one commit for everything" either. + assert _calculate_adaptive_batch_size(50_000, 0) == 2000 + + +def test_batch_plan_never_exceeds_the_server_ceiling(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr(upload_module, "COMMIT_MAX_ACTIONS_PER_REQUEST", 10) + files = [(f"f{i}.bin", str(tmp_path / f"f{i}.bin")) for i in range(35)] + sizes = dict.fromkeys((path for _, path in files), 1) + + plan = _plan_commit_batches( + files, + "dataset", + max_operations=1000, + max_inline_bytes=0, + sizes=sizes, + ) + + assert sum(plan) == 35 + assert max(plan) <= 10 + + +def test_delete_files_splits_at_the_server_ceiling(monkeypatch) -> None: + monkeypatch.setattr(upload_module, "COMMIT_MAX_ACTIONS_PER_REQUEST", 100) + manager, client = _make_manager() + paths = [f"junk/file-{i}.bin" for i in range(250)] + + result = manager.delete_files(repo_id="owner/repo", repo_type="dataset", file_paths=paths) + + assert client.create_commit.call_count == 3 + sent = [len(call.kwargs["operations"]) for call in client.create_commit.call_args_list] + assert sent == [100, 100, 50] + assert result["total_files"] == 250 + assert result["deleted_files"] == paths + + +def test_delete_files_keeps_one_commit_when_it_fits(monkeypatch) -> None: + monkeypatch.setattr(upload_module, "COMMIT_MAX_ACTIONS_PER_REQUEST", 2000) + manager, client = _make_manager() + + manager.delete_files(repo_id="owner/repo", repo_type="dataset", file_paths=["a.bin", "b.bin"]) + + client.create_commit.assert_called_once() + assert len(client.create_commit.call_args.kwargs["operations"]) == 2 + + +def test_delete_files_reports_what_a_partial_split_removed(monkeypatch) -> None: + # Across a split the deletion is no longer atomic. Failing loudly beats + # returning a success that claims paths which are still present. + monkeypatch.setattr(upload_module, "COMMIT_MAX_ACTIONS_PER_REQUEST", 2) + manager, client = _make_manager() + client.create_commit.side_effect = [{"ok": True}, APIError("commit rejected", status_code=422)] + + with pytest.raises(APIError): + manager.delete_files( + repo_id="owner/repo", + repo_type="dataset", + file_paths=["a", "b", "c", "d"], + ) + + assert client.create_commit.call_count == 2 + + +def test_server_ceiling_is_env_tunable() -> None: + from modelscope_hub.constants import COMMIT_MAX_ACTIONS_PER_REQUEST + + assert COMMIT_MAX_ACTIONS_PER_REQUEST == 2000 diff --git a/tests/test_repo_tree_page_retry.py b/tests/test_repo_tree_page_retry.py new file mode 100644 index 0000000..c81ba9f --- /dev/null +++ b/tests/test_repo_tree_page_retry.py @@ -0,0 +1,143 @@ +"""File-tree listing resilience against spurious denials. + +The server intermittently answers a tree request with ``403 无权访问该数据集`` on a +repository the caller has just read successfully. Observed live on a 25k-file +dataset (paginated listing: page 7 denied, then page 145 denied, then success) +and again on a 10k-file dataset (``Root``-scoped listing denied outright). +Enumerating a large repo takes hundreds of such requests, so a per-request +failure rate becomes a near-certain whole-listing failure. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +import modelscope_hub._legacy_api as legacy_module +from modelscope_hub._legacy_api import LegacyClient +from modelscope_hub.errors import NetworkError, PermissionDeniedError + + +def _client() -> LegacyClient: + return LegacyClient(endpoint="https://modelscope.cn", token="ms-test") + + +def _page(count: int, offset: int = 0) -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.headers = {} + response.json.return_value = {"Data": {"Files": [{"Path": f"f{offset + i}"} for i in range(count)]}} + return response + + +def _denied() -> PermissionDeniedError: + return PermissionDeniedError("无权访问该数据集", status_code=403) + + +def test_transient_denial_on_a_later_page_is_retried(monkeypatch) -> None: + client = _client() + monkeypatch.setattr(legacy_module.time, "sleep", lambda _s: None) + calls: list[int] = [] + + def request(method: str, path: str, **kwargs): + page = kwargs["params"]["PageNumber"] + calls.append(page) + if page == 2 and calls.count(2) == 1: + raise _denied() + return _page(200, offset=page * 200) if page < 3 else _page(7, offset=600) + + with patch.object(client, "_request", side_effect=request): + files = client.list_dataset_files_paginated("owner/ds", page_size=200) + + # Page 2 was retried rather than aborting and discarding page 1. + assert calls == [1, 2, 2, 3] + assert len(files) == 407 + + +def test_denial_before_any_successful_tree_read_fails_fast() -> None: + # Nothing has proven the credential yet, so this really is an authorization + # answer and must not be retried into a slow, misleading failure. + client = _client() + calls: list[int] = [] + + def request(method: str, path: str, **kwargs): + calls.append(kwargs["params"]["PageNumber"]) + raise _denied() + + with patch.object(client, "_request", side_effect=request): + with pytest.raises(PermissionDeniedError): + client.list_dataset_files_paginated("owner/ds", page_size=200) + + assert calls == [1] + + +def test_root_scoped_listing_also_retries_a_spurious_denial(monkeypatch) -> None: + # The per-directory walk that works around the 3000-entry cap issues the same + # kind of request, and was seen to be denied the same way. + client = _client() + monkeypatch.setattr(legacy_module.time, "sleep", lambda _s: None) + client._tree_reads_ok = True # an earlier read already proved the credential + calls: list[str | None] = [] + + def request(method: str, path: str, **kwargs): + root = kwargs["params"].get("Root") + calls.append(root) + if calls.count(root) == 1: + raise _denied() + return _page(3) + + with patch.object(client, "_request", side_effect=request): + entries = client._list_files_page("owner/ds", "dataset", "master", recursive=False, root="level1_000") + + assert calls == ["level1_000", "level1_000"] + assert len(entries) == 3 + + +def test_a_successful_tree_read_arms_the_retry_for_later_requests(monkeypatch) -> None: + client = _client() + monkeypatch.setattr(legacy_module.time, "sleep", lambda _s: None) + assert client._tree_reads_ok is False + outcomes = [_page(2), _denied(), _page(2)] + + def request(method: str, path: str, **kwargs): + result = outcomes.pop(0) + if isinstance(result, Exception): + raise result + return result + + with patch.object(client, "_request", side_effect=request): + client._list_files_page("owner/ds", "dataset", "master", recursive=False, root="a") + assert client._tree_reads_ok is True + # The denial that follows is now retried instead of raised. + entries = client._list_files_page("owner/ds", "dataset", "master", recursive=False, root="b") + + assert len(entries) == 2 + assert not outcomes + + +def test_a_persistently_denied_listing_is_reported_as_a_transport_failure(monkeypatch) -> None: + client = _client() + monkeypatch.setattr(legacy_module.time, "sleep", lambda _s: None) + monkeypatch.setattr(legacy_module, "REPO_TREE_PAGE_MAX_ATTEMPTS", 3) + + def request(method: str, path: str, **kwargs): + if kwargs["params"]["PageNumber"] == 1: + return _page(200) + raise _denied() + + with patch.object(client, "_request", side_effect=request): + # Surfaced as a transport failure, not as "permission denied": the caller + # can read the repo, so reporting a denial would misdirect the diagnosis. + with pytest.raises(NetworkError, match="already-authorized"): + client.list_dataset_files_paginated("owner/ds", page_size=200) + + +def test_retry_budget_is_env_tunable() -> None: + from modelscope_hub.constants import ( + REPO_TREE_PAGE_MAX_ATTEMPTS, + REPO_TREE_PAGE_RETRY_MAX_DELAY_SECONDS, + ) + + assert REPO_TREE_PAGE_MAX_ATTEMPTS >= 2 + assert REPO_TREE_PAGE_RETRY_MAX_DELAY_SECONDS >= 1 diff --git a/tests/test_upload_batching.py b/tests/test_upload_batching.py new file mode 100644 index 0000000..4d36949 --- /dev/null +++ b/tests/test_upload_batching.py @@ -0,0 +1,785 @@ +"""Commit-batch planning, LFS routing and commit throttling behaviour.""" + +from __future__ import annotations + +import json +import threading +import time +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +import modelscope_hub._upload as upload_module +from modelscope_hub._upload import ( + BatchTracker, + UploadManager, + _calculate_adaptive_batch_size, + _is_inline_metadata, + _plan_commit_batches, + _upload_mode, +) +from modelscope_hub.errors import InvalidParameter, NetworkError, RateLimitError + + +def _make_manager() -> tuple[UploadManager, MagicMock]: + client = MagicMock() + client.create_commit.return_value = {"ok": True} + + def validate_blobs(*, repo_id: str, repo_type: str, objects: list[dict]) -> dict[str, str]: + return {obj["oid"]: f"https://upload/{obj['oid']}" for obj in objects} + + def upload_blob(*, upload_url: str, data, size: int) -> None: + while data.read(1024 * 1024): + pass + + client.validate_blobs.side_effect = validate_blobs + client.upload_blob.side_effect = upload_blob + return UploadManager(client, MagicMock()), client + + +# ---------------------------------------------------------------- batch sizing + + +@pytest.mark.parametrize( + ("total_files", "cap", "expected"), + [ + (0, 256, 1), + (1, 256, 1), + (100, 256, 100), + (2_000, 256, 256), + (60_000, 256, 256), + (60_000, 512, 512), + # "No cap" still respects the server's action ceiling, not the file count. + (5_000, 0, 2_000), + (1_500, 0, 1_500), + # An explicit cap above the ceiling cannot be honored either. + (60_000, 4_096, 2_000), + ], +) +def test_adaptive_batch_size_fills_up_to_the_cap(total_files: int, cap: int, expected: int) -> None: + assert _calculate_adaptive_batch_size(total_files, cap) == expected + + +def test_adaptive_batch_size_is_monotonic_and_never_exceeds_cap() -> None: + sizes = [_calculate_adaptive_batch_size(n, 256) for n in range(1, 3_000)] + assert sizes == sorted(sizes) + assert max(sizes) == 256 + + +def test_batch_plan_closes_on_inline_bytes_before_operation_cap(tmp_path: Path) -> None: + # 8 inline files of 100 KiB: the operation cap would take all 8 in one + # commit, but their base64 form exceeds a 512 KiB inline budget. + files = [] + sizes = {} + for index in range(8): + path = tmp_path / f"part-{index}.txt" + path.write_bytes(b"x" * (100 * 1024)) + files.append((f"part-{index}.txt", str(path))) + sizes[str(path)] = 100 * 1024 + + plan = _plan_commit_batches( + files, + "dataset", + max_operations=256, + max_inline_bytes=512 * 1024, + sizes=sizes, + ) + + assert sum(plan) == 8 + assert max(plan) < 8 + encoded_per_file = (100 * 1024 + 2) // 3 * 4 + assert all(count * encoded_per_file <= 512 * 1024 for count in plan) + + +def test_batch_plan_ignores_lfs_bytes_in_the_inline_budget(tmp_path: Path) -> None: + # .parquet is an LFS suffix for datasets, so these contribute a pointer + # rather than inline content and must not split the batch. + files = [] + sizes = {} + for index in range(8): + path = tmp_path / f"shard-{index}.parquet" + files.append((f"shard-{index}.parquet", str(path))) + sizes[str(path)] = 100 * 1024 + + plan = _plan_commit_batches( + files, + "dataset", + max_operations=256, + max_inline_bytes=512 * 1024, + sizes=sizes, + ) + + assert plan == [8] + + +def test_batch_plan_keeps_one_oversized_inline_file_per_batch(tmp_path: Path) -> None: + # Below the 1 MiB LFS threshold, so these really do travel inline; each one + # alone blows the inline budget, which must not stall the plan. + files = [] + sizes = {} + for index in range(3): + path = tmp_path / f"big-{index}.txt" + files.append((f"big-{index}.txt", str(path))) + sizes[str(path)] = 512 * 1024 + + plan = _plan_commit_batches( + files, + "dataset", + max_operations=256, + max_inline_bytes=1024, + sizes=sizes, + ) + + assert plan == [1, 1, 1] + + +def test_batch_tracker_handles_uneven_batches() -> None: + tracker = BatchTracker(7, [3, 1, 3]) + + assert tracker.num_batches == 3 + assert tracker.batch_range(0) == (0, 3) + assert tracker.batch_range(1) == (3, 4) + assert tracker.batch_range(2) == (4, 7) + assert [tracker.batch_index(i) for i in range(7)] == [0, 0, 0, 1, 2, 2, 2] + + +def test_batch_tracker_absorbs_a_short_plan() -> None: + tracker = BatchTracker(5, [2]) + + assert tracker.num_batches == 2 + assert tracker.batch_range(1) == (2, 5) + + +def test_batch_tracker_accepts_a_uniform_size() -> None: + tracker = BatchTracker(5, 2) + + assert tracker.num_batches == 3 + assert [tracker.batch_index(i) for i in range(5)] == [0, 0, 1, 1, 2] + + +# ------------------------------------------------------- inline metadata gate + + +@pytest.mark.parametrize( + "path", + ["README.md", "nested/README.md", ".gitattributes", "configuration.json", "sub/dir/config.json"], +) +def test_inline_metadata_is_recognized_at_any_depth(path: str) -> None: + assert _is_inline_metadata(path) + + +def test_metadata_stays_inline_even_below_a_zero_threshold(monkeypatch) -> None: + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 0) + + assert _upload_mode("README.md", 500 * 1024, "dataset") == "normal" + assert _upload_mode("configuration.json", 10, "model") == "normal" + # Everything that is not repository metadata does move to LFS. + assert _upload_mode("data/sample.txt", 1, "dataset") == "lfs" + + +def test_small_files_move_to_lfs_when_the_threshold_is_lowered(monkeypatch) -> None: + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 32 * 1024) + + assert _upload_mode("data/sample.txt", 115 * 1024, "dataset") == "lfs" + # A small card or config still rides inline at this threshold. + assert _upload_mode("data/notes.txt", 2 * 1024, "dataset") == "normal" + + +def test_upload_folder_routes_small_files_to_blob_storage(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 32 * 1024) + manager, client = _make_manager() + (tmp_path / "README.md").write_bytes(b"# card") + for index in range(4): + (tmp_path / f"sample-{index}.txt").write_bytes(bytes([index]) * (64 * 1024)) + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=2, + use_cache=False, + disable_tqdm=True, + ) + + operations = client.create_commit.call_args.kwargs["operations"] + modes = {op["path"]: op["type"] for op in operations} + assert modes == { + "README.md": "normal", + "sample-0.txt": "lfs", + "sample-1.txt": "lfs", + "sample-2.txt": "lfs", + "sample-3.txt": "lfs", + } + # The four data files were pre-signed together, not one request each. + assert client.validate_blobs.call_count == 1 + assert len(client.validate_blobs.call_args.kwargs["objects"]) == 4 + assert client.upload_blob.call_count == 4 + + +def test_upload_folder_batch_presigns_in_configured_group_size(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 0) + monkeypatch.setattr(upload_module, "UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS", 3) + manager, client = _make_manager() + for index in range(7): + (tmp_path / f"sample-{index}.txt").write_bytes(bytes([index]) * 128) + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=2, + use_cache=False, + disable_tqdm=True, + ) + + assert client.validate_blobs.call_count == 3 + assert client.upload_blob.call_count == 7 + + +def test_presign_groups_run_concurrently(tmp_path: Path, monkeypatch) -> None: + # The groups are independent requests; serialising them made the pre-sign + # phase scale with the file count in pure round-trip latency. + monkeypatch.setattr(upload_module, "UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS", 1) + manager, client = _make_manager() + peak = 0 + inflight = 0 + lock = threading.Lock() + + def validate_blobs(*, repo_id: str, repo_type: str, objects: list[dict]) -> dict[str, str]: + nonlocal peak, inflight + with lock: + inflight += 1 + peak = max(peak, inflight) + time.sleep(0.05) + with lock: + inflight -= 1 + return {obj["oid"]: f"https://upload/{obj['oid']}" for obj in objects} + + client.validate_blobs.side_effect = validate_blobs + objects = [{"oid": f"{index:064x}", "size": 1} for index in range(8)] + + result = manager._validate_blobs_batch(repo_id="owner/repo", repo_type="dataset", objects=objects, max_workers=8) + + assert len(result) == 8 + assert peak > 1, "pre-sign groups must overlap" + + +def test_presign_concurrency_stays_within_the_connection_pool(tmp_path: Path, monkeypatch) -> None: + # More in-flight requests than pooled connections just trades latency for + # discarded connections and fresh TLS handshakes. + monkeypatch.setattr(upload_module, "UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS", 1) + monkeypatch.setattr(upload_module, "API_CONNECTION_POOL_MAXSIZE", 2) + manager, client = _make_manager() + peak = 0 + inflight = 0 + lock = threading.Lock() + + def validate_blobs(*, repo_id: str, repo_type: str, objects: list[dict]) -> dict[str, str]: + nonlocal peak, inflight + with lock: + inflight += 1 + peak = max(peak, inflight) + time.sleep(0.02) + with lock: + inflight -= 1 + return {obj["oid"]: "https://upload/x" for obj in objects} + + client.validate_blobs.side_effect = validate_blobs + objects = [{"oid": f"{index:064x}", "size": 1} for index in range(12)] + + manager._validate_blobs_batch(repo_id="owner/repo", repo_type="dataset", objects=objects, max_workers=32) + + assert peak <= 2 + + +def test_presign_normalizes_unmentioned_objects_as_existing(monkeypatch) -> None: + # The batch endpoint answers only about objects that need uploading and + # returns empty arrays for ones that already exist. Leaving those out of the + # map would send every file of a re-run back to per-file negotiation. + monkeypatch.setattr(upload_module, "UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS", 2) + manager, client = _make_manager() + objects = [{"oid": f"{index:064x}", "size": 1} for index in range(4)] + # Server mentions only the last object; the rest already exist. + wanted = objects[-1]["oid"] + client.validate_blobs.side_effect = lambda **kw: ( + {wanted: "https://upload/x"} if any(o["oid"] == wanted for o in kw["objects"]) else {} + ) + + result = manager._validate_blobs_batch(repo_id="owner/repo", repo_type="dataset", objects=objects, max_workers=4) + + assert set(result) == {o["oid"] for o in objects} + assert result[wanted] == "https://upload/x" + assert all(result[o["oid"]] is None for o in objects[:-1]) + + +def test_a_failed_presign_group_falls_back_to_per_file_negotiation(tmp_path: Path, monkeypatch) -> None: + # Losing the optimisation must never escalate into losing the upload -- and + # critically, an oid missing from the map must not read as "already exists", + # which would skip the transfer and commit a pointer to a blob that was + # never stored. + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 0) + monkeypatch.setattr(upload_module, "UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS", 2) + manager, client = _make_manager() + for index in range(6): + (tmp_path / f"blob-{index}.dat").write_bytes(bytes([index]) * 512) + + seen: list[int] = [] + + def validate_blobs(*, repo_id: str, repo_type: str, objects: list[dict]) -> dict[str, str]: + seen.append(len(objects)) + if len(objects) > 1 and len(seen) == 1: + raise NetworkError("pre-sign group failed") + return {obj["oid"]: f"https://upload/{obj['oid']}" for obj in objects} + + client.validate_blobs.side_effect = validate_blobs + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=1, + use_cache=False, + disable_tqdm=True, + ) + + # Every file still reached object storage, none was silently treated as reused. + assert client.upload_blob.call_count == 6 + operations = client.create_commit.call_args.kwargs["operations"] + assert len(operations) == 6 + assert all(op["type"] == "lfs" and op["sha256"] for op in operations) + + +@pytest.mark.parametrize("use_cache", [True, False]) +def test_pre_hashed_files_are_not_hashed_twice(tmp_path: Path, monkeypatch, use_cache: bool) -> None: + # Pre-hashing feeds group pre-signing; without threading the digest through + # to the upload worker it would read and hash every file a second time. + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 0) + manager, _ = _make_manager() + for index in range(5): + (tmp_path / f"sample-{index}.txt").write_bytes(bytes([index]) * 256) + + calls: list[object] = [] + original = upload_module._compute_file_hash + + def counting_hash(*args, **kwargs): + calls.append(kwargs.get("file_path_or_obj", args[0] if args else None)) + return original(*args, **kwargs) + + monkeypatch.setattr(upload_module, "_compute_file_hash", counting_hash) + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=2, + use_cache=use_cache, + disable_tqdm=True, + ) + + assert len(calls) == 5 + assert len(set(calls)) == 5 + + +# ------------------------------------------------------------- tracker path + + +def test_tracker_path_keeps_resume_state_outside_the_uploaded_tree(tmp_path: Path) -> None: + manager, _ = _make_manager() + staging = tmp_path / "staging" + staging.mkdir() + (staging / "README.md").write_bytes(b"hello") + cache_path = tmp_path / "state" / "upload.json" + cache_path.parent.mkdir() + + manager.upload_folder( + repo_id="owner/repo", + repo_type="model", + folder_path=staging, + max_workers=1, + use_cache=True, + disable_tqdm=True, + tracker_path=cache_path, + ) + + assert not (staging / ".ms_upload_cache").exists() + assert cache_path.exists() + assert json.loads(cache_path.read_text(encoding="utf-8"))["repo_id"] == "owner/repo" + + +def test_external_tracker_lets_a_second_run_skip_committed_files(tmp_path: Path) -> None: + manager, client = _make_manager() + staging = tmp_path / "staging" + staging.mkdir() + (staging / "README.md").write_bytes(b"hello") + cache_path = tmp_path / "upload.json" + + for _ in range(2): + manager.upload_folder( + repo_id="owner/repo", + repo_type="model", + folder_path=staging, + max_workers=1, + use_cache=True, + disable_tqdm=True, + tracker_path=cache_path, + ) + + # The second run recognises the committed file and issues no new commit. + assert client.create_commit.call_count == 1 + + +# ------------------------------------------------------------ progress events + + +def test_progress_callback_reports_every_batch(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr(upload_module, "UPLOAD_COMMIT_BATCH_MAX_OPERATIONS", 2) + monkeypatch.setattr(upload_module, "UPLOAD_ADAPTIVE_BATCHING_ENABLED", False) + manager, _ = _make_manager() + for index in range(4): + (tmp_path / f"file-{index}.txt").write_bytes(bytes([index]) * (index + 1)) + events: list[dict] = [] + + manager.upload_folder( + repo_id="owner/repo", + repo_type="model", + folder_path=tmp_path, + max_workers=1, + use_cache=False, + disable_tqdm=True, + progress_callback=events.append, + ) + + commits = [e for e in events if e["event"] == "batch_committed"] + assert [e["batch_index"] for e in commits] == [0, 1] + assert commits[-1]["committed_files"] == 4 + assert commits[-1]["num_batches"] == 2 + assert all(e["total_files"] == 4 for e in commits) + + +def test_progress_events_carry_byte_counts(tmp_path: Path, monkeypatch) -> None: + # A consumer that only learns file counts cannot derive a throughput rate or + # an ETA, which is most of what a progress feed is for. + monkeypatch.setattr(upload_module, "UPLOAD_COMMIT_BATCH_MAX_OPERATIONS", 2) + monkeypatch.setattr(upload_module, "UPLOAD_ADAPTIVE_BATCHING_ENABLED", False) + manager, _ = _make_manager() + expected_total = 0 + for index in range(4): + payload = bytes([index]) * (1024 * (index + 1)) + (tmp_path / f"file-{index}.txt").write_bytes(payload) + expected_total += len(payload) + events: list[dict] = [] + + manager.upload_folder( + repo_id="owner/repo", + repo_type="model", + folder_path=tmp_path, + max_workers=1, + use_cache=False, + disable_tqdm=True, + progress_callback=events.append, + ) + + commits = [e for e in events if e["event"] == "batch_committed"] + assert all(e["total_bytes"] == expected_total for e in commits) + assert sum(e["batch_bytes"] for e in commits) == expected_total + assert commits[-1]["committed_bytes"] == expected_total + # Cumulative counters advance monotonically alongside the per-batch ones. + assert commits[0]["committed_bytes"] == commits[0]["batch_bytes"] + # These files ride inline, so all of their volume is attributed to the commit + # rather than to an earlier object-storage transfer. + assert sum(e["batch_inline_bytes"] for e in commits) == expected_total + + +def test_wire_progress_tracks_object_storage_transfers(tmp_path: Path, monkeypatch) -> None: + # Commits land in lumps, so a rate built from them alone reads as a spike + # followed by zero. Blob uploads finish continuously and are the honest + # source; only bytes that really moved are counted. + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 0) + monkeypatch.setattr(upload_module, "UPLOAD_PROGRESS_MIN_INTERVAL_SECONDS", 0) + manager, _ = _make_manager() + expected_total = 0 + for index in range(6): + payload = bytes([index]) * (2048 * (index + 1)) + (tmp_path / f"blob-{index}.dat").write_bytes(payload) + expected_total += len(payload) + events: list[dict] = [] + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=2, + use_cache=False, + disable_tqdm=True, + progress_callback=events.append, + ) + + wire = [e for e in events if e["event"] == "upload_progress"] + assert wire, "per-file transfers must report progress before the commit lands" + # Cumulative and never decreasing. + assert [e["uploaded_bytes"] for e in wire] == sorted(e["uploaded_bytes"] for e in wire) + assert wire[-1]["uploaded_bytes"] == expected_total + assert wire[-1]["uploaded_files"] == 6 + # Deltas partition the total exactly, so a consumer can sum them into a rate. + assert sum(e["uploaded_bytes_delta"] for e in wire) == expected_total + # Nothing rode inline, so the commit attributes no wire traffic to itself. + commits = [e for e in events if e["event"] == "batch_committed"] + assert sum(e["batch_inline_bytes"] for e in commits) == 0 + + +def test_reused_blobs_report_no_wire_bytes(tmp_path: Path, monkeypatch) -> None: + # A deduplicated blob transfers nothing; counting its size would inflate the + # reported throughput above what the link actually carried. + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 0) + monkeypatch.setattr(upload_module, "UPLOAD_PROGRESS_MIN_INTERVAL_SECONDS", 0) + manager, client = _make_manager() + client.validate_blobs.side_effect = lambda **kwargs: {obj["oid"]: None for obj in kwargs["objects"]} + for index in range(4): + (tmp_path / f"blob-{index}.dat").write_bytes(bytes([index]) * 4096) + events: list[dict] = [] + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=2, + use_cache=False, + disable_tqdm=True, + progress_callback=events.append, + ) + + client.upload_blob.assert_not_called() + wire = [e for e in events if e["event"] == "upload_progress"] + assert all(e["uploaded_bytes"] == 0 for e in wire) + assert wire[-1]["uploaded_files"] == 4 + # The files are still committed, and the commit still reports their volume. + commits = [e for e in events if e["event"] == "batch_committed"] + assert commits[-1]["committed_bytes"] == 4 * 4096 + + +def test_progress_callback_failure_does_not_abort_the_upload(tmp_path: Path) -> None: + manager, client = _make_manager() + (tmp_path / "README.md").write_bytes(b"hello") + + def explode(_event: dict) -> None: + raise RuntimeError("reporter is broken") + + manager.upload_folder( + repo_id="owner/repo", + repo_type="model", + folder_path=tmp_path, + max_workers=1, + use_cache=False, + disable_tqdm=True, + progress_callback=explode, + ) + + client.create_commit.assert_called_once() + + +# ------------------------------------------------------------ commit throttle + + +def test_commit_honors_server_retry_after(monkeypatch) -> None: + manager, client = _make_manager() + slept: list[float] = [] + monkeypatch.setattr(upload_module.time, "sleep", slept.append) + client.create_commit.side_effect = [ + RateLimitError("commit budget exhausted", retry_after=42), + {"ok": True}, + ] + + result = manager._commit_with_retry( + repo_id="owner/repo", + repo_type="model", + operations=[{"action": "create", "path": "a.txt"}], + commit_message="retry", + ) + + assert result == {"ok": True} + assert slept == [42.0] + + +def test_commit_retry_after_above_the_ceiling_aborts(monkeypatch) -> None: + manager, client = _make_manager() + slept: list[float] = [] + monkeypatch.setattr(upload_module.time, "sleep", slept.append) + monkeypatch.setattr(upload_module, "UPLOAD_COMMIT_MAX_RETRY_AFTER_SECONDS", 60) + client.create_commit.side_effect = RateLimitError("commit budget exhausted", retry_after=3600) + + with pytest.raises(RateLimitError): + manager._commit_with_retry( + repo_id="owner/repo", + repo_type="model", + operations=[{"action": "create", "path": "a.txt"}], + commit_message="retry", + ) + + assert slept == [] + + +def test_throttled_wait_does_not_consume_the_transient_error_budget(monkeypatch) -> None: + # A long, server-declared wait must not exhaust the allowance reserved for + # failures of unknown duration, or a throttle would look like a hard error. + manager, client = _make_manager() + monkeypatch.setattr(upload_module, "UPLOAD_COMMIT_RETRY_TOTAL_WAIT_SECONDS", 10) + monkeypatch.setattr(upload_module.time, "sleep", lambda _s: None) + client.create_commit.side_effect = [ + RateLimitError("throttled", retry_after=600), + RateLimitError("throttled", retry_after=600), + {"ok": True}, + ] + + assert manager._commit_with_retry( + repo_id="owner/repo", + repo_type="model", + operations=[{"action": "create", "path": "a.txt"}], + commit_message="retry", + max_attempts=4, + ) == {"ok": True} + + +def test_commit_rate_governor_paces_once_the_budget_is_spent(monkeypatch) -> None: + governor = upload_module._CommitRateGovernor(2) + clock = {"now": 0.0} + slept: list[float] = [] + monkeypatch.setattr(upload_module.time, "monotonic", lambda: clock["now"]) + + def fake_sleep(seconds: float) -> None: + slept.append(seconds) + clock["now"] += seconds + + monkeypatch.setattr(upload_module.time, "sleep", fake_sleep) + + assert governor.acquire() == 0.0 + assert governor.acquire() == 0.0 + waited = governor.acquire() + + assert slept, "the third commit within the window must wait" + assert waited == pytest.approx(3600.0, abs=1.0) + + +def test_commit_rate_governor_is_inert_when_disabled() -> None: + governor = upload_module._CommitRateGovernor(0) + + assert not governor.enabled + assert [governor.acquire() for _ in range(50)] == [0.0] * 50 + + +def test_every_commit_path_goes_through_the_shared_governor(tmp_path: Path, monkeypatch) -> None: + # The budget belongs to the repository, not to one call. Pacing only the + # happy path would leave recovery rounds free to hammer a throttled server. + monkeypatch.setattr(upload_module, "UPLOAD_COMMIT_MAX_PER_HOUR", 500) + monkeypatch.setattr(upload_module, "UPLOAD_COMMIT_BATCH_MAX_OPERATIONS", 2) + monkeypatch.setattr(upload_module, "UPLOAD_ADAPTIVE_BATCHING_ENABLED", False) + manager, client = _make_manager() + assert manager._commit_governor.enabled + + acquired: list[int] = [] + original_acquire = manager._commit_governor.acquire + monkeypatch.setattr( + manager._commit_governor, + "acquire", + lambda: (acquired.append(1), original_acquire())[1], + ) + + for index in range(4): + (tmp_path / f"file-{index}.txt").write_bytes(bytes([index])) + + manager.upload_folder( + repo_id="owner/repo", + repo_type="model", + folder_path=tmp_path, + max_workers=1, + use_cache=False, + disable_tqdm=True, + ) + manager.upload_file( + repo_id="owner/repo", + repo_type="model", + path_or_fileobj=b"single", + path_in_repo="extra.txt", + disable_tqdm=True, + ) + + # Two batch commits plus the single-file commit. + assert client.create_commit.call_count == 3 + assert len(acquired) == 3 + + +def test_upload_file_retries_a_transient_commit_failure(monkeypatch) -> None: + manager, client = _make_manager() + monkeypatch.setattr(upload_module.time, "sleep", lambda _s: None) + client.create_commit.side_effect = [ + NetworkError("commit temporarily unavailable"), + {"ok": True}, + ] + + result = manager.upload_file( + repo_id="owner/repo", + repo_type="model", + path_or_fileobj=b"hello", + path_in_repo="README.md", + disable_tqdm=True, + ) + + assert result == {"ok": True} + assert client.create_commit.call_count == 2 + + +def test_upload_file_honors_retry_after(monkeypatch) -> None: + manager, client = _make_manager() + slept: list[float] = [] + monkeypatch.setattr(upload_module.time, "sleep", slept.append) + client.create_commit.side_effect = [ + RateLimitError("commit budget exhausted", retry_after=7), + {"ok": True}, + ] + + manager.upload_file( + repo_id="owner/repo", + repo_type="model", + path_or_fileobj=b"hello", + path_in_repo="README.md", + disable_tqdm=True, + ) + + assert slept == [7.0] + + +def test_upload_file_still_fails_fast_on_a_permanent_error() -> None: + manager, client = _make_manager() + client.create_commit.side_effect = InvalidParameter("path is not allowed") + + with pytest.raises(InvalidParameter): + manager.upload_file( + repo_id="owner/repo", + repo_type="model", + path_or_fileobj=b"hello", + path_in_repo="README.md", + disable_tqdm=True, + ) + + assert client.create_commit.call_count == 1 + + +def test_prepare_upload_folder_reports_sizes_for_reuse(tmp_path: Path) -> None: + # Planning needs these sizes immediately afterwards; re-stating the tree + # costs one syscall per file and yields nothing new. + manager, _ = _make_manager() + (tmp_path / "a.txt").write_bytes(b"x" * 10) + nested = tmp_path / "sub" + nested.mkdir() + (nested / "b.txt").write_bytes(b"y" * 20) + + sizes: dict[str, int] = {} + prepared = manager._prepare_upload_folder( + folder_path=tmp_path, + path_in_repo="", + repo_type="model", + sizes_out=sizes, + ) + + assert sizes == {str(tmp_path / "a.txt"): 10, str(nested / "b.txt"): 20} + assert {path for _, path in prepared} == set(sizes) diff --git a/tests/test_upload_config.py b/tests/test_upload_config.py index 6176727..acaf9ab 100644 --- a/tests/test_upload_config.py +++ b/tests/test_upload_config.py @@ -280,6 +280,146 @@ def test_dead_lfs_threshold_is_not_registered() -> None: assert "UPLOAD_LFS_THRESHOLD" not in names +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("32KiB", 32 * 1024), + ("512K", 512 * 1024), + ("65536", 65536), + ("1MiB", 1024 * 1024), + ("2MB", 2 * 1000 * 1000), + ("0", 0), + ], +) +def test_lfs_threshold_accepts_byte_sizes_and_unit_suffixes(raw: str, expected: int) -> None: + # A megabyte-only knob cannot express the sub-MB range that decides whether + # small files ride inline in a commit or go to object storage. + result = _run_constants( + "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", + env={"MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD": raw}, + ) + assert json.loads(result.stdout)["UPLOAD_LFS_FORCE_THRESHOLD_BYTES"] == expected + assert "FutureWarning" not in result.stderr + + +def test_deprecated_lfs_threshold_mb_alias_keeps_megabyte_units() -> None: + result = _run_constants( + "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", + env={"MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD_MB": "4"}, + ) + assert json.loads(result.stdout)["UPLOAD_LFS_FORCE_THRESHOLD_BYTES"] == 4 * 1024 * 1024 + assert "MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD_MB" in result.stderr + assert "expects bytes" in result.stderr + + +def test_canonical_lfs_threshold_wins_over_deprecated_aliases() -> None: + result = _run_constants( + "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", + env={ + "MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD": "32KiB", + "MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD_MB": "4", + "UPLOAD_LFS_ENFORCE_THRESHOLD": "999", + }, + ) + assert json.loads(result.stdout)["UPLOAD_LFS_FORCE_THRESHOLD_BYTES"] == 32 * 1024 + assert "FutureWarning" not in result.stderr + + +@pytest.mark.parametrize( + ("env", "constant", "expected", "needle"), + [ + ( + {"MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD": "32ZB"}, + "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", + 1024 * 1024, + "unknown size unit", + ), + ( + {"MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD": "-1"}, + "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", + 1024 * 1024, + "must not be negative", + ), + ( + {"MODELSCOPE_UPLOAD_COMMIT_BATCH_MAX_OPERATIONS": "lots"}, + "UPLOAD_COMMIT_BATCH_MAX_OPERATIONS", + 256, + "not an integer", + ), + ( + {"MODELSCOPE_UPLOAD_COMMIT_BATCH_MAX_OPERATIONS": "0"}, + "UPLOAD_COMMIT_BATCH_MAX_OPERATIONS", + 256, + "must be a positive integer", + ), + ( + {"MODELSCOPE_UPLOAD_COMMIT_MAX_INLINE_BYTES": "8bogus"}, + "UPLOAD_COMMIT_MAX_INLINE_BYTES", + 8 * 1024 * 1024, + "unknown size unit", + ), + ], +) +def test_invalid_upload_env_values_warn_instead_of_silently_reverting( + env: dict[str, str], + constant: str, + expected: int, + needle: str, +) -> None: + # Silently falling back made misconfiguration invisible: the value had no + # effect and nothing said so. + result = _run_constants(constant, env=env) + assert json.loads(result.stdout)[constant] == expected + assert needle in result.stderr + assert "is invalid" in result.stderr + + +def test_commit_rate_governor_accepts_explicit_zero_without_warning() -> None: + result = _run_constants( + "UPLOAD_COMMIT_MAX_PER_HOUR", + env={"MODELSCOPE_UPLOAD_COMMIT_MAX_PER_HOUR": "0"}, + ) + assert json.loads(result.stdout)["UPLOAD_COMMIT_MAX_PER_HOUR"] == 0 + assert "is invalid" not in result.stderr + + +def test_new_upload_knobs_are_registered_under_upload() -> None: + result = _run_constants("ENV_REGISTRY") + registry = {item["name"]: item for item in json.loads(result.stdout)["ENV_REGISTRY"]} + for name in ( + "MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD", + "MODELSCOPE_UPLOAD_COMMIT_MAX_INLINE_BYTES", + "MODELSCOPE_UPLOAD_COMMIT_MAX_PER_HOUR", + "MODELSCOPE_UPLOAD_COMMIT_MAX_RETRY_AFTER_SECONDS", + "MODELSCOPE_UPLOAD_INLINE_METADATA_PATHS", + ): + assert registry[name]["category"] == "Upload" + assert registry["MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD"]["default"] == "1MiB" + assert "MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD_MB" not in registry + + +def test_http_sessions_size_the_pool_for_concurrent_workers() -> None: + # urllib3 defaults the pool to 10. A folder upload runs max_workers requests + # at once, so the default discarded the excess connections and paid for a + # fresh TLS handshake on their next use. + from modelscope_hub.constants import API_CONNECTION_POOL_MAXSIZE + + api = HubApi(config=HubConfig(token="ms-test")) + for session in (api.legacy._session, api.openapi._session): + adapter = session.get_adapter("https://modelscope.cn") + pool_kw = adapter.poolmanager.connection_pool_kw + assert pool_kw["maxsize"] == API_CONNECTION_POOL_MAXSIZE + assert API_CONNECTION_POOL_MAXSIZE >= 16 + + +def test_connection_pool_size_is_env_tunable() -> None: + result = _run_constants( + "API_CONNECTION_POOL_MAXSIZE", + env={"MODELSCOPE_API_CONNECTION_POOL_MAXSIZE": "64"}, + ) + assert json.loads(result.stdout)["API_CONNECTION_POOL_MAXSIZE"] == 64 + + def _response(data: dict | None = None) -> MagicMock: response = MagicMock() response.status_code = 200 diff --git a/tests/test_upload_lfs_gate.py b/tests/test_upload_lfs_gate.py index 6532b8c..c01c16a 100644 --- a/tests/test_upload_lfs_gate.py +++ b/tests/test_upload_lfs_gate.py @@ -80,17 +80,14 @@ def test_delete_files_rejects_empty_paths() -> None: manager, client = _make_manager() with pytest.raises(InvalidParameter, match="at least one"): - manager.delete_files( - repo_id="owner/repo", repo_type="model", file_paths=["", ""]) + manager.delete_files(repo_id="owner/repo", repo_type="model", file_paths=["", ""]) client.create_commit.assert_not_called() @pytest.mark.parametrize("repo_type", ["model", "dataset"]) @pytest.mark.parametrize("file_paths", [["", "old.bin"], "old.bin"]) -def test_hub_api_delete_files_delegates_to_upload_manager( - repo_type: str, file_paths: list[str] | str -) -> None: +def test_hub_api_delete_files_delegates_to_upload_manager(repo_type: str, file_paths: list[str] | str) -> None: api = HubApi(token="test-token") api._uploader = MagicMock() api._uploader.delete_files.return_value = {"deleted_files": ["old.bin"]} @@ -128,7 +125,8 @@ def test_hub_api_delete_patterns_resolve_remote_paths(repo_type: str) -> None: SimpleNamespace(path="nested/metadata.json", type="blob"), SimpleNamespace(path="weights.bin", type="blob"), SimpleNamespace(path="nested", type="tree"), - ]) + ] + ) result = api.delete_files( "owner/repo", @@ -138,8 +136,7 @@ def test_hub_api_delete_patterns_resolve_remote_paths(repo_type: str) -> None: revision="main", ) - api.list_repo_files.assert_called_once_with( - "owner/repo", repo_type, revision="main", recursive=True) + api.list_repo_files.assert_called_once_with("owner/repo", repo_type, revision="main", recursive=True) api._uploader.delete_files.assert_called_once_with( repo_id="owner/repo", repo_type=repo_type, @@ -153,11 +150,9 @@ def test_hub_api_delete_patterns_resolve_remote_paths(repo_type: str) -> None: def test_hub_api_delete_patterns_with_no_match_is_noop() -> None: api = HubApi(token="test-token") api._uploader = MagicMock() - api.list_repo_files = MagicMock( - return_value=[SimpleNamespace(path="weights.bin", type="blob")]) + api.list_repo_files = MagicMock(return_value=[SimpleNamespace(path="weights.bin", type="blob")]) - result = api.delete_files( - "owner/repo", "model", delete_patterns="*.json") + result = api.delete_files("owner/repo", "model", delete_patterns="*.json") api._uploader.delete_files.assert_not_called() assert result == { From bd9f58ca1514b0642f50fc0832b381e2c22392c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Fri, 18 Sep 2026 00:19:01 +0800 Subject: [PATCH 11/22] feat(upload): transfer identical content once per run, and report recovered work Batch pre-signing asks the server about every distinct oid before any upload starts, so the server cannot answer "already stored" for a duplicate -- it is not stored yet. Confirmed against the Hub: pre-signing the same fresh oid twice returns two upload URLs, and only after the blob lands does the endpoint report it as existing. Every occurrence of identical content therefore PUT the same bytes, which the previous per-file pre-sign path had avoided by accident of timing. Owner election, decided before any worker starts: - The first file holding an oid owns its transfer; the rest carry a marker and skip the PUT. `files_to_upload` is in ascending file index and a batch owns a contiguous ascending range, so an owner always lands in a batch no later than its copies. Batches commit in order and each waits for its own files, so a copy's blob is guaranteed present by the time its batch commits -- no locks, no waiting, and no way for the worker pool to block on itself. - Content the server already holds needs no owner at all. - If an owner's upload fails, its copies are withheld from the commit and pushed to the retry queue rather than published as pointers to a blob that was never stored. - Inline files are untouched: their content travels in the commit body, so there is no transfer to skip. Recovery accounting: - The ReAct and simple retry paths now report their uploads and commits. They previously committed silently, so any rescued volume never reached the metrics. An 8 GiB run that lost one 512-file batch to a rejected commit finished with all 40000 files on the Hub but 91 MB missing from done_bytes. Verified against the Hub with 40000 files / 8.00 GiB (37543 distinct contents, largest duplicate group 1272): - predicted saving 51,369,983 bytes; the run transferred 8,538,564,609 of 8,589,934,592 committed bytes -- short by exactly 51,369,983. - 12 of the 385 files that skipped their own PUT were downloaded and hashed: all matched, so no pointer was left dangling. - listing check: 40000 remote blobs, 0 missing, 0 extra, 0 size mismatches. - a fresh-state re-upload moved 64.81 MiB instead of 8 GiB and reported done_bytes exactly, confirming the recovery accounting fix. --- src/modelscope_hub/_upload.py | 173 ++++++++++++++++++++- src/modelscope_hub/version.py | 2 +- tests/test_upload_batching.py | 48 ++++++ tests/test_upload_blob_dedup.py | 256 ++++++++++++++++++++++++++++++++ 4 files changed, 476 insertions(+), 3 deletions(-) create mode 100644 tests/test_upload_blob_dedup.py diff --git a/src/modelscope_hub/_upload.py b/src/modelscope_hub/_upload.py index 19749a9..ec07890 100644 --- a/src/modelscope_hub/_upload.py +++ b/src/modelscope_hub/_upload.py @@ -90,6 +90,25 @@ _TRACKER_VERSION = 3 +class _DuplicateBlob: + """Marker: an earlier file in this run uploads this exact content. + + Identical content hashes to one oid, and the batch pre-sign step hands every + occurrence the same upload URL -- so without this marker each occurrence + would PUT the same bytes again. The server only reports "already stored" + once a blob has landed, which cannot help when all the pre-signing happens + before any upload starts. + """ + + __slots__ = () + + def __repr__(self) -> str: # pragma: no cover - diagnostic only + return "" + + +DUPLICATE_BLOB = _DuplicateBlob() + + # ==================================================================== # Helpers # ==================================================================== @@ -1057,6 +1076,42 @@ def upload_folder( f", {unresolved} unresolved (will negotiate per file)" if unresolved else "", ) + # Elect one owner per distinct blob. + # + # `files_to_upload` is in ascending file index, and a batch owns a + # contiguous ascending index range, so the first file holding a given oid + # always lands in a batch no later than any of its duplicates. Batches are + # committed in order and each waits for its own files, so by the time a + # duplicate's batch commits, its owner has already finished -- which is + # what lets the duplicates skip the transfer with no locking and no risk + # of a worker pool deadlocking on itself. + blob_owner: dict[str, int] = {} + blob_ready: dict[str, bool] = {} + blob_state_lock = threading.Lock() + deduped_files = 0 + deduped_bytes = 0 + for file_idx, _file_info in files_to_upload: + info = lfs_hash_info_map.get(file_idx) + if info is None: + continue + oid = info["file_hash"] + if pre_validated_map.get(oid, "") is None: + # Already stored server-side; nobody needs to transfer it. + blob_ready[oid] = True + continue + if oid not in blob_owner: + blob_owner[oid] = file_idx + else: + deduped_files += 1 + deduped_bytes += info["file_size"] + if deduped_files: + logger.info( + "Deduplicated %d file(s) sharing content with an earlier file: %d byte(s) that " + "would otherwise be uploaded twice.", + deduped_files, + deduped_bytes, + ) + skipped_count = len(skipped_indices) if skipped_count > 0: logger.info("%d file(s) already committed, skipping.", skipped_count) @@ -1077,6 +1132,10 @@ def upload_folder( # Pipeline: upload workers def _upload_worker(file_idx: int, file_info: tuple, pre_validated: Any = None) -> None: path_in_repo_w, file_path_w = file_info + owned_oid: str | None = None + info = lfs_hash_info_map.get(file_idx) + if info is not None and blob_owner.get(info["file_hash"]) == file_idx: + owned_oid = info["file_hash"] try: logger.debug("Uploading: %s ...", path_in_repo_w) result = self._upload_single_file( @@ -1090,10 +1149,18 @@ def _upload_worker(file_idx: int, file_info: tuple, pre_validated: Any = None) - disable_tqdm=disable_tqdm, ) logger.debug("Uploaded: %s", path_in_repo_w) + # Publish the blob outcome before the batch is marked complete: + # the consumer reads it as soon as the batch event fires. + if owned_oid is not None: + with blob_state_lock: + blob_ready[owned_oid] = True batch_tracker.record_success(file_idx, result) _report_wire(result) except Exception as e: logger.error("Upload failed: %s - %s", path_in_repo_w, e) + if owned_oid is not None: + with blob_state_lock: + blob_ready[owned_oid] = False batch_tracker.record_failure(file_idx, file_info, e) # Pipeline: consume batches in order @@ -1229,7 +1296,7 @@ def _flush_wire() -> None: try: with ThreadPoolExecutor(max_workers=max_workers) as executor: for file_idx, file_info in files_to_upload: - pv: str | bool | None = None + pv: str | bool | _DuplicateBlob | None = None if file_idx in lfs_hash_info_map: cached_hash = lfs_hash_info_map[file_idx]["file_hash"] # "Answered, and the server did not ask for an upload" @@ -1240,7 +1307,12 @@ def _flush_wire() -> None: # was never stored. if cached_hash in pre_validated_map: url = pre_validated_map[cached_hash] - pv = True if url is None else url + if url is None: + pv = True + elif blob_owner.get(cached_hash) == file_idx: + pv = url + else: + pv = DUPLICATE_BLOB executor.submit(_upload_worker, file_idx, file_info, pv) consecutive_failures = 0 @@ -1270,6 +1342,43 @@ def _flush_wire() -> None: for item, err in failures: logger.error(" Failed: %s - %s", item[0], err) + # A file that skipped its transfer because a duplicate owned + # it must not be committed if that owner's upload failed: + # the commit would reference a blob that was never stored. + # Its owner is in this batch or an earlier one, both already + # resolved, so the outcome is known here. + orphaned: list[dict] = [] + if blob_owner: + with blob_state_lock: + ready_snapshot = dict(blob_ready) + committable = [] + for item_r in results: + oid_r = item_r["file_hash_info"]["file_hash"] + if ( + item_r.get("upload_mode") == "lfs" + and oid_r in blob_owner + and not ready_snapshot.get(oid_r, False) + ): + orphaned.append(item_r) + continue + committable.append(item_r) + if orphaned: + logger.warning( + "Batch %d/%d: %d file(s) deferred, the upload of the content they " + "share failed; they will be retried on their own.", + batch_idx + 1, + num_batches, + len(orphaned), + ) + total_failed_files.extend( + ( + (item_r["file_path_in_repo"], item_r["file_path"]), + StorageError("shared blob upload failed"), + ) + for item_r in orphaned + ) + results = committable + self._track_uploaded_batch(tracker, results) operations = self._build_batch_operations(results, repo_type) @@ -1365,6 +1474,39 @@ def _flush_wire() -> None: tracker.save() # ReAct progressive retry fallback + # + # Recovery has to report progress too. It is exactly the moment an + # operator is watching, and a run whose recovered volume never reaches + # the metrics under-reports by however much it rescued: an 8 GiB run that + # lost one 512-file batch to a rejected commit finished with 40000 files + # on the Hub but 91 MB missing from done_bytes. + def _report_recovery(results: list[dict], label: str) -> None: + nonlocal committed_files, committed_bytes + recovered_bytes = sum(r["file_size_on_disk"] for r in results) + inline_bytes = sum(r["file_size_on_disk"] for r in results if r.get("upload_mode") != "lfs") + committed_files += len(results) + committed_bytes += recovered_bytes + _flush_wire() + _emit( + { + "event": "recovery_committed", + "repo_id": repo_id, + "stage": label, + "batch_index": -1, + "num_batches": num_batches, + "batch_files": len(results), + "batch_bytes": recovered_bytes, + "batch_inline_bytes": inline_bytes, + "committed_files": committed_files, + "committed_bytes": committed_bytes, + "total_files": len(sorted_files), + "total_bytes": total_bytes, + "skipped_files": skipped_count, + "elapsed": time.time() - start_time, + "error": None, + } + ) + if total_failed_files and UPLOAD_RECOVERY_ENABLED: total_failed_files, react_commits, react_results = self._retry_failed_files_react( failed_files=total_failed_files, @@ -1375,6 +1517,8 @@ def _flush_wire() -> None: revision=revision, max_workers=max_workers, disable_tqdm=disable_tqdm, + on_uploaded=_report_wire, + on_committed=_report_recovery, ) commit_infos.extend(react_commits) all_results.extend(react_results) @@ -1390,6 +1534,8 @@ def _flush_wire() -> None: commit_infos=commit_infos, all_results=all_results, disable_tqdm=disable_tqdm, + on_uploaded=_report_wire, + on_committed=_report_recovery, ) tracker.save() @@ -1698,6 +1844,15 @@ def _upload_blob( res_d["is_reused"] = True return res_d + if pre_validated is DUPLICATE_BLOB: + # An earlier file in this run owns the transfer for this content. + # Batch ordering guarantees it has finished before any commit that + # references this file, so nothing has to be waited on here. + logger.debug("Blob %s is uploaded by an earlier duplicate, skipping transfer.", sha256[:8]) + res_d["is_uploaded"] = True + res_d["is_reused"] = True + return res_d + if isinstance(pre_validated, str): upload_url: str = pre_validated else: @@ -2090,6 +2245,8 @@ def _retry_failed_files_react( revision: str, max_workers: int, disable_tqdm: bool = False, + on_uploaded: Any = None, + on_committed: Any = None, ) -> tuple[list[tuple], list[dict], list[dict]]: commit_infos: list[dict] = [] all_successes: list[dict] = [] @@ -2180,6 +2337,8 @@ def _retry_failed_files_react( try: result = future.result() round_successes.append(result) + if on_uploaded is not None: + on_uploaded(result) except Exception as e: round_failures.append(((path_in_repo_r, file_path_r), e)) else: @@ -2207,6 +2366,8 @@ def _retry_failed_files_react( disable_tqdm=disable_tqdm, ) round_successes.append(result) + if on_uploaded is not None: + on_uploaded(result) except Exception as e: logger.error( "[ReAct] %s: failed %s - %s", @@ -2236,6 +2397,8 @@ def _retry_failed_files_react( ) commit_infos.append(commit_info) self._track_committed_batch(tracker, batch) + if on_committed is not None: + on_committed(batch, round_name) logger.info( "[ReAct] %s: committed %d file(s).", round_name, @@ -2335,6 +2498,8 @@ def _retry_failed_simple( commit_infos: list[dict], all_results: list[dict], disable_tqdm: bool = False, + on_uploaded: Any = None, + on_committed: Any = None, ) -> list[tuple]: total_failed_files = list(failed_files) for retry_round in range(UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS): @@ -2359,6 +2524,8 @@ def _retry_failed_simple( disable_tqdm=disable_tqdm, ) retry_successes.append(result) + if on_uploaded is not None: + on_uploaded(result) except Exception as e: logger.error(" Retry failed: %s - %s", path_in_repo_r, e) retry_failures.append(((path_in_repo_r, file_path_r), e)) @@ -2377,6 +2544,8 @@ def _retry_failed_simple( commit_infos.append(commit_info) all_results.extend(retry_successes) self._track_committed_batch(tracker, retry_successes) + if on_committed is not None: + on_committed(retry_successes, f"retry round {retry_round + 1}") logger.info( " Retry round %d: committed %d file(s).", retry_round + 1, diff --git a/src/modelscope_hub/version.py b/src/modelscope_hub/version.py index a59471b..764c69a 100644 --- a/src/modelscope_hub/version.py +++ b/src/modelscope_hub/version.py @@ -1,3 +1,3 @@ """Version information for modelscope_hub.""" -__version__ = "0.4.6+main" +__version__ = "0.4.7+main" diff --git a/tests/test_upload_batching.py b/tests/test_upload_batching.py index 4d36949..9114176 100644 --- a/tests/test_upload_batching.py +++ b/tests/test_upload_batching.py @@ -558,6 +558,54 @@ def test_reused_blobs_report_no_wire_bytes(tmp_path: Path, monkeypatch) -> None: assert commits[-1]["committed_bytes"] == 4 * 4096 +def test_recovered_files_are_reported_so_totals_stay_complete(tmp_path: Path, monkeypatch) -> None: + # Recovery is exactly when an operator is watching. An 8 GiB run that lost one + # 512-file batch to a rejected commit put all 40000 files on the Hub but left + # 91 MB missing from done_bytes, because the recovery path emitted nothing. + monkeypatch.setattr(upload_module, "UPLOAD_COMMIT_BATCH_MAX_OPERATIONS", 2) + monkeypatch.setattr(upload_module, "UPLOAD_ADAPTIVE_BATCHING_ENABLED", False) + monkeypatch.setattr(upload_module.time, "sleep", lambda _s: None) + manager, client = _make_manager() + expected_total = 0 + for index in range(4): + payload = bytes([index]) * (1024 * (index + 1)) + (tmp_path / f"file-{index}.txt").write_bytes(payload) + expected_total += len(payload) + events: list[dict] = [] + + # Exhaust every in-commit attempt for the first batch so it really falls + # through to the recovery path instead of being absorbed earlier. The attempt + # budget is a default argument, so it is spelled out rather than patched. + attempts = upload_module.UPLOAD_COMMIT_MAX_ATTEMPTS + calls = {"n": 0} + + def create_commit(**kwargs): + calls["n"] += 1 + if calls["n"] <= attempts: + raise NetworkError("commit rejected") + return {"ok": True} + + client.create_commit.side_effect = create_commit + + manager.upload_folder( + repo_id="owner/repo", + repo_type="model", + folder_path=tmp_path, + max_workers=1, + use_cache=False, + disable_tqdm=True, + progress_callback=events.append, + ) + + recovery = [e for e in events if e["event"] == "recovery_committed"] + assert recovery, "a recovered commit must report its progress" + accounted = [e for e in events if e["event"] in ("batch_committed", "recovery_committed")] + # Every byte is accounted for exactly once across the two commit paths. + assert sum(e["batch_bytes"] for e in accounted) == expected_total + assert accounted[-1]["committed_bytes"] == expected_total + assert accounted[-1]["committed_files"] == 4 + + def test_progress_callback_failure_does_not_abort_the_upload(tmp_path: Path) -> None: manager, client = _make_manager() (tmp_path / "README.md").write_bytes(b"hello") diff --git a/tests/test_upload_blob_dedup.py b/tests/test_upload_blob_dedup.py new file mode 100644 index 0000000..13c79fe --- /dev/null +++ b/tests/test_upload_blob_dedup.py @@ -0,0 +1,256 @@ +"""Identical content inside one upload must be transferred once. + +The batch pre-sign step asks the server about every distinct oid *before* any +upload starts, so the server cannot answer "already stored" for a duplicate -- +it is not stored yet. Verified against the Hub: pre-signing the same fresh oid +twice yields two upload URLs, and only after the blob lands does the endpoint +report it as existing. Without an owner election, every occurrence therefore +PUTs the same bytes. +""" + +from __future__ import annotations + +import threading +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +import modelscope_hub._upload as upload_module +from modelscope_hub._upload import DUPLICATE_BLOB, UploadManager +from modelscope_hub.errors import NetworkError, StorageError + + +def _make_manager() -> tuple[UploadManager, MagicMock, list[str]]: + """Manager whose blob PUTs are recorded by oid.""" + client = MagicMock() + client.create_commit.return_value = {"ok": True} + put_oids: list[str] = [] + lock = threading.Lock() + + def validate_blobs(*, repo_id: str, repo_type: str, objects: list[dict]) -> dict[str, str]: + return {obj["oid"]: f"https://upload/{obj['oid']}" for obj in objects} + + def upload_blob(*, upload_url: str, data, size: int) -> None: + with lock: + put_oids.append(upload_url.rsplit("/", 1)[-1]) + while data.read(1024 * 1024): + pass + + client.validate_blobs.side_effect = validate_blobs + client.upload_blob.side_effect = upload_blob + return UploadManager(client, MagicMock()), client, put_oids + + +def _write_duplicates(root: Path, groups: dict[str, int], size: int = 4096) -> int: + """Write ``{content_tag: copies}``; returns the number of distinct contents.""" + index = 0 + for tag, copies in groups.items(): + payload = tag.encode() * (size // len(tag.encode())) + for _ in range(copies): + (root / f"file-{index:04d}.dat").write_bytes(payload) + index += 1 + return len(groups) + + +def test_duplicate_content_is_uploaded_once(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 0) + manager, client, put_oids = _make_manager() + distinct = _write_duplicates(tmp_path, {"aa": 5, "bb": 3, "cc": 1}) + total_files = 9 + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=4, + use_cache=False, + disable_tqdm=True, + ) + + assert len(put_oids) == distinct, f"expected one PUT per distinct content, got {len(put_oids)}" + assert len(set(put_oids)) == distinct + # All nine files are still committed, each as an LFS pointer. + operations = client.create_commit.call_args.kwargs["operations"] + assert len(operations) == total_files + assert all(op["type"] == "lfs" and op["sha256"] for op in operations) + # The pointers cover exactly the distinct contents that were uploaded. + assert {op["sha256"] for op in operations} == set(put_oids) + + +def test_duplicates_report_no_wire_bytes(tmp_path: Path, monkeypatch) -> None: + # Only the owner moves bytes, so throughput must not count the copies. + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 0) + monkeypatch.setattr(upload_module, "UPLOAD_PROGRESS_MIN_INTERVAL_SECONDS", 0) + manager, _, put_oids = _make_manager() + _write_duplicates(tmp_path, {"aa": 4, "bb": 4}, size=8192) + events: list[dict] = [] + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=4, + use_cache=False, + disable_tqdm=True, + progress_callback=events.append, + ) + + wire = [e for e in events if e["event"] == "upload_progress"] + assert len(put_oids) == 2 + # Two owners at 8192 bytes each; the six copies contribute nothing. + assert wire[-1]["uploaded_bytes"] == 2 * 8192 + assert wire[-1]["uploaded_files"] == 8 + + +def test_owner_is_the_earliest_file_so_no_commit_precedes_its_blob(tmp_path: Path, monkeypatch) -> None: + # Correctness rests on the owner never landing in a later batch than a copy: + # batches commit in order, so a copy in batch j needs its owner in batch <= j. + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 0) + monkeypatch.setattr(upload_module, "UPLOAD_COMMIT_BATCH_MAX_OPERATIONS", 2) + monkeypatch.setattr(upload_module, "UPLOAD_ADAPTIVE_BATCHING_ENABLED", False) + manager, client, put_oids = _make_manager() + # Same content in the first and last file: owner in batch 0, copy in batch 2. + payload = b"x" * 4096 + for name in ("a.dat", "b.dat", "c.dat", "d.dat", "e.dat"): + (tmp_path / name).write_bytes(payload if name in ("a.dat", "e.dat") else name.encode() * 512) + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=1, + use_cache=False, + disable_tqdm=True, + ) + + # Four distinct contents for five files. + assert len(put_oids) == 4 + committed = [op["path"] for call in client.create_commit.call_args_list for op in call.kwargs["operations"]] + assert committed == ["a.dat", "b.dat", "c.dat", "d.dat", "e.dat"] + + +def test_a_copy_is_not_committed_when_its_owner_fails(tmp_path: Path, monkeypatch) -> None: + # The whole point of the owner election is that copies skip the transfer. + # If the owner fails, committing a copy would publish an LFS pointer to a + # blob that was never stored. + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 0) + monkeypatch.setattr(upload_module, "UPLOAD_RECOVERY_ENABLED", False) + monkeypatch.setattr(upload_module, "UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS", 0) + manager, client, _ = _make_manager() + payload = b"shared" * 700 + for name in ("dup-0.dat", "dup-1.dat", "dup-2.dat"): + (tmp_path / name).write_bytes(payload) + (tmp_path / "solo.dat").write_bytes(b"solo" * 900) + + def failing_upload(*, upload_url: str, data, size: int) -> None: + if size == len(payload): + raise NetworkError("owner transfer failed") + while data.read(1024 * 1024): + pass + + client.upload_blob.side_effect = failing_upload + monkeypatch.setattr(upload_module.time, "sleep", lambda _s: None) + + # The owner and both copies count as failures, so the upload reports itself + # as failed rather than quietly publishing broken pointers. + with pytest.raises(StorageError, match="3 file"): + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=1, + use_cache=False, + disable_tqdm=True, + ) + + committed = {op["path"] for call in client.create_commit.call_args_list for op in call.kwargs["operations"]} + # Only the independent file is committed; no copy of the failed content is. + assert committed == {"solo.dat"} + assert not any(path.startswith("dup-") for path in committed) + + +def test_globally_existing_content_needs_no_owner(tmp_path: Path, monkeypatch) -> None: + # When the server already holds the blob, nobody transfers it and the copies + # are not deferred either. + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 0) + manager, client, put_oids = _make_manager() + client.validate_blobs.side_effect = lambda **kw: {obj["oid"]: None for obj in kw["objects"]} + _write_duplicates(tmp_path, {"aa": 3, "bb": 2}) + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=2, + use_cache=False, + disable_tqdm=True, + ) + + assert put_oids == [] + operations = client.create_commit.call_args.kwargs["operations"] + assert len(operations) == 5 + + +def test_inline_files_are_not_deduplicated(tmp_path: Path, monkeypatch) -> None: + # Inline content travels in the commit body, not as a blob, so there is no + # transfer to skip and every occurrence must carry its own bytes. + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 1024 * 1024) + manager, client, put_oids = _make_manager() + for name in ("a.txt", "b.txt", "c.txt"): + (tmp_path / name).write_bytes(b"identical") + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=2, + use_cache=False, + disable_tqdm=True, + ) + + assert put_oids == [] + operations = client.create_commit.call_args.kwargs["operations"] + assert len(operations) == 3 + assert all(op["type"] == "normal" and op["content"] for op in operations) + + +def test_duplicate_marker_skips_the_transfer_without_claiming_global_reuse() -> None: + manager, client, put_oids = _make_manager() + + result = manager._upload_blob( + repo_id="owner/repo", + repo_type="dataset", + sha256="a" * 64, + size=123, + data=b"x" * 123, + disable_tqdm=True, + pre_validated=DUPLICATE_BLOB, + ) + + assert put_oids == [] + client.validate_blobs.assert_not_called() + assert result["is_uploaded"] is True + assert result["is_reused"] is True + # Nothing went over the wire, so throughput accounting must see zero. + assert result["is_blob_uploaded"] is False + + +@pytest.mark.parametrize("workers", [1, 4, 16]) +def test_dedup_holds_at_any_concurrency(tmp_path: Path, monkeypatch, workers: int) -> None: + # Election happens before any worker starts, so the outcome must not depend + # on scheduling -- and no worker may ever block on another. + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 0) + manager, _, put_oids = _make_manager() + distinct = _write_duplicates(tmp_path, {"aa": 8, "bb": 8, "cc": 8, "dd": 1}) + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=workers, + use_cache=False, + disable_tqdm=True, + ) + + assert len(put_oids) == distinct From a537370fe321843e38aa88f44b4bfde4bbbfd491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Fri, 18 Sep 2026 10:35:04 +0800 Subject: [PATCH 12/22] bump version --- src/modelscope_hub/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modelscope_hub/version.py b/src/modelscope_hub/version.py index 764c69a..62ee45e 100644 --- a/src/modelscope_hub/version.py +++ b/src/modelscope_hub/version.py @@ -1,3 +1,3 @@ """Version information for modelscope_hub.""" -__version__ = "0.4.7+main" +__version__ = "0.4.4+main" From 1c2c2f16e36d98e0616da649a1c96db7b4c0e700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Sun, 20 Sep 2026 15:02:50 +0800 Subject: [PATCH 13/22] fix: normalize path_in_repo so "." maps to repo root, not a "./" prefix path_in_repo="." (e.g. `ms upload REPO DIR .`) became a literal "./" prefix on every file, which the Hub rejects with E3021 "invalid commit action" on every commit, aborting the upload after 3 consecutive failed batches. Collapse ".", "./", "" and "/" to the repo root, normalize separators and ".", ".." segments, and refuse paths that escape the root. Applied in both upload_folder and upload_file; added regression tests. Verified end-to-end with a real 14131-file / 2.4GB dataset upload using path_in_repo=".": all files committed, 0 E3021, 0 aborts, server paths carry no "./" prefix. --- src/modelscope_hub/_upload.py | 31 ++++++++++++++++++++- tests/test_upload_batching.py | 52 +++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/modelscope_hub/_upload.py b/src/modelscope_hub/_upload.py index ec07890..0b4f0de 100644 --- a/src/modelscope_hub/_upload.py +++ b/src/modelscope_hub/_upload.py @@ -20,6 +20,7 @@ import io import json import os +import posixpath import tempfile import threading import time @@ -240,6 +241,26 @@ def _safe_size(file_path: str) -> int: return 0 +def _normalize_path_in_repo(path_in_repo: str | None) -> str: + """Collapse a repo destination prefix to a clean, root-relative form. + + ``"."``, ``"./"``, ``""`` and ``"/"`` all denote the repository root and + must yield no prefix. Left literal, a value like ``"."`` becomes a ``"./"`` + prefix on every file and rides into each commit action's ``path``, which the + Hub rejects wholesale as an invalid commit action (E3021). Separators are + normalized and ``.``/``..`` segments resolved; a path that escapes the root + is refused rather than silently rewritten. + """ + if not path_in_repo: + return "" + cleaned = posixpath.normpath(path_in_repo.strip().replace("\\", "/")).strip("/") + if cleaned in ("", "."): + return "" + if cleaned == ".." or cleaned.startswith("../"): + raise InvalidParameter(f"path_in_repo must stay within the repository root, got {path_in_repo!r}") + return cleaned + + def _compute_file_hash( file_path_or_obj: str | Path | bytes | BinaryIO | IO[bytes], buffer_size_mb: int = 16, @@ -746,6 +767,8 @@ def upload_file( if path_or_fileobj is None: raise InvalidParameter("Path or file object cannot be None!") + path_in_repo = _normalize_path_in_repo(path_in_repo) + if isinstance(path_or_fileobj, (str, Path)): path_or_fileobj = os.path.abspath(os.path.expanduser(str(path_or_fileobj))) path_in_repo = path_in_repo or os.path.basename(path_or_fileobj) @@ -2226,7 +2249,13 @@ def _prepare_upload_folder( ignore_patterns=ignore_patterns, ) - prefix = f"{path_in_repo.strip('/')}/" if path_in_repo else "" + # ``path_in_repo`` is a destination prefix, and "." / "./" / "" / "/" + # all mean the repo root. Collapsing them is not cosmetic: a literal + # value like "." otherwise rides into every commit action's ``path`` as + # a "./" prefix, which the Hub rejects wholesale with E3021 "invalid + # commit action". + norm_prefix = _normalize_path_in_repo(path_in_repo) + prefix = f"{norm_prefix}/" if norm_prefix else "" prepared = [(prefix + relpath, relpath_to_abspath[relpath]) for relpath in filtered_keys] logger.info("Prepared %d files for upload.", len(prepared)) diff --git a/tests/test_upload_batching.py b/tests/test_upload_batching.py index 9114176..157a458 100644 --- a/tests/test_upload_batching.py +++ b/tests/test_upload_batching.py @@ -16,6 +16,7 @@ UploadManager, _calculate_adaptive_batch_size, _is_inline_metadata, + _normalize_path_in_repo, _plan_commit_batches, _upload_mode, ) @@ -217,6 +218,57 @@ def test_upload_folder_routes_small_files_to_blob_storage(tmp_path: Path, monkey assert client.upload_blob.call_count == 4 +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("", ""), + (".", ""), + ("./", ""), + ("/", ""), + (" . ", ""), + ("sub", "sub"), + ("./sub", "sub"), + ("sub/", "sub"), + ("/sub/", "sub"), + ("a/./b", "a/b"), + ("a/../b", "b"), + ("data\\shard", "data/shard"), + ], +) +def test_normalize_path_in_repo_collapses_root_aliases(raw: str, expected: str) -> None: + assert _normalize_path_in_repo(raw) == expected + + +@pytest.mark.parametrize("escaping", ["..", "../x", "a/../../b"]) +def test_normalize_path_in_repo_refuses_escaping_root(escaping: str) -> None: + with pytest.raises(InvalidParameter): + _normalize_path_in_repo(escaping) + + +def test_upload_folder_dot_path_in_repo_commits_root_relative_paths(tmp_path: Path, monkeypatch) -> None: + # `ms upload REPO LOCAL .` passes path_in_repo=".". Left literal it became a + # "./" prefix on every commit action path, which the Hub rejects wholesale + # as an invalid commit action (E3021). "." must map to the repo root. + manager, client = _make_manager() + (tmp_path / "README.md").write_bytes(b"# card") + (tmp_path / "data").mkdir() + (tmp_path / "data" / "train.txt").write_bytes(b"rows") + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + path_in_repo=".", + max_workers=2, + use_cache=False, + disable_tqdm=True, + ) + + operations = client.create_commit.call_args.kwargs["operations"] + assert sorted(op["path"] for op in operations) == ["README.md", "data/train.txt"] + assert not any(op["path"].startswith("./") for op in operations) + + def test_upload_folder_batch_presigns_in_configured_group_size(tmp_path: Path, monkeypatch) -> None: monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 0) monkeypatch.setattr(upload_module, "UPLOAD_BLOB_VALIDATION_BATCH_MAX_OBJECTS", 3) From 85bea576a3d5acd80ed816c9bb620aba66f4c57d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Mon, 21 Sep 2026 16:08:16 +0800 Subject: [PATCH 14/22] enhance upload pipeline --- AGENTS.md | 140 +++++++ README.md | 14 +- src/modelscope_hub/_legacy_api.py | 15 +- src/modelscope_hub/_upload.py | 633 +++++++++++++++++++---------- src/modelscope_hub/constants.py | 26 +- src/modelscope_hub/errors.py | 14 +- tests/test_errors_openapi_codes.py | 52 +++ tests/test_upload_batching.py | 279 ++++++++++++- tests/test_upload_config.py | 6 +- 9 files changed, 943 insertions(+), 236 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..2b94ee3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,140 @@ +# AGENTS.md — modelscope_hub Engineering Conventions + +This document defines repository-level engineering conventions that apply to every automated or manual change in this repository. In addition to the general requirements, it **strictly governs changes to the `upload_folder` upload path**: every change that touches this path must perform the specified scenario-based simulation review and include an Upload Simulation Regression Report. + +--- + +## 1. General Engineering Requirements + +- This repository uses a src-layout. Always run code and tests against local sources: `PYTHONPATH=src` (a package installed with `pip install` may be outdated). +- Before committing, run the following gates in order; all must pass: + 1. Targeted unit tests: `PYTHONPATH=src pytest -q ` + 2. Full non-remote test suite: `PYTHONPATH=src pytest -q tests/ -k 'not remote' --ignore=tests/integration` + 3. Style checks: `ruff check src/ tests/` and `ruff format --check src/ tests/` + 4. Type checks: `mypy src/modelscope_hub/` +- Never commit secrets. `~/.modelscope/credentials/` is private (it contains a pickled cookie jar and `m_session_id`); never expose its contents in logs or reports. +- When changing the default value of a configuration constant, update the corresponding tests for its default value and registration table, as well as the environment-variable table in `README.md`. + +--- + +## 2. Controlled Scope: `upload_folder` Path + +Changing any of the following files or symbols constitutes a change to the upload path and requires the mandatory simulation regression in Section 5: + +- `src/modelscope_hub/_upload.py` + - Batching: `_calculate_adaptive_batch_size`, `_plan_commit_batches`, `_estimate_commit_operation_bytes`, `_plan_result_batches` + - Routing: `_is_lfs`, `_is_inline_metadata`, `_upload_mode` + - Main flow and recovery: `upload_folder`, `_retry_failed_commits`, `_retry_failed_files_react`, `_retry_failed_simple`, `_commit_with_retry` + - Failure classification: `classify_error`, `_ErrorCategory`, `_is_retryable_commit_error` + - Preflight validation: `_warn_advisory_upload_limits`, `_prepare_upload_folder` + - State tracking: `UploadTracker` (`begin_attempt` / `mark_*`) +- `src/modelscope_hub/errors.py`: `raise_for_status`, `_COMMIT_RETRYABLE_BUSINESS_CODES`, and HTTP/business-code mappings +- `src/modelscope_hub/_legacy_api.py`: `create_commit`, including the HTTP 200 / `Success:false` check +- `src/modelscope_hub/constants.py`: upload-related constants such as `UPLOAD_*` and `COMMIT_MAX_ACTIONS_PER_REQUEST` + +--- + +## 3. Commit Batching Semantics That Must Be Preserved + +Only files that are **pending** (not `COMMITTED`) for the current run are planned into batches. This prevents a resumed upload from turning a small number of remaining files into multiple tiny commits. + +Apply the following three constraints; start a new batch as soon as any constraint is reached: + +1. **Operation target (primary constraint):** `UPLOAD_COMMIT_BATCH_MAX_OPERATIONS`, default `256`. `_calculate_adaptive_batch_size` must return `min(target, 2000, pending_count)`—for fewer files, it converges to the pending count and by default never exceeds `256`. +2. **Request-body budget (secondary constraint):** `UPLOAD_COMMIT_MAX_INLINE_BYTES`, default `8 MiB`. Count only normal files, using base64 expansion (`×4/3`) plus JSON overhead. For LFS files, count only pointer metadata (about `370 B/op`); **never include blob payload bytes**. +3. **Server hard limit:** `COMMIT_MAX_ACTIONS_PER_REQUEST=2000`. Clamp to this limit in every case; it must never be exceeded. + +The following behavior must also be preserved: + +- **Tail-batch rebalancing:** when `last_batch × 4 < previous_batch` and both rebalanced halves meet all constraints, rebalance the final two batches (for example, `256/32 → 144/144`). Do not add commits, and preserve deterministic path order. +- **Oversized inline fallback:** if one inline file alone exceeds the budget, place it in its own batch and emit a warning; the upload must not stall. +- **Recovery uses the same planner:** `_retry_failed_commits`, ReAct recovery, and simple retry must all use `_plan_result_batches` (and therefore the same `_plan_commit_batches`). +- **`delete_files` batches independently:** split only at the hard limit of `2000`, not at `256`. +- **`upload_file`:** one file and one commit per call; no batching and no tracker. + +### Routing Rules: LFS vs. Inline + +- Files matched by `_is_inline_metadata` (such as `README.md`, `.gitattributes`, `.gitignore`, and `config*.json`) are **always inline**. +- Otherwise, `size > UPLOAD_LFS_FORCE_THRESHOLD_BYTES` (default **64 KiB**) routes to LFS. +- Otherwise, files matching `DATASET_LFS_SUFFIX` or `MODEL_LFS_SUFFIX` route to LFS. +- All other files are normal inline files. + +--- + +## 4. Failure and Retry Model That Must Be Preserved + +- Failures have three categories, and **all of them must be included in final statistics**: + - `retry_failed_files`: transport failure → ReAct recovery (parallel or serial retries, backoff, then single-file handling). + - `retry_commit_batches`: retryable commit failure → **commit-only recovery** (do not re-upload blobs). + - `terminal_failures`: non-retryable in the current run; record as `FAILED` in the tracker and include in the final `Failed` count. +- **Retryability in commit context** (`_is_retryable_commit_error` / `classify_error(commit_context=True)`): + - Business code `10030000001` ("branch changed, please retry" / "commit rejected by repository policy") is retryable only at a `/commit/` endpoint; `raise_for_status` maps it to `ServerError`. + - During commit-only recovery, policy-related rejections recursively split batches in half by file count, down to individual files. + - Genuine parameter errors (for example, an invalid-path E3021, a 400 without a policy code, or 401/403/404) remain **non-retryable**. +- **HTTP 200 with `Success:false`:** `LegacyClient.create_commit` must raise `APIError` and enter the unified failure path; false successes are prohibited. +- **Abort after consecutive failures:** after `UPLOAD_COMMIT_MAX_CONSECUTIVE_FAILED_BATCHES` (default `3`) batches fail completely in succession, the main flow must raise `RuntimeError`. The tracker must already be persisted so that the upload can be resumed. +- **Manual rerun semantics (mandatory):** regardless of the prior error, skip only `COMMITTED` entries. `FAILED`, `UPLOADED`, and missing entries must re-enter the full upload-and-commit path; `begin_attempt` clears the previous `error_type`. A terminal failure must raise `StorageError` and suppress synchronous deletion. +- **Advisory preflight limits:** capacity-related thresholds (total file count, files per directory, individual file size, and total normal-file bytes) must only warn and never block. Structural errors (an empty `repo_id`, empty input, a nonexistent or out-of-bound path, invalid `repo_type`, or a file modified during upload), together with the server's hard limit of `2000`, remain enforced constraints. + +--- + +## 5. Mandatory Simulation Regression for Upload-Path Changes + +> Rule: **Before committing any change within the scope in Section 2, perform a scenario-based simulation review of every case below and include an Upload Simulation Regression Report in the delivery notes.** A simulation is a logical derivation from the code's current strategy and must match the implementation. If a conclusion changes because of the current change, explicitly document the before/after difference and its cause. + +### 5.1 Required Dataset-Scale Scenarios + +| ID | Dataset characteristics | Required simulated conclusions | +|---|---|---| +| D1 | 1 TB / 2 million files, from tens of KiB to tens of MiB | Routing distribution, blob PUT order of magnitude, presign request count, commit-batch order of magnitude, rate-limit risk, memory and hashing overhead | +| D2 | 50 GB / 20 thousand files | Nearly all LFS, presign count, commit-batch count (including tail-batch rebalancing), dominant costs | +| D3 | 5 GB / 2 thousand files | Commit-batch count and tail behavior (including whether rebalancing applies), blob-dominated cost | +| D4 | 200 GB / 10 files | All LFS, **one commit**, blob-transfer dominated, no capacity warning | +| DX | 1,000 files × 1 GB | All LFS, the 8 MiB budget does not apply, commit batches (`256×3+232=4`), approximately 1,000 PUTs | + +For every scenario, derive at least: the LFS/normal routing result, the `_calculate_adaptive_batch_size` target, `_plan_commit_batches` count plus operations and estimated body size per batch, blob PUT count, presign request count, advisory warnings triggered, and resume behavior. + +### 5.2 Required Error-Type Scenarios + +For each error below, describe its classification, whether it is automatically retried in the current run, its handling path, and its final outcome: + +- Transport-layer network failure or timeout (`E1020` / `E1001`, blob stage) +- Commit 5xx (`E1002`) +- 429 rate limiting (`E1021`), including `Retry-After` and the configured limit +- 409 branch changed / 400 repository policy (business code `10030000001`, `/commit/`) +- HTTP 200 + `Success:false` +- Genuine parameter error (invalid-path E3021 without a policy code) +- Authentication, authorization, or nonexistent resource (`E3001` / `E3002` / `E3020`) +- File modified during upload (size mismatch) +- Shared blob owner failure (replicas are deferred) +- Three consecutive batches failing completely (protective abort and resumable rerun) + +### 5.3 Upload Simulation Regression Report Template + +``` +# upload_folder Simulation Regression Report +- Change summary / affected Section 2 symbols: +- Baseline configuration snapshot: LFS threshold / 256 / 8 MiB / 2000 / recovery switches / consecutive-failure threshold +- Dataset scenario derivations: D1 / D2 / D3 / D4 / DX (routing, blob PUTs, presigning, commit batches with operations and body sizes, warnings, resume behavior) +- Error-type behavior: each item (classification / automatic retry / handling path / final outcome) +- Difference from the previous version: behavior changed by this modification and the reason (write "None" if there is no difference) +- Automated validation: pytest / ruff / mypy results +- Conclusion: compliance with the semantics in Sections 3 and 4; risks and tuning recommendations +``` + +### 5.4 Required Automated Validation (Run in Parallel with the Simulation) + +- `PYTHONPATH=src pytest -q tests/test_upload_batching.py tests/test_commit_action_limit.py tests/test_errors_openapi_codes.py tests/test_upload_config.py tests/test_upload_lfs_gate.py tests/test_upload_blob_dedup.py` +- If error mappings changed, additionally run `tests/test_login_failure_paths.py`. +- The full non-remote suite, ruff, and mypy must pass. + +--- + +## 6. Change Checklist + +After changing the upload path, check whether the following must be updated as well: + +- `tests/`: assertions for batching, errors, configuration, deduplication, LFS gating, and related behavior—especially assertions for default values. +- The upload environment-variable table and semantic documentation in `README.md`. +- `__all__` exports and compatibility aliases in `src/modelscope_hub/compat/` and `constants.py`. +- This file, if new constants, error codes, or scenarios were introduced. diff --git a/README.md b/README.md index 18294d0..45b922d 100644 --- a/README.md +++ b/README.md @@ -931,13 +931,21 @@ Token is persisted locally after `ms-hub login` and auto-loaded in subsequent se | Variable | Default | Description | |----------|---------|-------------| | `MODELSCOPE_UPLOAD_MAX_CONCURRENT_WORKERS` | `min(8, cpu+4)` | Default parallel worker threads | -| `MODELSCOPE_UPLOAD_CACHE_ENABLED` | `true` | Enable resumable upload cache | +| `MODELSCOPE_UPLOAD_CACHE_ENABLED` | `true` | Enable resumable upload cache; only committed files are skipped on a later run | | `MODELSCOPE_UPLOAD_IGNORE_FILE_PATTERN` | — | File pattern excluded by legacy `push_to_hub` uploads | -| `MODELSCOPE_UPLOAD_MAX_FILE_SIZE_MB` | `102400` | Max single file size (MB, default 100 GB) | -| `MODELSCOPE_UPLOAD_MAX_FILE_COUNT` | `100000` | Max total files per upload | +| `MODELSCOPE_UPLOAD_MAX_FILE_SIZE_MB` | `102400` | Advisory single-file warning threshold (MB); uploads continue above it | +| `MODELSCOPE_UPLOAD_MAX_FILE_COUNT` | `100000` | Advisory total file-count warning threshold; uploads continue above it | +| `MODELSCOPE_UPLOAD_MAX_FILES_PER_DIRECTORY` | `50000` | Advisory per-directory file-count warning threshold | +| `MODELSCOPE_UPLOAD_NORMAL_FILES_TOTAL_SIZE_MB` | `500` | Advisory total inline-file size warning threshold (MB) | +| `MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD` | `1MiB` | Route larger non-metadata files through LFS; accepts byte-unit suffixes | +| `MODELSCOPE_UPLOAD_COMMIT_BATCH_MAX_OPERATIONS` | `256` | Target actions per commit, clamped to the server hard ceiling | +| `MODELSCOPE_UPLOAD_COMMIT_MAX_INLINE_BYTES` | `8MiB` | Estimated commit request-body budget used as the secondary batch constraint | +| `MODELSCOPE_UPLOAD_COMMIT_MAX_ATTEMPTS` | `5` | Maximum attempts for one transient commit failure | | `MODELSCOPE_UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS` | `30` | Blob upload connect timeout (seconds) | | `MODELSCOPE_UPLOAD_BLOB_READ_TIMEOUT_SECONDS` | `3600` | Blob upload read timeout (seconds) | +Capacity thresholds are advisory and emit warnings without blocking upload. Structural errors (invalid paths, missing inputs, changed files) still fail immediately, while the server's per-commit action ceiling is always enforced by splitting. A manual rerun retries every file not marked committed, including files whose previous run ended with a non-retryable error. + **Logging:** | Variable | Default | Description | diff --git a/src/modelscope_hub/_legacy_api.py b/src/modelscope_hub/_legacy_api.py index aa3d217..211614d 100644 --- a/src/modelscope_hub/_legacy_api.py +++ b/src/modelscope_hub/_legacy_api.py @@ -40,6 +40,7 @@ RepoType, ) from .errors import ( + APIError, InvalidParameter, NetworkError, PermissionDeniedError, @@ -815,7 +816,19 @@ def create_commit( f"repos/{segment}/{repo_id}/commit/{revision}", json_body=payload, ) - return self._json_data(resp) + body = resp.json() + if isinstance(body, dict) and (body.get("Success") is False or body.get("success") is False): + message = body.get("Message") or body.get("message") or "Commit was rejected by the server" + request_id = body.get("RequestId") or body.get("requestId") or body.get("request_id") + raise APIError( + str(message), + status_code=resp.status_code, + request_id=request_id, + response_body=body, + url=resp.url, + method="POST", + ) + return body.get("Data", body) if isinstance(body, dict) else body # ------------------------------------------------------------------ # Blob Upload (LFS) diff --git a/src/modelscope_hub/_upload.py b/src/modelscope_hub/_upload.py index 0b4f0de..a97cced 100644 --- a/src/modelscope_hub/_upload.py +++ b/src/modelscope_hub/_upload.py @@ -178,15 +178,11 @@ def _upload_mode(path: str | Path, size: int, repo_type: str) -> str: def _calculate_adaptive_batch_size(total_files: int, max_operations: int) -> int: - """Commit batch size from the file count alone, capped by *max_operations*. + """Return the operation target, always below the server's hard ceiling. - Fewer, fuller commits are strictly better: the Hub throttles commits per - repository, so commit count -- not commit size -- is the scarce resource. - The only reason to stop growing a batch is the operation cap, or the inlined - content limit that :func:`_plan_commit_batches` applies on top of this. - - The cap is itself clamped to :data:`COMMIT_MAX_ACTIONS_PER_REQUEST`, which - the server rejects outright rather than truncating. + File count is the primary constraint because repository publishing is more + sensitive to action count than to blob volume. Request bytes are enforced + independently by :func:`_plan_commit_batches`. """ if total_files <= 0: return 1 @@ -195,6 +191,21 @@ def _calculate_adaptive_batch_size(total_files: int, max_operations: int) -> int return max(1, min(cap, ceiling, total_files)) +_COMMIT_ENVELOPE_ESTIMATED_BYTES = 512 +_COMMIT_OPERATION_ESTIMATED_BYTES = 192 +_LFS_POINTER_ESTIMATED_BYTES = 128 + + +def _estimate_commit_operation_bytes(path_in_repo: str, size: int, repo_type: str) -> int: + """Estimate the JSON bytes one create operation contributes to a commit.""" + path_bytes = len(path_in_repo.encode("utf-8")) + scalar_bytes = len(str(max(0, size))) + if _is_lfs(path_in_repo, size, repo_type): + return _COMMIT_OPERATION_ESTIMATED_BYTES + _LFS_POINTER_ESTIMATED_BYTES + path_bytes + scalar_bytes + encoded_bytes = (max(0, size) + 2) // 3 * 4 + return _COMMIT_OPERATION_ESTIMATED_BYTES + path_bytes + scalar_bytes + encoded_bytes + + def _plan_commit_batches( files: list[tuple[str, str]], repo_type: str, @@ -203,35 +214,60 @@ def _plan_commit_batches( max_inline_bytes: int, sizes: dict[str, int] | None = None, ) -> list[int]: - """Split *files* into commit batches, returning each batch's file count. - - Two limits close a batch, whichever is reached first: ``max_operations`` - files, or ``max_inline_bytes`` of content that will ride *inside* the commit - request. Only non-LFS files contribute to the byte total -- an LFS file adds - a fixed-size pointer -- and base64 expansion is accounted for, because the - request carries the encoded form. A batch always holds at least one file, so - a single oversized inline file still makes progress instead of deadlocking. + """Split files by action count first and estimated request bytes second. + + Normal content is counted after base64 expansion. LFS blob bytes are not in + the commit request, but their pointer/action JSON still consumes a small + amount of space. A single oversized inline operation is allowed to make + progress and emits a warning. A tiny final count-only batch is rebalanced + with its predecessor when both balanced batches still fit all constraints. """ if not files: return [] cap = max_operations if max_operations > 0 else len(files) cap = max(1, min(cap, max(1, COMMIT_MAX_ACTIONS_PER_REQUEST))) - - batches: list[int] = [] - count = 0 - inline_bytes = 0 + weights: list[int] = [] for path_in_repo, file_path in files: size = sizes.get(file_path, 0) if sizes is not None else _safe_size(file_path) - encoded = 0 if _is_lfs(path_in_repo, size, repo_type) else (size + 2) // 3 * 4 - if count > 0 and (count >= cap or (max_inline_bytes > 0 and inline_bytes + encoded > max_inline_bytes)): - batches.append(count) - count = 0 - inline_bytes = 0 - count += 1 - inline_bytes += encoded - if count: - batches.append(count) - return batches + is_lfs = _is_lfs(path_in_repo, size, repo_type) + weight = _estimate_commit_operation_bytes(path_in_repo, size, repo_type) + weights.append(weight) + if not is_lfs and max_inline_bytes > 0 and _COMMIT_ENVELOPE_ESTIMATED_BYTES + weight > max_inline_bytes: + logger.warning( + "Inline file %s alone exceeds the advisory commit request budget (%d > %d bytes); " + "sending it in a dedicated commit.", + path_in_repo, + _COMMIT_ENVELOPE_ESTIMATED_BYTES + weight, + max_inline_bytes, + ) + + batch_indexes: list[list[int]] = [] + current: list[int] = [] + request_bytes = _COMMIT_ENVELOPE_ESTIMATED_BYTES + for index, weight in enumerate(weights): + exceeds_bytes = max_inline_bytes > 0 and request_bytes + weight > max_inline_bytes + if current and (len(current) >= cap or exceeds_bytes): + batch_indexes.append(current) + current = [] + request_bytes = _COMMIT_ENVELOPE_ESTIMATED_BYTES + current.append(index) + request_bytes += weight + if current: + batch_indexes.append(current) + + if len(batch_indexes) >= 2 and len(batch_indexes[-1]) * 4 < len(batch_indexes[-2]): + combined = batch_indexes[-2] + batch_indexes[-1] + midpoint = (len(combined) + 1) // 2 + candidates = (combined[:midpoint], combined[midpoint:]) + + def _fits(candidate: list[int]) -> bool: + estimated = _COMMIT_ENVELOPE_ESTIMATED_BYTES + sum(weights[i] for i in candidate) + return len(candidate) <= cap and (max_inline_bytes <= 0 or estimated <= max_inline_bytes) + + if all(_fits(candidate) for candidate in candidates): + batch_indexes[-2:] = [list(candidates[0]), list(candidates[1])] + + return [len(batch) for batch in batch_indexes] def _safe_size(file_path: str) -> int: @@ -241,6 +277,61 @@ def _safe_size(file_path: str) -> int: return 0 +def _warn_advisory_upload_limits( + files: list[tuple[str, str]], + sizes: dict[str, int], + repo_type: str, +) -> None: + """Warn about advisory scale limits without refusing a valid upload.""" + if len(files) > UPLOAD_MAX_FILE_COUNT: + logger.warning( + "Upload contains %d files, above the advisory limit of %d; continuing.", + len(files), + UPLOAD_MAX_FILE_COUNT, + ) + + directory_counts: dict[str, int] = {} + oversized: list[tuple[str, int]] = [] + normal_size = 0 + for path_in_repo, file_path in files: + directory = posixpath.dirname(path_in_repo) + directory_counts[directory] = directory_counts.get(directory, 0) + 1 + size = sizes.get(file_path, 0) + if size > UPLOAD_MAX_FILE_SIZE_BYTES: + oversized.append((path_in_repo, size)) + if not _is_lfs(path_in_repo, size, repo_type): + normal_size += size + + crowded = [ + (path or "/", count) for path, count in directory_counts.items() if count > UPLOAD_MAX_FILES_PER_DIRECTORY + ] + if crowded: + preview = ", ".join(f"{path} ({count})" for path, count in crowded[:3]) + suffix = f" and {len(crowded) - 3} more" if len(crowded) > 3 else "" + logger.warning( + "Upload has directories above the advisory %d-file limit: %s%s; continuing.", + UPLOAD_MAX_FILES_PER_DIRECTORY, + preview, + suffix, + ) + if oversized: + preview = ", ".join(f"{path} ({size} bytes)" for path, size in oversized[:3]) + suffix = f" and {len(oversized) - 3} more" if len(oversized) > 3 else "" + logger.warning( + "Upload has files above the advisory %d-byte single-file limit: %s%s; continuing.", + UPLOAD_MAX_FILE_SIZE_BYTES, + preview, + suffix, + ) + if normal_size > UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES: + logger.warning( + "Total normal (non-LFS) content is %d bytes, above the advisory limit of %d; continuing. " + "Consider lowering MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD.", + normal_size, + UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES, + ) + + def _normalize_path_in_repo(path_in_repo: str | None) -> str: """Collapse a repo destination prefix to a clean, root-relative form. @@ -349,13 +440,38 @@ def is_retryable(cls, category: str) -> bool: } -def classify_error(error: Exception) -> str: - """Classify an exception for retry strategy using the SDK error hierarchy. +_RETRYABLE_COMMIT_BUSINESS_CODES = frozenset({10030000001}) +_RETRYABLE_COMMIT_MESSAGE_MARKERS = ( + "branch changed", + "could not update refs", + "please retry", + "try again", + "commit rejected by repository policy", +) + - For :class:`HubError` instances, classification is driven by - :attr:`~HubError.error_code` via a lookup table — no ``isinstance`` - chains or string-matching heuristics needed for known error types. - """ +def _is_retryable_commit_error(error: Exception) -> bool: + """Return whether a commit failure is transient in commit context.""" + if isinstance(error, HubError) and error.retryable: + return True + body = getattr(error, "response_body", None) + if isinstance(body, dict): + raw_code = body.get("Code") if body.get("Code") is not None else body.get("code") + try: + if raw_code is not None and int(raw_code) in _RETRYABLE_COMMIT_BUSINESS_CODES: + return True + except (TypeError, ValueError): + pass + message = getattr(error, "message", None) or str(error) + lowered = str(message).lower() + status_code = getattr(error, "status_code", None) + return status_code in (400, 409) and any(marker in lowered for marker in _RETRYABLE_COMMIT_MESSAGE_MARKERS) + + +def classify_error(error: Exception, *, commit_context: bool = False) -> str: + """Classify an exception for retry strategy using the SDK error hierarchy.""" + if commit_context and _is_retryable_commit_error(error): + return _ErrorCategory.TRANSIENT_SERVER if isinstance(error, HubError): code = getattr(error, "error_code", None) if code and code in _CATEGORY_BY_ERROR_CODE: @@ -438,11 +554,24 @@ def get_status(self, rel_path: str, mtime: float, size: int) -> str | None: entry = self._files.get(key) return entry.get("status") if entry else None + def begin_attempt(self, rel_path: str, mtime: float, size: int) -> None: + """Clear stale failure metadata when a later run retries a file.""" + key = self._make_key(rel_path, mtime, size) + with self._lock: + entry = self._files.get(key) + if entry is None or entry.get("status") == FileStatus.COMMITTED: + return + if entry.get("status") == FileStatus.FAILED: + entry.pop("status", None) + entry.pop("error_type", None) + self._dirty = True + def mark_uploaded(self, rel_path: str, mtime: float, size: int) -> None: key = self._make_key(rel_path, mtime, size) with self._lock: if key in self._files: self._files[key]["status"] = FileStatus.UPLOADED + self._files[key].pop("error_type", None) self._dirty = True def mark_committed_batch(self, file_keys: list[tuple[str, float, int]]) -> None: @@ -451,6 +580,7 @@ def mark_committed_batch(self, file_keys: list[tuple[str, float, int]]) -> None: key = self._make_key(rel_path, mtime, size) if key in self._files: self._files[key]["status"] = FileStatus.COMMITTED + self._files[key].pop("error_type", None) self._dirty = True def mark_failed(self, rel_path: str, mtime: float, size: int, error_type: str = "") -> None: @@ -574,6 +704,9 @@ def is_committed(self, rel_path: str, mtime: float, size: int) -> bool: def get_status(self, rel_path: str, mtime: float, size: int) -> None: return None + def begin_attempt(self, rel_path: str, mtime: float, size: int) -> None: + pass + def mark_uploaded(self, rel_path: str, mtime: float, size: int) -> None: pass @@ -779,6 +912,12 @@ def upload_file( hash_info = _compute_file_hash(path_or_fileobj, buffer_size_mb) file_hash = hash_info["file_hash"] file_size = hash_info["file_size"] + warning_source = str(path_or_fileobj) if isinstance(path_or_fileobj, (str, Path)) else path_in_repo + _warn_advisory_upload_limits( + [(path_in_repo, warning_source)], + {warning_source: file_size}, + repo_type, + ) # If BinaryIO was consumed, _compute_file_hash returns the bytes if not isinstance(path_or_fileobj, (str, Path, bytes)): path_or_fileobj = hash_info["file_path_or_obj"] @@ -971,6 +1110,7 @@ def upload_folder( repo_type=repo_type, allow_patterns=None, ignore_patterns=None, + warn_limits=False, ) all_local_paths_in_repo = {p for p, _ in all_local_files_in_repo} else: @@ -984,61 +1124,73 @@ def upload_folder( if self._create_repo_fn is not None: self._create_repo_fn(repo_id, repo_type) - # Sort for deterministic batch assignment + # Sort for deterministic assignment, then remove committed files before + # planning. Otherwise a resume with four pending files in four old batch + # ranges needlessly creates four one-file commits. sorted_files = sorted(sorted_files, key=lambda x: x[0]) - # Plan commit batches. The operation cap bounds the file count; the - # inline-content cap bounds how many bytes a commit body carries, which - # only non-LFS files add to. - max_operations = ( - _calculate_adaptive_batch_size(len(sorted_files), UPLOAD_COMMIT_BATCH_MAX_OPERATIONS) - if UPLOAD_ADAPTIVE_BATCHING_ENABLED - else (UPLOAD_COMMIT_BATCH_MAX_OPERATIONS if UPLOAD_COMMIT_BATCH_MAX_OPERATIONS > 0 else len(sorted_files)) - ) - batch_plan = _plan_commit_batches( - sorted_files, - repo_type, - max_operations=max_operations, - max_inline_bytes=UPLOAD_COMMIT_MAX_INLINE_BYTES, - sizes=file_sizes, - ) - logger.info( - "Commit plan: %d batch(es) for %d file(s) (max %d ops, max %d inline bytes per commit).", - len(batch_plan), - len(sorted_files), - max_operations, - UPLOAD_COMMIT_MAX_INLINE_BYTES, - ) - - # Initialize tracker. The cache normally lives in the uploaded folder, - # but a caller that stages files into a throwaway tree (a link tree, a - # per-chunk directory) must be able to keep it outside, or every run - # rediscovers hashes and re-commits what was already committed. + # The cache normally lives in the uploaded folder, but a caller that + # stages files into a throwaway tree can keep it outside. folder_path_resolved = Path(folder_path).resolve() if use_cache: cache_path = Path(tracker_path).expanduser() if tracker_path else folder_path_resolved / UPLOAD_CACHE_FILE tracker: UploadTracker | NullTracker = UploadTracker(cache_path, repo_id=repo_id) else: tracker = NullTracker() - batch_tracker = BatchTracker(len(sorted_files), batch_plan) - # Skip individually committed files - files_to_upload: list[tuple[int, tuple[str, str]]] = [] - skipped_indices: set[int] = set() - for file_idx, (file_path_in_repo, file_path) in enumerate(sorted_files): + pending_files: list[tuple[str, str]] = [] + skipped_count = 0 + for file_path_in_repo, file_path in sorted_files: try: st = os.stat(file_path) if tracker.is_committed(file_path_in_repo, st.st_mtime, st.st_size): - skipped_indices.add(file_idx) - batch_tracker.mark_file_skipped(file_idx) + skipped_count += 1 continue + tracker.begin_attempt(file_path_in_repo, st.st_mtime, st.st_size) except OSError as e: logger.warning( "Cannot stat file %s, will re-upload: %s", file_path_in_repo, e, ) - files_to_upload.append((file_idx, (file_path_in_repo, file_path))) + pending_files.append((file_path_in_repo, file_path)) + tracker.save() + + max_operations = ( + _calculate_adaptive_batch_size(len(pending_files), UPLOAD_COMMIT_BATCH_MAX_OPERATIONS) + if UPLOAD_ADAPTIVE_BATCHING_ENABLED + else (UPLOAD_COMMIT_BATCH_MAX_OPERATIONS if UPLOAD_COMMIT_BATCH_MAX_OPERATIONS > 0 else len(pending_files)) + ) + batch_plan = _plan_commit_batches( + pending_files, + repo_type, + max_operations=max_operations, + max_inline_bytes=UPLOAD_COMMIT_MAX_INLINE_BYTES, + sizes=file_sizes, + ) + batch_tracker = BatchTracker(len(pending_files), batch_plan) + files_to_upload = list(enumerate(pending_files)) + + plan_details: list[str] = [] + offset = 0 + for count in batch_plan: + batch_files = pending_files[offset : offset + count] + estimated = _COMMIT_ENVELOPE_ESTIMATED_BYTES + sum( + _estimate_commit_operation_bytes(path, file_sizes.get(local, 0), repo_type) + for path, local in batch_files + ) + lfs_count = sum(1 for path, local in batch_files if _is_lfs(path, file_sizes.get(local, 0), repo_type)) + plan_details.append(f"{count} ops/{estimated} bytes/{lfs_count} LFS") + offset += count + logger.info( + "Commit plan: %d batch(es), %d pending, %d skipped (target %d ops, request budget %d bytes): %s", + len(batch_plan), + len(pending_files), + skipped_count, + max_operations, + UPLOAD_COMMIT_MAX_INLINE_BYTES, + "; ".join(plan_details[:8]) + (f"; ... {len(plan_details) - 8} more" if len(plan_details) > 8 else ""), + ) # Batch pre-validation for every LFS candidate. # @@ -1135,7 +1287,6 @@ def upload_folder( deduped_bytes, ) - skipped_count = len(skipped_indices) if skipped_count > 0: logger.info("%d file(s) already committed, skipping.", skipped_count) @@ -1189,7 +1340,9 @@ def _upload_worker(file_idx: int, file_info: tuple, pre_validated: Any = None) - # Pipeline: consume batches in order commit_infos: list[dict] = [] all_results: list[dict] = [] - total_failed_files: list[tuple] = [] + retry_failed_files: list[tuple] = [] + terminal_failures: list[tuple] = [] + retry_commit_batches: list[tuple[list[dict], Exception]] = [] num_batches = batch_tracker.num_batches committed_files = 0 committed_bytes = 0 @@ -1345,15 +1498,6 @@ def _flush_wire() -> None: total=num_batches, disable=disable_tqdm, ): - batch_start, batch_end = batch_tracker.batch_range(batch_idx) - if all(i in skipped_indices for i in range(batch_start, batch_end)): - logger.info( - "Batch %d/%d fully committed, skipping.", - batch_idx + 1, - num_batches, - ) - continue - results, failures = batch_tracker.wait_for_batch(batch_idx) # Every file of this batch has finished its transfer by now, # so publish the bytes the throttle may still be holding @@ -1361,7 +1505,7 @@ def _flush_wire() -> None: _flush_wire() if failures: - total_failed_files.extend(failures) + retry_failed_files.extend(failures) for item, err in failures: logger.error(" Failed: %s - %s", item[0], err) @@ -1393,7 +1537,7 @@ def _flush_wire() -> None: num_batches, len(orphaned), ) - total_failed_files.extend( + retry_failed_files.extend( ( (item_r["file_path_in_repo"], item_r["file_path"]), StorageError("shared blob upload failed"), @@ -1454,7 +1598,7 @@ def _flush_wire() -> None: sum(r["file_size_on_disk"] for r in results), error=str(e), ) - category = classify_error(e) + category = classify_error(e, commit_context=True) if not _ErrorCategory.is_retryable(category): for r in results: tracker.mark_failed( @@ -1463,30 +1607,26 @@ def _flush_wire() -> None: r["file_size_on_disk"], error_type="commit_" + category, ) + terminal_failures.append(((r["file_path_in_repo"], r["file_path"]), e)) logger.error( - "Batch %d/%d: permanent failure (%s), %d file(s) will not be retried.", + "Batch %d/%d: terminal failure for this run (%s); %d file(s) will not be retried " + "automatically. Rerun upload to retry them.", batch_idx + 1, num_batches, category, len(results), ) - consecutive_failures += 1 else: - for r in results: - total_failed_files.append( - ( - (r["file_path_in_repo"], r["file_path"]), - e, - ) - ) + retry_commit_batches.append((list(results), e)) logger.warning( - "Batch %d/%d: %d file(s) recovered to retry queue (error_category=%s).", + "Batch %d/%d: retained %d uploaded file(s) for commit-only recovery " + "(error_category=%s).", batch_idx + 1, num_batches, len(results), category, ) - consecutive_failures += 1 + consecutive_failures += 1 if consecutive_failures >= UPLOAD_COMMIT_MAX_CONSECUTIVE_FAILED_BATCHES: raise RuntimeError( @@ -1530,9 +1670,23 @@ def _report_recovery(results: list[dict], label: str) -> None: } ) - if total_failed_files and UPLOAD_RECOVERY_ENABLED: - total_failed_files, react_commits, react_results = self._retry_failed_files_react( - failed_files=total_failed_files, + if retry_commit_batches: + commit_failures, recovered_commits, recovered_results = self._retry_failed_commits( + failed_batches=retry_commit_batches, + tracker=tracker, + repo_id=repo_id, + repo_type=repo_type, + commit_message=commit_message, + revision=revision, + on_committed=_report_recovery, + ) + terminal_failures.extend(commit_failures) + commit_infos.extend(recovered_commits) + all_results.extend(recovered_results) + + if retry_failed_files and UPLOAD_RECOVERY_ENABLED: + retry_failed_files, react_commits, react_results = self._retry_failed_files_react( + failed_files=retry_failed_files, tracker=tracker, repo_id=repo_id, repo_type=repo_type, @@ -1545,9 +1699,9 @@ def _report_recovery(results: list[dict], label: str) -> None: ) commit_infos.extend(react_commits) all_results.extend(react_results) - elif total_failed_files: - total_failed_files = self._retry_failed_simple( - failed_files=total_failed_files, + elif retry_failed_files: + retry_failed_files = self._retry_failed_simple( + failed_files=retry_failed_files, tracker=tracker, repo_id=repo_id, repo_type=repo_type, @@ -1562,10 +1716,11 @@ def _report_recovery(results: list[dict], label: str) -> None: ) tracker.save() + all_failures = terminal_failures + retry_failed_files - # Sync: delete remote orphan files + # Sync: delete remote orphan files only after every upload commit landed. deleted_count = 0 - if sync_remote_repo and not total_failed_files: + if sync_remote_repo and not all_failures: prefix = path_in_repo.strip("/") if path_in_repo else "" orphans = self._compute_remote_orphans( repo_id=repo_id, @@ -1597,7 +1752,7 @@ def _report_recovery(results: list[dict], label: str) -> None: # Upload report elapsed = time.time() - start_time total_files = len(sorted_files) - failed_count = len(total_failed_files) + failed_count = len(all_failures) lfs_reused_count = sum(1 for r in all_results if r.get("upload_mode") == "lfs" and r.get("is_reused")) lfs_uploaded_count = sum(1 for r in all_results if r.get("is_blob_uploaded")) normal_count = sum(1 for r in all_results if r.get("upload_mode") == "normal") @@ -1617,8 +1772,8 @@ def _report_recovery(results: list[dict], label: str) -> None: print(f" Elapsed : {elapsed:.1f}s") print("=" * 60) - if total_failed_files: - for (path_in_repo_f, _), err in total_failed_files: + if all_failures: + for (path_in_repo_f, _), err in all_failures: logger.error(" - %s: %s: %s", path_in_repo_f, type(err).__name__, err) succeeded = total_files - failed_count raise StorageError( @@ -2043,11 +2198,8 @@ def _commit_with_retry( revision=revision, ) except HubError as e: - if not e.retryable: - error_str = str(e) - retryable_patterns = ["Could not update refs", "try again"] - if not any(p in error_str for p in retryable_patterns): - raise + if not _is_retryable_commit_error(e): + raise last_error = e except (ConnectionError, TimeoutError) as e: last_error = e @@ -2182,6 +2334,118 @@ def _track_committed_batch( ) tracker.save() + def _plan_result_batches(self, results: list[dict], repo_type: str) -> list[list[dict]]: + """Apply the normal commit planner to already-uploaded results.""" + if not results: + return [] + files = [(r["file_path_in_repo"], r["file_path"]) for r in results] + sizes = {r["file_path"]: r["file_size_on_disk"] for r in results} + max_operations = _calculate_adaptive_batch_size(len(files), UPLOAD_COMMIT_BATCH_MAX_OPERATIONS) + counts = _plan_commit_batches( + files, + repo_type, + max_operations=max_operations, + max_inline_bytes=UPLOAD_COMMIT_MAX_INLINE_BYTES, + sizes=sizes, + ) + batches: list[list[dict]] = [] + offset = 0 + for count in counts: + batches.append(results[offset : offset + count]) + offset += count + return batches + + @staticmethod + def _should_split_commit_failure(error: Exception) -> bool: + body = getattr(error, "response_body", None) + raw_code = body.get("Code") if isinstance(body, dict) else None + if raw_code is None and isinstance(body, dict): + raw_code = body.get("code") + try: + policy_code = raw_code is not None and int(raw_code) in _RETRYABLE_COMMIT_BUSINESS_CODES + except (TypeError, ValueError): + policy_code = False + message = str(getattr(error, "message", None) or error).lower() + return "repository policy" in message or (policy_code and getattr(error, "status_code", None) == 400) + + def _retry_failed_commits( + self, + *, + failed_batches: list[tuple[list[dict], Exception]], + tracker: UploadTracker | NullTracker, + repo_id: str, + repo_type: str, + commit_message: str, + revision: str, + on_committed: Any = None, + ) -> tuple[list[tuple], list[dict], list[dict]]: + """Retry commits without re-uploading blobs, splitting policy rejects.""" + failures: list[tuple] = [] + commit_infos: list[dict] = [] + committed_results: list[dict] = [] + + def commit_batch( + batch: list[dict], + label: str, + prior_error: Exception, + *, + split_before_attempt: bool = False, + ) -> None: + if split_before_attempt and self._should_split_commit_failure(prior_error) and len(batch) > 1: + midpoint = (len(batch) + 1) // 2 + logger.warning( + "Commit-only recovery is splitting a policy-rejected batch of %d into %d and %d files.", + len(batch), + midpoint, + len(batch) - midpoint, + ) + commit_batch(batch[:midpoint], label + "a", prior_error) + commit_batch(batch[midpoint:], label + "b", prior_error) + return + + try: + commit_info = self._commit_with_retry( + repo_id=repo_id, + repo_type=repo_type, + operations=self._build_batch_operations(batch, repo_type), + commit_message=f"{commit_message} (commit recovery {label})", + revision=revision, + ) + except Exception as error: + if self._should_split_commit_failure(error) and len(batch) > 1: + midpoint = (len(batch) + 1) // 2 + commit_batch(batch[:midpoint], label + "a", error) + commit_batch(batch[midpoint:], label + "b", error) + return + category = classify_error(error, commit_context=True) + for result in batch: + tracker.mark_failed( + result["file_path_in_repo"], + result["file_mtime"], + result["file_size_on_disk"], + error_type="commit_" + category, + ) + failures.append(((result["file_path_in_repo"], result["file_path"]), error)) + logger.error("Commit-only recovery %s failed for %d file(s): %s", label, len(batch), error) + return + + commit_infos.append(commit_info) + committed_results.extend(batch) + self._track_committed_batch(tracker, batch) + if on_committed is not None: + on_committed(batch, f"commit recovery {label}") + logger.info("Commit-only recovery %s committed %d file(s).", label, len(batch)) + + for batch_index, (results, error) in enumerate(failed_batches, start=1): + for part_index, batch in enumerate(self._plan_result_batches(results, repo_type), start=1): + commit_batch( + batch, + f"{batch_index}.{part_index}", + error, + split_before_attempt=True, + ) + return failures, commit_infos, committed_results + # ------------------------------------------------------------------ # Internal: file collection # ------------------------------------------------------------------ @@ -2193,56 +2457,14 @@ def _prepare_upload_folder( allow_patterns: list[str] | None = None, ignore_patterns: list[str] | None = None, sizes_out: dict[str, int] | None = None, + warn_limits: bool = True, ) -> list[tuple[str, str]]: folder = Path(folder_path).expanduser().resolve() if not folder.is_dir(): raise InvalidParameter(f"Provided path: '{folder}' is not a directory") all_files = sorted(path for path in folder.glob("**/*") if path.is_file()) - - if len(all_files) > UPLOAD_MAX_FILE_COUNT: - raise InvalidParameter(f"Too many files ({len(all_files)}) in folder, max allowed: {UPLOAD_MAX_FILE_COUNT}") - - # Per-directory file count check - dir_counts: dict[str, int] = {} - for path in all_files: - parent = str(path.parent) - dir_counts[parent] = dir_counts.get(parent, 0) + 1 - for dir_path, count in dir_counts.items(): - if count > UPLOAD_MAX_FILES_PER_DIRECTORY: - raise InvalidParameter( - f"Too many files ({count}) in directory {dir_path}, " - f"max allowed per directory: {UPLOAD_MAX_FILES_PER_DIRECTORY}" - ) - - # File size checks. Sizes are handed back through ``sizes_out`` because - # batch planning needs them next; re-stating a large tree costs one - # syscall per file for no new information. - total_size = 0 - normal_size = 0 - for path in all_files: - fsize = path.stat().st_size - if fsize > UPLOAD_MAX_FILE_SIZE_BYTES: - raise InvalidParameter( - f"File too large: {path} ({fsize / 1024 / 1024:.1f} MB), " - f"max allowed: {UPLOAD_MAX_FILE_SIZE_BYTES / 1024 / 1024:.0f} MB" - ) - total_size += fsize - if not _is_lfs(str(path), fsize, repo_type): - normal_size += fsize - if sizes_out is not None: - sizes_out[str(path)] = fsize - - if normal_size > UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES: - logger.warning( - "Total normal (non-LFS) file size %d bytes exceeds soft " - "limit %d bytes. Consider using LFS for large files.", - normal_size, - UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES, - ) - relpath_to_abspath = {path.relative_to(folder).as_posix(): str(path) for path in all_files} - filtered_keys = _filter_repo_objects( list(relpath_to_abspath.keys()), allow_patterns=allow_patterns, @@ -2250,14 +2472,19 @@ def _prepare_upload_folder( ) # ``path_in_repo`` is a destination prefix, and "." / "./" / "" / "/" - # all mean the repo root. Collapsing them is not cosmetic: a literal - # value like "." otherwise rides into every commit action's ``path`` as - # a "./" prefix, which the Hub rejects wholesale with E3021 "invalid - # commit action". + # all mean the repository root. norm_prefix = _normalize_path_in_repo(path_in_repo) prefix = f"{norm_prefix}/" if norm_prefix else "" prepared = [(prefix + relpath, relpath_to_abspath[relpath]) for relpath in filtered_keys] + selected_sizes: dict[str, int] = {} + for _path_in_repo, file_path in prepared: + selected_sizes[file_path] = Path(file_path).stat().st_size + if sizes_out is not None: + sizes_out.update(selected_sizes) + if warn_limits: + _warn_advisory_upload_limits(prepared, selected_sizes, repo_type) + logger.info("Prepared %d files for upload.", len(prepared)) return prepared @@ -2314,21 +2541,18 @@ def _retry_failed_files_react( "name": "Round 1 (parallel)", "parallel": True, "workers": max(1, max_workers // 2), - "batch_size": 16, "delay": 0, }, { "name": "Round 2 (serial+backoff)", "parallel": False, "workers": 1, - "batch_size": 8, "delay": UPLOAD_RECOVERY_SERIAL_BACKOFF_BASE_SECONDS, }, { "name": "Round 3 (single-file)", "parallel": False, "workers": 1, - "batch_size": 1, "delay": UPLOAD_RECOVERY_SINGLE_FILE_DELAY_SECONDS, }, ] @@ -2406,13 +2630,8 @@ def _retry_failed_files_react( ) round_failures.append(((path_in_repo_r, file_path_r), e)) - all_successes.extend(round_successes) - - batch_size = min(cfg["batch_size"], max(1, len(round_successes))) - for batch_start in range(0, len(round_successes), batch_size): - batch = round_successes[batch_start : batch_start + batch_size] - self._track_uploaded_batch(tracker, batch) - + self._track_uploaded_batch(tracker, round_successes) + for batch in self._plan_result_batches(round_successes, repo_type): operations = self._build_batch_operations(batch, repo_type) if not operations: continue @@ -2425,6 +2644,7 @@ def _retry_failed_files_react( revision=revision, ) commit_infos.append(commit_info) + all_successes.extend(batch) self._track_committed_batch(tracker, batch) if on_committed is not None: on_committed(batch, round_name) @@ -2435,18 +2655,18 @@ def _retry_failed_files_react( ) except Exception as e: logger.error("[ReAct] %s commit failed: %s", round_name, e) - category = classify_error(e) - if not _ErrorCategory.is_retryable(category): - for r in batch: - tracker.mark_failed( - r["file_path_in_repo"], - r["file_mtime"], - r["file_size_on_disk"], - error_type="commit_" + category, - ) - else: - for r in batch: - round_failures.append(((r["file_path_in_repo"], r["file_path"]), e)) + commit_failures, recovered_commits, recovered_results = self._retry_failed_commits( + failed_batches=[(batch, e)], + tracker=tracker, + repo_id=repo_id, + repo_type=repo_type, + commit_message=commit_message, + revision=revision, + on_committed=on_committed, + ) + permanent_failures.extend(commit_failures) + commit_infos.extend(recovered_commits) + all_successes.extend(recovered_results) new_retryable = [] for item_err in round_failures: @@ -2531,6 +2751,7 @@ def _retry_failed_simple( on_committed: Any = None, ) -> list[tuple]: total_failed_files = list(failed_files) + terminal_failures: list[tuple] = [] for retry_round in range(UPLOAD_FAILED_FILE_MAX_RETRY_ROUNDS): if not total_failed_files: break @@ -2560,8 +2781,10 @@ def _retry_failed_simple( retry_failures.append(((path_in_repo_r, file_path_r), e)) if retry_successes: self._track_uploaded_batch(tracker, retry_successes) - operations = self._build_batch_operations(retry_successes, repo_type) - if operations: + for batch in self._plan_result_batches(retry_successes, repo_type): + operations = self._build_batch_operations(batch, repo_type) + if not operations: + continue try: commit_info = self._commit_with_retry( repo_id=repo_id, @@ -2571,14 +2794,14 @@ def _retry_failed_simple( revision=revision, ) commit_infos.append(commit_info) - all_results.extend(retry_successes) - self._track_committed_batch(tracker, retry_successes) + all_results.extend(batch) + self._track_committed_batch(tracker, batch) if on_committed is not None: - on_committed(retry_successes, f"retry round {retry_round + 1}") + on_committed(batch, f"retry round {retry_round + 1}") logger.info( " Retry round %d: committed %d file(s).", retry_round + 1, - len(retry_successes), + len(batch), ) except Exception as e: logger.error( @@ -2586,25 +2809,17 @@ def _retry_failed_simple( retry_round + 1, e, ) - category = classify_error(e) - if not _ErrorCategory.is_retryable(category): - for result in retry_successes: - tracker.mark_failed( - result["file_path_in_repo"], - result["file_mtime"], - result["file_size_on_disk"], - error_type="commit_" + category, - ) - else: - for result in retry_successes: - retry_failures.append( - ( - ( - result["file_path_in_repo"], - result.get("file_path", ""), - ), - e, - ) - ) + commit_failures, recovered_commits, recovered_results = self._retry_failed_commits( + failed_batches=[(batch, e)], + tracker=tracker, + repo_id=repo_id, + repo_type=repo_type, + commit_message=commit_message, + revision=revision, + on_committed=on_committed, + ) + terminal_failures.extend(commit_failures) + commit_infos.extend(recovered_commits) + all_results.extend(recovered_results) total_failed_files = retry_failures - return total_failed_files + return terminal_failures + total_failed_files diff --git a/src/modelscope_hub/constants.py b/src/modelscope_hub/constants.py index cf1b76b..e317f99 100644 --- a/src/modelscope_hub/constants.py +++ b/src/modelscope_hub/constants.py @@ -839,17 +839,15 @@ def _env_register( UPLOAD_COMMIT_MAX_INLINE_BYTES: int = _env_bytes( "MODELSCOPE_UPLOAD_COMMIT_MAX_INLINE_BYTES", 8 * 1024 * 1024, - "Maximum inlined (non-LFS) content carried by one commit request", + "Advisory estimated request-body budget for one upload commit", "Upload", ) -"""Byte ceiling on the base64 content a single commit may carry. - -Non-LFS files travel *inside* the commit request body, so a batch sized purely -by file count can produce a request tens of megabytes large -- big enough for -the server to time out mid-read, which surfaces on the client as an unrelated -write timeout. Batching therefore closes a batch on whichever limit is reached -first, this one or :data:`UPLOAD_COMMIT_BATCH_MAX_OPERATIONS`. LFS files -contribute only a pointer, so they do not count against it. +"""Advisory byte budget for an estimated commit request body. + +Normal files contribute their base64-expanded content plus operation JSON; LFS +files contribute only pointer/action metadata. Batching closes on this budget or +the operation target, whichever comes first. A single oversized inline file is +still sent alone so this advisory budget never blocks an upload. """ UPLOAD_COMMIT_MAX_PER_HOUR: int = _env_int( "MODELSCOPE_UPLOAD_COMMIT_MAX_PER_HOUR", @@ -1012,7 +1010,7 @@ def get_upload_ignore_file_pattern() -> str | None: # Upload: limits UPLOAD_LFS_FORCE_THRESHOLD_BYTES: int = _env_bytes( "MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD", - 1024 * 1024, + 64 * 1024, "File size above which LFS mode is forced (bytes; accepts a unit suffix, 0 forces LFS for every file)", "Upload", deprecated_mb_names=("MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD_MB",), @@ -1049,7 +1047,7 @@ def get_upload_ignore_file_pattern() -> str | None: UPLOAD_MAX_FILE_SIZE_BYTES: int = _env_int_mb_with_deprecated_units( "MODELSCOPE_UPLOAD_MAX_FILE_SIZE_MB", 100 * 1024, - "Maximum single upload file size (MB, default 100 GB)", + "Advisory single upload file size warning threshold (MB, default 100 GB)", "Upload", deprecated_mb_names=("UPLOAD_MAX_FILE_SIZE_MB",), deprecated_byte_names=("UPLOAD_MAX_FILE_SIZE",), @@ -1057,21 +1055,21 @@ def get_upload_ignore_file_pattern() -> str | None: UPLOAD_MAX_FILE_COUNT: int = _env_int( "MODELSCOPE_UPLOAD_MAX_FILE_COUNT", 100_000, - "Maximum total files per upload", + "Advisory total file-count warning threshold per upload", "Upload", "UPLOAD_MAX_FILE_COUNT", ) UPLOAD_MAX_FILES_PER_DIRECTORY: int = _env_int( "MODELSCOPE_UPLOAD_MAX_FILES_PER_DIRECTORY", 50_000, - "Maximum files in one uploaded directory", + "Advisory per-directory file-count warning threshold", "Upload", "UPLOAD_MAX_FILE_COUNT_IN_DIR", ) UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES: int = _env_int_mb_with_deprecated_units( "MODELSCOPE_UPLOAD_NORMAL_FILES_TOTAL_SIZE_MB", 500, - "Maximum total size of normal (non-LFS) files (MB)", + "Advisory total normal-file size warning threshold (MB)", "Upload", deprecated_byte_names=("UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT",), ) diff --git a/src/modelscope_hub/errors.py b/src/modelscope_hub/errors.py index d3a3799..c2e1623 100644 --- a/src/modelscope_hub/errors.py +++ b/src/modelscope_hub/errors.py @@ -440,6 +440,12 @@ class NotSupportedError(HubError): 10010202004: AlreadyExistsError, # 国际站 - 名称已被使用 } +# The legacy commit endpoint overloads 400/409 for optimistic branch conflicts +# and repository-side commit policy throttling. The numeric code is scoped to +# that endpoint so it must not globally turn unrelated invalid requests into +# retryable server failures. +_COMMIT_RETRYABLE_BUSINESS_CODES: frozenset[int] = frozenset({10030000001}) + # --------------------------------------------------------------------------- # OpenAPI string error-code -> exception mapping @@ -583,7 +589,13 @@ def raise_for_status(response: Response) -> None: business_cls: type[APIError] | None = None if status < 500: code = _business_code(body) - business_cls = _BUSINESS_CODE_MAP.get(code) if code is not None else None + is_commit_request = isinstance(url, str) and "/commit/" in urlparse(url).path + if code in _COMMIT_RETRYABLE_BUSINESS_CODES and is_commit_request: + # A branch race or repository-side policy gate is transient even + # though the endpoint reports it with a client-error HTTP status. + business_cls = ServerError + else: + business_cls = _BUSINESS_CODE_MAP.get(code) if code is not None else None if business_cls is None: # The OpenAPI surface publishes a string code instead of a numeric one. openapi_code = _openapi_code(body) diff --git a/tests/test_errors_openapi_codes.py b/tests/test_errors_openapi_codes.py index fdfd14e..c80ef38 100644 --- a/tests/test_errors_openapi_codes.py +++ b/tests/test_errors_openapi_codes.py @@ -15,6 +15,7 @@ import pytest import requests +from modelscope_hub._legacy_api import LegacyClient from modelscope_hub._openapi import _RETRYABLE_EXC from modelscope_hub.errors import ( AlreadyExistsError, @@ -128,6 +129,57 @@ def test_413_maps_to_invalid_parameter(): raise_for_status(_plain_error(413)) +def test_commit_policy_business_code_is_retryable_only_on_commit_endpoint(): + body = { + "Code": 10030000001, + "Message": "commit rejected by repository policy", + "Success": False, + } + commit_response = requests.Response() + commit_response.status_code = 400 + commit_response._content = json.dumps(body).encode() + commit_response.headers["Content-Type"] = "application/json" + commit_response.url = "https://modelscope.cn/api/v1/repos/datasets/owner/repo/commit/master" + + with pytest.raises(ServerError) as excinfo: + raise_for_status(commit_response) + assert excinfo.value.retryable is True + assert excinfo.value.status_code == 400 + + non_commit_response = requests.Response() + non_commit_response.status_code = 400 + non_commit_response._content = json.dumps(body).encode() + non_commit_response.headers["Content-Type"] = "application/json" + non_commit_response.url = "https://modelscope.cn/api/v1/login" + with pytest.raises(InvalidParameter): + raise_for_status(non_commit_response) + + +def test_legacy_commit_rejects_http_200_success_false(monkeypatch): + response = requests.Response() + response.status_code = 200 + response._content = json.dumps( + { + "Code": 10030000001, + "Message": "commit rejected by repository policy", + "Success": False, + } + ).encode() + response.headers["Content-Type"] = "application/json" + response.url = "https://modelscope.cn/api/v1/repos/datasets/owner/repo/commit/master" + client = LegacyClient(endpoint="https://modelscope.cn", token="test") + monkeypatch.setattr(client, "_request", lambda *args, **kwargs: response) + + with pytest.raises(APIError, match="repository policy") as excinfo: + client.create_commit( + repo_id="owner/repo", + repo_type="dataset", + operations=[{"action": "create", "path": "a.txt"}], + commit_message="test", + ) + assert excinfo.value.response_body["Success"] is False + + # --------------------------------------------------------------------------- # Retry policy # --------------------------------------------------------------------------- diff --git a/tests/test_upload_batching.py b/tests/test_upload_batching.py index 157a458..1c42c81 100644 --- a/tests/test_upload_batching.py +++ b/tests/test_upload_batching.py @@ -14,13 +14,21 @@ from modelscope_hub._upload import ( BatchTracker, UploadManager, + UploadTracker, _calculate_adaptive_batch_size, _is_inline_metadata, _normalize_path_in_repo, _plan_commit_batches, _upload_mode, ) -from modelscope_hub.errors import InvalidParameter, NetworkError, RateLimitError +from modelscope_hub.errors import ( + AlreadyExistsError, + APIError, + InvalidParameter, + NetworkError, + RateLimitError, + StorageError, +) def _make_manager() -> tuple[UploadManager, MagicMock]: @@ -68,9 +76,11 @@ def test_adaptive_batch_size_is_monotonic_and_never_exceeds_cap() -> None: assert max(sizes) == 256 -def test_batch_plan_closes_on_inline_bytes_before_operation_cap(tmp_path: Path) -> None: +def test_batch_plan_closes_on_inline_bytes_before_operation_cap(tmp_path: Path, monkeypatch) -> None: # 8 inline files of 100 KiB: the operation cap would take all 8 in one - # commit, but their base64 form exceeds a 512 KiB inline budget. + # commit, but their base64 form exceeds a 512 KiB inline budget. Pin the LFS + # threshold above their size so they really travel inline under any default. + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 2 * 1024 * 1024) files = [] sizes = {} for index in range(8): @@ -114,9 +124,72 @@ def test_batch_plan_ignores_lfs_bytes_in_the_inline_budget(tmp_path: Path) -> No assert plan == [8] -def test_batch_plan_keeps_one_oversized_inline_file_per_batch(tmp_path: Path) -> None: - # Below the 1 MiB LFS threshold, so these really do travel inline; each one +def test_four_25gb_lfs_files_share_one_commit_plan() -> None: + files = [(f"shard-{index}.bin", f"/virtual/shard-{index}.bin") for index in range(4)] + sizes = {local: 25 * 1024**3 for _, local in files} + + plan = _plan_commit_batches( + files, + "dataset", + max_operations=256, + max_inline_bytes=8 * 1024 * 1024, + sizes=sizes, + ) + + assert plan == [4] + + +def test_four_oversized_inline_metadata_files_require_four_commits() -> None: + files = [(f"part-{index}/README.md", f"/virtual/part-{index}/README.md") for index in range(4)] + sizes = {local: 10 * 1024**2 for _, local in files} + + plan = _plan_commit_batches( + files, + "dataset", + max_operations=256, + max_inline_bytes=8 * 1024 * 1024, + sizes=sizes, + ) + + assert plan == [1, 1, 1, 1] + + +def test_forty_thousand_small_files_use_count_target() -> None: + files = [(f"part-{index}.txt", f"/virtual/part-{index}.txt") for index in range(40_000)] + sizes = dict.fromkeys((local for _, local in files), 1) + + plan = _plan_commit_batches( + files, + "dataset", + max_operations=256, + max_inline_bytes=8 * 1024 * 1024, + sizes=sizes, + ) + + assert len(plan) == 157 + assert sum(plan) == 40_000 + assert max(plan) == 256 + + +def test_tiny_tail_batch_is_rebalanced_without_adding_commits() -> None: + files = [(f"part-{index}.txt", f"/virtual/part-{index}.txt") for index in range(257)] + sizes = dict.fromkeys((local for _, local in files), 1) + + plan = _plan_commit_batches( + files, + "dataset", + max_operations=256, + max_inline_bytes=8 * 1024 * 1024, + sizes=sizes, + ) + + assert plan == [129, 128] + + +def test_batch_plan_keeps_one_oversized_inline_file_per_batch(tmp_path: Path, monkeypatch) -> None: + # Pinned above the LFS threshold, so these really do travel inline; each one # alone blows the inline budget, which must not stall the plan. + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 2 * 1024 * 1024) files = [] sizes = {} for index in range(3): @@ -481,6 +554,77 @@ def test_external_tracker_lets_a_second_run_skip_committed_files(tmp_path: Path) assert client.create_commit.call_count == 1 +def test_resume_replans_only_pending_files_into_compact_batches(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr(upload_module, "UPLOAD_COMMIT_BATCH_MAX_OPERATIONS", 4) + monkeypatch.setattr(upload_module, "UPLOAD_ADAPTIVE_BATCHING_ENABLED", False) + manager, client = _make_manager() + cache_path = tmp_path / ".ms_upload_cache" + tracker = UploadTracker(cache_path, repo_id="owner/repo") + + pending = [] + for index in range(8): + path = tmp_path / f"file-{index}.txt" + path.write_bytes(bytes([index])) + stat = path.stat() + tracker.put_hash( + path.name, + stat.st_mtime, + stat.st_size, + {"file_hash": f"{index:064x}", "file_size": stat.st_size}, + ) + if index % 2 == 0: + tracker.mark_committed_batch([(path.name, stat.st_mtime, stat.st_size)]) + else: + pending.append(path.name) + tracker.mark_failed(path.name, stat.st_mtime, stat.st_size, error_type="commit_permanent") + tracker.save() + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=1, + use_cache=True, + disable_tqdm=True, + ) + + client.create_commit.assert_called_once() + assert [op["path"] for op in client.create_commit.call_args.kwargs["operations"]] == pending + saved = json.loads(cache_path.read_text(encoding="utf-8")) + assert all(entry.get("status") == "c" for entry in saved["files"].values()) + assert all("error_type" not in entry for entry in saved["files"].values()) + + +def test_terminal_commit_failure_is_reported_and_manual_rerun_retries(tmp_path: Path) -> None: + manager, client = _make_manager() + (tmp_path / "README.md").write_bytes(b"hello") + client.create_commit.side_effect = [InvalidParameter("path is not allowed"), {"ok": True}] + + with pytest.raises(StorageError, match="1 file"): + manager.upload_folder( + repo_id="owner/repo", + repo_type="model", + folder_path=tmp_path, + max_workers=1, + use_cache=True, + disable_tqdm=True, + sync_remote_repo=True, + ) + + client.list_repo_files.assert_not_called() + result = manager.upload_folder( + repo_id="owner/repo", + repo_type="model", + folder_path=tmp_path, + max_workers=1, + use_cache=True, + disable_tqdm=True, + ) + + assert result == {"ok": True} + assert client.create_commit.call_count == 2 + + # ------------------------------------------------------------ progress events @@ -848,6 +992,84 @@ def test_upload_file_honors_retry_after(monkeypatch) -> None: assert slept == [7.0] +def test_upload_file_manual_rerun_retries_a_permanent_failure() -> None: + manager, client = _make_manager() + client.create_commit.side_effect = [InvalidParameter("path is not allowed"), {"ok": True}] + + with pytest.raises(InvalidParameter): + manager.upload_file( + repo_id="owner/repo", + repo_type="model", + path_or_fileobj=b"hello", + path_in_repo="README.md", + disable_tqdm=True, + ) + + assert manager.upload_file( + repo_id="owner/repo", + repo_type="model", + path_or_fileobj=b"hello", + path_in_repo="README.md", + disable_tqdm=True, + ) == {"ok": True} + assert client.create_commit.call_count == 2 + + +def test_branch_changed_409_is_retried_in_commit_context(monkeypatch) -> None: + manager, client = _make_manager() + monkeypatch.setattr(upload_module.time, "sleep", lambda _seconds: None) + client.create_commit.side_effect = [ + AlreadyExistsError( + "branch changed, please retry", + status_code=409, + response_body={"Code": 10030000001, "Message": "branch changed, please retry"}, + ), + {"ok": True}, + ] + + assert manager._commit_with_retry( + repo_id="owner/repo", + repo_type="dataset", + operations=[{"action": "create", "path": "a.txt"}], + commit_message="retry branch race", + ) == {"ok": True} + assert client.create_commit.call_count == 2 + + +def test_policy_rejected_commit_splits_without_reuploading_blobs(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr(upload_module, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", 0) + monkeypatch.setattr(upload_module, "UPLOAD_COMMIT_MAX_ATTEMPTS", 2) + monkeypatch.setattr(upload_module.time, "sleep", lambda _seconds: None) + manager, client = _make_manager() + for index in range(4): + (tmp_path / f"blob-{index}.dat").write_bytes(bytes([index + 1]) * 32) + + def create_commit(**kwargs): + operations = kwargs["operations"] + if len(operations) == 4: + raise APIError( + "commit rejected by repository policy", + status_code=400, + response_body={"Code": 10030000001}, + ) + return {"ok": True} + + client.create_commit.side_effect = create_commit + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=1, + use_cache=False, + disable_tqdm=True, + ) + + assert client.upload_blob.call_count == 4 + sent_sizes = [len(call.kwargs["operations"]) for call in client.create_commit.call_args_list] + assert sent_sizes[-2:] == [2, 2] + assert all(size == 4 for size in sent_sizes[:-2]) + + def test_upload_file_still_fails_fast_on_a_permanent_error() -> None: manager, client = _make_manager() client.create_commit.side_effect = InvalidParameter("path is not allowed") @@ -864,6 +1086,53 @@ def test_upload_file_still_fails_fast_on_a_permanent_error() -> None: assert client.create_commit.call_count == 1 +def test_folder_capacity_limits_warn_but_do_not_block_upload(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr(upload_module, "UPLOAD_MAX_FILE_COUNT", 1) + monkeypatch.setattr(upload_module, "UPLOAD_MAX_FILES_PER_DIRECTORY", 1) + monkeypatch.setattr(upload_module, "UPLOAD_MAX_FILE_SIZE_BYTES", 1) + monkeypatch.setattr(upload_module, "UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES", 1) + manager, client = _make_manager() + warnings = MagicMock() + monkeypatch.setattr(upload_module.logger, "warning", warnings) + (tmp_path / "a.txt").write_bytes(b"aa") + (tmp_path / "b.txt").write_bytes(b"bb") + + manager.upload_folder( + repo_id="owner/repo", + repo_type="dataset", + folder_path=tmp_path, + max_workers=1, + use_cache=False, + disable_tqdm=True, + ) + + client.create_commit.assert_called_once() + messages = " ".join(call.args[0] for call in warnings.call_args_list) + assert "advisory limit" in messages + assert "continuing" in messages + + +def test_upload_file_capacity_limit_warns_but_does_not_block(monkeypatch) -> None: + monkeypatch.setattr(upload_module, "UPLOAD_MAX_FILE_SIZE_BYTES", 1) + monkeypatch.setattr(upload_module, "UPLOAD_NORMAL_FILES_TOTAL_SIZE_BYTES", 1) + manager, client = _make_manager() + warnings = MagicMock() + monkeypatch.setattr(upload_module.logger, "warning", warnings) + + manager.upload_file( + repo_id="owner/repo", + repo_type="model", + path_or_fileobj=b"hello", + path_in_repo="notes.txt", + disable_tqdm=True, + ) + + client.create_commit.assert_called_once() + messages = " ".join(call.args[0] for call in warnings.call_args_list) + assert "single-file limit" in messages + assert "normal (non-LFS)" in messages + + def test_prepare_upload_folder_reports_sizes_for_reuse(tmp_path: Path) -> None: # Planning needs these sizes immediately afterwards; re-stating the tree # costs one syscall per file and yields nothing new. diff --git a/tests/test_upload_config.py b/tests/test_upload_config.py index acaf9ab..05df365 100644 --- a/tests/test_upload_config.py +++ b/tests/test_upload_config.py @@ -331,13 +331,13 @@ def test_canonical_lfs_threshold_wins_over_deprecated_aliases() -> None: ( {"MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD": "32ZB"}, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", - 1024 * 1024, + 64 * 1024, "unknown size unit", ), ( {"MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD": "-1"}, "UPLOAD_LFS_FORCE_THRESHOLD_BYTES", - 1024 * 1024, + 64 * 1024, "must not be negative", ), ( @@ -394,7 +394,7 @@ def test_new_upload_knobs_are_registered_under_upload() -> None: "MODELSCOPE_UPLOAD_INLINE_METADATA_PATHS", ): assert registry[name]["category"] == "Upload" - assert registry["MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD"]["default"] == "1MiB" + assert registry["MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD"]["default"] == "64KiB" assert "MODELSCOPE_UPLOAD_LFS_FORCE_THRESHOLD_MB" not in registry From f782b17c29195aa73c003dca8a6854bd1aa0790a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Mon, 21 Sep 2026 16:20:11 +0800 Subject: [PATCH 15/22] update readme --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 45b922d..4ae6df1 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,6 @@ The official Python SDK & CLI for [ModelScope Hub](https://modelscope.cn) — do ## News -**Unreleased** -- **Feature**: `ms-hub agent install -r owner/name` fetches the official framework plugin from a model repository and delegates to the entry operation it declares, plus the `modelscope_hub.agent.install_agent` SDK entry. The hub gains no framework knowledge. -- **Quality**: loading a plugin executes code this package did not ship, so it is gated by a compile-time owner allow-list checked before any download and a `content_sha256` manifest verified against every file before import. - **v0.4.0** (2026-09-01) - **Feature**: complete OpenAPI coverage for Agent-IDP, MCP, and Studios — Agent Ed25519 identities, OIDC discovery/JWKS and signed JWT issuance (`HubApi`, `ms-hub agent-idp`); Studio lists, variables and configuration options; hosted MCP discovery; protected visibility and runtime metadata; read-only tokens can log in and rejected writes name the required tier. Agent private JWKs are only written to an explicitly requested owner-only file. - **Fix**: Studio compat calls no longer leak connection options or API tokens, or drop cover images; errors distinguish permission, quota and conflicts; anonymous Studio info and log pagination work From 751fc6f0f6d1d179ab6ba68570737d94af48b8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Fri, 23 Oct 2026 18:24:48 +0800 Subject: [PATCH 16/22] Reliable Downloads & Consistent SDK Identification --- AGENTS.md | 9 + README.md | 17 +- src/modelscope_hub/__init__.py | 4 + src/modelscope_hub/_download.py | 1061 +++++++++++++---- src/modelscope_hub/_download_control.py | 280 +++++ src/modelscope_hub/_legacy_api.py | 245 +++- src/modelscope_hub/_openapi.py | 73 +- src/modelscope_hub/agent/_api.py | 6 +- src/modelscope_hub/api.py | 72 +- src/modelscope_hub/cli/agent.py | 8 +- src/modelscope_hub/cli/base.py | 1 + src/modelscope_hub/cli/download.py | 4 +- src/modelscope_hub/compat/file_download.py | 9 +- src/modelscope_hub/compat/hub_api.py | 43 +- .../compat/snapshot_download.py | 9 +- src/modelscope_hub/constants.py | 58 +- src/modelscope_hub/errors.py | 72 +- src/modelscope_hub/utils/__init__.py | 39 +- tests/cli/test_base.py | 8 +- tests/test_agent_plugin.py | 4 +- tests/test_download_integrity.py | 802 +++++++++++++ tests/test_user_agent.py | 331 +++++ 22 files changed, 2830 insertions(+), 325 deletions(-) create mode 100644 src/modelscope_hub/_download_control.py create mode 100644 tests/test_download_integrity.py create mode 100644 tests/test_user_agent.py diff --git a/AGENTS.md b/AGENTS.md index 2b94ee3..995d2e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -138,3 +138,12 @@ After changing the upload path, check whether the following must be updated as w - The upload environment-variable table and semantic documentation in `README.md`. - `__all__` exports and compatibility aliases in `src/modelscope_hub/compat/` and `constants.py`. - This file, if new constants, error codes, or scenarios were introduced. + +--- + +## 7. User-Agent (UA) Requirements + +- Every SDK HTTP request must include a `User-Agent` starting with `modelscope_hub/`. Use `utils.build_user_agent` / `append_user_agent` and enforce coverage at client send boundaries across OpenAPI, Legacy, Agent, both sessions, anonymous requests, retries, and probes. +- Custom UA information must be appended, never replacing the SDK identifier. Handle header names case-insensitively; empty values must not remove the SDK UA. Propagate CLI / compat markers (`invoked_by/cli` / `invoked_by/compat`) to derived clients and download requests. +- Prefer a persisted `session_id`; fall back to a process-stable ID when storage is not writable. Telemetry persistence failures must not block business requests. UA must not contain tokens, cookies, or other credentials. +- Changes to HTTP transport, client construction, or delegating entry points must verify these invariants against actual `PreparedRequest` objects; checking session default headers alone is insufficient. diff --git a/README.md b/README.md index 4ae6df1..12be847 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,8 @@ snapshot = api.download_repo( path = api.download_file("Qwen/Qwen3-0.6B", "model", "config.json", local_files_only=True) ``` +Downloads validate available size and SHA-256 metadata before replacing files, with safe resume, bounded retries, timeouts, and cancellation. Incomplete snapshots raise `FileDownloadError` (CLI exit code `1`), preserving completed files for subsequent retries. + ### Upload ```bash @@ -905,7 +907,11 @@ Token is persisted locally after `ms-hub login` and auto-loaded in subsequent se |----------|---------|-------------| | `MODELSCOPE_API_TIMEOUT` | `60` | HTTP request timeout (seconds) | | `MODELSCOPE_API_CONNECT_TIMEOUT` | `10` | HTTP connect timeout (seconds) | -| `MODELSCOPE_API_MAX_RETRIES` | `5` | Max retry attempts for transient failures | +| `MODELSCOPE_API_MAX_RETRIES` | `5` | Regular API retries; downloads and file listings use separate retry policies | +| `MODELSCOPE_REPO_TREE_PAGE_TIMEOUT_SECONDS` | `15` | Per-page listing timeout (seconds) | +| `MODELSCOPE_REPO_TREE_PAGE_MAX_ATTEMPTS` | `4` | Maximum attempts per listing page, including the initial request | +| `MODELSCOPE_REPO_TREE_PAGE_RETRY_MAX_DELAY_SECONDS` | `8` | Maximum listing retry delay (seconds) | +| `MODELSCOPE_REPO_TREE_LIST_TIMEOUT_SECONDS` | `3600` | Total repository listing time budget (seconds) | **Download:** @@ -915,8 +921,13 @@ Token is persisted locally after `ms-hub login` and auto-loaded in subsequent se | `MODELSCOPE_DOWNLOAD_PARALLEL_THRESHOLD_MB` | `500` | Parallel download threshold (MB) | | `MODELSCOPE_DOWNLOAD_CHUNK_SIZE_MB` | `1` | Streaming chunk size (MB) | | `MODELSCOPE_DOWNLOAD_PART_SIZE_MB` | `160` | Parallel range chunk size (MB) | -| `MODELSCOPE_DOWNLOAD_MAX_RETRIES` | `5` | Per-file download retry count | -| `MODELSCOPE_DOWNLOAD_TIMEOUT` | `60` | Per-file download timeout (seconds) | +| `MODELSCOPE_DOWNLOAD_MAX_RETRIES` | `5` | Single-layer transport retries; `0` disables them; integrity retries are separately bounded | +| `MODELSCOPE_DOWNLOAD_TIMEOUT` | `60` | Idle read timeout for streams and range parts, not the whole file (seconds) | +| `MODELSCOPE_DOWNLOAD_TOTAL_TIMEOUT` | `3600` | Per-file time budget, including locks, probes, transfers, retries, and validation (seconds) | +| `MODELSCOPE_DOWNLOAD_MIN_BYTES_PER_SECOND` | `1024` | Minimum rate per connection (B/s); `0` disables low-speed detection | +| `MODELSCOPE_DOWNLOAD_LOW_SPEED_WINDOW_SECONDS` | `60` | Minimum-rate observation window (seconds) | +| `MODELSCOPE_DOWNLOAD_RETRY_MAX_DELAY_SECONDS` | `60` | Maximum download Retry-After or backoff delay (seconds) | +| `MODELSCOPE_DOWNLOAD_FILE_LOCK_TIMEOUT_SECONDS` | `600` | Maximum wait for another process's file lock (seconds) | | `MODELSCOPE_DOWNLOAD_FILE_LOCK` | `true` | File lock for multiprocess download safety | | `MODELSCOPE_DOWNLOAD_INTRA_CLOUD` | `true` | Alibaba cloud intra-cloud acceleration | | `MODELSCOPE_DOWNLOAD_INTRA_CLOUD_REGION` | (auto) | Override intra-cloud region ID | diff --git a/src/modelscope_hub/__init__.py b/src/modelscope_hub/__init__.py index 46f2d95..4db8011 100644 --- a/src/modelscope_hub/__init__.py +++ b/src/modelscope_hub/__init__.py @@ -26,7 +26,9 @@ CacheError, CacheNotFound, CorruptedCacheException, + FileDownloadError, FileIntegrityError, + FileSizeMismatchError, HubError, InvalidParameter, NetworkError, @@ -106,7 +108,9 @@ "CacheError", "CacheNotFound", "CorruptedCacheException", + "FileDownloadError", "FileIntegrityError", + "FileSizeMismatchError", "HubError", "InvalidParameter", "NetworkError", diff --git a/src/modelscope_hub/_download.py b/src/modelscope_hub/_download.py index bb5fab5..c1becfa 100644 --- a/src/modelscope_hub/_download.py +++ b/src/modelscope_hub/_download.py @@ -22,36 +22,54 @@ import errno import fnmatch import hashlib -import io import os import re import threading import time import uuid -from concurrent.futures import ThreadPoolExecutor, as_completed +from collections.abc import Callable +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait from pathlib import Path from typing import TYPE_CHECKING from urllib.parse import urlparse import requests from tqdm.auto import tqdm +from urllib3.exceptions import MaxRetryError from urllib3.util.retry import Retry +from ._download_control import ( + _DownloadCancelledError, + _DownloadControl, + _DownloadDeadlineError, + _network_context, + _ReadHTTPAdapter, +) from .constants import ( DOWNLOAD_CHUNK_SIZE, + DOWNLOAD_FILE_LOCK_TIMEOUT_SECONDS, + DOWNLOAD_LOW_SPEED_WINDOW_SECONDS, + DOWNLOAD_MIN_BYTES_PER_SECOND, DOWNLOAD_PARALLEL_THRESHOLD, DOWNLOAD_PARALLELS, DOWNLOAD_PART_SIZE, + DOWNLOAD_RETRY_MAX_DELAY_SECONDS, DOWNLOAD_RETRY_TIMES, - DOWNLOAD_TIMEOUT, + DOWNLOAD_TOTAL_TIMEOUT, ENV_FILE_LOCK, ENV_INTER_CLOUD_REGIONS, ENV_INTRA_CLOUD_ACCELERATION, ENV_INTRA_CLOUD_REGION, + REPO_TREE_LIST_TIMEOUT_SECONDS, ) from .errors import ( + APIError, + AuthenticationError, CacheNotFound, + FileDownloadError, FileIntegrityError, + FileSizeMismatchError, + HubError, NetworkError, NotExistError, PermissionDeniedError, @@ -59,6 +77,7 @@ ) from .utils.file_utils import compute_hash, ensure_dir from .utils.logger import get_logger +from .utils.patterns import extract_common_prefix if TYPE_CHECKING: from ._legacy_api import LegacyClient @@ -67,6 +86,18 @@ logger = get_logger("download") DOWNLOAD_HASH_RETRY_TIMES = 3 +# Attempts for a file whose delivered size contradicts the repository listing. +# The server answered consistently, so one more try over the default route is +# enough to rule out a stale replica or CDN node before reporting the file. +DOWNLOAD_SIZE_RETRY_TIMES = 2 +# Seconds before the first integrity retry, doubled for each further one, so a +# transient CDN or replica inconsistency has time to settle. +DOWNLOAD_INTEGRITY_RETRY_BACKOFF = 1.0 + +# Response headers identifying the request and the node that served it. They +# are quoted in integrity errors so a broken object can be traced server-side. +_DIAGNOSTIC_HEADERS = ("x-oss-request-id", "x-request-id", "eagleid", "x-cache", "x-linked-etag") +_CONTENT_RANGE_RE = re.compile(r"bytes\s+(\d+)-(\d+)/(\d+|\*)", re.IGNORECASE) # --------------------------------------------------------------------------- @@ -120,7 +151,12 @@ def end(self) -> None: @contextlib.contextmanager -def _optional_file_lock(lock_path: Path | None, *, enabled: bool = True): +def _optional_file_lock( + lock_path: Path | None, + *, + enabled: bool = True, + control: _DownloadControl | None = None, +): """Acquire a file lock when *enabled*, otherwise no-op.""" if not enabled or lock_path is None: yield @@ -129,12 +165,18 @@ def _optional_file_lock(lock_path: Path | None, *, enabled: bool = True): from filelock import FileLock, SoftFileLock, Timeout lock_path.parent.mkdir(parents=True, exist_ok=True) - default_interval = 60 + default_interval = 0.2 if control is not None else 60 lock = FileLock(str(lock_path), timeout=default_interval) is_soft = False - waited = 0 + waited = 0.0 + started = time.monotonic() + next_notice = 60.0 while True: + if control is not None: + control.check() + if time.monotonic() - started >= DOWNLOAD_FILE_LOCK_TIMEOUT_SECONDS: + raise _DownloadDeadlineError(f"等待下载文件锁超过 {DOWNLOAD_FILE_LOCK_TIMEOUT_SECONDS}s：{lock_path}") try: lock.acquire(timeout=default_interval) except Timeout: @@ -156,10 +198,9 @@ def _optional_file_lock(lock_path: Path | None, *, enabled: bool = True): pass waited = 0 continue - logger.info( - "Still waiting to acquire lock on %s", - lock_path, - ) + if waited >= next_notice: + logger.info("Still waiting to acquire lock on %s", lock_path) + next_notice = waited + 60 except NotImplementedError as exc: if "use SoftFileLock instead" in str(exc): logger.warning( @@ -224,12 +265,148 @@ def _matches_patterns(path: str, patterns: list[str] | None) -> bool: return False +# --------------------------------------------------------------------------- +# Response validation +# --------------------------------------------------------------------------- +class _IncompleteBodyError(Exception): + """The body ended before the announced length arrived; safe to resume.""" + + +class _RangeNotHonouredError(Exception): + """A ranged request was answered with something other than that range.""" + + +def _listing_roots(patterns: list[str] | None) -> list[str | None]: + """Use only literal directory prefixes; keep local glob filtering as the final selection check.""" + if not patterns or any(".*" in p or "\\" in p or p.startswith("/") for p in patterns): + return [None] + roots = [extract_common_prefix([pattern]) for pattern in patterns] + if any(root is None or ".." in root.split("/") for root in roots): + return [None] + result: list[str | None] = [] + for root in sorted(set(roots), key=lambda value: (len(value or ""), value or "")): + if root is not None and not any(root == parent or root.startswith(f"{parent}/") for parent in result): + result.append(root) + return result + + +def _parse_content_range(value: str | None) -> tuple[int, int, int | None] | None: + """Parse ``Content-Range: bytes start-end/total``; *total* may be ``*``.""" + match = _CONTENT_RANGE_RE.fullmatch(value.strip()) if value else None + if match is None: + return None + total = None if match.group(3) == "*" else int(match.group(3)) + start, end = int(match.group(1)), int(match.group(2)) + if end < start or (total is not None and end >= total): + return None + return start, end, total + + +def _unsatisfied_range_total(value: str | None) -> int | None: + """Return the object size from a 416 ``Content-Range: bytes */total`` header.""" + total = (value or "").rpartition("/")[2].strip() + return int(total) if total.isdigit() else None + + +def _checkable_content_length(resp: requests.Response) -> int | None: + """Return the announced body length when it can be compared with the bytes read. + + Under a ``Content-Encoding`` the header counts encoded bytes while + ``iter_content`` yields decoded ones, so the two cannot be compared. + """ + raw = resp.headers.get("Content-Length") + if raw is None or resp.headers.get("Content-Encoding", "identity").strip().lower() not in ("", "identity"): + return None + try: + return int(raw) + except ValueError: + return None + + +def _describe_response(resp: requests.Response, request_id: str) -> str: + """Summarise a download response for diagnostics without leaking the signed URL.""" + headers = resp.headers + parts = [f"HTTP {resp.status_code}", f"Content-Length={headers.get('Content-Length', 'absent')}"] + if headers.get("Content-Range"): + parts.append(f"Content-Range={headers['Content-Range']}") + parts.append(f"host={urlparse(resp.url or '').hostname or 'unknown'}") + parts.append(f"X-Request-ID={request_id}") + parts.extend(f"{name}={headers[name]}" for name in _DIAGNOSTIC_HEADERS if headers.get(name)) + return ", ".join(parts) + + +def _incomplete_path(target: Path) -> Path: + """Return the partial-download path used while *target* is being fetched.""" + return target.with_suffix(target.suffix + ".incomplete") + + +def _wait_download_retry(retry: Retry, exc: Exception, path: str, control: _DownloadControl) -> Retry: + control.check() + if isinstance(exc, HubError) and not exc.retryable: + raise exc + try: + retry = retry.increment("GET", path, error=exc) + except MaxRetryError: + if isinstance(exc, HubError): + raise exc + raise NetworkError(f"下载 {path} 在 {DOWNLOAD_RETRY_TIMES} 次重试后失败：{type(exc).__name__}") from exc + requested = getattr(exc, "retry_after", None) + delay = min( + max(0, requested) if requested is not None else retry.get_backoff_time(), DOWNLOAD_RETRY_MAX_DELAY_SECONDS + ) + logger.warning("Download failed for %s (%s), retrying in %ss", path, type(exc).__name__, delay) + control.sleep(delay) + return retry + + +def _finish_download( + tmp: Path, + target: Path, + expected_hash: str | None, + control: _DownloadControl, + actual_hash: str | None = None, +) -> None: + if expected_hash: + if actual_hash is None: + digest = hashlib.sha256() + with tmp.open("rb") as fh: + while block := fh.read(8 * DOWNLOAD_CHUNK_SIZE): + control.check() + digest.update(block) + actual_hash = digest.hexdigest() + if actual_hash != expected_hash: + raise FileIntegrityError( + f"Hash mismatch for {target.name}: expected {expected_hash[:16]}..., got {actual_hash[:16]}..." + ) + control.check() + tmp.replace(target) + + # --------------------------------------------------------------------------- # Parallel range download # --------------------------------------------------------------------------- def _download_part_with_retry(params: tuple) -> None: - """Download a byte range with retry and resume support.""" - file_path, progress_callbacks, start, end, url, file_name, headers, cookies = params + """Download one byte range into its part file, resuming after transport errors. + + Every response must be exactly the requested range: a server that ignores + ``Range`` raises :class:`_RangeNotHonouredError` so the caller can fall back to a + single stream instead of splicing whole-file bodies into the parts. + """ + ( + file_path, + progress_callbacks, + start, + end, + url, + file_name, + headers, + cookies, + file_size, + request, + control, + part_stop, + ) = params + expected = end - start + 1 get_headers = {} if headers is None else copy.deepcopy(headers) get_headers["X-Request-ID"] = uuid.uuid4().hex retry = Retry( @@ -238,37 +415,81 @@ def _download_part_with_retry(params: tuple) -> None: allowed_methods=["GET"], ) part_file_name = f"{file_path}_{start}_{end}" + reported = 0 while True: try: - partial_length = 0 - if os.path.exists(part_file_name): - with open(part_file_name, "rb") as f: - partial_length = f.seek(0, io.SEEK_END) - for cb in progress_callbacks: - cb.update(partial_length) + control.check() + if part_stop.is_set(): + raise _DownloadCancelledError("分片组已取消") + partial_length = os.path.getsize(part_file_name) if os.path.exists(part_file_name) else 0 + if partial_length > expected: + # Left over from a different split of the file; it cannot be resumed. + os.remove(part_file_name) + partial_length = 0 + if partial_length > reported: + for cb in progress_callbacks: + cb.update(partial_length - reported) + reported = partial_length download_start = start + partial_length if download_start > end: break get_headers["Range"] = f"bytes={download_start}-{end}" - with open(part_file_name, "ab+") as f: - r = requests.get( - url, - stream=True, - headers=get_headers, - cookies=cookies, - timeout=DOWNLOAD_TIMEOUT, - ) - r.raise_for_status() - for chunk in r.iter_content(chunk_size=DOWNLOAD_CHUNK_SIZE): - if chunk: - f.write(chunk) - for cb in progress_callbacks: - cb.update(len(chunk)) + try: + r = request(get_headers) + except APIError as exc: + if exc.status_code == 416: + response = getattr(exc, "response", None) + total = ( + _unsatisfied_range_total(response.headers.get("Content-Range")) + if response is not None + else None + ) + raise FileSizeMismatchError( + f"Server reports {total if total is not None else 'fewer'} bytes for {file_name} " + f"but the repository lists {file_size} bytes (HTTP 416)" + ) from exc + raise + with control.response(r): + if part_stop.is_set(): + raise _DownloadCancelledError("分片组已取消") + content_range = _parse_content_range(r.headers.get("Content-Range")) + if content_range is not None and content_range[2] is not None and content_range[2] != file_size: + raise FileSizeMismatchError( + f"Server reports {content_range[2]} bytes for {file_name} but the repository lists " + f"{file_size} bytes ({_describe_response(r, get_headers['X-Request-ID'])})" + ) + if content_range is None or content_range[0] != download_start or content_range[1] != end: + raise _RangeNotHonouredError( + f"asked for bytes={download_start}-{end}, got " + f"{_describe_response(r, get_headers['X-Request-ID'])}" + ) + with open(part_file_name, "ab") as f: + for chunk in control.chunks(r, DOWNLOAD_CHUNK_SIZE): + if chunk: + if f.tell() + len(chunk) > expected: + raise FileSizeMismatchError(f"Server exceeded range {start}-{end} for {file_name}") + f.write(chunk) + reported += len(chunk) + for cb in progress_callbacks: + cb.update(len(chunk)) + written = os.path.getsize(part_file_name) + if written != expected: + if written > expected: + os.remove(part_file_name) + raise _IncompleteBodyError(f"part {start}-{end} holds {written} of {expected} bytes") break + except ( + _RangeNotHonouredError, + FileIntegrityError, + _DownloadCancelledError, + _DownloadDeadlineError, + NotExistError, + PermissionDeniedError, + AuthenticationError, + ): + raise except Exception as exc: - retry = retry.increment("GET", url, error=exc) - logger.warning("Downloading part %s-%s failed: %s, will retry", start, end, exc) - retry.sleep() + retry = _wait_download_retry(retry, exc, f"{file_name}:{start}-{end}", control) def _parallel_download( @@ -278,13 +499,20 @@ def _parallel_download( headers: dict[str, str] | None, cookies: dict | None, progress_callbacks: list[ProgressCallback] | None = None, + *, + request: Callable[[dict[str, str]], requests.Response], + control: _DownloadControl, + expected_sha256: str | None = None, ) -> str: """Split a large file into parts and download in parallel. - Returns the SHA256 hex digest of the merged file. + Returns the SHA256 hex digest of the merged file. When the server ignores + ranged requests, or serves an object of a different size, the parts are + removed and the error is re-raised for the caller to handle. """ callbacks = progress_callbacks or [] part_size = DOWNLOAD_PART_SIZE + part_stop = threading.Event() tasks: list[tuple] = [] file_path = str(target) target.parent.mkdir(parents=True, exist_ok=True) @@ -293,33 +521,92 @@ def _parallel_download( for idx in range(num_full_parts): start = idx * part_size end = (idx + 1) * part_size - 1 - tasks.append((file_path, callbacks, start, end, url, target.name, headers, cookies)) + tasks.append( + ( + file_path, + callbacks, + start, + end, + url, + target.name, + headers, + cookies, + file_size, + request, + control, + part_stop, + ) + ) remainder_start = num_full_parts * part_size if remainder_start < file_size: - tasks.append((file_path, callbacks, remainder_start, file_size - 1, url, target.name, headers, cookies)) + tasks.append( + ( + file_path, + callbacks, + remainder_start, + file_size - 1, + url, + target.name, + headers, + cookies, + file_size, + request, + control, + part_stop, + ) + ) + part_names = [f"{file_path}_{task[2]}_{task[3]}" for task in tasks] parallels = min(DOWNLOAD_PARALLELS, 16) - with ThreadPoolExecutor(max_workers=parallels, thread_name_prefix="download") as executor: - list(executor.map(_download_part_with_retry, tasks)) - - for cb in callbacks: - cb.end() + executor = ThreadPoolExecutor(max_workers=parallels, thread_name_prefix="download") + futures = [executor.submit(_download_part_with_retry, task) for task in tasks] + try: + for future in as_completed(futures): + future.result() + except BaseException as exc: + part_stop.set() + control.abort_responses() + for future in futures: + future.cancel() + executor.shutdown(wait=True, cancel_futures=True) + if isinstance(exc, (_RangeNotHonouredError, FileIntegrityError)): + # Parts of a whole-file body or of the wrong object cannot be resumed. + for name in part_names: + with contextlib.suppress(OSError): + os.remove(name) + raise + finally: + executor.shutdown(wait=True, cancel_futures=True) + for cb in callbacks: + cb.end() hash_sha256 = hashlib.sha256() + merged = 0 tmp_target = target.with_suffix(target.suffix + ".parallel_tmp") with open(tmp_target, "wb") as output_file: - for task in tasks: - part_file_name = f"{task[0]}_{task[2]}_{task[3]}" + for part_file_name in part_names: with open(part_file_name, "rb") as part_file: while True: chunk = part_file.read(16 * DOWNLOAD_CHUNK_SIZE) if not chunk: break + control.check() output_file.write(chunk) hash_sha256.update(chunk) + merged += len(chunk) os.remove(part_file_name) + if merged != file_size: + tmp_target.unlink(missing_ok=True) + raise FileSizeMismatchError(f"Merged {merged} of {file_size} bytes for {target.name} from ranged parts") + actual = hash_sha256.hexdigest() + if expected_sha256 and actual != expected_sha256: + tmp_target.unlink(missing_ok=True) + raise FileIntegrityError( + f"Hash mismatch for {target.name}: expected {expected_sha256[:16]}..., got {actual[:16]}..." + ) + control.check() tmp_target.replace(target) - return hash_sha256.hexdigest() + return actual # --------------------------------------------------------------------------- @@ -331,10 +618,14 @@ class DownloadManager: Dependencies are injected via constructor to keep this class testable. """ - def __init__(self, legacy_client: LegacyClient, config: HubConfig) -> None: + def __init__(self, legacy_client: LegacyClient, config: HubConfig, *, user_agent: dict | str | None = None) -> None: + from .utils import build_user_agent + self._client = legacy_client self._config = config + self._user_agent = build_user_agent(config=config, extra=user_agent) self._cached_region: str | None = None + self._region_lock = threading.Lock() # Cache only confirmed OSS-internal resolutions. A repository can contain # Git metadata and LFS objects behind different download routes, so cache # entries are scoped to repo/type/revision plus the redirect endpoint. @@ -349,27 +640,34 @@ def __init__(self, legacy_client: LegacyClient, config: HubConfig) -> None: # User-agent & headers # ------------------------------------------------------------------ def _build_user_agent(self, user_agent: dict | str | None = None) -> str: - from .utils import build_user_agent + from .utils import append_user_agent - return build_user_agent( - session_id=self._config.get_session_id(), - extra=user_agent, - ) + return append_user_agent(self._user_agent, user_agent) def _detect_region(self) -> str: - """Detect Alibaba cloud region ID for intra-cloud acceleration.""" - if self._cached_region is not None: + """Probe the region only once per DownloadManager instance.""" + with self._region_lock: + if self._cached_region is None: + self._cached_region = "" + self._cached_region = self._detect_region_uncached() return self._cached_region + def _detect_region_uncached(self) -> str: endpoint = self._client.endpoint internal_url = f"{endpoint}/api/v1/repos/internalAccelerationInfo" def _get(url: str, timeout: float): try: - resp = requests.get(url, timeout=timeout) - resp.raise_for_status() - return resp - except requests.exceptions.RequestException: + with _DownloadControl(total_timeout=timeout) as probe: + with self._client.read_context(probe): + resp = self._client._read_session.get(url, stream=True, timeout=timeout) + probe.track(resp) + with probe.response(resp): + resp.raise_for_status() + resp._content = b"".join(probe.chunks(resp, 8192)) + resp._content_consumed = True + return resp + except (requests.exceptions.RequestException, _DownloadDeadlineError): return None region_id = "" @@ -411,19 +709,27 @@ def _probe_redirect_url( headers: dict[str, str], cookies: dict | None = None, timeout: float = 5.0, + control: _DownloadControl | None = None, ) -> str: """Send a HEAD request without following redirects to get the 302 Location.""" + if control is not None: + control.check() + timeout = min(timeout, max(0.001, control.deadline - time.monotonic())) try: - r = requests.head( - url, - headers=headers, - cookies=cookies, - allow_redirects=False, - timeout=timeout, - ) - if r.status_code in (301, 302, 303, 307, 308): - return r.headers.get("Location", "") - except requests.exceptions.RequestException: + with ( + _DownloadControl( + total_timeout=timeout, + cancel_event=control.cancel_event if control is not None else None, + ) as probe, + _network_context(probe), + requests.Session() as session, + ): + session.mount("https://", _ReadHTTPAdapter(max_retries=0)) + session.mount("http://", _ReadHTTPAdapter(max_retries=0)) + with session.head(url, headers=headers, cookies=cookies, allow_redirects=False, timeout=timeout) as r: + if r.status_code in (301, 302, 303, 307, 308): + return r.headers.get("Location", "") + except (requests.exceptions.RequestException, _DownloadDeadlineError): pass return "" @@ -459,6 +765,7 @@ def _resolve_inter_region_headers( cookies: dict | None = None, peer_regions: list[str] | None = None, initial_redirect_url: str | None = None, + control: _DownloadControl | None = None, ) -> tuple[dict[str, str], str]: """Probe peer regions and return headers with the best region for OSS internal download. @@ -481,7 +788,9 @@ def _resolve_inter_region_headers( # established the storage route pass the initial redirect to avoid an # identical second HEAD request. redirect_url = ( - self._probe_redirect_url(url, headers, cookies) if initial_redirect_url is None else initial_redirect_url + self._probe_redirect_url(url, headers, cookies, control=control) + if initial_redirect_url is None + else initial_redirect_url ) if self._is_oss_internal_url(redirect_url): logger.debug("Inter-region: local region already yields OSS internal URL, skipping.") @@ -492,7 +801,7 @@ def _resolve_inter_region_headers( if peer_region == current_region: continue probe_headers = {**headers, "x-aliyun-region-id": peer_region} - redirect_url = self._probe_redirect_url(url, probe_headers, cookies) + redirect_url = self._probe_redirect_url(url, probe_headers, cookies, control=control) if self._is_oss_internal_url(redirect_url): logger.debug( 'Inter-region acceleration: using peer region "%s" (OSS internal).', @@ -507,14 +816,17 @@ def _resolve_inter_region_headers( def _build_download_headers( self, user_agent: dict | str | None = None, + *, + accelerate: bool = True, ) -> dict[str, str]: from .constants import _env, _env_bool headers: dict[str, str] = { "user-agent": self._build_user_agent(user_agent), "snapshot-identifier": uuid.uuid4().hex, + "Accept-Encoding": "identity", } - if _env_bool(ENV_INTRA_CLOUD_ACCELERATION, True): + if accelerate and _env_bool(ENV_INTRA_CLOUD_ACCELERATION, True): region = (_env(ENV_INTRA_CLOUD_REGION, "INTRA_CLOUD_ACCELERATION_REGION") or "").strip() if not region: try: @@ -542,6 +854,48 @@ def download_file( file_size: int | None = None, user_agent: dict | str | None = None, progress_callbacks: list[type[ProgressCallback]] | None = None, + _cancel_event: threading.Event | None = None, + ) -> Path: + """Download one file with cache-first lookup and a shared budget for locks, probes, and retries.""" + with _DownloadControl( + total_timeout=DOWNLOAD_TOTAL_TIMEOUT, + cancel_event=_cancel_event, + min_rate=DOWNLOAD_MIN_BYTES_PER_SECOND, + speed_window=DOWNLOAD_LOW_SPEED_WINDOW_SECONDS, + ) as control: + with self._client.read_context(control): + return self._download_file( + repo_id, + repo_type, + file_path, + revision, + cache_dir, + local_dir, + force, + expected_sha256, + local_files_only, + file_size, + user_agent, + progress_callbacks, + _control=control, + ) + + def _download_file( + self, + repo_id: str, + repo_type: str, + file_path: str, + revision: str | None = None, + cache_dir: Path | None = None, + local_dir: Path | None = None, + force: bool = False, + expected_sha256: str | None = None, + local_files_only: bool = False, + file_size: int | None = None, + user_agent: dict | str | None = None, + progress_callbacks: list[type[ProgressCallback]] | None = None, + *, + _control: _DownloadControl, ) -> Path: """Download a single file from a repository. @@ -574,7 +928,9 @@ def download_file( When ``True``, return the cached path without network access. Raises ``ValueError`` if the file is not cached. file_size: - Known file size (enables parallel range download for large files). + Known file size, e.g. from the repository listing. Received bytes + and cached files are validated against it, and it enables parallel + range download for large files. user_agent: Custom user-agent info appended to the default UA string. progress_callbacks: @@ -585,6 +941,14 @@ def download_file( ------- Path Absolute path to the downloaded (or cached) file on disk. + + Raises + ------ + FileSizeMismatchError + When the server keeps delivering a size other than *file_size*. + FileIntegrityError + When the content keeps failing the *expected_sha256* check. A + rejected download never remains at the target path. """ if local_files_only: return self._resolve_cached_file( @@ -608,47 +972,67 @@ def download_file( root = self._repo_cache_dir(repo_id, repo_type, cache_dir) target = root / "snapshots" / effective_revision / file_path - if not force and self._cache_hit(target, expected_sha256): + if not force and self._cache_hit(target, expected_sha256, file_size, control=_control): return target use_lock = _file_lock_enabled() lock_path = self._lock_path(repo_id, repo_type, cache_dir=cache_dir, file_path=file_path) if use_lock else None - with _optional_file_lock(lock_path, enabled=use_lock): + with _optional_file_lock(lock_path, enabled=use_lock, control=_control): # Re-check after acquiring lock — another process may have finished. - if not force and self._cache_hit(target, expected_sha256): + if not force and self._cache_hit(target, expected_sha256, file_size, control=_control): return target ensure_dir(target.parent) - - for attempt in range(DOWNLOAD_HASH_RETRY_TIMES): - self._download_with_resume( - repo_id, - repo_type, - file_path, - effective_revision, - target, - file_size=file_size, - user_agent=user_agent, - progress_callbacks=progress_callbacks, - ) - - if not expected_sha256: - break - + if not force: + # Whatever is at target failed validation. Drop it now so a failed + # download cannot leave it looking finished. + target.unlink(missing_ok=True) + + # A hash mismatch gets the full retry budget. A size mismatch means + # the server consistently served the wrong object, so it only gets + # one more attempt, over the default route. + failures = {"hash": 0, "size": 0} + limits = {"hash": DOWNLOAD_HASH_RETRY_TIMES, "size": DOWNLOAD_SIZE_RETRY_TIMES} + prefer_default_route = False + while True: + _control.check() try: - self.verify_file(target, expected_sha256) + self._download_with_resume( + repo_id, + repo_type, + file_path, + effective_revision, + target, + file_size=file_size, + user_agent=user_agent, + progress_callbacks=progress_callbacks, + prefer_default_route=prefer_default_route, + control=_control, + expected_sha256=expected_sha256, + ) break - except FileIntegrityError: - if attempt < DOWNLOAD_HASH_RETRY_TIMES - 1: - logger.warning( - "Hash validation failed for %s, retrying (%d/%d)", - file_path, - attempt + 1, - DOWNLOAD_HASH_RETRY_TIMES, - ) - target.unlink(missing_ok=True) - else: + except FileIntegrityError as exc: + kind = "size" if isinstance(exc, FileSizeMismatchError) else "hash" + # A size mismatch is raised before the rename, so the target is + # still the previous copy (only possible with force). + self._discard_download(target, keep_target=True) + target.with_suffix(target.suffix + ".parallel_tmp").unlink(missing_ok=True) + failures[kind] += 1 + if failures[kind] >= limits[kind]: raise + if kind == "size": + prefer_default_route = True + delay = DOWNLOAD_INTEGRITY_RETRY_BACKOFF * 2 ** (sum(failures.values()) - 1) + logger.warning( + "%s validation failed for %s, retrying in %.0fs (%d/%d): %s", + kind.capitalize(), + file_path, + delay, + failures[kind], + limits[kind], + exc.message, + ) + _control.sleep(delay) return target @@ -665,6 +1049,7 @@ def download_repo( local_files_only: bool = False, user_agent: dict | str | None = None, progress_callbacks: list[type[ProgressCallback]] | None = None, + file_list: list[dict] | None = None, ) -> Path: """Download an entire repository (snapshot download). @@ -697,6 +1082,13 @@ def download_repo( ------- Path Absolute path to the snapshot/local directory. + + Raises + ------ + FileDownloadError + After every file was attempted, when any of them failed. Completed + files stay in place; ``failed_files`` maps each failed path to its + error. """ if local_files_only: return self._resolve_cached_snapshot( @@ -732,18 +1124,42 @@ def download_repo( output_dir=output_dir, ) - if repo_type in ("dataset", "datasets"): - files = self._client.list_dataset_files_paginated( - repo_id=repo_id, - revision=effective_revision, - ) + cancel_event = threading.Event() + if file_list is not None: + files = file_list else: - files = self._client.list_repo_files( - repo_id=repo_id, - repo_type=repo_type, - revision=effective_revision, - recursive=True, - ) + files = [] + with _DownloadControl( + total_timeout=REPO_TREE_LIST_TIMEOUT_SECONDS, + cancel_event=cancel_event, + ) as listing_control: + with self._client.read_context(listing_control): + for listing_root in _listing_roots(allow_patterns): + listing_control.check() + try: + if repo_type in ("dataset", "datasets"): + files.extend( + self._client.list_dataset_files_paginated( + repo_id=repo_id, + revision=effective_revision, + root_path=listing_root or "/", + ) + ) + else: + files.extend( + self._client.list_repo_files( + repo_id=repo_id, + repo_type=repo_type, + revision=effective_revision, + recursive=True, + root=listing_root, + ) + ) + except NotExistError: + # A missing literal directory has no matches; a missing repository must still raise. + if listing_root is None: + raise + self._client.get_repo_info(repo_id, repo_type) download_items: list[tuple[str, str | None, int | None]] = [] for f in files: @@ -758,8 +1174,8 @@ def download_repo( if ignore_patterns and _matches_patterns(path, ignore_patterns): continue sha256 = f.get("Sha256") or f.get("sha256") or None - raw_size = f.get("Size") or f.get("size") - size = int(raw_size) if raw_size else None + raw_size = f.get("Size", f.get("size")) + size = int(raw_size) if raw_size is not None else None download_items.append((path, sha256, size)) if not download_items: @@ -768,38 +1184,68 @@ def download_repo( logger.info("Downloading %d files from %s@%s", len(download_items), repo_id, effective_revision) - errors: list[str] = [] - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit( - self.download_file, - repo_id=repo_id, - repo_type=repo_type, - file_path=fp, - revision=effective_revision, - cache_dir=cache_dir, - local_dir=local_dir, - expected_sha256=sha256, - file_size=size, - user_agent=user_agent, - progress_callbacks=progress_callbacks, - ): fp - for fp, sha256, size in download_items - } + failures: dict[str, BaseException] = {} + executor = ThreadPoolExecutor(max_workers=max_workers) + items = iter(download_items) + futures = {} + + def submit_next() -> None: + item = next(items, None) + if item is None: + return + fp, sha256, size = item + future = executor.submit( + self.download_file, + repo_id=repo_id, + repo_type=repo_type, + file_path=fp, + revision=effective_revision, + cache_dir=cache_dir, + local_dir=local_dir, + expected_sha256=sha256, + file_size=size, + user_agent=user_agent, + progress_callbacks=progress_callbacks, + _cancel_event=cancel_event, + ) + futures[future] = fp + try: + # Bound pending tasks instead of creating millions of Futures; stop scheduling after cancellation. + for _ in range(max_workers * 2): + submit_next() with tqdm(total=len(download_items), desc="Downloading", unit="file") as pbar: - for future in as_completed(futures): - fp = futures[future] - try: - future.result() - except Exception as exc: - errors.append(f"{fp}: {exc}") - logger.error("Failed to download %s: %s", fp, exc) - finally: - pbar.update(1) - - if errors: - logger.warning("%d file(s) failed to download", len(errors)) + while futures: + completed, _ = wait(futures, return_when=FIRST_COMPLETED) + for future in completed: + fp = futures.pop(future) + try: + future.result() + except Exception as exc: + failures[fp] = exc + logger.error("Failed to download %s: %s", fp, exc) + finally: + pbar.update(1) + submit_next() + except BaseException: + cancel_event.set() + for future in futures: + future.cancel() + raise + finally: + executor.shutdown(wait=True, cancel_futures=True) + + if failures: + # Returning normally would let callers treat a partial snapshot as + # complete. Every file was attempted first, so a re-run only fetches + # the failed ones. + raise FileDownloadError( + f"{len(failures)} of {len(download_items)} file(s) failed to download from " + f"{repo_id}@{effective_revision}", + failed_files=failures, + total=len(download_items), + output_dir=output_dir, + ) return output_dir @@ -824,17 +1270,36 @@ def _download_archive( tmp_path: Path | None = None try: - resp = self._client.download_archive( - repo_id=repo_id, - repo_type=repo_type, - revision=revision, - ) - - with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tmp: - tmp_path = Path(tmp.name) - for chunk in resp.iter_content(chunk_size=DOWNLOAD_CHUNK_SIZE): - if chunk: - tmp.write(chunk) + with ( + _DownloadControl( + total_timeout=DOWNLOAD_TOTAL_TIMEOUT, + min_rate=DOWNLOAD_MIN_BYTES_PER_SECOND, + speed_window=DOWNLOAD_LOW_SPEED_WINDOW_SECONDS, + ) as control, + self._client.read_context(control), + ): + retry = Retry(total=DOWNLOAD_RETRY_TIMES, backoff_factor=1, allowed_methods=["GET"]) + while True: + try: + resp = self._client.download_archive( + repo_id=repo_id, + repo_type=repo_type, + revision=revision, + ) + with control.response(resp), tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tmp: + tmp_path = Path(tmp.name) + for chunk in control.chunks(resp, DOWNLOAD_CHUNK_SIZE): + tmp.write(chunk) + announced = _checkable_content_length(resp) + if announced is not None and tmp_path.stat().st_size != announced: + raise _IncompleteBodyError("归档内容长度与响应头不一致") + break + except (_DownloadCancelledError, _DownloadDeadlineError): + raise + except Exception as exc: + if tmp_path is not None: + tmp_path.unlink(missing_ok=True) + retry = _wait_download_retry(retry, exc, repo_id, control) with zipfile.ZipFile(tmp_path, "r") as zf: zf.extractall(output_dir) @@ -857,12 +1322,29 @@ def _download_archive( # Internal helpers # ------------------------------------------------------------------ @staticmethod - def _cache_hit(target: Path, expected_sha256: str | None) -> bool: - """Return True if *target* exists and passes optional hash check.""" + def _cache_hit( + target: Path, + expected_sha256: str | None, + expected_size: int | None = None, + *, + control: _DownloadControl | None = None, + ) -> bool: + """Return True if *target* exists and passes the optional size and hash checks.""" if not target.exists(): return False + if expected_size is not None and target.stat().st_size != expected_size: + logger.debug("Cache stale (size mismatch): %s", target) + return False if expected_sha256: - actual = compute_hash(target, "sha256") + if control is None: + actual = compute_hash(target, "sha256") + else: + digest = hashlib.sha256() + with target.open("rb") as cached: + while block := cached.read(8 * DOWNLOAD_CHUNK_SIZE): + control.check() + digest.update(block) + actual = digest.hexdigest() if actual != expected_sha256: logger.debug("Cache stale (hash mismatch): %s", target) return False @@ -871,6 +1353,13 @@ def _cache_hit(target: Path, expected_sha256: str | None) -> bool: logger.debug("Cache hit: %s", target) return True + @staticmethod + def _discard_download(target: Path, *, keep_target: bool = False) -> None: + """Remove a rejected download so neither it nor its partial file looks reusable.""" + _incomplete_path(target).unlink(missing_ok=True) + if not keep_target: + target.unlink(missing_ok=True) + @staticmethod def _dir_has_entries(path: Path) -> bool: """Return True when a directory exists and contains at least one entry.""" @@ -1149,11 +1638,23 @@ def _download_with_resume( file_size: int | None = None, user_agent: dict | str | None = None, progress_callbacks: list[type[ProgressCallback]] | None = None, + prefer_default_route: bool = False, + control: _DownloadControl, + expected_sha256: str | None = None, ) -> Path: - """Download a file with HTTP Range resume support and retry.""" + """Download a file with HTTP Range resume support and retry. + + Before the partial file is moved into place, the bytes received are + checked against the announced ``Content-Length`` and, when *file_size* + is known, against the repository listing. A short body is resumed; a + body whose size contradicts the listing raises + :class:`~.errors.FileSizeMismatchError` and leaves no partial file. + *prefer_default_route* skips intra-cloud and peer-region routing so a + retry can avoid a replica that served bad content. + """ use_parallel = file_size is not None and file_size > DOWNLOAD_PARALLEL_THRESHOLD and DOWNLOAD_PARALLELS > 1 - download_headers = self._build_download_headers(user_agent) + download_headers = self._build_download_headers(user_agent, accelerate=not prefer_default_route) # Inter-region acceleration: first discover the file's storage route, # then cache only a confirmed internal resolution for that exact route. @@ -1161,7 +1662,7 @@ def _download_with_resume( # OSS; a metadata miss must therefore never suppress an LFS probe. # Progress bar prefix: "⚡ " = local OSS, "⇄ " = peer OSS, " " = CDN, "" = not configured source_prefix = "" - peer_regions = self._get_inter_cloud_regions() + peer_regions = [] if prefer_default_route else self._get_inter_cloud_regions() if peer_regions: source = "default" cached: tuple[str | None, str] | None = None @@ -1169,7 +1670,7 @@ def _download_with_resume( try: probe_url = self._client.get_download_url(repo_id, repo_type, file_path, revision) cookies = {"m_session_id": self._client.token} if self._client.token else None - initial_redirect_url = self._probe_redirect_url(probe_url, download_headers, cookies) + initial_redirect_url = self._probe_redirect_url(probe_url, download_headers, cookies, control=control) route_key = self._storage_route_key(initial_redirect_url) if route_key is not None: @@ -1188,7 +1689,8 @@ def _download_with_resume( if not is_probe_owner: assert probe_event is not None - probe_event.wait() + while not probe_event.wait(0.1): + control.check() continue try: @@ -1198,6 +1700,7 @@ def _download_with_resume( cookies, peer_regions=peer_regions, initial_redirect_url=initial_redirect_url, + control=control, ) resolved_region = download_headers.get("x-aliyun-region-id") if source != "default": @@ -1220,6 +1723,8 @@ def _download_with_resume( resolved_region, source = cached if resolved_region is not None: download_headers["x-aliyun-region-id"] = resolved_region + except (_DownloadCancelledError, _DownloadDeadlineError): + raise except Exception as exc: logger.warning( "Failed to resolve inter-region acceleration: %s. Falling back to default.", @@ -1233,6 +1738,17 @@ def _download_with_resume( "default": " ", }[source] + def request(headers: dict[str, str]) -> requests.Response: + control.check() + with self._client.read_context(control): + return self._client.download_stream( + repo_id, + repo_type, + file_path, + revision, + headers=headers, + ) + if use_parallel: assert file_size is not None # guaranteed by use_parallel above url = self._client.get_download_url( @@ -1249,95 +1765,206 @@ def _download_with_resume( if progress_callbacks: cb_instances = [cls(file_path, file_size) for cls in progress_callbacks] - _parallel_download( - url=url, - target=target, - file_size=file_size, - headers=download_headers, - cookies=cookies, - progress_callbacks=cb_instances, - ) - return target + try: + _parallel_download( + url=url, + target=target, + file_size=file_size, + headers=download_headers, + cookies=cookies, + progress_callbacks=cb_instances, + request=request, + control=control, + expected_sha256=expected_sha256, + ) + return target + except _RangeNotHonouredError as exc: + logger.warning("Ranged requests are not honoured for %s (%s); using a single stream", file_path, exc) # Single-stream download with retry cb_classes = list(progress_callbacks or []) cb_instances = [cls(file_path, file_size or 0) for cls in cb_classes] + reported = 0 + # The listing reports 0 both for empty files and for unknown sizes. + expected_size = file_size retry = Retry( total=DOWNLOAD_RETRY_TIMES, backoff_factor=1, allowed_methods=["GET"], ) - tmp_path = target.with_suffix(target.suffix + ".incomplete") + tmp_path = _incomplete_path(target) while True: try: - existing_size = 0 - if tmp_path.exists(): - existing_size = tmp_path.stat().st_size + control.check() + existing_size = tmp_path.stat().st_size if tmp_path.exists() else 0 + if expected_size is not None and existing_size > expected_size: + # Longer than the file itself, so it cannot be a prefix of it. + tmp_path.unlink() + existing_size = 0 + if existing_size > reported: for cb in cb_instances: - cb.update(existing_size) + cb.update(existing_size - reported) + reported = existing_size + if tmp_path.exists() and expected_size is not None and existing_size == expected_size: + # An earlier run received every byte but stopped before the + # rename. Resuming would only earn a 416 from the CDN; the + # caller's hash check decides whether the bytes are right. + _finish_download(tmp_path, target, expected_sha256, control) + for cb in cb_instances: + cb.end() + return target + request_id = uuid.uuid4().hex extra_headers: dict[str, str] = copy.deepcopy(download_headers) - extra_headers["X-Request-ID"] = uuid.uuid4().hex + extra_headers["X-Request-ID"] = request_id if existing_size > 0: extra_headers["Range"] = f"bytes={existing_size}-" logger.debug("Resuming download from byte %d", existing_size) try: - resp = self._client.download_stream( - repo_id=repo_id, - repo_type=repo_type, - file_path=file_path, - revision=revision, - headers=extra_headers, - ) + resp = request(extra_headers) except requests.Timeout as exc: raise RequestTimeoutError(f"Download timed out for {file_path}: {exc}") from exc except requests.ConnectionError as exc: raise NetworkError(f"Download connection failed for {file_path}: {exc}") from exc except requests.RequestException as exc: raise NetworkError(f"Download failed for {file_path}: {exc}") from exc + except APIError as exc: + if exc.status_code == 416 and existing_size > 0: + # The object does not extend past the partial file, so the + # partial file is stale. Start over. + tmp_path.unlink(missing_ok=True) + raise _IncompleteBodyError( + f"server refused to resume from byte {existing_size} (HTTP 416)" + ) from exc + raise + + with control.response(resp): + # The LFS CDN returns the object SHA-256, enabling validation without caller metadata. + linked_hash = resp.headers.get("X-Linked-ETag", "").strip('"') + if expected_sha256 is None and re.fullmatch(r"[0-9a-fA-F]{64}", linked_hash): + expected_sha256 = linked_hash.lower() + content_length = _checkable_content_length(resp) + content_range = _parse_content_range(resp.headers.get("Content-Range")) + if existing_size > 0 and (resp.status_code == 206 or content_range is not None): + if content_range is None or content_range[0] != existing_size: + tmp_path.unlink(missing_ok=True) + raise _IncompleteBodyError( + f"resume from byte {existing_size} was answered with " + f"{_describe_response(resp, request_id)}; restarting" + ) + mode, offset = "ab", existing_size + else: + # Anything but a matching 206 carries the whole file: files + # served without a redirect ignore Range entirely. + mode, offset = "wb", 0 + if reported: + for cb in cb_instances: + cb.update(-reported) + reported = 0 + + if resp.status_code == 206 and (content_range is None or content_range[0] != offset): + tmp_path.unlink(missing_ok=True) + raise _IncompleteBodyError("服务器返回了不匹配请求起点的分段响应") + announced: int | None = None + if content_range is not None and content_range[2] is not None: + announced = content_range[2] + elif content_length is not None: + announced = offset + content_length + if expected_size is not None and announced is not None and announced != expected_size: + tmp_path.unlink(missing_ok=True) + raise FileSizeMismatchError( + f"Server announced {announced} bytes for {file_path} but the repository lists " + f"{expected_size} bytes ({_describe_response(resp, request_id)}); the remote object " + "is empty, truncated or out of date" + ) - content_length = resp.headers.get("Content-Length") - total_size = int(content_length) if content_length else None - is_resumed = resp.status_code == 206 - - if is_resumed and total_size: - total_size += existing_size - - mode = "ab" if is_resumed else "wb" - if not is_resumed: - existing_size = 0 + received = 0 + digest = hashlib.sha256() if expected_sha256 else None + if digest is not None and offset: + with tmp_path.open("rb") as prefix: + while block := prefix.read(8 * DOWNLOAD_CHUNK_SIZE): + control.check() + digest.update(block) + with tqdm( + total=announced, + initial=offset, + unit="B", + unit_scale=True, + desc=f"{source_prefix}{Path(file_path).name}", + leave=False, + ) as pbar: + with open(tmp_path, mode) as fh: + for chunk in control.chunks(resp, DOWNLOAD_CHUNK_SIZE): + if chunk: + if expected_size is not None and offset + received + len(chunk) > expected_size: + raise FileSizeMismatchError( + f"Server exceeded {expected_size} bytes for {file_path} " + f"({_describe_response(resp, request_id)})" + ) + fh.write(chunk) + if digest is not None: + digest.update(chunk) + received += len(chunk) + reported += len(chunk) + pbar.update(len(chunk)) + for cb in cb_instances: + cb.update(len(chunk)) + + if content_length is not None and received < content_length: + # urllib3 2.x raises on a short read, but 1.x ends the body + # silently; keep the partial file and resume from it. + raise _IncompleteBodyError( + f"connection closed after {received} of {content_length} bytes " + f"({_describe_response(resp, request_id)})" + ) + if expected_size is not None and offset + received != expected_size: + tmp_path.unlink(missing_ok=True) + raise FileSizeMismatchError( + f"Server returned {offset + received} of {expected_size} bytes for {file_path} " + f"({_describe_response(resp, request_id)}); the remote object is empty, truncated " + "or out of date" + ) - with tqdm( - total=total_size, - initial=existing_size, - unit="B", - unit_scale=True, - desc=f"{source_prefix}{Path(file_path).name}", - leave=False, - ) as pbar: - with open(tmp_path, mode) as fh: - for chunk in resp.iter_content(chunk_size=DOWNLOAD_CHUNK_SIZE): - if chunk: - fh.write(chunk) - pbar.update(len(chunk)) - for cb in cb_instances: - cb.update(len(chunk)) - - tmp_path.replace(target) + _finish_download( + tmp_path, + target, + expected_sha256, + control, + actual_hash=digest.hexdigest() if digest is not None else None, + ) for cb in cb_instances: cb.end() logger.debug("Downloaded: %s", target) return target - except (NotExistError, PermissionDeniedError): + except ( + NotExistError, + PermissionDeniedError, + AuthenticationError, + FileIntegrityError, + _DownloadCancelledError, + _DownloadDeadlineError, + ): + for cb in cb_instances: + cb.end() raise except Exception as exc: - retry = retry.increment("GET", file_path, error=exc) - logger.warning("Download failed for %s: %s, will retry", file_path, exc) - retry.sleep() + if not prefer_default_route: + # Fall back to the public route for this file only, without changing other files' cached routes. + download_headers.pop("x-aliyun-region-id", None) + try: + retry = _wait_download_retry(retry, exc, file_path, control) + except BaseException: + for cb in cb_instances: + cb.end() + raise + except BaseException: + for cb in cb_instances: + cb.end() + raise def verify_file(self, file_path: Path, expected_sha256: str) -> bool: """Verify a downloaded file's SHA256 hash. diff --git a/src/modelscope_hub/_download_control.py b/src/modelscope_hub/_download_control.py new file mode 100644 index 0000000..f98b120 --- /dev/null +++ b/src/modelscope_hub/_download_control.py @@ -0,0 +1,280 @@ +"""Download cancellation, deadlines, and stream monitoring without changing uploads.""" + +from __future__ import annotations + +import contextlib +import socket +import threading +import time +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Any + +import requests +from requests.adapters import HTTPAdapter +from urllib3.exceptions import HTTPError as Urllib3HTTPError + +from .errors import NetworkError, RequestTimeoutError + + +class _DownloadCancelledError(Exception): + """The parent download was cancelled; do not retry or report a remote failure.""" + + +class _DownloadDeadlineError(RequestTimeoutError): + """The total download time budget has been exhausted.""" + + retryable = False + + +class _DownloadTooSlowError(NetworkError): + """Download progress stayed below the minimum rate for a full observation window.""" + + +@dataclass +class _WatchedResponse: + sock: Any + started: float + received: int = 0 + error: Exception | None = None + monitor_speed: bool = True + + +def _response_socket(response: requests.Response) -> Any: + # urllib3 may release connection.sock while HTTPResponse still retains + # SocketIO. Only shut down interrupted reads; leave socket timeouts and data unchanged. + raw = response.raw + connection = getattr(raw, "_connection", None) + sock = getattr(connection, "sock", None) + if sock is None: + fp = getattr(getattr(raw, "_fp", None), "fp", None) + sock = getattr(getattr(fp, "raw", None), "_sock", None) + return sock + + +class _DownloadControl: + """A per-call budget whose workers share a parent cancellation event, not global state.""" + + def __init__( + self, + *, + total_timeout: float, + cancel_event: threading.Event | None = None, + min_rate: int = 0, + speed_window: float = 60, + ) -> None: + self.cancel_event = cancel_event if cancel_event is not None else threading.Event() + self.deadline = time.monotonic() + total_timeout + self.min_rate = min_rate + self.speed_window = speed_window + self._responses: dict[requests.Response, _WatchedResponse] = {} + self._transports: dict[Any, _WatchedResponse] = {} + self._lock = threading.Lock() + self._finished = threading.Event() + self._monitor: threading.Thread | None = None + + def __enter__(self) -> _DownloadControl: + return self + + def __exit__(self, *args: Any) -> None: + self._finished.set() + if self._monitor is not None: + self._monitor.join(timeout=1) + with self._lock: + responses = list(self._responses) + self._responses.clear() + for response in responses: + response.close() + + def check(self) -> None: + if self.cancel_event.is_set(): + raise _DownloadCancelledError("下载已取消") + if time.monotonic() >= self.deadline: + raise _DownloadDeadlineError("下载总时限已耗尽；已保存可继续下载的临时文件") + + def timeouts(self, connect: float, read: float) -> tuple[float, float]: + self.check() + remaining = max(0.001, self.deadline - time.monotonic()) + return min(connect, remaining), min(read, remaining) + + def sleep(self, seconds: float) -> None: + self.check() + self.cancel_event.wait(min(max(0, seconds), max(0, self.deadline - time.monotonic()))) + self.check() + + def _start_monitor(self) -> None: + if self._monitor is None: + self._monitor = threading.Thread(target=self._watch, name="download-watchdog", daemon=True) + self._monitor.start() + + def track(self, response: requests.Response) -> None: + self.check() + with self._lock: + self._responses[response] = _WatchedResponse(_response_socket(response), time.monotonic()) + self._start_monitor() + + @contextlib.contextmanager + def transport(self, sock: Any) -> Iterator[None]: + """Interrupt sockets while reading headers so trickling headers cannot bypass deadlines.""" + self.check() + state = _WatchedResponse(sock, time.monotonic(), monitor_speed=False) + with self._lock: + self._transports[sock] = state + self._start_monitor() + try: + yield + finally: + with self._lock: + self._transports.pop(sock, None) + self.check() + if state.error is not None: + raise state.error + + def release(self, response: requests.Response) -> None: + with self._lock: + self._responses.pop(response, None) + + @contextlib.contextmanager + def response(self, response: requests.Response) -> Iterator[requests.Response]: + try: + yield response + finally: + self.release(response) + response.close() + + def _check_response(self, response: requests.Response, received: int = 0) -> None: + self.check() + with self._lock: + state = self._responses.get(response) + if state is not None: + state.received += received + if state.error is not None: + raise state.error + + def _watch(self) -> None: + # A separate watchdog interrupts reads that keep receiving tiny chunks, + # including with urllib3 1.26. Do not call response.close() here: the + # reader may hold the buffered stream lock. + while not self._finished.wait(0.1): + now = time.monotonic() + with self._lock: + for state in [*self._responses.values(), *self._transports.values()]: + if state.error is not None: + continue + if self.cancel_event.is_set(): + state.error = _DownloadCancelledError("下载已取消") + elif now >= self.deadline: + state.error = _DownloadDeadlineError("下载总时限已耗尽") + elif state.monitor_speed and self.min_rate and now - state.started >= self.speed_window: + rate = state.received / (now - state.started) + if rate < self.min_rate: + state.error = _DownloadTooSlowError( + f"下载速率 {rate:.0f} B/s 低于 {self.min_rate} B/s，观察窗口 {self.speed_window:g}s" + ) + else: + state.started, state.received = now, 0 + if state.error is not None and state.sock is not None: + with contextlib.suppress(OSError): + state.sock.shutdown(socket.SHUT_RDWR) + + def abort_responses(self) -> None: + """Abort active responses without cancelling the parent, allowing single-stream fallback.""" + with self._lock: + for state in [*self._responses.values(), *self._transports.values()]: + state.error = _DownloadCancelledError("分片组已取消") + if state.sock is not None: + with contextlib.suppress(OSError): + state.sock.shutdown(socket.SHUT_RDWR) + + def chunks(self, response: requests.Response, chunk_size: int) -> Iterator[bytes]: + """Yield received bytes promptly instead of waiting for a full chunk to report progress.""" + raw = response.raw + try: + if hasattr(raw, "stream"): + read1 = getattr(raw, "read1", None) + if read1 is None and not response.headers.get("Content-Encoding"): + read1 = getattr(getattr(raw, "_fp", None), "read1", None) + decoded = False + else: + decoded = True + while True: + self._check_response(response) + if read1 is not None: + data = read1(chunk_size, decode_content=True) if decoded else read1(chunk_size) + else: + data = raw.read(chunk_size, decode_content=True) + self._check_response(response, len(data)) + if not data: + break + yield data + else: + for chunk in response.iter_content(chunk_size=chunk_size): + self._check_response(response, len(chunk)) + if chunk: + yield chunk + except (Urllib3HTTPError, OSError) as exc: + self._check_response(response) + raise requests.ConnectionError(f"下载流读取失败：{type(exc).__name__}") from exc + finally: + self.release(response) + + +_network_state = threading.local() + + +@contextlib.contextmanager +def _network_context(control: _DownloadControl) -> Iterator[None]: + previous = getattr(_network_state, "control", None) + _network_state.control = control + try: + yield + finally: + _network_state.control = previous + + +class _WatchedConnectionMixin: + def _watched(self, method: str) -> Any: + original = getattr(super(), method) + control = getattr(_network_state, "control", None) + if control is None: + return original() + with control.transport(getattr(self, "sock", None)): + return original() + + def getresponse(self) -> Any: + return self._watched("getresponse") + + def _tunnel(self) -> Any: + # HTTP CONNECT tunnel headers can also arrive as an endless trickle. + return self._watched("_tunnel") + + +class _ReadHTTPAdapter(HTTPAdapter): + """Download/listing pools with instance-local connection classes, leaving urllib3 globals intact.""" + + @staticmethod + def _configure(manager: Any) -> None: + classes = dict(manager.pool_classes_by_scheme) + for scheme, pool in classes.items(): + if getattr(pool, "_download_controlled", False): + continue + connection = getattr(pool, "ConnectionCls", None) + if connection is None: + continue + # Preserve the original TLS, HTTP proxy, and SOCKS connection classes and arguments. + watched = type(f"_Download{connection.__name__}", (_WatchedConnectionMixin, connection), {}) + classes[scheme] = type( + f"_Download{pool.__name__}", + (pool,), + {"ConnectionCls": watched, "_download_controlled": True}, + ) + manager.pool_classes_by_scheme = classes + + def init_poolmanager(self, connections: int, maxsize: int, block: bool = False, **kwargs: Any) -> None: + super().init_poolmanager(connections, maxsize, block=block, **kwargs) + self._configure(self.poolmanager) + + def proxy_manager_for(self, proxy: str, **kwargs: Any) -> Any: + manager = super().proxy_manager_for(proxy, **kwargs) + self._configure(manager) + return manager diff --git a/src/modelscope_hub/_legacy_api.py b/src/modelscope_hub/_legacy_api.py index 211614d..b7844b5 100644 --- a/src/modelscope_hub/_legacy_api.py +++ b/src/modelscope_hub/_legacy_api.py @@ -14,25 +14,35 @@ from __future__ import annotations +import contextlib +import threading import time import uuid +from collections.abc import Iterator from concurrent.futures import ThreadPoolExecutor, as_completed from typing import IO, Any, BinaryIO from urllib.parse import quote_plus, urlparse import requests from requests.adapters import HTTPAdapter, Retry +from requests.structures import CaseInsensitiveDict +from urllib3.exceptions import InvalidHeader +from ._download_control import _DownloadControl, _DownloadDeadlineError, _network_context, _ReadHTTPAdapter +from .config import get_default_config from .constants import ( API_CONNECT_TIMEOUT, API_CONNECTION_POOL_MAXSIZE, API_MAX_RETRIES, API_TIMEOUT, + DOWNLOAD_CHUNK_SIZE, + DOWNLOAD_TIMEOUT, LEGACY_API_PREFIX, REPO_FILES_TRUNCATION_LIMIT, REPO_TREE_MAX_REQUESTS, REPO_TREE_PAGE_MAX_ATTEMPTS, REPO_TREE_PAGE_RETRY_MAX_DELAY_SECONDS, + REPO_TREE_PAGE_TIMEOUT_SECONDS, REPO_TREE_WALK_WORKERS, UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS, UPLOAD_BLOB_READ_TIMEOUT_SECONDS, @@ -44,10 +54,12 @@ InvalidParameter, NetworkError, PermissionDeniedError, + RateLimitError, RequestTimeoutError, ServerError, raise_for_status, ) +from .utils import append_user_agent, build_user_agent from .utils.logger import get_logger logger = get_logger("legacy_api") @@ -118,8 +130,12 @@ def __init__( self._tree_reads_ok = False self._session = requests.Session() - if user_agent: - self._session.headers["User-Agent"] = user_agent + self._user_agent = ( + user_agent + if user_agent and user_agent.startswith("modelscope_hub/") + else build_user_agent(config=get_default_config(), extra=user_agent) + ) + self._session.headers["User-Agent"] = self._user_agent retry = Retry( total=max_retries, backoff_factor=0.5, @@ -133,6 +149,27 @@ def __init__( ) self._session.mount("https://", adapter) self._session.mount("http://", adapter) + # Callers manage download/listing retries; this pool leaves upload and regular API policies unchanged. + self._read_session = requests.Session() + self._read_session.headers.update(self._session.headers) + read_adapter = _ReadHTTPAdapter( + max_retries=0, + pool_connections=API_CONNECTION_POOL_MAXSIZE, + pool_maxsize=API_CONNECTION_POOL_MAXSIZE, + ) + self._read_session.mount("https://", read_adapter) + self._read_session.mount("http://", read_adapter) + self._read_state = threading.local() + + @contextlib.contextmanager + def read_context(self, control: _DownloadControl) -> Iterator[None]: + previous = getattr(self._read_state, "control", None) + self._read_state.control = control + try: + with _network_context(control): + yield + finally: + self._read_state.control = previous # ------------------------------------------------------------------ # Properties @@ -150,6 +187,7 @@ def token(self, value: str | None) -> None: self._token = value self._session_authenticated = False self._session.cookies.clear() + self._read_session.cookies.clear() # ------------------------------------------------------------------ # Transport @@ -165,9 +203,11 @@ def _headers(self, extra: dict[str, str] | None = None) -> dict[str, str]: } if self._token: headers["Authorization"] = f"Bearer {self._token}" + merged = CaseInsensitiveDict(headers) if extra: - headers.update(extra) - return headers + merged.update(extra) + merged["User-Agent"] = append_user_agent(self._user_agent, merged.get("User-Agent")) + return dict(merged) def _ensure_session_auth(self) -> None: """Set the ``m_session_id`` cookie on the session for legacy API auth. @@ -181,6 +221,7 @@ def _ensure_session_auth(self) -> None: return domain = urlparse(self._endpoint).hostname or "" self._session.cookies.set("m_session_id", self._token, domain=domain, path="/") + self._read_session.cookies.set("m_session_id", self._token, domain=domain, path="/") self._session_authenticated = True logger.debug("Legacy session cookie set for domain=%s", domain) @@ -194,8 +235,9 @@ def _request( data: Any | None = None, files: dict[str, Any] | None = None, headers: dict[str, str] | None = None, - timeout: int | None = None, + timeout: int | tuple[float, float] | None = None, stream: bool = False, + single_attempt: bool = False, ) -> requests.Response: """Unified request dispatcher with auth, retry, and error handling.""" self._ensure_session_auth() @@ -207,8 +249,32 @@ def _request( merged_headers.pop("Content-Type", None) logger.debug("%s %s", method, url) + control = getattr(self._read_state, "control", None) + if single_attempt and control is None: + # Bound both headers and body reads for single-page calls without an external controller. + with _DownloadControl(total_timeout=REPO_TREE_PAGE_TIMEOUT_SECONDS) as page_control: + with self.read_context(page_control): + return self._request( + method, + path, + params=params, + json_body=json_body, + data=data, + files=files, + headers=headers, + timeout=timeout, + stream=stream, + single_attempt=True, + ) + session = self._read_session if control is not None or single_attempt else self._session + request_timeout = timeout or self._timeout + if control is not None: + connect, read = ( + (API_CONNECT_TIMEOUT, request_timeout) if isinstance(request_timeout, int) else request_timeout + ) + request_timeout = control.timeouts(connect, read) try: - resp = self._session.request( + resp = session.request( method=method, url=url, params=params, @@ -216,8 +282,8 @@ def _request( data=data, files=files, headers=merged_headers, - timeout=timeout or self._timeout, - stream=stream, + timeout=request_timeout, + stream=stream or control is not None, ) except requests.exceptions.RetryError as exc: raise ServerError(f"Max retries exceeded: {exc}") from exc @@ -227,7 +293,7 @@ def _request( raise RequestTimeoutError(f"Request timed out: {exc}") from exc logger.debug("%s %s -> %s", method, url, resp.status_code) - if resp.status_code >= 400: + if resp.status_code >= 400 and control is None: logger.debug( "Request failed: %s %s params=%s status=%s body=%s", method, @@ -236,6 +302,38 @@ def _request( resp.status_code, resp.text[:500] if resp.text else "", ) + if control is not None: + try: + control.track(resp) + if not stream or resp.status_code >= 400: + # Bound error-body reads too, avoiding unbounded reads inside raise_for_status. + body = bytearray() + for chunk in control.chunks(resp, DOWNLOAD_CHUNK_SIZE): + body.extend(chunk) + if resp.status_code >= 400 and len(body) >= 65536: + break + if resp.status_code >= 400 and len(body) >= 65536: + resp.raw.close() + resp._content = bytes(body) + resp._content_consumed = True + control.release(resp) + resp.close() + try: + raise_for_status(resp) + except APIError as exc: + exc.response = resp # type: ignore[attr-defined] + retry_after = resp.headers.get("Retry-After") + if retry_after: + try: + exc.retry_after = Retry().parse_retry_after(retry_after) # type: ignore[attr-defined] + except (ValueError, InvalidHeader): + pass + raise + return resp + except BaseException: + control.release(resp) + resp.close() + raise raise_for_status(resp) return resp @@ -488,22 +586,37 @@ def fetch_level(roots: list[str | None], *, recursive: bool) -> dict[str | None, budget_spent = True requests_made += len(roots) + control = getattr(self._read_state, "control", None) + def one(subroot: str | None) -> list[dict]: - return self._list_files_page( - repo_id, - repo_type, - revision, - recursive=recursive, - root=subroot, - ) + scope = self.read_context(control) if control is not None else contextlib.nullcontext() + with scope: + if control is not None: + control.check() + return self._list_files_page( + repo_id, + repo_type, + revision, + recursive=recursive, + root=subroot, + ) if len(roots) == 1: listings = {roots[0]: one(roots[0])} else: workers = min(REPO_TREE_WALK_WORKERS, len(roots)) - with ThreadPoolExecutor(max_workers=workers) as pool: - futures = {pool.submit(one, subroot): subroot for subroot in roots} + pool = ThreadPoolExecutor(max_workers=workers) + futures = {pool.submit(one, subroot): subroot for subroot in roots} + try: listings = {futures[future]: future.result() for future in as_completed(futures)} + except BaseException: + if control is not None: + control.cancel_event.set() + for future in futures: + future.cancel() + raise + finally: + pool.shutdown(wait=True, cancel_futures=True) if requests_made >= progress_mark: logger.info( @@ -593,7 +706,7 @@ def list_dataset_files_paginated( ``GET /api/v1/datasets/{repo_id}/repo/tree`` with ``PageNumber``/``PageSize`` params. - A page occasionally answers ``403 无权访问该数据集`` on a repository the + A page occasionally answers ``403 access denied`` on a repository the caller demonstrably can read. Once any page has succeeded the credential is proven, so a later-page denial is a server-side hiccup rather than an authorization result, and it is retried instead of discarding every page @@ -602,6 +715,7 @@ def list_dataset_files_paginated( """ all_files: list[dict] = [] page_number = 1 + logger.info("Listing %s@%s Root=%s ...", repo_id, revision, root_path) while True: params: dict[str, Any] = { "Revision": revision, @@ -626,11 +740,37 @@ def list_dataset_files_paginated( files = [] all_files.extend(files) + if page_number % 10 == 0 or len(files) < page_size: + logger.info("Listing %s Root=%s: page %d, %d entries", repo_id, root_path, page_number, len(all_files)) if len(files) < page_size: break page_number += 1 return all_files + def _fetch_tree_page(self, path: str, params: dict[str, Any]) -> requests.Response: + parent = getattr(self._read_state, "control", None) + budget = float(REPO_TREE_PAGE_TIMEOUT_SECONDS) + if parent is not None: + parent.check() + budget = min(budget, max(0.001, parent.deadline - time.monotonic())) + try: + with _DownloadControl( + total_timeout=budget, + cancel_event=parent.cancel_event if parent is not None else None, + ) as page_control: + with self.read_context(page_control): + return self._request( + "GET", + path, + params=params, + timeout=(API_CONNECT_TIMEOUT, REPO_TREE_PAGE_TIMEOUT_SECONDS), + single_attempt=True, + ) + except _DownloadDeadlineError as exc: + if parent is not None: + parent.check() + raise RequestTimeoutError(f"列表单页请求超过 {REPO_TREE_PAGE_TIMEOUT_SECONDS}s") from exc + def _request_repo_tree( self, path: str, @@ -641,8 +781,8 @@ def _request_repo_tree( ) -> Any: """Fetch one file-tree listing, retrying a spurious denial. - The server intermittently answers a tree request with ``403 无权访问该数据 - 集`` on a repository the caller has just read successfully -- observed on + The server intermittently answers a tree request with ``403 access + denied`` on a repository the caller has just read successfully -- observed on both ``PageNumber``-paginated and ``Root``-scoped listings. Enumerating a large repository takes hundreds of such requests, so at that scale a single-request failure rate is effectively a guaranteed whole-listing @@ -653,31 +793,48 @@ def _request_repo_tree( Until then a 403 is taken at face value, keeping a real permission error fast and honest. """ - last_error: PermissionDeniedError | None = None + last_error: NetworkError | ServerError | RateLimitError | PermissionDeniedError | None = None for attempt in range(REPO_TREE_PAGE_MAX_ATTEMPTS): try: - resp = self._request("GET", path, params=params) - except PermissionDeniedError as error: - if not authorized: + resp = self._fetch_tree_page(path, params) + except (NetworkError, ServerError, RateLimitError, PermissionDeniedError) as error: + if isinstance(error, PermissionDeniedError) and not authorized: + raise + if not error.retryable and not isinstance(error, PermissionDeniedError): raise last_error = error if attempt < REPO_TREE_PAGE_MAX_ATTEMPTS - 1: - wait = min(2**attempt, REPO_TREE_PAGE_RETRY_MAX_DELAY_SECONDS) + requested_wait = getattr(error, "retry_after", None) + wait = min( + max(0, requested_wait) if requested_wait is not None else 2**attempt, + REPO_TREE_PAGE_RETRY_MAX_DELAY_SECONDS, + ) logger.warning( - "Repo %s: tree listing (%s) denied on an already-authorized repo, retrying in %ds ...", + "Repo %s: Root=%s page=%s listing failed (%s), retry in %ss (%s/%s)", repo_id, - params.get("Root") or params.get("PageNumber") or "/", + params.get("Root") or "/", + params.get("PageNumber", 1), + type(error).__name__, wait, + attempt + 1, + REPO_TREE_PAGE_MAX_ATTEMPTS, ) - time.sleep(wait) + control = getattr(self._read_state, "control", None) + if control is None: + time.sleep(wait) + else: + control.sleep(wait) continue self._tree_reads_ok = True return resp - raise NetworkError( - f"Repo {repo_id}: tree listing ({params.get('Root') or params.get('PageNumber') or '/'}) kept " - f"returning a denial after {REPO_TREE_PAGE_MAX_ATTEMPTS} attempts on an already-authorized " - f"repo: {last_error}" - ) from last_error + if isinstance(last_error, PermissionDeniedError): + raise NetworkError( + f"Repo {repo_id}: tree listing ({params.get('Root') or params.get('PageNumber') or '/'}) kept " + f"returning a denial after {REPO_TREE_PAGE_MAX_ATTEMPTS} attempts on an already-authorized " + f"repo: {last_error}" + ) from last_error + assert last_error is not None + raise last_error # ------------------------------------------------------------------ # Revisions @@ -890,14 +1047,16 @@ def upload_blob( if self._token: upload_headers["Authorization"] = f"Bearer {self._token}" upload_headers["Cookie"] = f"m_session_id={self._token}" + merged_headers = CaseInsensitiveDict(upload_headers) if headers: - upload_headers.update(headers) + merged_headers.update(headers) + merged_headers["User-Agent"] = append_user_agent(self._user_agent, merged_headers.get("User-Agent")) try: resp = self._session.put( upload_url, data=data, - headers=upload_headers, + headers=merged_headers, timeout=timeout or (UPLOAD_BLOB_CONNECT_TIMEOUT_SECONDS, UPLOAD_BLOB_READ_TIMEOUT_SECONDS), ) except requests.ConnectionError as exc: @@ -992,6 +1151,7 @@ def download_archive( f"{segment}/{repo_id}/archive/zip/{revision}", headers=headers, stream=True, + timeout=(API_CONNECT_TIMEOUT, DOWNLOAD_TIMEOUT), ) # ------------------------------------------------------------------ @@ -1011,10 +1171,23 @@ def download_stream( """ segment = _resolve_segment(repo_type) params = {"Revision": revision, "FilePath": file_path} + control = getattr(self._read_state, "control", None) + if control is None: + # Low-level calls do not add adapter retries; higher-level callers handle retries and integrity checks. + self._ensure_session_auth() + resp = self._read_session.get( + self.get_download_url(repo_id, repo_type, file_path, revision), + headers=self._headers(headers), + timeout=(API_CONNECT_TIMEOUT, DOWNLOAD_TIMEOUT), + stream=True, + ) + raise_for_status(resp) + return resp return self._request( "GET", f"{segment}/{repo_id}/repo", params=params, headers=headers, stream=True, + timeout=(API_CONNECT_TIMEOUT, DOWNLOAD_TIMEOUT), ) diff --git a/src/modelscope_hub/_openapi.py b/src/modelscope_hub/_openapi.py index ebdd71f..7217f09 100644 --- a/src/modelscope_hub/_openapi.py +++ b/src/modelscope_hub/_openapi.py @@ -22,22 +22,28 @@ from __future__ import annotations +import contextlib import random +import threading import time -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Iterator, Mapping from pathlib import Path from typing import Any, BinaryIO from urllib.parse import urljoin, urlsplit import requests from requests.adapters import HTTPAdapter +from requests.structures import CaseInsensitiveDict +from ._download_control import _DownloadControl, _network_context, _ReadHTTPAdapter from .config import HubConfig, get_default_config from .constants import ( API_CONNECT_TIMEOUT, API_CONNECTION_POOL_MAXSIZE, API_MAX_RETRIES, API_TIMEOUT, + DOWNLOAD_CHUNK_SIZE, + DOWNLOAD_RETRY_MAX_DELAY_SECONDS, OPENAPI_PREFIX, TokenScope, ) @@ -64,6 +70,7 @@ UpdateSkillSettingsPayload, UpdateStudioSettingsPayload, ) +from .utils import append_user_agent, build_user_agent from .utils.logger import get_logger __all__ = ["OPERATION_REGISTRY", "OpenAPIClient"] @@ -196,6 +203,10 @@ class OpenAPIClient: ``(API_CONNECT_TIMEOUT, API_TIMEOUT)`` as ``(connect, read)`` tuple. max_retries: Maximum number of retry attempts for transient failures. + user_agent: + Additional information appended to the standard SDK UA. Custom UAs from + an injected session or request headers are appended too, preserving the + SDK identifier. Both sessions share the same base UA. """ def __init__( @@ -205,9 +216,14 @@ def __init__( session: requests.Session | None = None, timeout: float | None = None, max_retries: int | None = None, + user_agent: dict | str | None = None, ) -> None: self._config = config or get_default_config() + self._user_agent = build_user_agent(config=self._config, extra=user_agent) self._session = session or requests.Session() + session_ua = self._session.headers.get("User-Agent") + if session_ua and session_ua != requests.utils.default_user_agent(): + self._user_agent = append_user_agent(self._user_agent, session_ua) if session is None: # A bare Session caps the pool at urllib3's default of 10, which is # below the concurrency bulk transfers use; the excess connections @@ -225,6 +241,27 @@ def __init__( # Tri-state cache for whether this deployment serves ``GET /mcp/servers``. # ``None`` means "not probed yet"; see :meth:`list_mcp_servers`. self._mcp_list_supports_get: bool | None = None + self._read_state = threading.local() + self._read_session = requests.Session() + self._read_session.headers.update(self._session.headers) + adapter = _ReadHTTPAdapter( + max_retries=0, + pool_connections=API_CONNECTION_POOL_MAXSIZE, + pool_maxsize=API_CONNECTION_POOL_MAXSIZE, + ) + self._read_session.mount("https://", adapter) + self._read_session.mount("http://", adapter) + + @contextlib.contextmanager + def read_context(self, control: _DownloadControl) -> Iterator[None]: + """Use deadline-controlled transport for site probes, preserving regular OpenAPI policies.""" + previous = getattr(self._read_state, "control", None) + self._read_state.control = control + try: + with _network_context(control): + yield + finally: + self._read_state.control = previous # ------------------------------------------------------------------ # Lifecycle @@ -232,6 +269,7 @@ def __init__( def close(self) -> None: """Release the underlying HTTP session.""" self._session.close() + self._read_session.close() def __enter__(self) -> OpenAPIClient: return self @@ -493,27 +531,47 @@ def _send( else: merged_headers = {} request_cookies = {} + request_headers = CaseInsensitiveDict(merged_headers) if headers: - merged_headers.update(headers) + request_headers.update(headers) + # Inject UA at the sole send boundary for both sessions, anonymous requests and retries. + request_headers["User-Agent"] = append_user_agent(self._user_agent, request_headers.get("User-Agent")) method_upper = method.upper() attempts = max(1, self._max_retries) last_exc: BaseException | None = None + control = getattr(self._read_state, "control", None) for attempt in range(1, attempts + 1): _logger.debug("%s %s", method_upper, final_url) + request_timeout = timeout if timeout is not None else self._timeout + if control is not None: + connect, read = ( + (API_CONNECT_TIMEOUT, request_timeout) + if isinstance(request_timeout, (int, float)) + else request_timeout + ) + request_timeout = control.timeouts(connect, read) try: - response = self._session.request( + session = self._read_session if control is not None else self._session + stream_options: dict[str, Any] = {"stream": True} if control is not None else {} + response = session.request( method=method_upper, url=final_url, params=params, json=json_body, data=data, files=files, - headers=merged_headers, + headers=request_headers, cookies=request_cookies, - timeout=timeout if timeout is not None else self._timeout, + timeout=request_timeout, + **stream_options, ) + if control is not None: + control.track(response) + with control.response(response): + response._content = b"".join(control.chunks(response, DOWNLOAD_CHUNK_SIZE)) + response._content_consumed = True except requests.Timeout as exc: last_exc = RequestTimeoutError(f"Request timed out: {exc}") except requests.ConnectionError as exc: @@ -568,7 +626,10 @@ def _send( attempt, attempts, ) - time.sleep(backoff) + if control is None: + time.sleep(backoff) + else: + control.sleep(min(backoff, DOWNLOAD_RETRY_MAX_DELAY_SECONDS)) assert last_exc is not None # for type-checkers raise last_exc diff --git a/src/modelscope_hub/agent/_api.py b/src/modelscope_hub/agent/_api.py index 86c2d0f..8d9eb97 100644 --- a/src/modelscope_hub/agent/_api.py +++ b/src/modelscope_hub/agent/_api.py @@ -181,6 +181,9 @@ class AgentApi: API token (fallback: HubConfig default / ``ms login``). timeout : int HTTP request timeout in seconds. + user_agent : dict, str or None + Application or channel information appended to the SDK UA across + OpenAPI, legacy and blob requests. """ def __init__( @@ -190,12 +193,13 @@ def __init__( timeout: int = 60, *, config: HubConfig | None = None, + user_agent: dict | str | None = None, ): self._config = config or HubConfig(endpoint=endpoint, token=token) self.server = (self._config.endpoint or "").rstrip("/") self.token = self._config.token self.timeout = timeout - self._openapi = OpenAPIClient(config=self._config, timeout=float(timeout)) + self._openapi = OpenAPIClient(config=self._config, timeout=float(timeout), user_agent=user_agent) # ---- repository ---- diff --git a/src/modelscope_hub/api.py b/src/modelscope_hub/api.py index 2718b29..1453215 100644 --- a/src/modelscope_hub/api.py +++ b/src/modelscope_hub/api.py @@ -65,6 +65,7 @@ RepoInfo, UserInfo, ) +from .utils import build_user_agent from .utils.logger import get_logger __all__ = ["HubApi"] @@ -171,6 +172,9 @@ class HubApi: Defaults to ``https://modelscope.cn``. token : str or None, optional Override the API token. Takes precedence over ``config.token``. + user_agent : dict, str or None, optional + Application or channel information appended to the SDK UA and propagated + to OpenAPI, legacy, probe and download requests. Must not contain credentials. Examples -------- @@ -193,7 +197,9 @@ def __init__( *, endpoint: str | None = None, token: str | None = None, + user_agent: dict | str | None = None, ) -> None: + self._user_agent_extra = dict(user_agent) if isinstance(user_agent, dict) else user_agent base = config or get_default_config() if config is None and (endpoint is not None or token is not None): from dataclasses import replace @@ -224,19 +230,17 @@ def __init__( def openapi(self) -> OpenAPIClient: """Lazily-constructed OpenAPI client.""" if self._openapi is None: - self._openapi = OpenAPIClient(self._config) + self._openapi = OpenAPIClient(self._config, user_agent=self._user_agent_extra) return self._openapi @property def legacy(self) -> LegacyClient: """Lazily-constructed legacy ``/api/v1`` client.""" if self._legacy is None: - from .utils import build_user_agent - self._legacy = LegacyClient( token=self._config.token, endpoint=self._config.endpoint or DEFAULT_ENDPOINT, - user_agent=build_user_agent(self._config.get_session_id()), + user_agent=build_user_agent(config=self._config, extra=self._user_agent_extra), ) elif self._legacy.token != self._config.token: # Clears propagate as well as changes: a cached client left holding a @@ -248,7 +252,7 @@ def legacy(self) -> LegacyClient: def downloader(self) -> DownloadManager: """Lazily-constructed :class:`DownloadManager`.""" if self._downloader is None: - self._downloader = DownloadManager(self.legacy, self._config) + self._downloader = DownloadManager(self.legacy, self._config, user_agent=self._user_agent_extra) return self._downloader @property @@ -644,7 +648,9 @@ def _login_with_scoped_token(self, token: str) -> UserInfo | None: """ from .constants import API_CONNECT_TIMEOUT - probe = OpenAPIClient(self._config, timeout=API_CONNECT_TIMEOUT, max_retries=0) + probe = OpenAPIClient( + self._config, timeout=API_CONNECT_TIMEOUT, max_retries=0, user_agent=self._user_agent_extra + ) try: payload = probe.get_current_user() except Exception: # advisory only -- never mask the original failure @@ -694,7 +700,9 @@ def _explain_login_failure(self, token: str, exc: HubError) -> HubError: if not isinstance(exc, AuthenticationError): return exc peer = self._peer_site_endpoint() - if peer is None or not self._token_valid_on(token, peer): + if peer is None or not self._token_valid_on( + token, peer, user_agent=build_user_agent(config=self._config, extra=self._user_agent_extra) + ): return exc return AuthenticationError( f"{exc.message} This token is valid on {peer} instead; retry with " @@ -728,7 +736,7 @@ def site_key(url: str) -> str: return None @staticmethod - def _token_valid_on(token: str, endpoint: str) -> bool: + def _token_valid_on(token: str, endpoint: str, *, user_agent: str | None = None) -> bool: """Best-effort check of whether *token* authenticates against *endpoint*. Runs on the failure path only and is strictly advisory: any error means @@ -743,6 +751,7 @@ def _token_valid_on(token: str, endpoint: str) -> bool: endpoint=endpoint, timeout=API_CONNECT_TIMEOUT, max_retries=0, + user_agent=user_agent or build_user_agent(config=get_default_config()), ) try: probe.login(token) @@ -1248,12 +1257,29 @@ def resolve_endpoint_for_read( primary = DEFAULT_INTL_ENDPOINT if prefer_ai else DEFAULT_ENDPOINT fallback = DEFAULT_ENDPOINT if prefer_ai else DEFAULT_INTL_ENDPOINT - primary_probe = HubApi(endpoint=primary, token=effective_token) - if primary_probe.repo_exists(repo_id, repo_type): + from ._download_control import _DownloadControl + from .constants import REPO_TREE_PAGE_TIMEOUT_SECONDS + + def exists(endpoint: str) -> bool: + from dataclasses import replace + + probe = HubApi( + config=replace(self._config), + endpoint=endpoint, + token=effective_token, + user_agent=self._user_agent_extra, + ) + try: + with _DownloadControl(total_timeout=REPO_TREE_PAGE_TIMEOUT_SECONDS) as control: + with probe.openapi.read_context(control): + return probe.repo_exists(repo_id, repo_type) + finally: + probe.openapi.close() + + if exists(primary): return primary - fallback_probe = HubApi(endpoint=fallback, token=effective_token) - if fallback_probe.repo_exists(repo_id, repo_type): + if exists(fallback): logger.warning( "Repo %s not found on %s, using %s instead.", repo_id, @@ -1472,6 +1498,7 @@ def download_file( expected_sha256: str | None = None, local_files_only: bool = False, user_agent: dict | str | None = None, + file_size: int | None = None, ) -> Path: """Download a single file from a repository. @@ -1503,6 +1530,10 @@ def download_file( Raises :class:`CacheNotFound` if the file is not cached. user_agent : dict, str or None, optional Custom user-agent info appended to the default UA string. + file_size : int, optional + Expected size in bytes, e.g. from a file listing. The download is + rejected when the server delivers a different size, and files + larger than the parallel threshold use ranged parallel streams. Returns ------- @@ -1513,6 +1544,9 @@ def download_file( ------ NotExistError When the file or repository does not exist. + FileIntegrityError + When the content keeps failing ``expected_sha256`` or (as + :class:`FileSizeMismatchError`) ``file_size``. Examples -------- @@ -1542,6 +1576,7 @@ def download_file( expected_sha256=expected_sha256, local_files_only=local_files_only, user_agent=user_agent, + file_size=file_size, ) def download_repo( @@ -1558,6 +1593,7 @@ def download_repo( local_files_only: bool = False, user_agent: dict | str | None = None, progress_callbacks: list[type[ProgressCallback]] | None = None, + file_list: list[dict] | None = None, ) -> Path: """Download an entire repository snapshot. @@ -1589,12 +1625,23 @@ def download_repo( progress_callbacks : list of ProgressCallback subclasses, optional Callback *classes* (not instances); each is instantiated per file to report byte-level download progress. + file_list : list of dict, optional + Previously fetched legacy file entries (Path, Size, Sha256). When + provided, listing requests are skipped. The caller must ensure the + entries belong to the same repository and revision; filters still apply. Returns ------- Path Absolute path to the snapshot/local directory. + Raises + ------ + FileDownloadError + When any file failed after all files were attempted. Completed + files are kept, so calling again fetches only the failed ones; + ``failed_files`` maps each failed path to its error. + Examples -------- Download only the tokenizer assets of a model: @@ -1627,6 +1674,7 @@ def download_repo( local_files_only=local_files_only, user_agent=user_agent, progress_callbacks=progress_callbacks, + file_list=file_list, ) def list_repo_files( diff --git a/src/modelscope_hub/cli/agent.py b/src/modelscope_hub/cli/agent.py index 0aced6c..f8a1153 100644 --- a/src/modelscope_hub/cli/agent.py +++ b/src/modelscope_hub/cli/agent.py @@ -92,7 +92,7 @@ def _cmd_list(owner, page_number, page_size, *, endpoint, token) -> int: """List remote agent repositories.""" if not endpoint: return _fail("not logged in. Provide endpoint.") - client = AgentApi(endpoint=endpoint, token=token or "") + client = AgentApi(endpoint=endpoint, token=token or "", user_agent={"invoked_by": "cli"}) try: result = client.list_agents(owner=owner, page_number=page_number, page_size=page_size) except APIError as e: @@ -144,7 +144,7 @@ def _cmd_download(repo, local_dir, revision, *, endpoint, token, username) -> in ) group, name = _resolve_repo(repo, username or "") - client = AgentApi(endpoint=endpoint, token=token or "") + client = AgentApi(endpoint=endpoint, token=token or "", user_agent={"invoked_by": "cli"}) try: if client.repo_info(group, name) is None: return _fail(f"repository {group}/{name} not found.") @@ -209,7 +209,7 @@ def _cmd_upload(repo, local_dir, revision, dry_run, *, endpoint, token, username ) group, name = _resolve_repo(repo, username or "") - client = AgentApi(endpoint=endpoint, token=token) + client = AgentApi(endpoint=endpoint, token=token, user_agent={"invoked_by": "cli"}) try: if not client.check_repo(group, name): client.create_repo(group, name, visibility=visibility) @@ -507,7 +507,7 @@ def execute(self) -> None: from .._openapi import OpenAPIClient try: - openapi = OpenAPIClient(config=config) + openapi = OpenAPIClient(config=config, user_agent={"invoked_by": "cli"}) user_data = openapi.get_current_user() or {} username = user_data.get("username") or user_data.get("Username") or "" except Exception: diff --git a/src/modelscope_hub/cli/base.py b/src/modelscope_hub/cli/base.py index f2de5f6..cf66030 100644 --- a/src/modelscope_hub/cli/base.py +++ b/src/modelscope_hub/cli/base.py @@ -75,6 +75,7 @@ def make_api(args: Namespace) -> HubApi: return HubApi( token=getattr(args, "token", None), endpoint=getattr(args, "endpoint", None), + user_agent={"invoked_by": "cli"}, ) diff --git a/src/modelscope_hub/cli/download.py b/src/modelscope_hub/cli/download.py index 328fc17..e041662 100644 --- a/src/modelscope_hub/cli/download.py +++ b/src/modelscope_hub/cli/download.py @@ -146,7 +146,7 @@ def _make_api_with_endpoint(self) -> HubApi: if endpoint: endpoint = _resolve_cli_endpoint(endpoint) - return HubApi(token=token, endpoint=endpoint) + return HubApi(token=token, endpoint=endpoint, user_agent={"invoked_by": "cli"}) api = make_api(self.args) try: @@ -154,7 +154,7 @@ def _make_api_with_endpoint(self) -> HubApi: self.args.repo_id, repo_type=self.args.repo_type, ) - return HubApi(token=token, endpoint=resolved) + return HubApi(token=token, endpoint=resolved, user_agent={"invoked_by": "cli"}) except Exception: return api diff --git a/src/modelscope_hub/compat/file_download.py b/src/modelscope_hub/compat/file_download.py index 5d81739..1494fa1 100644 --- a/src/modelscope_hub/compat/file_download.py +++ b/src/modelscope_hub/compat/file_download.py @@ -13,6 +13,7 @@ from ..api import HubApi from ..constants import RepoType from ..errors import AuthenticationError, NotExistError, PermissionDeniedError +from ..utils import append_user_agent def _resolve_legacy_paths( @@ -54,11 +55,11 @@ def model_file_download( stacklevel=2, ) - api = HubApi(token=token, endpoint=endpoint) + api = HubApi(token=token, endpoint=endpoint, user_agent=append_user_agent("invoked_by/compat", user_agent)) if endpoint is None and not local_files_only: try: endpoint = api.resolve_endpoint_for_read(model_id, repo_type="model") - api = HubApi(token=token, endpoint=endpoint) + api = HubApi(token=token, endpoint=endpoint, user_agent=append_user_agent("invoked_by/compat", user_agent)) except Exception: pass effective_cache, effective_local = _resolve_legacy_paths( @@ -104,11 +105,11 @@ def dataset_file_download( stacklevel=2, ) - api = HubApi(token=token, endpoint=endpoint) + api = HubApi(token=token, endpoint=endpoint, user_agent=append_user_agent("invoked_by/compat", user_agent)) if endpoint is None and not local_files_only: try: endpoint = api.resolve_endpoint_for_read(dataset_id, repo_type="dataset") - api = HubApi(token=token, endpoint=endpoint) + api = HubApi(token=token, endpoint=endpoint, user_agent=append_user_agent("invoked_by/compat", user_agent)) except Exception: pass effective_cache, effective_local = _resolve_legacy_paths( diff --git a/src/modelscope_hub/compat/hub_api.py b/src/modelscope_hub/compat/hub_api.py index f011f91..5d91634 100644 --- a/src/modelscope_hub/compat/hub_api.py +++ b/src/modelscope_hub/compat/hub_api.py @@ -125,7 +125,7 @@ def __init__( if endpoint and not endpoint.startswith("http"): endpoint = f"https://{endpoint}" self._endpoint = endpoint - self._api = HubApi(endpoint=endpoint, token=token) + self._api = HubApi(endpoint=endpoint, token=token, user_agent={"invoked_by": "compat"}) # ------------------------------------------------------------------ # Authentication @@ -215,7 +215,7 @@ def delete_files( """ api = self._api if token or endpoint: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) return api.delete_files( repo_id, repo_type, @@ -242,7 +242,7 @@ def create_repo( """Create a repository (legacy signature).""" api = self._api if token or endpoint: - api = HubApi(token=token, endpoint=endpoint or self._endpoint) + api = HubApi(token=token, endpoint=endpoint or self._endpoint, user_agent={"invoked_by": "compat"}) if create_default_config: kwargs["create_default_config"] = True try: @@ -301,7 +301,7 @@ def _create_aigc_model(self, model_id: str, aigc_model: Any, kwargs: dict[str, A api = self._api if token or endpoint: - api = HubApi(token=token, endpoint=endpoint or self._endpoint) + api = HubApi(token=token, endpoint=endpoint or self._endpoint, user_agent={"invoked_by": "compat"}) owner, name = api._parse_repo_id(model_id) normalised_visibility = api._normalize_visibility(visibility) if normalised_visibility is None: @@ -376,7 +376,7 @@ def create_model_tag( api = self._api if token or endpoint: - api = HubApi(token=token, endpoint=endpoint or self._endpoint) + api = HubApi(token=token, endpoint=endpoint or self._endpoint, user_agent={"invoked_by": "compat"}) if aigc_model is None: if readme_content is not None: raise InvalidParameter("readme_content is only supported for AIGC model tags") @@ -507,6 +507,7 @@ def repo_exists( api = self._api if endpoint is not None or token is not None: api = HubApi( + user_agent={"invoked_by": "compat"}, endpoint=endpoint or self._api._config.endpoint, token=token or self._api._config.token, ) @@ -595,7 +596,9 @@ def _api_for(self, control: dict[str, Any]) -> HubApi: endpoint = control.get("endpoint") config = self._api._config if (token and token != config.token) or (endpoint and endpoint != config.endpoint): - return HubApi(token=token or config.token, endpoint=endpoint or config.endpoint) + return HubApi( + token=token or config.token, endpoint=endpoint or config.endpoint, user_agent={"invoked_by": "compat"} + ) return self._api def deploy_studio(self, studio_id: str, **kwargs: Any) -> dict: @@ -826,7 +829,7 @@ def list_models( """List models owned by a user/org.""" api = self._api if token: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) page = api.list_repos( RepoType.MODEL, owner=owner_or_group, @@ -883,7 +886,7 @@ def list_datasets( actual_owner = owner or owner_or_group api = self._api if token: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) return api.list_repos( RepoType.DATASET, owner=actual_owner, @@ -908,7 +911,7 @@ def create_dataset( repo_id = f"{namespace}/{dataset_name}" api = self._api if token: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) api.create_repo( repo_id, repo_type=RepoType.DATASET, @@ -950,7 +953,7 @@ def get_dataset( ) api = self._api if token: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) return api.get_repo(dataset_id, RepoType.DATASET, revision=revision) def delete_model( @@ -962,7 +965,7 @@ def delete_model( """Delete a model repository.""" api = self._api if token: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) api.delete_repo(model_id, RepoType.MODEL) def delete_dataset( @@ -980,7 +983,7 @@ def delete_dataset( ) api = self._api if token: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) api.delete_repo(dataset_id, RepoType.DATASET) def get_dataset_files( @@ -999,7 +1002,7 @@ def get_dataset_files( """Get dataset file tree via the datahub API.""" api = self._api if token: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) if dataset_hub_id is None: if "/" in repo_id: @@ -1038,7 +1041,7 @@ def get_dataset_id_and_type( """Get the dataset hub-internal id and type.""" api = self._api if token: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) cache_key = (namespace, dataset_name, endpoint or self._endpoint) cached = LegacyHubApi._dataset_id_type_cache.get(cache_key) @@ -1063,7 +1066,7 @@ def get_dataset_meta_file_list( """Get the meta file-list of the dataset.""" api = self._api if token: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) params = {"Revision": revision} resp = api.legacy._request("GET", f"datasets/{dataset_id}/repo/tree", params=params) @@ -1107,7 +1110,7 @@ def get_dataset_meta_files_local_paths( api = self._api if token: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) local_paths: dict[str, list] = defaultdict(list) dataset_formation = DatasetFormations(dataset_type) @@ -1197,7 +1200,7 @@ def get_dataset_access_config( """Get STS token config for dataset OSS access.""" api = self._api if token: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) resp = api.legacy._request( "GET", f"datasets/{namespace}/{dataset_name}/ststoken", @@ -1217,7 +1220,7 @@ def get_dataset_access_config_session( """Get STS token config with session-based auth.""" api = self._api if token: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) resp = api.legacy._request( "GET", f"datasets/{namespace}/{dataset_name}/ststoken", @@ -1237,7 +1240,7 @@ def get_dataset_access_config_for_unzipped( """Get STS config for unzipped dataset files.""" api = self._api if token: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) # Get visibility data = api.legacy.get_repo_info(f"{namespace}/{dataset_name}", RepoType.DATASET) @@ -1270,7 +1273,7 @@ def dataset_download_statistics( try: api = self._api if token: - api = HubApi(endpoint=endpoint or self._endpoint, token=token) + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) api.legacy._request( "POST", f"datasets/{namespace}/{dataset_name}/download/increase", diff --git a/src/modelscope_hub/compat/snapshot_download.py b/src/modelscope_hub/compat/snapshot_download.py index 3981972..b06208d 100644 --- a/src/modelscope_hub/compat/snapshot_download.py +++ b/src/modelscope_hub/compat/snapshot_download.py @@ -16,6 +16,7 @@ from ..api import HubApi from ..constants import RepoType from ..errors import AuthenticationError, NotExistError, PermissionDeniedError +from ..utils import append_user_agent from ..utils.patterns import normalize_patterns from .file_download import _resolve_legacy_paths @@ -69,14 +70,14 @@ def snapshot_download( include = _normalize_pattern(allow_patterns) or _normalize_pattern(allow_file_pattern) exclude = _normalize_pattern(ignore_patterns) or _normalize_pattern(ignore_file_pattern) - api = HubApi(token=token, endpoint=endpoint) + api = HubApi(token=token, endpoint=endpoint, user_agent=append_user_agent("invoked_by/compat", user_agent)) if endpoint is None and not local_files_only: try: endpoint = api.resolve_endpoint_for_read( effective_id, repo_type=effective_type, ) - api = HubApi(token=token, endpoint=endpoint) + api = HubApi(token=token, endpoint=endpoint, user_agent=append_user_agent("invoked_by/compat", user_agent)) except Exception: pass effective_cache, effective_local = _resolve_legacy_paths( @@ -132,11 +133,11 @@ def dataset_snapshot_download( include = _normalize_pattern(allow_patterns) or _normalize_pattern(allow_file_pattern) exclude = _normalize_pattern(ignore_patterns) or _normalize_pattern(ignore_file_pattern) - api = HubApi(token=token, endpoint=endpoint) + api = HubApi(token=token, endpoint=endpoint, user_agent=append_user_agent("invoked_by/compat", user_agent)) if endpoint is None and not local_files_only: try: endpoint = api.resolve_endpoint_for_read(dataset_id, repo_type="dataset") - api = HubApi(token=token, endpoint=endpoint) + api = HubApi(token=token, endpoint=endpoint, user_agent=append_user_agent("invoked_by/compat", user_agent)) except Exception: pass effective_cache, effective_local = _resolve_legacy_paths( diff --git a/src/modelscope_hub/constants.py b/src/modelscope_hub/constants.py index e317f99..c4ca763 100644 --- a/src/modelscope_hub/constants.py +++ b/src/modelscope_hub/constants.py @@ -614,7 +614,7 @@ def _env_register( """Retries for a spurious ``403`` on a single page of a paginated tree listing. A large dataset listing spans hundreds of pages and an occasional page answers -``403 无权访问该数据集`` on a repository the caller has just read successfully. +``403 access denied`` on a repository the caller has just read successfully. Treating that as an authorization result throws away every page already collected, so it is retried once the credential has been proven by an earlier page. @@ -696,19 +696,64 @@ def _env_register( DOWNLOAD_RETRY_TIMES: int = _env_int( "MODELSCOPE_DOWNLOAD_MAX_RETRIES", 5, - "Per-file download retry count", + "Per-file download retry count (one retry layer; zero disables retries)", "Download", "DOWNLOAD_RETRY_TIMES", + allow_zero=True, ) DOWNLOAD_TIMEOUT: int = _env_int( "MODELSCOPE_DOWNLOAD_TIMEOUT", 60, - "Per-file download timeout (seconds)", + "Download idle read timeout (seconds; single stream and range parts)", "Download", "DOWNLOAD_TIMEOUT", ) +DOWNLOAD_TOTAL_TIMEOUT: int = _env_int( + "MODELSCOPE_DOWNLOAD_TOTAL_TIMEOUT", + 3600, + "Total time budget per file including retries, probes and lock wait (seconds)", + "Download", +) +DOWNLOAD_MIN_BYTES_PER_SECOND: int = _env_int( + "MODELSCOPE_DOWNLOAD_MIN_BYTES_PER_SECOND", + 1024, + "Minimum transfer rate per connection (B/s; zero disables rate checking)", + "Download", + allow_zero=True, +) +DOWNLOAD_LOW_SPEED_WINDOW_SECONDS: int = _env_int( + "MODELSCOPE_DOWNLOAD_LOW_SPEED_WINDOW_SECONDS", + 60, + "Observation window for minimum download rate (seconds)", + "Download", +) +DOWNLOAD_RETRY_MAX_DELAY_SECONDS: int = _env_int( + "MODELSCOPE_DOWNLOAD_RETRY_MAX_DELAY_SECONDS", + 60, + "Maximum download Retry-After or backoff delay (seconds)", + "Download", +) +DOWNLOAD_FILE_LOCK_TIMEOUT_SECONDS: int = _env_int( + "MODELSCOPE_DOWNLOAD_FILE_LOCK_TIMEOUT_SECONDS", + 600, + "Maximum wait for another downloader's file lock (seconds)", + "Download", +) +REPO_TREE_PAGE_TIMEOUT_SECONDS: int = _env_int( + "MODELSCOPE_REPO_TREE_PAGE_TIMEOUT_SECONDS", + 15, + "Idle timeout per file-tree page (seconds; no nested HTTP retries)", + "Network", +) +REPO_TREE_LIST_TIMEOUT_SECONDS: int = _env_int( + "MODELSCOPE_REPO_TREE_LIST_TIMEOUT_SECONDS", + 3600, + "Total budget for enumerating a repository (seconds)", + "Network", +) + DOWNLOAD_PART_SIZE: int = _env_int_mb( "MODELSCOPE_DOWNLOAD_PART_SIZE_MB", 160, @@ -1302,6 +1347,13 @@ def get_upload_ignore_file_pattern() -> str | None: "DOWNLOAD_PART_SIZE", "DOWNLOAD_RETRY_TIMES", "DOWNLOAD_TIMEOUT", + "DOWNLOAD_TOTAL_TIMEOUT", + "DOWNLOAD_MIN_BYTES_PER_SECOND", + "DOWNLOAD_LOW_SPEED_WINDOW_SECONDS", + "DOWNLOAD_RETRY_MAX_DELAY_SECONDS", + "DOWNLOAD_FILE_LOCK_TIMEOUT_SECONDS", + "REPO_TREE_PAGE_TIMEOUT_SECONDS", + "REPO_TREE_LIST_TIMEOUT_SECONDS", "ENV_AGENT_PLUGIN_REPO", "ENV_FILE_LOCK", "ENV_CACHE", diff --git a/src/modelscope_hub/errors.py b/src/modelscope_hub/errors.py index c2e1623..23b821c 100644 --- a/src/modelscope_hub/errors.py +++ b/src/modelscope_hub/errors.py @@ -18,10 +18,13 @@ from __future__ import annotations import re +from collections.abc import Mapping from typing import TYPE_CHECKING, Any from urllib.parse import parse_qs, urlparse, urlunparse if TYPE_CHECKING: # pragma: no cover - type-only imports + from os import PathLike + from requests import Response # --------------------------------------------------------------------------- @@ -347,6 +350,48 @@ class StorageError(HubError): suggestion = "File upload/download failed (storage service error). Please retry later." +class FileDownloadError(StorageError): + """Raised after a snapshot download attempted every file and some failed (E1003). + + Files that downloaded successfully stay in place, so re-running the same + call fetches only the paths in :attr:`failed_files`. :attr:`retryable` is + true only when every individual failure is itself retryable. + """ + + suggestion = ( + "Some files failed to download; completed files are kept. " + "Re-run the same command to retry only the failed files." + ) + _MAX_LISTED = 10 + + def __init__( + self, + message: str, + *, + failed_files: Mapping[str, BaseException] | None = None, + total: int | None = None, + output_dir: str | PathLike[str] | None = None, + ) -> None: + super().__init__(message) + self.failed_files: dict[str, BaseException] = dict(failed_files or {}) + self.total = total + self.output_dir = output_dir + self.retryable = bool(self.failed_files) and all( + getattr(exc, "retryable", False) for exc in self.failed_files.values() + ) + + def __str__(self) -> str: + lines = [super().__str__()] + for path, exc in list(self.failed_files.items())[: self._MAX_LISTED]: + # Only the headline: API errors append multi-line request detail. + text = str(exc) + lines.append(f" - {path}: {text.splitlines()[0] if text else exc.__class__.__name__}") + hidden = len(self.failed_files) - self._MAX_LISTED + if hidden > 0: + lines.append(f" ... and {hidden} more") + return "\n".join(lines) + + class FileIntegrityError(HubError): """Raised when a downloaded or uploaded file fails integrity validation (E2020).""" @@ -355,6 +400,23 @@ class FileIntegrityError(HubError): suggestion = "File SHA256 checksum mismatch. Will retry automatically." +class FileSizeMismatchError(FileIntegrityError): + """Raised when the server delivers a different number of bytes than the repository lists (E2020). + + The transfer itself completed, so the remote object (or the replica / CDN + node serving it) is empty, truncated or out of date rather than the + connection being cut. Repeating the same request rarely helps, so it is not + marked retryable: the download makes one more attempt over the default + route and then reports the file. + """ + + retryable = False + suggestion = ( + "The server returned content that does not match the repository's file size. " + "Report the file path and request ids in this message to the repository owner or ModelScope." + ) + + # -- Cache errors (E1022) --------------------------------------------------- class CacheError(HubError): """Base for local cache filesystem or metadata corruption.""" @@ -433,11 +495,11 @@ class NotSupportedError(HubError): # --------------------------------------------------------------------------- _BUSINESS_CODE_MAP: dict[int, type[APIError]] = { # -> E3001, served with HTTP 400 by POST /api/v1/login on both sites - 10010103009: AuthenticationError, # AccessToken 无效或过期 + 10010103009: AuthenticationError, # AccessToken is invalid or expired # -> E3026 - 10020101001: AlreadyExistsError, # 国内站 - 数据集已存在 - 10010101001: AlreadyExistsError, # 国内站 - 模型已存在 - 10010202004: AlreadyExistsError, # 国际站 - 名称已被使用 + 10020101001: AlreadyExistsError, # China site: dataset already exists + 10010101001: AlreadyExistsError, # China site: model already exists + 10010202004: AlreadyExistsError, # International site: name is already in use } # The legacy commit endpoint overloads 400/409 for optimistic branch conflicts @@ -699,7 +761,9 @@ def is_repo_exists_error(exc: BaseException) -> bool: "CacheError", "CacheNotFound", "CorruptedCacheException", + "FileDownloadError", "FileIntegrityError", + "FileSizeMismatchError", "NetworkError", "RequestTimeoutError", "StorageError", diff --git a/src/modelscope_hub/utils/__init__.py b/src/modelscope_hub/utils/__init__.py index d5363ae..2117b67 100644 --- a/src/modelscope_hub/utils/__init__.py +++ b/src/modelscope_hub/utils/__init__.py @@ -2,6 +2,9 @@ from __future__ import annotations +import uuid +from typing import TYPE_CHECKING + from .file_utils import compute_hash, ensure_dir, get_cache_dir, get_file_size from .format import format_size, format_timesince, tabulate from .logger import get_logger @@ -9,7 +12,14 @@ from .patterns import extract_common_prefix, normalize_patterns from .time_utils import parse_timestamp +if TYPE_CHECKING: + from ..config import HubConfig + +# Keep retries and transports in one session when credential storage is unavailable. +_FALLBACK_SESSION_ID = uuid.uuid4().hex + __all__ = [ + "append_user_agent", "build_user_agent", "compute_hash", "encode_media_to_base64", @@ -29,6 +39,8 @@ def build_user_agent( session_id: str | None = None, extra: dict | str | None = None, + *, + config: HubConfig | None = None, ) -> str: """Build the standard ModelScope Hub SDK User-Agent string. @@ -39,12 +51,21 @@ def build_user_agent( Falls back to a random UUID if not provided. extra : dict, str or None Additional key/value pairs or free-form string appended to the UA. + config : HubConfig, optional + Resolve the persisted session_id from this config; use a process-stable + ID if storage is unavailable. An explicit session_id avoids storage access. """ import os import platform - import uuid from .. import __version__ + from ..errors import CacheError + + if config is not None and session_id is None: + try: + session_id = config.get_session_id() + except (CacheError, OSError): + session_id = _FALLBACK_SESSION_ID env = os.environ.get("MODELSCOPE_CLOUD_ENVIRONMENT", "custom") user_name = os.environ.get("MODELSCOPE_CLOUD_USERNAME", "unknown") @@ -55,8 +76,16 @@ def build_user_agent( f"session_id/{sid}; platform/{platform.platform()}; " f"processor/{platform.processor()}; env/{env}; user/{user_name}" ) + return append_user_agent(ua, extra) + + +def append_user_agent(base: str, extra: dict | str | None = None) -> str: + """Append custom UA information without removing the SDK prefix or duplicating fields.""" if isinstance(extra, dict): - ua += "; " + "; ".join(f"{k}/{v}" for k, v in extra.items()) - elif isinstance(extra, str): - ua += "; " + extra - return ua + extra = "; ".join(f"{k}/{v}" for k, v in extra.items()) + parts = base.split("; ") + if extra: + for part in extra.split("; "): + if part and part not in parts: + parts.append(part) + return "; ".join(parts) diff --git a/tests/cli/test_base.py b/tests/cli/test_base.py index 5bd2d3b..0ef7da3 100644 --- a/tests/cli/test_base.py +++ b/tests/cli/test_base.py @@ -97,21 +97,23 @@ def test_make_api_with_token(self): args = Namespace(token="my_token", endpoint=None) with patch("modelscope_hub.cli.base.HubApi") as mock_hub: make_api(args) - mock_hub.assert_called_once_with(token="my_token", endpoint=None) + mock_hub.assert_called_once_with(token="my_token", endpoint=None, user_agent={"invoked_by": "cli"}) def test_make_api_without_token(self): """make_api passes None when no token.""" args = Namespace(token=None, endpoint=None) with patch("modelscope_hub.cli.base.HubApi") as mock_hub: make_api(args) - mock_hub.assert_called_once_with(token=None, endpoint=None) + mock_hub.assert_called_once_with(token=None, endpoint=None, user_agent={"invoked_by": "cli"}) def test_make_api_with_endpoint(self): """make_api passes endpoint to HubApi.""" args = Namespace(token=None, endpoint="https://custom.endpoint.com") with patch("modelscope_hub.cli.base.HubApi") as mock_hub: make_api(args) - mock_hub.assert_called_once_with(token=None, endpoint="https://custom.endpoint.com") + mock_hub.assert_called_once_with( + token=None, endpoint="https://custom.endpoint.com", user_agent={"invoked_by": "cli"} + ) class TestAddRepoTypeArg: diff --git a/tests/test_agent_plugin.py b/tests/test_agent_plugin.py index f32e6c0..f07817a 100644 --- a/tests/test_agent_plugin.py +++ b/tests/test_agent_plugin.py @@ -325,12 +325,14 @@ def test_verify_manifest_refuses_keys_pointing_outside_the_package(tmp_path, key # --------------------------------------------------------------------------- # execution audit / no opt-in # --------------------------------------------------------------------------- -def test_log_execution_records_the_build_before_import(tmp_path, caplog): +def test_log_execution_records_the_build_before_import(tmp_path, caplog, monkeypatch): """There is no opt-in to wait for any more -- the allow-list is the authorisation -- but executing downloaded code still deserves an audit line naming the exact build, emitted before the import so a crash during it leaves a trace of what was being loaded.""" spec = spec_for(make_plugin(tmp_path)) + # Capture on the target logger because SDK logs do not propagate to root; monkeypatch restores its handlers. + monkeypatch.setattr(_plugin.logger, "handlers", [*_plugin.logger.handlers, caplog.handler]) with caplog.at_level("INFO", logger="modelscope_hub.agent"): _plugin.log_execution(spec) for expected in (spec.repo_id, spec.revision, spec.entry_module, "9.9.9"): diff --git a/tests/test_download_integrity.py b/tests/test_download_integrity.py new file mode 100644 index 0000000..9c7a6d7 --- /dev/null +++ b/tests/test_download_integrity.py @@ -0,0 +1,802 @@ +"""Download integrity hardening: size validation, resume, cleanup and failure reporting. + +A real HTTP server on 127.0.0.1 stands in for the hub and its LFS CDN, so the +tests exercise requests/urllib3 streaming, redirects, ``Range`` handling and +chunked bodies end to end. It mirrors what ModelScope does in production: +LFS files redirect to a CDN that honours ``Range`` (206 + ``Content-Range``, +416 past the end), while files served directly by the API ignore ``Range`` and +answer 200 with a chunked body. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import threading +import time +from collections.abc import Iterator +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, quote, unquote, urlparse + +import pytest + +from modelscope_hub import _download +from modelscope_hub.api import HubApi +from modelscope_hub.cli.main import run_cmd +from modelscope_hub.errors import ( + FileDownloadError, + FileIntegrityError, + FileSizeMismatchError, + NetworkError, +) + +REPO = "owner/data" + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +@dataclass +class _RemoteFile: + content: bytes + # "lfs": 302 to a Range-capable CDN; "direct": 200 chunked, Range ignored. + mode: str = "lfs" + # Bytes actually served when they differ from what the listing describes. + served: bytes | None = None + # Served instead while the intra-cloud region header is present. + served_with_region: bytes | None = None + # The first body response stops after this many bytes and drops the socket. + truncate_first: int | None = None + header_delay: float = 0 + trickle_interval: float = 0 + header_trickle: bool = False + status: int = 200 + retry_after: str | None = None + + def body(self, region: str | None) -> bytes: + if region and self.served_with_region is not None: + return self.served_with_region + return self.content if self.served is None else self.served + + +@dataclass +class _Request: + kind: str + path: str + range: str | None + region: str | None + + +@dataclass +class _FakeHub: + url: str = "" + files: dict[str, _RemoteFile] = field(default_factory=dict) + requests: list[_Request] = field(default_factory=list) + truncated: set[str] = field(default_factory=set) + lock: threading.Lock = field(default_factory=threading.Lock) + + def add(self, path: str, content: bytes, **kwargs) -> bytes: + self.files[path] = _RemoteFile(content, **kwargs) + return content + + def seen(self, path: str, kind: str) -> list[_Request]: + return [r for r in self.requests if r.path == path and r.kind == kind] + + +def _handler(hub: _FakeHub) -> type[BaseHTTPRequestHandler]: + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args) -> None: + pass + + def _record(self, kind: str, path: str) -> None: + with hub.lock: + hub.requests.append( + _Request(kind, path, self.headers.get("Range"), self.headers.get("x-aliyun-region-id")) + ) + + def _truncate_now(self, path: str) -> bool: + with hub.lock: + if hub.files[path].truncate_first is None or path in hub.truncated: + return False + hub.truncated.add(path) + return True + + def _send(self, status: int, body: bytes, headers: dict[str, str] | None = None) -> None: + self.send_response(status) + for name, value in (headers or {}).items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 - http.server API + url = urlparse(self.path) + if url.path.endswith("/repo/tree"): + entries = [ + {"Path": p, "Type": "blob", "Size": len(f.content), "Sha256": _sha256(f.content)} + for p, f in sorted(hub.files.items()) + if not parse_qs(url.query).get("Root") + or p.startswith(parse_qs(url.query)["Root"][0].rstrip("/") + "/") + ] + body = json.dumps({"Code": 200, "Data": {"Files": entries}}).encode() + return self._send(200, body, {"Content-Type": "application/json"}) + if url.path.endswith("/repo"): + path = parse_qs(url.query)["FilePath"][0] + self._record("api", path) + if hub.files[path].mode == "direct": + return self._serve_whole(path) + self.send_response(302) + self.send_header("Location", f"/cdn/{quote(path)}") + self.send_header("Content-Length", "0") + self.end_headers() + return None + if url.path.startswith("/cdn/"): + path = unquote(url.path[len("/cdn/") :]) + self._record("cdn", path) + return self._serve_ranged(path) + return self._send(404, b'{"Code": 404}', {"Content-Type": "application/json"}) + + def _serve_whole(self, path: str) -> None: + body = hub.files[path].body(self.headers.get("x-aliyun-region-id")) + self.send_response(200) + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + for start in range(0, len(body), 8192): + piece = body[start : start + 8192] + self.wfile.write(f"{len(piece):x}\r\n".encode() + piece + b"\r\n") + self.wfile.write(b"0\r\n\r\n") + + def _serve_ranged(self, path: str) -> None: + remote = hub.files[path] + if remote.header_delay: + time.sleep(remote.header_delay) + if remote.header_trickle: + for byte in b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n": + self.wfile.write(bytes([byte])) + self.wfile.flush() + time.sleep(0.05) + return None + if remote.status >= 400: + return self._send(remote.status, b"{}", {"Retry-After": remote.retry_after or "0"}) + body = remote.body(self.headers.get("x-aliyun-region-id")) + headers = { + "Accept-Ranges": "bytes", + "X-Cache": "HIT TCP_MEM_HIT", + "EagleId": "test-eagle", + "X-Linked-ETag": _sha256(remote.content), + } + status, payload = 200, body + requested = self.headers.get("Range") + if requested: + first, _, last = requested.removeprefix("bytes=").partition("-") + start = int(first) + if start >= len(body): + return self._send(416, b"", {"Content-Range": f"bytes */{len(body)}"}) + end = min(int(last), len(body) - 1) if last else len(body) - 1 + status, payload = 206, body[start : end + 1] + headers["Content-Range"] = f"bytes {start}-{end}/{len(body)}" + if self._truncate_now(path): + cut = hub.files[path].truncate_first or 0 + self.send_response(status) + for name, value in headers.items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload[:cut]) + self.close_connection = True + return None + if remote.trickle_interval: + self.send_response(status) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + for byte in payload: + self.wfile.write(bytes([byte])) + self.wfile.flush() + time.sleep(remote.trickle_interval) + return None + return self._send(status, payload, headers) + + return Handler + + +class _QuietServer(ThreadingHTTPServer): + daemon_threads = True + + def handle_error(self, request, client_address) -> None: + # Clients abandoning a body they rejected are expected here. + pass + + +@pytest.fixture() +def hub(monkeypatch) -> Iterator[_FakeHub]: + monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost") + monkeypatch.setenv("MODELSCOPE_DOWNLOAD_INTRA_CLOUD", "false") + monkeypatch.delenv("MODELSCOPE_DOWNLOAD_INTRA_CLOUD_REGION", raising=False) + monkeypatch.delenv("MODELSCOPE_DOWNLOAD_INTER_CLOUD_REGIONS", raising=False) + monkeypatch.setattr(_download, "DOWNLOAD_INTEGRITY_RETRY_BACKOFF", 0) + fake = _FakeHub() + server = _QuietServer(("127.0.0.1", 0), _handler(fake)) + fake.url = f"http://127.0.0.1:{server.server_port}" + thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True) + thread.start() + try: + yield fake + finally: + server.shutdown() + server.server_close() + + +@pytest.fixture() +def api(hub) -> HubApi: + return HubApi(endpoint=hub.url, token="") + + +@pytest.fixture() +def dirs(tmp_path) -> tuple[Path, Path]: + return tmp_path / "out", tmp_path / "cache" + + +class TestSnapshotFailureReporting: + def test_empty_body_for_listed_file_is_reported_and_leaves_nothing(self, hub, api, dirs): + out, cache = dirs + good = hub.add("shards/a.tar", os.urandom(200_000)) + hub.add("shards/b.tar", os.urandom(200_000), served=b"") + hub.add("shards/c.tar", os.urandom(150_000)) + + with pytest.raises(FileDownloadError) as excinfo: + api.download_repo(REPO, "dataset", local_dir=out, cache_dir=cache) + + error = excinfo.value + assert list(error.failed_files) == ["shards/b.tar"] + assert error.total == 3 + assert Path(error.output_dir) == out + assert error.retryable is False + assert "1 of 3 file(s) failed" in str(error) + assert " - shards/b.tar: [E2020] Server announced 0 bytes" in str(error) + + cause = error.failed_files["shards/b.tar"] + assert isinstance(cause, FileSizeMismatchError) + assert "repository lists 200000 bytes" in str(cause) + assert "Content-Length=0" in str(cause) + assert "host=127.0.0.1" in str(cause) + assert "eagleid=test-eagle" in str(cause) + + assert (out / "shards/a.tar").read_bytes() == good + assert (out / "shards/c.tar").stat().st_size == 150_000 + assert not (out / "shards/b.tar").exists() + assert not (out / "shards/b.tar.incomplete").exists() + # One retry over the default route instead of three full re-downloads. + assert len(hub.seen("shards/b.tar", "cdn")) == 2 + + def test_complete_snapshot_returns_output_dir(self, hub, api, dirs): + out, cache = dirs + files = {f"part-{i}.bin": hub.add(f"part-{i}.bin", os.urandom(30_000 + i)) for i in range(4)} + files["README.md"] = hub.add("README.md", b"# data\n", mode="direct") + + assert api.download_repo(REPO, "dataset", local_dir=out, cache_dir=cache) == out + for path, content in files.items(): + assert (out / path).read_bytes() == content + + def test_cli_exits_non_zero_when_a_file_fails(self, hub, dirs, monkeypatch): + out, cache = dirs + monkeypatch.setenv("MODELSCOPE_API_TOKEN", "") + hub.add("a.bin", os.urandom(10_000)) + hub.add("b.bin", os.urandom(10_000), served=b"") + + argv = ["download", REPO, "--repo-type", "dataset", "--local-dir", str(out), "--cache-dir", str(cache)] + assert run_cmd(["--endpoint", hub.url, *argv]) == 1 + assert (out / "a.bin").exists() + assert not (out / "b.bin").exists() + + +class TestSizeValidation: + def test_short_chunked_body_is_rejected(self, hub, api, dirs): + out, cache = dirs + hub.add("notes.txt", b"x" * 5000, mode="direct", served=b"x" * 100) + + with pytest.raises(FileSizeMismatchError, match="returned 100 of 5000 bytes"): + api.download_file(REPO, "dataset", "notes.txt", local_dir=out, cache_dir=cache, file_size=5000) + + assert not (out / "notes.txt").exists() + assert not (out / "notes.txt.incomplete").exists() + + def test_zero_byte_leftover_is_not_a_cache_hit(self, hub, api, dirs): + out, cache = dirs + content = hub.add("z.bin", os.urandom(4096)) + (out).mkdir(parents=True) + (out / "z.bin").write_bytes(b"") + + path = api.download_file(REPO, "dataset", "z.bin", local_dir=out, cache_dir=cache, file_size=len(content)) + + assert path.read_bytes() == content + + def test_size_mismatch_retries_once_without_region_routing(self, hub, api, dirs, monkeypatch): + out, cache = dirs + monkeypatch.setenv("MODELSCOPE_DOWNLOAD_INTRA_CLOUD", "true") + monkeypatch.setenv("MODELSCOPE_DOWNLOAD_INTRA_CLOUD_REGION", "cn-test") + content = hub.add("r.bin", os.urandom(8192), served_with_region=b"") + + path = api.download_file( + REPO, + "dataset", + "r.bin", + local_dir=out, + cache_dir=cache, + file_size=len(content), + expected_sha256=_sha256(content), + ) + + assert path.read_bytes() == content + assert [r.region for r in hub.seen("r.bin", "api")] == ["cn-test", None] + + def test_forced_refresh_keeps_previous_copy_on_wrong_size(self, hub, api, dirs): + out, cache = dirs + content = hub.add("keep.bin", os.urandom(2048), served=b"") + out.mkdir(parents=True) + (out / "keep.bin").write_bytes(content) + + with pytest.raises(FileSizeMismatchError): + api.download_file( + REPO, "dataset", "keep.bin", local_dir=out, cache_dir=cache, force=True, file_size=len(content) + ) + + assert (out / "keep.bin").read_bytes() == content + + +class TestHashValidation: + def test_cdn_hash_detects_empty_body_without_caller_metadata(self, hub, api, dirs): + hub.add("empty.bin", b"original", served=b"") + with pytest.raises(FileIntegrityError): + api.download_file(REPO, "dataset", "empty.bin", local_dir=dirs[0], cache_dir=dirs[1]) + assert not (dirs[0] / "empty.bin").exists() + assert len(hub.seen("empty.bin", "cdn")) == 3 + + def test_hash_mismatch_is_retried_then_leaves_nothing(self, hub, api, dirs): + out, cache = dirs + content = hub.add("w.bin", os.urandom(10_000), served=os.urandom(10_000)) + + with pytest.raises(FileIntegrityError) as excinfo: + api.download_file( + REPO, + "dataset", + "w.bin", + local_dir=out, + cache_dir=cache, + file_size=len(content), + expected_sha256=_sha256(content), + ) + + assert not isinstance(excinfo.value, FileSizeMismatchError) + assert len(hub.seen("w.bin", "cdn")) == _download.DOWNLOAD_HASH_RETRY_TIMES + assert not (out / "w.bin").exists() + assert not (out / "w.bin.incomplete").exists() + + +class TestResume: + def test_truncated_transfer_resumes_from_received_bytes(self, hub, api, dirs, monkeypatch): + out, cache = dirs + # read1 preserves partial chunks so interrupted transfers resume from the exact byte offset. + monkeypatch.setattr(_download, "DOWNLOAD_CHUNK_SIZE", 8192) + content = hub.add("big.bin", os.urandom(200_000), truncate_first=70_000) + + path = api.download_file( + REPO, + "dataset", + "big.bin", + local_dir=out, + cache_dir=cache, + file_size=len(content), + expected_sha256=_sha256(content), + ) + + assert path.read_bytes() == content + assert [r.range for r in hub.seen("big.bin", "cdn")] == [None, "bytes=70000-"] + + def test_server_ignoring_range_rewrites_instead_of_splicing(self, hub, api, dirs): + out, cache = dirs + content = hub.add("plain.txt", os.urandom(50_000), mode="direct") + out.mkdir(parents=True) + (out / "plain.txt.incomplete").write_bytes(content[:20_000]) + + path = api.download_file( + REPO, + "dataset", + "plain.txt", + local_dir=out, + cache_dir=cache, + file_size=len(content), + expected_sha256=_sha256(content), + ) + + assert path.read_bytes() == content + assert [r.range for r in hub.seen("plain.txt", "api")] == ["bytes=20000-"] + + def test_complete_partial_is_finalised_without_a_request(self, hub, api, dirs): + out, cache = dirs + content = hub.add("done.bin", os.urandom(30_000)) + out.mkdir(parents=True) + (out / "done.bin.incomplete").write_bytes(content) + + path = api.download_file( + REPO, + "dataset", + "done.bin", + local_dir=out, + cache_dir=cache, + file_size=len(content), + expected_sha256=_sha256(content), + ) + + assert path.read_bytes() == content + assert hub.requests == [] + + def test_unresumable_partial_restarts_after_416(self, hub, api, dirs): + out, cache = dirs + content = hub.add("obj.bin", os.urandom(30_000)) + out.mkdir(parents=True) + (out / "obj.bin.incomplete").write_bytes(b"\0" * len(content)) + + # Without a listed size the stale partial file can only be detected by the 416. + path = api.download_file(REPO, "dataset", "obj.bin", local_dir=out, cache_dir=cache) + + assert path.read_bytes() == content + assert [r.range for r in hub.seen("obj.bin", "cdn")] == ["bytes=30000-", None] + + +class TestParallelDownload: + @pytest.fixture(autouse=True) + def _parallel(self, monkeypatch): + monkeypatch.setattr(_download, "DOWNLOAD_PARALLELS", 4) + monkeypatch.setattr(_download, "DOWNLOAD_PARALLEL_THRESHOLD", 1024) + monkeypatch.setattr(_download, "DOWNLOAD_PART_SIZE", 64 * 1024) + + def _fetch(self, api, dirs, path: str, content: bytes) -> Path: + out, cache = dirs + return api.download_file( + REPO, + "dataset", + path, + local_dir=out, + cache_dir=cache, + file_size=len(content), + expected_sha256=_sha256(content), + ) + + def test_parts_are_fetched_by_range_and_merged(self, hub, api, dirs): + content = hub.add("p.bin", os.urandom(300_000)) + + assert self._fetch(api, dirs, "p.bin", content).read_bytes() == content + assert len(hub.seen("p.bin", "cdn")) == 5 + assert all(r.range for r in hub.seen("p.bin", "cdn")) + assert list(dirs[0].glob("p.bin_*")) == [] + + def test_truncated_part_resumes_within_its_range(self, hub, api, dirs, monkeypatch): + monkeypatch.setattr(_download, "DOWNLOAD_CHUNK_SIZE", 8192) + content = hub.add("p.bin", os.urandom(300_000), truncate_first=10_000) + + assert self._fetch(api, dirs, "p.bin", content).read_bytes() == content + starts = [int(r.range.removeprefix("bytes=").split("-")[0]) for r in hub.seen("p.bin", "cdn")] + assert any(start % (64 * 1024) == 10_000 for start in starts) + + def test_server_ignoring_range_falls_back_to_single_stream(self, hub, api, dirs): + content = hub.add("d.bin", os.urandom(300_000), mode="direct") + + assert self._fetch(api, dirs, "d.bin", content).read_bytes() == content + assert hub.seen("d.bin", "api")[-1].range is None + assert list(dirs[0].glob("d.bin_*")) == [] + + def test_part_from_object_of_another_size_is_rejected(self, hub, api, dirs): + content = hub.add("s.bin", os.urandom(300_000), served=os.urandom(100_000)) + + with pytest.raises(FileSizeMismatchError, match="reports 100000 bytes"): + self._fetch(api, dirs, "s.bin", content) + assert list(dirs[0].glob("s.bin*")) == [] + + +class TestFileDownloadError: + def test_lists_failures_and_derives_retryability(self): + transient = FileDownloadError("2 of 5 file(s) failed", failed_files={"a": NetworkError("reset")}, total=5) + mixed = FileDownloadError( + "2 of 5 file(s) failed", + failed_files={"a": NetworkError("reset"), "b": FileSizeMismatchError("short")}, + total=5, + ) + + assert str(mixed).splitlines() == [ + "[E1003] 2 of 5 file(s) failed", + " - a: [E1020] reset", + " - b: [E2020] short", + ] + assert transient.retryable is True + assert mixed.retryable is False + assert FileDownloadError("none").retryable is False + + def test_long_failure_lists_are_truncated(self): + failures = {f"f{i}": NetworkError("x") for i in range(12)} + + assert str(FileDownloadError("12 failed", failed_files=failures)).endswith(" ... and 2 more") + + +class TestBoundedDownloads: + def test_header_stall_has_one_retry_layer(self, hub, api, dirs, monkeypatch): + import modelscope_hub._legacy_api as legacy + + monkeypatch.setattr(legacy, "DOWNLOAD_TIMEOUT", 0.15) + monkeypatch.setattr(_download, "DOWNLOAD_RETRY_TIMES", 1) + hub.add("stall.bin", b"x" * 2048, header_delay=2) + started = time.monotonic() + with pytest.raises(NetworkError): + api.download_file(REPO, "dataset", "stall.bin", local_dir=dirs[0], cache_dir=dirs[1], file_size=2048) + assert len(hub.seen("stall.bin", "cdn")) == 2 + assert time.monotonic() - started < 1 + assert api.legacy._read_session.get_adapter(hub.url).max_retries.total == 0 + assert api.legacy._session.get_adapter(hub.url).max_retries.total == 5 + + @pytest.mark.parametrize("status", [429, 503]) + def test_retry_after_is_capped(self, hub, api, dirs, monkeypatch, status): + monkeypatch.setattr(_download, "DOWNLOAD_RETRY_TIMES", 1) + monkeypatch.setattr(_download, "DOWNLOAD_RETRY_MAX_DELAY_SECONDS", 0.15) + hub.add("busy.bin", b"x", status=status, retry_after="999999") + started = time.monotonic() + with pytest.raises(Exception): + api.download_file(REPO, "dataset", "busy.bin", local_dir=dirs[0], cache_dir=dirs[1]) + assert len(hub.seen("busy.bin", "cdn")) == 2 + assert time.monotonic() - started < 1 + + @pytest.mark.parametrize("headers", [False, True]) + def test_deadline_stops_trickling_headers_and_body(self, hub, api, dirs, monkeypatch, headers): + from modelscope_hub._download_control import _DownloadDeadlineError + + monkeypatch.setattr(_download, "DOWNLOAD_TOTAL_TIMEOUT", 0.35) + hub.add("slow.bin", b"x" * 100, header_trickle=headers, trickle_interval=0 if headers else 0.05) + started = time.monotonic() + with pytest.raises(_DownloadDeadlineError): + api.download_file(REPO, "dataset", "slow.bin", local_dir=dirs[0], cache_dir=dirs[1], file_size=100) + assert time.monotonic() - started < 1.2 + assert len(hub.seen("slow.bin", "cdn")) == 1 + assert not (dirs[0] / "slow.bin").exists() + + def test_minimum_rate_interrupts_a_connection_that_never_goes_idle(self, hub, api, dirs, monkeypatch): + from modelscope_hub._download_control import _DownloadTooSlowError + + monkeypatch.setattr(_download, "DOWNLOAD_LOW_SPEED_WINDOW_SECONDS", 0.25) + monkeypatch.setattr(_download, "DOWNLOAD_MIN_BYTES_PER_SECOND", 1000) + monkeypatch.setattr(_download, "DOWNLOAD_RETRY_TIMES", 0) + hub.add("slow.bin", b"x" * 100, trickle_interval=0.05) + with pytest.raises(_DownloadTooSlowError): + api.download_file(REPO, "dataset", "slow.bin", local_dir=dirs[0], cache_dir=dirs[1], file_size=100) + assert len(hub.seen("slow.bin", "cdn")) == 1 + + def test_ctrl_c_cancels_queued_and_active_downloads(self, hub, api, dirs, monkeypatch): + for i in range(10): + hub.add(f"f{i}.bin", b"x", header_delay=2) + + def interrupt(futures, **kwargs): + deadline = time.monotonic() + 1 + while len([r for r in hub.requests if r.kind == "cdn"]) < 2 and time.monotonic() < deadline: + time.sleep(0.01) + raise KeyboardInterrupt + + monkeypatch.setattr(_download, "wait", interrupt) + started = time.monotonic() + with pytest.raises(KeyboardInterrupt): + api.download_repo(REPO, "dataset", local_dir=dirs[0], cache_dir=dirs[1], max_workers=2) + assert time.monotonic() - started < 1.5 + assert len({r.path for r in hub.requests if r.kind == "api"}) == 2 + + def test_read_pool_does_not_change_global_or_proxy_connection_types(self): + from urllib3.poolmanager import pool_classes_by_scheme + + from modelscope_hub._download_control import _ReadHTTPAdapter + + original = dict(pool_classes_by_scheme) + adapter = _ReadHTTPAdapter(max_retries=0) + proxy = adapter.proxy_manager_for("http://127.0.0.1:9999") + assert pool_classes_by_scheme == original + for scheme, parent in original.items(): + pool = proxy.pool_classes_by_scheme[scheme] + assert issubclass(pool, parent) + assert issubclass(pool.ConnectionCls, parent.ConnectionCls) + adapter.close() + + def test_lock_wait_is_cancellable(self, hub, api, dirs): + from filelock import FileLock + + from modelscope_hub._download_control import _DownloadCancelledError + + stop = threading.Event() + lock_path = api.downloader._lock_path(REPO, "dataset", cache_dir=dirs[1], file_path="f.bin") + lock_path.parent.mkdir(parents=True) + with FileLock(str(lock_path)): + timer = threading.Timer(0.1, stop.set) + timer.start() + with pytest.raises(_DownloadCancelledError): + api.downloader.download_file( + REPO, + "dataset", + "f.bin", + local_dir=dirs[0], + cache_dir=dirs[1], + _cancel_event=stop, + ) + timer.join() + assert hub.requests == [] + + def test_corrupt_forced_refresh_does_not_replace_previous_valid_file(self, hub, api, dirs): + content = hub.add("f.bin", b"good", served=b"evil") + dirs[0].mkdir() + target = dirs[0] / "f.bin" + target.write_bytes(content) + with pytest.raises(FileIntegrityError): + api.download_file( + REPO, + "dataset", + "f.bin", + local_dir=dirs[0], + cache_dir=dirs[1], + force=True, + file_size=4, + expected_sha256=_sha256(content), + ) + assert target.read_bytes() == content + assert not (dirs[0] / "f.bin.incomplete").exists() + + +class TestListingOptimisation: + @pytest.mark.parametrize( + "patterns, expected", + [ + (["a/*.bin", "a/b/*.txt"], ["a"]), + (["a/*.bin", "b/f.json"], ["a", "b"]), + (["*.bin", "a/f.bin"], [None]), + ([r"a/.*\\.bin"], [None]), + (["../a/*.bin"], [None]), + ], + ) + def test_safe_roots(self, patterns, expected): + assert _download._listing_roots(patterns) == expected + + def test_scoped_listing_preserves_glob_selection(self, hub, api, dirs): + hub.add("a/good.bin", b"yes") + hub.add("a/no.txt", b"no") + hub.add("b/other.bin", b"no") + assert ( + api.download_repo( + REPO, + "dataset", + local_dir=dirs[0], + cache_dir=dirs[1], + allow_patterns=["a/*.bin"], + ) + == dirs[0] + ) + assert (dirs[0] / "a/good.bin").read_bytes() == b"yes" + assert not (dirs[0] / "b/other.bin").exists() + assert not (dirs[0] / "a/no.txt").exists() + + def test_file_list_reuse_skips_the_listing_endpoint(self, hub, api, dirs, monkeypatch): + content = hub.add("a/good.bin", b"yes") + manifest = [{"Path": "a/good.bin", "Size": 3, "Sha256": _sha256(content)}] + monkeypatch.setattr( + api.legacy, "list_dataset_files_paginated", lambda *args, **kwargs: pytest.fail("列举了清单") + ) + api.download_repo(REPO, "dataset", local_dir=dirs[0], cache_dir=dirs[1], file_list=manifest) + assert (dirs[0] / "a/good.bin").read_bytes() == content + + def test_endpoint_probe_obeys_its_total_budget(self, hub, monkeypatch): + import modelscope_hub.constants as constants + from modelscope_hub._download_control import _DownloadDeadlineError + + monkeypatch.setattr(constants, "REPO_TREE_PAGE_TIMEOUT_SECONDS", 0.2) + hub.add("metadata.bin", b"x", header_delay=2) + + def exists(probe, repo_id, repo_type): + probe.openapi.request( + "GET", + url=f"{hub.url}/cdn/metadata.bin", + require_token=False, + unwrap=False, + ) + return True + + monkeypatch.setattr(HubApi, "repo_exists", exists) + caller = HubApi(token="") + caller._config._endpoint_overridden = False + started = time.monotonic() + with pytest.raises(_DownloadDeadlineError): + caller.resolve_endpoint_for_read(REPO, repo_type="dataset") + assert time.monotonic() - started < 1 + + def test_region_detection_is_single_flight(self, api, monkeypatch): + calls = [] + + def probe(): + calls.append(1) + time.sleep(0.03) + return "cn-test" + + monkeypatch.setattr(api.downloader, "_detect_region_uncached", probe) + from concurrent.futures import ThreadPoolExecutor + + with ThreadPoolExecutor(max_workers=8) as pool: + assert list(pool.map(lambda _: api.downloader._detect_region(), range(16))) == ["cn-test"] * 16 + assert calls == [1] + + +class TestDownloadConfiguration: + def test_defaults_exports_registry_and_readme_are_aligned(self): + import modelscope_hub.constants as constants + + expected = { + "DOWNLOAD_TOTAL_TIMEOUT": 3600, + "DOWNLOAD_MIN_BYTES_PER_SECOND": 1024, + "DOWNLOAD_LOW_SPEED_WINDOW_SECONDS": 60, + "DOWNLOAD_RETRY_MAX_DELAY_SECONDS": 60, + "DOWNLOAD_FILE_LOCK_TIMEOUT_SECONDS": 600, + "REPO_TREE_PAGE_TIMEOUT_SECONDS": 15, + "REPO_TREE_LIST_TIMEOUT_SECONDS": 3600, + } + registry = {entry.name: entry.default for entry in constants.ENV_REGISTRY} + readme = (Path(__file__).parents[1] / "README.md").read_text() + for name, value in expected.items(): + assert getattr(constants, name) == value + assert name in constants.__all__ + env_name = f"MODELSCOPE_{name}" + assert registry[env_name] == str(value) + assert f"| `{env_name}` | `{value}` |" in readme + + def test_zero_can_disable_transport_retries(self, monkeypatch): + import subprocess + import sys + + monkeypatch.setenv("MODELSCOPE_DOWNLOAD_MAX_RETRIES", "0") + result = subprocess.run( + [ + sys.executable, + "-c", + "from modelscope_hub.constants import DOWNLOAD_RETRY_TIMES; print(DOWNLOAD_RETRY_TIMES)", + ], + capture_output=True, + text=True, + check=True, + ) + assert result.stdout.strip() == "0" + + +class TestDownloadCallbacks: + def test_resume_reports_each_byte_once_and_closes_callback(self, hub, api, dirs): + reports = [] + + class Callback(_download.ProgressCallback): + def __init__(self, filename, file_size): + super().__init__(filename, file_size) + self.size = 0 + self.closed = False + reports.append(self) + + def update(self, size): + self.size += size + + def end(self): + self.closed = True + + content = hub.add("resume.bin", b"x" * 5000, truncate_first=2000) + api.downloader.download_file( + REPO, + "dataset", + "resume.bin", + local_dir=dirs[0], + cache_dir=dirs[1], + file_size=5000, + expected_sha256=_sha256(content), + progress_callbacks=[Callback], + ) + assert len(reports) == 1 + assert reports[0].size == 5000 + assert reports[0].closed diff --git a/tests/test_user_agent.py b/tests/test_user_agent.py new file mode 100644 index 0000000..0e01658 --- /dev/null +++ b/tests/test_user_agent.py @@ -0,0 +1,331 @@ +"""Verify SDK UA, stable sessions and channel propagation using PreparedRequest, without remote access.""" + +from __future__ import annotations + +import importlib +import io +from argparse import Namespace +from unittest.mock import Mock + +import pytest +import requests +import responses + +from modelscope_hub import HubApi, __version__ +from modelscope_hub import config as config_module +from modelscope_hub._download_control import _DownloadControl +from modelscope_hub._legacy_api import LegacyClient +from modelscope_hub._openapi import OpenAPIClient +from modelscope_hub.agent import AgentApi +from modelscope_hub.cli.base import make_api +from modelscope_hub.cli.main import run_cmd +from modelscope_hub.compat import ( + dataset_file_download, + dataset_snapshot_download, + model_file_download, + snapshot_download, +) +from modelscope_hub.compat.hub_api import LegacyHubApi +from modelscope_hub.config import HubConfig +from modelscope_hub.constants import DEFAULT_ENDPOINT as CN +from modelscope_hub.constants import DEFAULT_INTL_ENDPOINT as AI +from modelscope_hub.errors import CacheError +from modelscope_hub.utils import append_user_agent, build_user_agent + +TOKEN = "ms-ua-test-dummy" +USER = {"success": True, "data": {"username": "alice"}} + + +@pytest.fixture(autouse=True) +def isolated_config(tmp_path, monkeypatch): + for name in ("MODELSCOPE_ENDPOINT", "MODELSCOPE_DOMAIN", "MODELSCOPE_API_TOKEN", "MODELSCOPE_PREFER_AI_SITE"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("MODELSCOPE_HOME", str(tmp_path / "home")) + monkeypatch.setenv("MODELSCOPE_CACHE", str(tmp_path / "cache")) + monkeypatch.setattr(config_module, "_default_config", None) + + +@pytest.fixture +def config(): + return HubConfig(token=TOKEN) + + +def _assert_sdk(ua): + assert ua.startswith(f"modelscope_hub/{__version__}; ") + assert "session_id/" in ua + assert "python/" in ua + assert TOKEN not in ua + return ua + + +def _session_id(ua): + return next(part.removeprefix("session_id/") for part in ua.split("; ") if part.startswith("session_id/")) + + +@pytest.mark.parametrize("extra", [None, {"application": "example"}, "example-client/1"]) +def test_build_user_agent_config_and_extra(config, extra): + first = _assert_sdk(build_user_agent(config=config, extra=extra)) + assert first == build_user_agent(config=config, extra=extra) + assert _session_id(first) == config.get_session_id() + if extra: + assert ("application/example" if isinstance(extra, dict) else extra) in first + assert append_user_agent(first, extra) == first + + +@pytest.mark.parametrize("failure", [CacheError("read-only"), PermissionError("read-only"), OSError("read-only")]) +@responses.activate +def test_readonly_home_keeps_requests_and_session_stable(config, monkeypatch, failure): + monkeypatch.setattr(HubConfig, "get_session_id", Mock(side_effect=failure)) + responses.add(responses.GET, f"{CN}/openapi/v1/users/me", json=USER) + api = HubApi(config=config) + api.whoami() + api.whoami() + uas = [call.request.headers["User-Agent"] for call in responses.calls] + uas.extend([api.legacy._user_agent, api.downloader._build_user_agent(), OpenAPIClient(config)._user_agent]) + assert len({_session_id(_assert_sdk(ua)) for ua in uas}) == 1 + assert not config.credentials_dir.exists() + + +def test_hubapi_constructor_remains_lazy(config, monkeypatch): + lookup = Mock(side_effect=AssertionError("不应提前访问用户目录")) + monkeypatch.setattr(HubConfig, "get_session_id", lookup) + HubApi(config=config, user_agent={"application": "example"}) + lookup.assert_not_called() + + +@pytest.mark.parametrize("read_session", [False, True]) +@pytest.mark.parametrize("method", ["GET", "POST", "PUT", "PATCH", "DELETE"]) +@responses.activate +def test_all_http_methods_and_both_sessions(config, method, read_session): + url = f"{CN}/openapi/v1/ua-test" + responses.add(method, url, json={"success": True, "data": {}}) + with OpenAPIClient(config, user_agent={"application": "example"}) as client: + if read_session: + with _DownloadControl(total_timeout=5) as control, client.read_context(control): + client.request(method, "/ua-test") + else: + client.request(method, "/ua-test") + ua = _assert_sdk(responses.calls[0].request.headers["User-Agent"]) + assert "application/example" in ua + assert "python-requests/" not in ua + + +@pytest.mark.parametrize("name", ["User-Agent", "user-agent", "USER-AGENT"]) +@pytest.mark.parametrize("value", [None, "", "other-client/2"]) +@responses.activate +def test_custom_header_is_appended_case_insensitively(config, name, value): + responses.add(responses.GET, f"{CN}/openapi/v1/users/me", json=USER) + headers = {name: value, "X-Example": "preserved"} + OpenAPIClient(config).request("GET", "/users/me", headers=headers) + prepared = responses.calls[0].request.headers + ua = _assert_sdk(prepared["User-Agent"]) + assert sum(key.lower() == "user-agent" for key in prepared) == 1 + assert prepared["X-Example"] == "preserved" + assert headers == {name: value, "X-Example": "preserved"} + if value: + assert value in ua + + +@pytest.mark.parametrize("session_ua", [None, "", "custom-session/1", requests.utils.default_user_agent()]) +@responses.activate +def test_injected_session_has_sdk_ua_without_mutating_headers(config, session_ua): + session = requests.Session() + if session_ua is None: + session.headers.pop("User-Agent", None) + else: + session.headers["User-Agent"] = session_ua + original = dict(session.headers) + responses.add(responses.GET, f"{CN}/openapi/v1/users/me", json=USER) + with OpenAPIClient(config, session=session) as client: + client.get_current_user() + with _DownloadControl(total_timeout=5) as control, client.read_context(control): + client.get_current_user() + uas = [_assert_sdk(call.request.headers["User-Agent"]) for call in responses.calls] + assert uas[0] == uas[1] + assert dict(session.headers) == original + if session_ua == "custom-session/1": + assert session_ua in uas[0] + + +@pytest.mark.parametrize("status", [429, 503]) +@responses.activate +def test_retry_ua_is_stable(config, monkeypatch, status): + sleeps = Mock() + monkeypatch.setattr("modelscope_hub._openapi.time.sleep", sleeps) + url = f"{CN}/openapi/v1/users/me" + responses.add(responses.GET, url, json={"message": "retry"}, status=status, headers={"Retry-After": "2"}) + responses.add(responses.GET, url, json=USER) + with OpenAPIClient(config, max_retries=2) as client: + client.get_current_user() + assert len(responses.calls) == 2 + assert ( + _assert_sdk(responses.calls[0].request.headers["User-Agent"]) + == responses.calls[1].request.headers["User-Agent"] + ) + sleeps.assert_called_once() + if status == 429: + sleeps.assert_called_with(2.0) + + +@responses.activate +def test_anonymous_fallback_keeps_ua_and_drops_auth(config): + url = f"{CN}/openapi/v1/studios" + responses.add(responses.GET, url, json={"message": "forbidden"}, status=403) + responses.add(responses.GET, url, json={"studios": []}) + OpenAPIClient(config).list_studios() + first, second = [call.request.headers for call in responses.calls] + assert _assert_sdk(first["User-Agent"]) == second["User-Agent"] + assert "Authorization" in first and "Cookie" in first + assert "Authorization" not in second and "Cookie" not in second + + +@pytest.mark.parametrize("path", ["/agent_id/.well-known/agentid-configuration", "/agent_id/.well-known/agentid-jwks"]) +@responses.activate +def test_anonymous_idp_metadata_has_ua(config, path): + responses.add(responses.GET, f"{CN}/openapi/v1{path}", json={"data": {}}) + client = OpenAPIClient(config) + if path.endswith("agentid-jwks"): + client.get_agent_id_jwks() + else: + client.get_agent_id_configuration() + headers = responses.calls[0].request.headers + _assert_sdk(headers["User-Agent"]) + assert "Authorization" not in headers and "Cookie" not in headers + + +@responses.activate +def test_multipart_absolute_and_raw_response_have_ua(config): + responses.add(responses.POST, f"{CN}/openapi/v1/files/upload", json={"data": {"id": "f"}}) + responses.add(responses.PUT, "https://storage.example.test/blob", status=204) + with OpenAPIClient(config) as client: + client.upload_file(file=io.BytesIO(b"x")) + response = client.request("PUT", url="https://storage.example.test/blob", data=b"x", unwrap=False) + assert response.status_code == 204 + assert "multipart/form-data" in responses.calls[0].request.headers["Content-Type"] + for call in responses.calls: + _assert_sdk(call.request.headers["User-Agent"]) + assert "Authorization" not in responses.calls[1].request.headers + assert "Cookie" not in responses.calls[1].request.headers + + +@responses.activate +def test_login_and_scoped_probe_share_channel_and_session(config): + responses.add(responses.POST, f"{CN}/api/v1/login", json={"message": "forbidden"}, status=403) + responses.add(responses.GET, f"{CN}/openapi/v1/users/me", json=USER) + assert HubApi(config=config, user_agent={"invoked_by": "cli"}).login(TOKEN).username == "alice" + assert len(responses.calls) == 2 + uas = [_assert_sdk(call.request.headers["User-Agent"]) for call in responses.calls] + assert uas[0] == uas[1] + assert "invoked_by/cli" in uas[0] + + +@responses.activate +def test_peer_login_probe_has_sdk_ua(): + responses.add(responses.POST, f"{AI}/api/v1/login", json={"Code": 200, "Data": {"Username": "alice"}}) + assert HubApi._token_valid_on(TOKEN, AI) + _assert_sdk(responses.calls[0].request.headers["User-Agent"]) + + +@responses.activate +def test_read_site_probes_keep_custom_config_channel_and_session(config): + responses.add(responses.GET, f"{CN}/openapi/v1/models/alice/m", status=404) + responses.add(responses.GET, f"{CN}/api/v1/models/alice/m", status=404) + responses.add(responses.GET, f"{AI}/openapi/v1/models/alice/m", json={"data": {"name": "m"}}) + api = HubApi(config=config, user_agent={"invoked_by": "compat", "application": "example"}) + assert api.resolve_endpoint_for_read("alice/m") == AI + assert len(responses.calls) == 3 + uas = [_assert_sdk(call.request.headers["User-Agent"]) for call in responses.calls] + assert len(set(uas)) == 1 + assert _session_id(uas[0]) == config.get_session_id() + assert "invoked_by/compat" in uas[0] and "application/example" in uas[0] + + +@responses.activate +def test_agent_openapi_legacy_and_blob_have_sdk_ua(config): + responses.add(responses.GET, f"{CN}/openapi/v1/agents/alice/a", json={"data": {}}) + responses.add(responses.GET, f"{CN}/api/v1/agents/alice/a/repo/files", json={"data": {"Trees": []}}) + responses.add(responses.PUT, "https://storage.example.test/blob", status=204) + api = AgentApi(config=config, user_agent={"invoked_by": "cli"}) + api.repo_info("alice", "a") + api.list_repo_files("alice", "a") + api.lfs_upload_blob("https://storage.example.test/blob", b"x") + for call in responses.calls: + assert "invoked_by/cli" in _assert_sdk(call.request.headers["User-Agent"]) + + +@pytest.mark.parametrize("read_session", [False, True]) +@responses.activate +def test_legacy_default_custom_headers_and_blob(config, read_session): + responses.add(responses.GET, f"{CN}/api/v1/models/alice/m", json={"Data": {}}) + responses.add(responses.PUT, f"{CN}/blob", json={}) + client = LegacyClient(token=TOKEN, endpoint=CN) + if read_session: + with _DownloadControl(total_timeout=5) as control, client.read_context(control): + client._request("GET", "models/alice/m", headers={"user-agent": "custom-read/1"}) + else: + client._request("GET", "models/alice/m", headers={"USER-AGENT": "custom-read/1"}) + client.upload_blob(f"{CN}/blob", b"x", size=1, headers={"user-agent": "custom-blob/1"}) + assert "custom-read/1" in _assert_sdk(responses.calls[0].request.headers["User-Agent"]) + assert "custom-blob/1" in _assert_sdk(responses.calls[1].request.headers["User-Agent"]) + + +@responses.activate +def test_cli_whoami_and_explicit_download_endpoint_have_channel(): + responses.add(responses.GET, f"{CN}/openapi/v1/users/me", json=USER) + assert run_cmd(["--token", TOKEN, "whoami"]) == 0 + assert "invoked_by/cli" in _assert_sdk(responses.calls[0].request.headers["User-Agent"]) + from modelscope_hub.cli.download import DownloadCommand + + command = DownloadCommand(Namespace(endpoint=CN, token=TOKEN)) + assert "invoked_by/cli" in _assert_sdk(command._make_api_with_endpoint().openapi._user_agent) + assert "invoked_by/cli" in make_api(Namespace(token=TOKEN)).legacy._user_agent + + +@responses.activate +def test_cli_agent_username_and_repo_calls_have_channel(monkeypatch): + cli_agent = importlib.import_module("modelscope_hub.cli.agent") + responses.add(responses.GET, f"{CN}/openapi/v1/users/me", json=USER) + responses.add(responses.PUT, f"{CN}/api/v1/dolphin/agents", json={"Data": {"AgentList": []}}) + + def list_after_username(**kwargs): + return cli_agent._cmd_list(owner="alice", page_number=1, page_size=10, endpoint=CN, token=TOKEN) + + monkeypatch.setattr(cli_agent, "_cmd_upload", list_after_username) + assert run_cmd(["--token", TOKEN, "agent", "upload", "--repo", "alice/a", "--local-dir", "."]) == 0 + assert responses.calls + for call in responses.calls: + assert "invoked_by/cli" in _assert_sdk(call.request.headers["User-Agent"]) + + +@responses.activate +def test_compat_default_and_override_clients_have_channel(): + responses.add(responses.GET, f"{CN}/openapi/v1/models/alice/m", json={"data": {"name": "m"}}) + api = LegacyHubApi(endpoint=CN, token=TOKEN) + api.repo_exists("alice/m", repo_type="model") + api.repo_exists("alice/m", repo_type="model", token="other-test-token") + for call in responses.calls: + assert "invoked_by/compat" in _assert_sdk(call.request.headers["User-Agent"]) + + +@pytest.mark.parametrize( + "download", [model_file_download, dataset_file_download, snapshot_download, dataset_snapshot_download] +) +def test_compat_download_user_agent_applies_to_all_transports(download, monkeypatch): + seen = [] + + def fake_download(self, *args, **kwargs): + ua = self.openapi._user_agent + assert "invoked_by/compat" in _assert_sdk(ua) and "application/example" in ua + assert self.legacy._user_agent == ua + assert self.downloader._build_user_agent(kwargs["user_agent"]) == ua + seen.append(ua) + return "local-path" + + monkeypatch.setattr(HubApi, "download_file", fake_download) + monkeypatch.setattr(HubApi, "download_repo", fake_download) + kwargs = {"local_files_only": True, "user_agent": {"application": "example"}} + if download in (model_file_download, dataset_file_download): + assert download("alice/m", "x", **kwargs) == "local-path" + else: + assert download("alice/m", **kwargs) == "local-path" + assert len(seen) == 1 From d9de27fdeb2d72d3f5c0ac55668d1b355265697b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Fri, 23 Oct 2026 18:30:20 +0800 Subject: [PATCH 17/22] bump version --- src/modelscope_hub/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modelscope_hub/version.py b/src/modelscope_hub/version.py index a59471b..8844a57 100644 --- a/src/modelscope_hub/version.py +++ b/src/modelscope_hub/version.py @@ -1,3 +1,3 @@ """Version information for modelscope_hub.""" -__version__ = "0.4.6+main" +__version__ = "0.5.0+main" From 3610b5949d1a0b15ee54e9a4ffbf743cea74990d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Mon, 26 Oct 2026 19:42:20 +0800 Subject: [PATCH 18/22] Continuous Downloads & Lightweight Recovery --- AGENTS.md | 1 + README.md | 34 ++ src/modelscope_hub/__init__.py | 6 + src/modelscope_hub/_download.py | 79 ++- src/modelscope_hub/_download_control.py | 84 ++- src/modelscope_hub/_download_state.py | 261 +++++++++ src/modelscope_hub/_legacy_api.py | 70 ++- src/modelscope_hub/_resilient_download.py | 361 ++++++++++++ src/modelscope_hub/api.py | 58 ++ src/modelscope_hub/cli/download.py | 35 ++ .../compat/snapshot_download.py | 58 +- src/modelscope_hub/types.py | 132 ++++- tests/download_corruption_harness.py | 473 ++++++++++++++++ tests/test_download_corruption.py | 38 ++ tests/test_download_lock_path.py | 9 +- tests/test_download_report.py | 526 ++++++++++++++++++ 16 files changed, 2206 insertions(+), 19 deletions(-) create mode 100644 src/modelscope_hub/_download_state.py create mode 100644 src/modelscope_hub/_resilient_download.py create mode 100644 tests/download_corruption_harness.py create mode 100644 tests/test_download_corruption.py create mode 100644 tests/test_download_report.py diff --git a/AGENTS.md b/AGENTS.md index 995d2e3..6192267 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,6 +14,7 @@ This document defines repository-level engineering conventions that apply to eve 4. Type checks: `mypy src/modelscope_hub/` - Never commit secrets. `~/.modelscope/credentials/` is private (it contains a pickled cookie jar and `m_session_id`); never expose its contents in logs or reports. - When changing the default value of a configuration constant, update the corresponding tests for its default value and registration table, as well as the environment-variable table in `README.md`. +- Local SDK state must use lightweight files, without embedded database modules or packages. Keep recovery hints sparse and advisory: verify actual files against available size/hash metadata before trusting completion or stale failure records. Missing, corrupt, or unwritable hints must not imply success or prevent valid files from being reused. Each download task must reuse one hidden JSON for sparse recovery hints and its latest summary, without persistent per-call report or result files; healthy files must not trigger per-file state rewrites. --- diff --git a/README.md b/README.md index 12be847..7b0922f 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,40 @@ path = api.download_file("Qwen/Qwen3-0.6B", "model", "config.json", local_files_ Downloads validate available size and SHA-256 metadata before replacing files, with safe resume, bounded retries, timeouts, and cancellation. Incomplete snapshots raise `FileDownloadError` (CLI exit code `1`), preserving completed files for subsequent retries. +### Continuous dataset downloads + +For migration jobs, `download_repo_report` continues past isolated file failures and returns an explicit `COMPLETE` or `PARTIAL` report. The original `download_repo` and `snapshot_download` remain strict by default. + +```python +from modelscope_hub import HubApi, DownloadRetryPolicy + +api = HubApi(endpoint="https://www.modelscope.ai") +report = api.download_repo_report( + "owner/dataset", "dataset", revision="master", local_dir="./data", + tracker_path="./.ms_download_state.json", max_workers=8, + retry_policy=DownloadRetryPolicy(), +) +for entry in report.iter_files("VERIFIED"): + consume_verified_file(entry.path) +print(report.status, report.succeeded, report.failed, report.deferred, report.report_path) +report.close() # Release temporary per-file results after consumption. +``` + +- Recovery defaults: at most **2 extra retries per file**, shared by range parts and integrity recovery; failed-request inactivity and waits have a **60-second cumulative budget**. Idle/header waits are limited to **15 seconds**, low-speed checks use a **10-second window**, and cooldowns longer than **5 seconds** defer the file instead of tying up a worker. Size/hash mismatches get at most one full re-download. Healthy transfer time still uses `MODELSCOPE_DOWNLOAD_TOTAL_TIMEOUT`; large files are not capped at 60 seconds. +- Each task uses **one hidden JSON file** outside downloaded contents (`cache_dir/.download_state/.ms_download_.json` by default), with no embedded database dependency. It combines sparse failures, recovery budgets, failed pages, and the latest call's summary under `report`; `report.report_path` equals `report.tracker_path`. Reruns and batches replace the same file, without extra report JSON or JSONL files. Successful files have no persistent per-file records; a healthy run writes only its scope header and final summary, not an update per file. Its scope is endpoint/repository/revision/output, independent of batch filters. Standard reusable lock files remain separate from JSON metadata. +- State is advisory, never the sole evidence of success. Every rerun validates actual files against current size/hash metadata, even if hints say `FAILED` or `DEFERRED`; a repaired valid file clears its stale hint without network IO. Otherwise failures remain isolated until metadata changes or `retry_failed=True`. Missing or corrupt hints allow bounded fresh recovery; load/write problems are logged and exposed in `report.state_warnings`, not treated as data corruption or success. A crash during recovery preserves its budget if hints were saved; healthy in-flight files need no markers and resume from actual partial files. Old database-format trackers are not read or migrated: select a new JSON path; existing correct data is reused through validation. +- Dataset pages are streamed with bounded queues. Failed pages are recorded and isolated across calls; later pages/roots can continue. Two consecutive failed pages stop that root, and `listing_complete=False` prevents false success. The next run can use `retry_failed=True` or an authoritative `file_list` iterable to recover incomplete enumeration. Model listings that hit an unrecoverable server cap also return a partial report. +- Counts and `report.iter_files()` cover **this call's selection**. Per-file results stream from an anonymous temporary spool owned by the live report, so earlier report objects remain independent of later batches without permanent manifests or an in-memory list of every file. Call `report.close()` after consumption; garbage collection also releases its spool. `results_path` is empty. The single JSON retains only the latest summary, not historical reports or a verified-file manifest: reconstructing a report from it cannot provide per-file results; rerun actual-file validation instead. Aggregate batch counts for the whole migration. Only `VERIFIED` entries may be uploaded; changed files must be checked again before use, and partial reports must not authorize deleting source data. A pinned revision or authoritative manifest is recommended. Reuse the same tracker and output across ten-file batches; a tracker cannot be shared by simultaneous writers. +- Credential loss or a full/read-only disk pauses further work and sets `global_error`; Ctrl+C writes a `CANCELLED` report and remains interruptible. DNS and blocked filesystem calls remain OS-constrained. Caller-provided iterables must also be responsive to cancellation. + +```bash +ms-hub download owner/dataset --repo-type dataset --local-dir ./data --continue-on-error +# Explicit repair run after the source is fixed +ms-hub download owner/dataset --repo-type dataset --local-dir ./data --continue-on-error --retry-failed +``` + +CLI exit codes: `0` complete, `3` partial after remaining known files have been processed, `1` global failure, `130` interrupted. Schedulers must treat `3` as partial completion, not an instruction to retry the whole task indefinitely. The SDK's `modelscope_hub.compat.snapshot_download(..., continue_on_error=True, tracker_path=...)` preserves a string return path and logs the report location; use `HubApi.download_repo_report` to read results directly. Older `modelscope.hub.snapshot_download` wrappers may not accept the new options: import the compat function from `modelscope_hub` directly until the wrapper is updated. + ### Upload ```bash diff --git a/src/modelscope_hub/__init__.py b/src/modelscope_hub/__init__.py index 4db8011..b6dc9c4 100644 --- a/src/modelscope_hub/__init__.py +++ b/src/modelscope_hub/__init__.py @@ -55,6 +55,9 @@ CacheInfo, CacheVerification, CommitInfo, + DownloadFileResult, + DownloadReport, + DownloadRetryPolicy, FileInfo, PagedResult, RepoInfo, @@ -97,6 +100,9 @@ "CacheVerification", "CachedRepoInfo", "CommitInfo", + "DownloadFileResult", + "DownloadReport", + "DownloadRetryPolicy", "FileInfo", "PagedResult", "RepoInfo", diff --git a/src/modelscope_hub/_download.py b/src/modelscope_hub/_download.py index c1becfa..900e9ea 100644 --- a/src/modelscope_hub/_download.py +++ b/src/modelscope_hub/_download.py @@ -27,7 +27,7 @@ import threading import time import uuid -from collections.abc import Callable +from collections.abc import Callable, Iterable from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait from pathlib import Path from typing import TYPE_CHECKING @@ -44,6 +44,7 @@ _DownloadDeadlineError, _network_context, _ReadHTTPAdapter, + _RecoveryBudget, ) from .constants import ( DOWNLOAD_CHUNK_SIZE, @@ -82,6 +83,7 @@ if TYPE_CHECKING: from ._legacy_api import LegacyClient from .config import HubConfig + from .types import DownloadReport, DownloadRetryPolicy logger = get_logger("download") @@ -175,8 +177,11 @@ def _optional_file_lock( while True: if control is not None: control.check() - if time.monotonic() - started >= DOWNLOAD_FILE_LOCK_TIMEOUT_SECONDS: - raise _DownloadDeadlineError(f"等待下载文件锁超过 {DOWNLOAD_FILE_LOCK_TIMEOUT_SECONDS}s：{lock_path}") + lock_limit = float(DOWNLOAD_FILE_LOCK_TIMEOUT_SECONDS) + if control.recovery is not None: + lock_limit = min(lock_limit, control.recovery.policy.idle_timeout) + if time.monotonic() - started >= lock_limit: + raise _DownloadDeadlineError(f"Download file lock timed out after {lock_limit}s: {lock_path}") try: lock.acquire(timeout=default_interval) except Timeout: @@ -342,6 +347,8 @@ def _incomplete_path(target: Path) -> Path: def _wait_download_retry(retry: Retry, exc: Exception, path: str, control: _DownloadControl) -> Retry: control.check() + if isinstance(exc, OSError) and exc.errno in (errno.ENOSPC, errno.EROFS, errno.EACCES): + raise exc if isinstance(exc, HubError) and not exc.retryable: raise exc try: @@ -351,6 +358,15 @@ def _wait_download_retry(retry: Retry, exc: Exception, path: str, control: _Down raise exc raise NetworkError(f"下载 {path} 在 {DOWNLOAD_RETRY_TIMES} 次重试后失败：{type(exc).__name__}") from exc requested = getattr(exc, "retry_after", None) + if control.recovery is not None: + delay = max(0, requested) if requested is not None else min(retry.get_backoff_time(), 2) + try: + control.recovery.retry(delay) + except Exception as budget_error: + raise budget_error from exc + logger.warning("Download recovery for %s (%s), retrying in %ss", path, type(exc).__name__, delay) + control.sleep(delay) + return retry delay = min( max(0, requested) if requested is not None else retry.get_backoff_time(), DOWNLOAD_RETRY_MAX_DELAY_SECONDS ) @@ -855,13 +871,17 @@ def download_file( user_agent: dict | str | None = None, progress_callbacks: list[type[ProgressCallback]] | None = None, _cancel_event: threading.Event | None = None, + _recovery: _RecoveryBudget | None = None, ) -> Path: """Download one file with cache-first lookup and a shared budget for locks, probes, and retries.""" with _DownloadControl( total_timeout=DOWNLOAD_TOTAL_TIMEOUT, cancel_event=_cancel_event, min_rate=DOWNLOAD_MIN_BYTES_PER_SECOND, - speed_window=DOWNLOAD_LOW_SPEED_WINDOW_SECONDS, + speed_window=_recovery.policy.low_speed_window + if _recovery is not None + else DOWNLOAD_LOW_SPEED_WINDOW_SECONDS, + recovery=_recovery, ) as control: with self._client.read_context(control): return self._download_file( @@ -992,7 +1012,12 @@ def _download_file( # the server consistently served the wrong object, so it only gets # one more attempt, over the default route. failures = {"hash": 0, "size": 0} - limits = {"hash": DOWNLOAD_HASH_RETRY_TIMES, "size": DOWNLOAD_SIZE_RETRY_TIMES} + limits = { + "hash": min(DOWNLOAD_HASH_RETRY_TIMES, 2) + if _control.recovery is not None + else DOWNLOAD_HASH_RETRY_TIMES, + "size": DOWNLOAD_SIZE_RETRY_TIMES, + } prefer_default_route = False while True: _control.check() @@ -1032,10 +1057,54 @@ def _download_file( limits[kind], exc.message, ) + if _control.recovery is not None: + try: + _control.recovery.retry(delay) + except Exception as budget_error: + raise budget_error from exc _control.sleep(delay) return target + def download_repo_report( + self, + repo_id: str, + repo_type: str, + *, + revision: str | None = None, + cache_dir: Path | None = None, + local_dir: Path | None = None, + allow_patterns: list[str] | None = None, + ignore_patterns: list[str] | None = None, + max_workers: int = 4, + file_list: Iterable[dict] | None = None, + tracker_path: Path | None = None, + retry_failed: bool = False, + retry_policy: DownloadRetryPolicy | None = None, + user_agent: dict | str | None = None, + progress_callbacks: list[type[ProgressCallback]] | None = None, + ) -> DownloadReport: + """Continue after isolated file failures and return a durable completion report.""" + from ._resilient_download import download_repo_report + + return download_repo_report( + self, + repo_id, + repo_type, + revision=revision, + cache_dir=cache_dir, + local_dir=local_dir, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + max_workers=max_workers, + file_list=file_list, + tracker_path=tracker_path, + retry_failed=retry_failed, + retry_policy=retry_policy, + user_agent=user_agent, + progress_callbacks=progress_callbacks, + ) + def download_repo( self, repo_id: str, diff --git a/src/modelscope_hub/_download_control.py b/src/modelscope_hub/_download_control.py index f98b120..8557ce2 100644 --- a/src/modelscope_hub/_download_control.py +++ b/src/modelscope_hub/_download_control.py @@ -6,9 +6,9 @@ import socket import threading import time -from collections.abc import Iterator +from collections.abc import Callable, Iterator from dataclasses import dataclass -from typing import Any +from typing import TYPE_CHECKING, Any import requests from requests.adapters import HTTPAdapter @@ -16,6 +16,50 @@ from .errors import NetworkError, RequestTimeoutError +if TYPE_CHECKING: + from .types import DownloadRetryPolicy + + +class _RecoveryBudget: + """One durable recovery budget per file, shared by range and integrity retries.""" + + def __init__( + self, + policy: DownloadRetryPolicy, + retries: int, + seconds: float, + persist: Callable[[int, float], None], + ) -> None: + self.policy = policy + self.retries = retries + self.seconds = seconds + self.persist = persist + self.lock = threading.Lock() + self.activity = threading.local() + + def progress(self) -> None: + self.activity.last = time.monotonic() + + def retry(self, delay: float) -> None: + with self.lock: + elapsed = max(0, time.monotonic() - getattr(self.activity, "last", time.monotonic())) + self.seconds += elapsed + self.persist(self.retries, self.seconds) + if delay > self.policy.max_retry_delay: + raise _DownloadDeferredError("Server cooldown exceeds the short recovery budget") + if self.retries >= self.policy.max_retries or self.seconds + delay >= self.policy.recovery_timeout: + raise _DownloadDeferredError("File recovery budget exhausted; automatic retry is disabled") + self.retries += 1 + self.seconds += delay + self.persist(self.retries, self.seconds) + self.progress() + + +class _DownloadDeferredError(NetworkError): + """Recovery is exhausted or a server cooldown requires a later explicit run.""" + + retryable = False + class _DownloadCancelledError(Exception): """The parent download was cancelled; do not retry or report a remote failure.""" @@ -38,6 +82,7 @@ class _WatchedResponse: received: int = 0 error: Exception | None = None monitor_speed: bool = True + header_deadline: float | None = None def _response_socket(response: requests.Response) -> Any: @@ -62,11 +107,13 @@ def __init__( cancel_event: threading.Event | None = None, min_rate: int = 0, speed_window: float = 60, + recovery: _RecoveryBudget | None = None, ) -> None: self.cancel_event = cancel_event if cancel_event is not None else threading.Event() self.deadline = time.monotonic() + total_timeout self.min_rate = min_rate self.speed_window = speed_window + self.recovery = recovery self._responses: dict[requests.Response, _WatchedResponse] = {} self._transports: dict[Any, _WatchedResponse] = {} self._lock = threading.Lock() @@ -95,6 +142,12 @@ def check(self) -> None: def timeouts(self, connect: float, read: float) -> tuple[float, float]: self.check() remaining = max(0.001, self.deadline - time.monotonic()) + if self.recovery is not None: + self.recovery.progress() + with self.recovery.lock: + remaining = min(remaining, max(0.001, self.recovery.policy.recovery_timeout - self.recovery.seconds)) + connect = min(connect, self.recovery.policy.idle_timeout) + read = min(read, self.recovery.policy.idle_timeout) return min(connect, remaining), min(read, remaining) def sleep(self, seconds: float) -> None: @@ -118,6 +171,9 @@ def transport(self, sock: Any) -> Iterator[None]: """Interrupt sockets while reading headers so trickling headers cannot bypass deadlines.""" self.check() state = _WatchedResponse(sock, time.monotonic(), monitor_speed=False) + if self.recovery is not None: + started = getattr(self.recovery.activity, "last", time.monotonic()) + state.header_deadline = min(self.deadline, started + self.recovery.policy.idle_timeout) with self._lock: self._transports[sock] = state self._start_monitor() @@ -148,6 +204,8 @@ def _check_response(self, response: requests.Response, received: int = 0) -> Non state = self._responses.get(response) if state is not None: state.received += received + if received and self.recovery is not None: + self.recovery.progress() if state.error is not None: raise state.error @@ -165,6 +223,8 @@ def _watch(self) -> None: state.error = _DownloadCancelledError("下载已取消") elif now >= self.deadline: state.error = _DownloadDeadlineError("下载总时限已耗尽") + elif state.header_deadline is not None and now >= state.header_deadline: + state.error = RequestTimeoutError("Response headers exceeded the short recovery time limit") elif state.monitor_speed and self.min_rate and now - state.started >= self.speed_window: rate = state.received / (now - state.started) if rate < self.min_rate: @@ -270,6 +330,26 @@ def _configure(manager: Any) -> None: ) manager.pool_classes_by_scheme = classes + def build_response(self, req: Any, resp: Any) -> requests.Response: + response = super().build_response(req, resp) + control = getattr(_network_state, "control", None) + if control is not None: + # Requests consumes intermediate redirect bodies outside chunks(). + # Track them here, and release before their connection can be reused. + original_close = response.close + + def close() -> None: + control.release(response) + original_close() + + response.close = close # type: ignore[method-assign] + try: + control.track(response) + except BaseException: + response.close() + raise + return response + def init_poolmanager(self, connections: int, maxsize: int, block: bool = False, **kwargs: Any) -> None: super().init_poolmanager(connections, maxsize, block=block, **kwargs) self._configure(self.poolmanager) diff --git a/src/modelscope_hub/_download_state.py b/src/modelscope_hub/_download_state.py new file mode 100644 index 0000000..48bf953 --- /dev/null +++ b/src/modelscope_hub/_download_state.py @@ -0,0 +1,261 @@ +"""Durable report-mode state; kept outside downloaded repository contents.""" + +from __future__ import annotations + +import contextlib +import hashlib +import json +import math +import re +import tempfile +import threading +from dataclasses import asdict, replace +from pathlib import Path +from typing import TextIO +from urllib.parse import urlsplit, urlunsplit + +from .errors import InvalidParameter +from .types import DownloadFileResult, DownloadReport +from .utils.logger import get_logger + +logger = get_logger("download") + + +def safe_error(error: BaseException) -> str: + """Keep diagnostics without persisting credentials or presigned queries.""" + text = str(error) + text = re.sub( + r"https?://[^\s)]+", + lambda m: urlunsplit(urlsplit(m[0])._replace(query="", fragment="", netloc=urlsplit(m[0]).hostname or "")), + text, + ) + text = re.sub(r"(?i)(token|signature|authorization|cookie|password|secret)\s*[=:]\s*[^\s,;]+", r"\1=***", text) + return text[:2000] + + +class _DownloadState: + """Sparse JSON recovery hints, never an authority for file completeness. + + Persist exceptional files, failed pages, and the latest summary together. + Per-file results stay in an anonymous spool owned by the returned report; + no permanent JSONL or per-run summary files are created. Healthy transfers + do not rewrite the tracker; actual partial files suffice after a crash. + """ + + def __init__(self, path: Path, identity: dict[str, str]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + self.path = path + self.identity = dict(identity) + self.lock = threading.Lock() + self._exceptions: dict[str, DownloadFileResult] = {} + self._pages: dict[str, dict[int, str]] = {} + self._active: dict[str, DownloadFileResult] = {} + self._seen: set[bytes] = set() + self._run: str | None = None + self._spool: TextIO | None = tempfile.TemporaryFile(mode="w+t", encoding="utf-8") + self._counts: dict[str, int] = {} + self._summary: dict | None = None + self.warnings: list[str] = [] + try: + self._load() + if not path.exists(): + self._save() + except BaseException: + self.close() + raise + + def _warn(self, message: str) -> None: + if message not in self.warnings and len(self.warnings) < 10: + self.warnings.append(message) + logger.warning("%s: %s", message, self.path) + + def _load(self) -> None: + if not self.path.exists(): + return + try: + data = json.loads(self.path.read_text(encoding="utf-8")) + if not isinstance(data, dict) or data.get("version") != 1 or not isinstance(data.get("identity"), dict): + raise ValueError("Invalid tracker header") + if data["identity"] != self.identity: + raise InvalidParameter( + "Download tracker belongs to a different endpoint/repository/revision/output directory" + ) + files, pages = data.get("files", {}), data.get("pages", {}) + if not isinstance(files, dict) or not isinstance(pages, dict): + raise ValueError("Invalid recovery records") + for path, raw in files.items(): + try: + if not isinstance(raw, dict) or raw.get("path") != path: + raise ValueError("Invalid file record") + result = DownloadFileResult.from_dict(raw) + if result.status not in ("FAILED", "DEFERRED", "DOWNLOADING", "PENDING"): + raise ValueError("Not a recovery hint") + if ( + not isinstance(result.retries, int) + or result.retries < 0 + or not isinstance(result.recovery_seconds, (int, float)) + or not math.isfinite(result.recovery_seconds) + or result.recovery_seconds < 0 + ): + raise ValueError("Invalid recovery budget") + if result.status == "DOWNLOADING": + result.status = "DEFERRED" + result.error = "Previous process stopped during recovery" + result.error_type = "InterruptedDownload" + self._exceptions[path] = result + except (ValueError, TypeError): + self._warn("Ignoring malformed file recovery hints; actual files will be revalidated") + for root, records in pages.items(): + try: + if not isinstance(records, dict): + raise ValueError("Invalid page records") + self._pages[root] = { + int(page): error + for page, error in records.items() + if str(page).isdigit() and int(page) > 0 and isinstance(error, str) + } + except (ValueError, TypeError): + self._warn("Ignoring malformed listing recovery hints") + except InvalidParameter: + raise + except (ValueError, UnicodeError, OSError): + self._exceptions.clear() + self._pages.clear() + self._warn( + "Cannot load download recovery hints; continuing with actual-file validation and bounded retries" + ) + + def _save(self) -> None: + """Atomically replace sparse hints; write failures do not invalidate data.""" + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=self.path.parent, + prefix=self.path.name + ".", + suffix=".tmp", + delete=False, + ) as stream: + temporary = Path(stream.name) + json.dump( + { + "version": 1, + "identity": self.identity, + "files": {path: asdict(result) for path, result in self._exceptions.items()}, + "pages": self._pages, + "report": self._summary, + }, + stream, + ensure_ascii=False, + separators=(",", ":"), + ) + temporary.replace(self.path) + except OSError: + self._warn("Cannot save download recovery hints; restart isolation may be unavailable") + finally: + if "temporary" in locals(): + with contextlib.suppress(OSError): + temporary.unlink(missing_ok=True) + + def close(self) -> None: + if self._spool is not None: + self._spool.close() + self._spool = None + + def failed_pages(self, root: str, *, reset: bool = False) -> dict[int, str]: + with self.lock: + if reset and self._pages.pop(root, None) is not None: + self._save() + return dict(self._pages.get(root, {})) + + def fail_page(self, root: str, page: int, error: Exception) -> None: + with self.lock: + message = safe_error(error) + if self._pages.setdefault(root, {}).get(page) != message: + self._pages[root][page] = message + self._save() + + def select(self, run: str, path: str, size: int | None, sha256: str | None) -> DownloadFileResult | None: + with self.lock: + if self._run is None: + self._run = run + if self._run != run: + raise InvalidParameter("A download state instance belongs to one invocation") + key = hashlib.sha256(path.encode()).digest() + if key in self._seen: + return None + self._seen.add(key) + old = self._exceptions.get(path) + if old is not None and (old.size, old.sha256) == (size, sha256): + result = replace(old) + else: + result = DownloadFileResult(path=path, size=size, sha256=sha256) + if self._exceptions.pop(path, None) is not None: + self._save() + self._active[path] = result + return replace(result) + + def begin(self, path: str, *, reset: bool = False) -> DownloadFileResult: + with self.lock: + result = self._active[path] + if reset: + result.retries, result.recovery_seconds = 0, 0 + result.status = "DOWNLOADING" + result.error, result.error_type, result.retryable = None, None, False + return replace(result) + + def budget(self, path: str, retries: int, seconds: float) -> None: + with self.lock: + result = self._active[path] + result.retries, result.recovery_seconds = retries, seconds + self._exceptions[path] = replace(result) + self._save() + + def finish(self, path: str, status: str, error: BaseException | None = None) -> None: + with self.lock: + result = self._active.pop(path, None) + if result is None: + return + result.status = status + if error is not None: + result.error, result.error_type = safe_error(error), type(error).__name__ + result.retryable = bool(getattr(error, "retryable", False)) + elif status in ("VERIFIED", "PENDING"): + result.error, result.error_type, result.retryable = None, None, False + old = self._exceptions.get(path) + keep = status in ("FAILED", "DEFERRED") or (status == "PENDING" and result.retries > 0) + if keep: + self._exceptions[path] = replace(result) + else: + self._exceptions.pop(path, None) + if old != self._exceptions.get(path): + self._save() + assert self._spool is not None + self._spool.write(json.dumps(asdict(result), ensure_ascii=False) + "\n") + self._counts[status] = self._counts.get(status, 0) + 1 + + def report(self, run: str, report: DownloadReport) -> DownloadReport: + for path in list(self._active): + self.finish(path, "PENDING") + report.total = sum(self._counts.values()) + report.succeeded = self._counts.get("VERIFIED", 0) + report.failed = self._counts.get("FAILED", 0) + report.deferred = self._counts.get("DEFERRED", 0) + report.pending = self._counts.get("PENDING", 0) + report.state_warnings = list(self.warnings) + if report.status != "CANCELLED": + report.status = ( + "COMPLETE" + if report.listing_complete and not report.global_error and report.succeeded == report.total + else "PARTIAL" + ) + report.report_path = str(self.path) + report.results_path = "" # No persistent per-file manifest. + assert self._spool is not None + self._spool.flush() + report._attach_file_results(self._spool) + self._spool = None + self._summary = {**asdict(report), "run_id": run} + self._save() + report.state_warnings = list(self.warnings) + return report diff --git a/src/modelscope_hub/_legacy_api.py b/src/modelscope_hub/_legacy_api.py index b7844b5..369387e 100644 --- a/src/modelscope_hub/_legacy_api.py +++ b/src/modelscope_hub/_legacy_api.py @@ -479,10 +479,12 @@ def list_repo_files( ) return self._list_files_page(repo_id, repo_type, revision, recursive=False, root=root) + self._read_state.listing_incomplete = False entries = self._list_files_page(repo_id, repo_type, revision, recursive=recursive, root=root) if len(entries) < REPO_FILES_TRUNCATION_LIMIT: return entries if not recursive: + self._read_state.listing_incomplete = True logger.warning( "Directory %r of %s holds at least %d entries and the server returns no more " "than that for a single listing; the result is incomplete.", @@ -666,6 +668,7 @@ def one(subroot: str | None) -> list[dict]: break frontier = list(dict.fromkeys(next_frontier)) + self._read_state.listing_incomplete = budget_spent or bool(oversized) if budget_spent: logger.warning( "Repo %s: stopped walking the file tree after %d listings " @@ -747,6 +750,63 @@ def list_dataset_files_paginated( page_number += 1 return all_files + def iter_dataset_file_pages( + self, + repo_id: str, + revision: str = "master", + root_path: str = "/", + page_size: int = 200, + isolated_pages: dict[int, str] | None = None, + ) -> Iterator[tuple[int, list[dict], Exception | None]]: + """Yield pages and bounded listing failures without buffering the dataset. + + One failed page does not suppress later pages; two consecutive failures + stop this root to avoid probing an unknown infinite tail. Consumers must + report every failed page and mark enumeration incomplete. + """ + failures = 0 + logger.info("Streaming listing %s@%s Root=%s", repo_id, revision, root_path) + for page in range(1, REPO_TREE_MAX_REQUESTS + 1): + params = {"Revision": revision, "Recursive": "True", "PageNumber": page, "PageSize": page_size} + if root_path != "/": + params["Root"] = root_path + try: + if isolated_pages and page in isolated_pages: + raise NetworkError(f"Previously isolated page: {isolated_pages[page]}") + response = self._request_repo_tree( + f"datasets/{repo_id}/repo/tree", + params, + repo_id=repo_id, + authorized=self._tree_reads_ok, + max_attempts=2, + ) + payload = response.json() + if isinstance(payload, dict) and ( + payload.get("Success") is False + or payload.get("success") is False + or payload.get("Code", 200) not in (200, "200", None) + ): + raise NetworkError("File listing returned a business failure with HTTP 200") + data = payload.get("Data", payload) if isinstance(payload, dict) else payload + if not isinstance(data, (list, dict)): + raise NetworkError("Malformed file listing response") + files = data if isinstance(data, list) else (data.get("Files") or data.get("files") or []) + if not isinstance(files, list): + raise NetworkError("Malformed file listing entries") + except (NetworkError, ServerError, RateLimitError) as exc: + yield page, [], exc + failures += 1 + if failures >= 2: + return + continue + failures = 0 + yield page, files, None + if page % 10 == 0 or len(files) < page_size: + logger.info("Streaming listing %s Root=%s: page %d", repo_id, root_path, page) + if len(files) < page_size: + return + yield REPO_TREE_MAX_REQUESTS + 1, [], NetworkError("File listing request budget exhausted") + def _fetch_tree_page(self, path: str, params: dict[str, Any]) -> requests.Response: parent = getattr(self._read_state, "control", None) budget = float(REPO_TREE_PAGE_TIMEOUT_SECONDS) @@ -778,6 +838,7 @@ def _request_repo_tree( *, repo_id: str, authorized: bool, + max_attempts: int | None = None, ) -> Any: """Fetch one file-tree listing, retrying a spurious denial. @@ -794,7 +855,8 @@ def _request_repo_tree( fast and honest. """ last_error: NetworkError | ServerError | RateLimitError | PermissionDeniedError | None = None - for attempt in range(REPO_TREE_PAGE_MAX_ATTEMPTS): + attempts = REPO_TREE_PAGE_MAX_ATTEMPTS if max_attempts is None else max_attempts + for attempt in range(attempts): try: resp = self._fetch_tree_page(path, params) except (NetworkError, ServerError, RateLimitError, PermissionDeniedError) as error: @@ -803,7 +865,7 @@ def _request_repo_tree( if not error.retryable and not isinstance(error, PermissionDeniedError): raise last_error = error - if attempt < REPO_TREE_PAGE_MAX_ATTEMPTS - 1: + if attempt < attempts - 1: requested_wait = getattr(error, "retry_after", None) wait = min( max(0, requested_wait) if requested_wait is not None else 2**attempt, @@ -817,7 +879,7 @@ def _request_repo_tree( type(error).__name__, wait, attempt + 1, - REPO_TREE_PAGE_MAX_ATTEMPTS, + attempts, ) control = getattr(self._read_state, "control", None) if control is None: @@ -830,7 +892,7 @@ def _request_repo_tree( if isinstance(last_error, PermissionDeniedError): raise NetworkError( f"Repo {repo_id}: tree listing ({params.get('Root') or params.get('PageNumber') or '/'}) kept " - f"returning a denial after {REPO_TREE_PAGE_MAX_ATTEMPTS} attempts on an already-authorized " + f"returning a denial after {attempts} attempts on an already-authorized " f"repo: {last_error}" ) from last_error assert last_error is not None diff --git a/src/modelscope_hub/_resilient_download.py b/src/modelscope_hub/_resilient_download.py new file mode 100644 index 0000000..dbc9f36 --- /dev/null +++ b/src/modelscope_hub/_resilient_download.py @@ -0,0 +1,361 @@ +"""Streaming report-mode downloads with durable file failure isolation.""" + +from __future__ import annotations + +import errno +import hashlib +import json +import queue +import threading +import uuid +from collections.abc import Iterable +from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING + +from filelock import FileLock, Timeout + +from ._download import _listing_roots, _matches_patterns +from ._download_control import _DownloadCancelledError, _DownloadControl, _DownloadDeferredError, _RecoveryBudget +from ._download_state import _DownloadState, safe_error +from .constants import DOWNLOAD_TOTAL_TIMEOUT, REPO_TREE_LIST_TIMEOUT_SECONDS +from .errors import AuthenticationError, FileIntegrityError, InvalidParameter, NotSupportedError +from .types import DownloadFileResult, DownloadReport, DownloadRetryPolicy +from .utils.logger import get_logger + +if TYPE_CHECKING: + from ._download import DownloadManager, ProgressCallback + +logger = get_logger("download") + + +def _validate_entry(entry: dict, output: Path) -> tuple[str, int | None, str | None]: + path = entry.get("Path") or entry.get("path") or entry.get("Name") + if not isinstance(path, str) or not path or "\\" in path or "\0" in path: + raise InvalidParameter("File entry has no safe repository-relative path") + parts = PurePosixPath(path) + if not parts.parts or parts.is_absolute() or ".." in parts.parts or path.endswith("/"): + raise InvalidParameter(f"Unsafe download path: {path}") + if not (output / path).resolve().is_relative_to(output.resolve()): + raise InvalidParameter(f"Download path escapes output directory: {path}") + raw_size = entry.get("Size", entry.get("size")) + size = int(raw_size) if raw_size is not None else None + if size is not None and size < 0: + raise InvalidParameter(f"Negative file size: {path}") + sha256 = entry.get("Sha256") or entry.get("sha256") + if sha256 is not None: + if not isinstance(sha256, str) or len(sha256) != 64 or any(c not in "0123456789abcdefABCDEF" for c in sha256): + raise InvalidParameter(f"Invalid SHA-256 metadata: {path}") + sha256 = sha256.lower() + if size is None and sha256 is None: + raise InvalidParameter(f"Missing size and SHA-256 metadata; refusing an unverifiable download: {path}") + return path, size, sha256 + + +def download_repo_report( + manager: DownloadManager, + repo_id: str, + repo_type: str, + *, + revision: str | None = None, + cache_dir: Path | None = None, + local_dir: Path | None = None, + allow_patterns: list[str] | None = None, + ignore_patterns: list[str] | None = None, + max_workers: int = 4, + file_list: Iterable[dict] | None = None, + tracker_path: Path | None = None, + retry_failed: bool = False, + retry_policy: DownloadRetryPolicy | None = None, + user_agent: dict | str | None = None, + progress_callbacks: list[type[ProgressCallback]] | None = None, +) -> DownloadReport: + """Ordinary file failures return PARTIAL after all known work has been processed. + + A tracker is scoped to endpoint, repo, revision and output, not include + patterns, so repeated ten-file batches share quarantine and retry budgets. + retry_failed is an explicit operator-authorized reset, never automatic. + """ + if not repo_id or max_workers < 1: + raise InvalidParameter("repo_id and a positive max_workers are required") + if repo_type not in ("model", "dataset"): + raise NotSupportedError("Report-mode downloads support model and dataset repositories") + revision = revision or "master" + revision_path = PurePosixPath(revision) + if revision_path.is_absolute() or ".." in revision_path.parts: + raise InvalidParameter("revision must not escape the snapshot cache directory") + output = local_dir or manager._find_legacy_repo_dir(repo_id, repo_type, cache_dir) + if output is None: + output = manager._repo_cache_dir_path(repo_id, repo_type, cache_dir) / "snapshots" / revision + output = output.absolute() + output.mkdir(parents=True, exist_ok=True) + identity = { + "endpoint": manager._client.endpoint, + "repo": repo_id, + "type": repo_type, + "revision": revision, + "output": str(output.resolve()), + "schema": "1", + } + key = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest() + tracker = tracker_path or (cache_dir or manager._config.cache_dir) / ".download_state" / f".ms_download_{key}.json" + tracker = tracker.absolute() + if tracker.resolve().is_relative_to(output.resolve()): + raise InvalidParameter("Keep the tracker outside the download directory to avoid uploading state files") + tracker.parent.mkdir(parents=True, exist_ok=True) + task_lock = FileLock(str(tracker) + ".lock") + try: + task_lock.acquire(timeout=1) + except Timeout as exc: + raise InvalidParameter( + "Another downloader is using this tracker; use separate outputs for parallel tasks" + ) from exc + try: + state = _DownloadState(tracker, identity) + try: + return _run( + manager, + state, + output, + repo_id, + repo_type, + revision, + allow_patterns, + ignore_patterns, + max_workers, + file_list, + retry_failed, + retry_policy or DownloadRetryPolicy(), + user_agent, + progress_callbacks, + cache_dir, + ) + finally: + state.close() + finally: + task_lock.release() + + +def _run( + manager: DownloadManager, + state: _DownloadState, + output: Path, + repo_id: str, + repo_type: str, + revision: str, + include: list[str] | None, + exclude: list[str] | None, + workers: int, + file_list: Iterable[dict] | None, + retry_failed: bool, + policy: DownloadRetryPolicy, + user_agent: dict | str | None, + callbacks: list[type[ProgressCallback]] | None, + cache_dir: Path | None, +) -> DownloadReport: + run = uuid.uuid4().hex + report = DownloadReport(repo_id=repo_id, revision=revision, output_dir=str(output), tracker_path=str(state.path)) + stop = threading.Event() + incoming: queue.Queue[tuple[dict | None, str | None]] = queue.Queue(maxsize=workers * 4) + producer_done = threading.Event() + pool = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="download-report") + futures: dict[Future[None], str] = {} + + def put(entry: dict | None, error: str | None = None) -> None: + while not stop.is_set(): + try: + incoming.put((entry, error), timeout=0.1) + return + except queue.Full: + pass + + def produce() -> None: + try: + if file_list is not None: + if retry_failed: + for root in _listing_roots(include): + state.failed_pages(root or "/", reset=True) + for entry in file_list: + if stop.is_set(): + break + put(entry) + else: + with _DownloadControl(total_timeout=REPO_TREE_LIST_TIMEOUT_SECONDS, cancel_event=stop) as control: + with manager._client.read_context(control): + for root in _listing_roots(include): + if repo_type == "dataset": + for page, entries, error in manager._client.iter_dataset_file_pages( + repo_id, + revision, + root or "/", + isolated_pages=state.failed_pages(root or "/", reset=retry_failed), + ): + if error is not None: + state.fail_page(root or "/", page, error) + put(None, f"Root={root or '/'} page={page}: {safe_error(error)}") + for entry in entries: + put(entry) + control.check() + else: + for entry in manager._client.list_repo_files(repo_id, repo_type, revision, root=root): + put(entry) + if getattr(manager._client._read_state, "listing_incomplete", False): + put(None, f"Root={root or '/'}: model file listing is truncated") + except _DownloadCancelledError: + pass + except Exception as exc: + if isinstance(exc, AuthenticationError): + report.global_error = safe_error(exc) + put(None, f"Enumeration stopped: {safe_error(exc)}") + finally: + producer_done.set() + + def transfer(result: DownloadFileResult) -> None: + try: + # State hints never override a file that passes current metadata checks. + _validate_entry({"Path": result.path, "Size": result.size, "Sha256": result.sha256}, output) + if result.status in ("FAILED", "DEFERRED") and not retry_failed: + target = output / result.path + with _DownloadControl(total_timeout=DOWNLOAD_TOTAL_TIMEOUT, cancel_event=stop) as control: + valid = target.is_file() and manager._cache_hit( + target, + result.sha256, + result.size, + control=control, + ) + if valid: + state.finish(result.path, "VERIFIED") + return + with state.lock: + report.skipped_failures += 1 + skipped = report.skipped_failures + if skipped <= 10 or skipped % 1000 == 0: + logger.warning( + "Skipping isolated file %s (%d skipped); use retry_failed=True after investigation", + result.path, + skipped, + ) + state.finish(result.path, result.status) + return + previous = state.begin(result.path, reset=retry_failed and result.status in ("FAILED", "DEFERRED")) + recovery = _RecoveryBudget( + policy, + previous.retries, + previous.recovery_seconds, + lambda count, seconds: state.budget(result.path, count, seconds), + ) + manager.download_file( + repo_id, + repo_type, + result.path, + revision=revision, + local_dir=output, + cache_dir=cache_dir, + file_size=result.size, + expected_sha256=result.sha256, + user_agent=user_agent, + progress_callbacks=callbacks, + _cancel_event=stop, + _recovery=recovery, + ) + except _DownloadCancelledError: + state.finish(result.path, result.status if result.status in ("FAILED", "DEFERRED") else "PENDING") + raise + except Exception as exc: + origin = exc.__cause__ if isinstance(exc, _DownloadDeferredError) and exc.__cause__ else exc + terminal = isinstance(origin, (FileIntegrityError, InvalidParameter)) or not getattr( + origin, "retryable", False + ) + state.finish(result.path, "FAILED" if terminal else "DEFERRED", exc) + if ( + isinstance(exc, AuthenticationError) + or isinstance(exc, OSError) + and exc.errno in (errno.ENOSPC, errno.EROFS) + ): + raise + logger.error("Isolated download failure %s; continuing remaining files: %s", result.path, safe_error(exc)) + else: + state.finish(result.path, "VERIFIED") + + producer = threading.Thread(target=produce, name="download-enumeration", daemon=True) + producer.start() + interrupted = False + entry_count = 0 + try: + while futures or not producer_done.is_set() or not incoming.empty(): + while len(futures) < workers * 2: + try: + entry, listing_error = incoming.get(timeout=0.05 if not futures else 0) + except queue.Empty: + break + if listing_error is not None: + report.listing_complete = False + if len(report.listing_errors) < 100: + report.listing_errors.append(listing_error) + logger.error("Incomplete listing; continuing known files: %s", listing_error) + continue + entry_count += 1 + try: + if not isinstance(entry, dict): + raise InvalidParameter("File entry must be a dictionary") + if (entry.get("Type") or entry.get("type")) == "tree": + continue + raw_path = entry.get("Path") or entry.get("path") or entry.get("Name") + if isinstance(raw_path, str): + if ( + include + and not _matches_patterns(raw_path, include) + or exclude + and _matches_patterns(raw_path, exclude) + ): + continue + path, size, sha256 = _validate_entry(entry, output) + result = state.select(run, path, size, sha256) + except (InvalidParameter, ValueError, TypeError, OverflowError) as exc: + path = ( + str(entry.get("Path", f"")) + if isinstance(entry, dict) + else f"" + ) + if state.select(run, path, None, None) is not None: + state.finish(path, "FAILED", exc) + logger.error("Invalid file metadata; continuing: %s", safe_error(exc)) + continue + if result is None: + continue + future = pool.submit(transfer, result) + futures[future] = path + if futures: + completed, _ = wait(futures, timeout=0.1, return_when=FIRST_COMPLETED) + for future in completed: + futures.pop(future) + future.result() + except KeyboardInterrupt: + interrupted = True + report.status = "CANCELLED" + stop.set() + except Exception as exc: + report.global_error = safe_error(exc) + report.listing_complete = False + stop.set() + finally: + if stop.is_set(): + for future, path in futures.items(): + if future.cancel(): + state.finish(path, "PENDING") + pool.shutdown(wait=True, cancel_futures=True) + stop.set() + producer.join() + state.report(run, report) + logger.info( + "Download %s: verified=%d failed=%d deferred=%d pending=%d; report=%s", + report.status, + report.succeeded, + report.failed, + report.deferred, + report.pending, + report.report_path, + ) + if interrupted: + raise KeyboardInterrupt + return report diff --git a/src/modelscope_hub/api.py b/src/modelscope_hub/api.py index 1453215..99035b0 100644 --- a/src/modelscope_hub/api.py +++ b/src/modelscope_hub/api.py @@ -60,6 +60,8 @@ AgentTokenRecord, CacheInfo, CacheVerification, + DownloadReport, + DownloadRetryPolicy, FileInfo, PagedResult, RepoInfo, @@ -1579,6 +1581,62 @@ def download_file( file_size=file_size, ) + def download_repo_report( + self, + repo_id: str, + repo_type: RepoTypeLike, + *, + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + allow_patterns: list[str] | None = None, + ignore_patterns: list[str] | None = None, + max_workers: int = 4, + file_list: Iterable[dict] | None = None, + tracker_path: str | Path | None = None, + retry_failed: bool = False, + retry_policy: DownloadRetryPolicy | None = None, + user_agent: dict | str | None = None, + progress_callbacks: list[type[ProgressCallback]] | None = None, + ) -> DownloadReport: + """Download known files despite ordinary failures; never label partial data complete. + + Unlike download_repo, file failures return a PARTIAL report instead of + raising an aggregate exception. The default hidden JSON tracker is in + cache_dir/.download_state, outside repository contents. The single JSON + combines exceptional files, failed pages, and the latest call's summary, + scoped to endpoint/repo/revision/ + output so batches share recovery hints. Actual size/hash validation + overrides stale failure hints; otherwise retry_failed=True or changed + metadata resets them. Missing/corrupt hints trigger bounded fresh + recovery, not false success. Inspect report.state_warnings for issues. + + file_list accepts an iterable of legacy Path/Size/Sha256 entries. Without + it, datasets are enumerated page by page with bounded backpressure. + Use a pinned revision or an authoritative manifest for migration jobs. + Inspect report.complete and report.iter_files('VERIFIED') before using + the downloaded data. report.report_path equals report.tracker_path; + its JSON has the latest summary under 'report', without per-run files. + iter_files streams temporary results owned by the live report; call + report.close() after consumption. Persisted hints cannot prove success. + """ + return self.downloader.download_repo_report( + repo_id, + str(self._normalize_repo_type(repo_type)), + revision=revision, + cache_dir=Path(cache_dir) if cache_dir else None, + local_dir=Path(local_dir) if local_dir else None, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + max_workers=max_workers, + file_list=file_list, + tracker_path=Path(tracker_path) if tracker_path else None, + retry_failed=retry_failed, + retry_policy=retry_policy, + user_agent=user_agent, + progress_callbacks=progress_callbacks, + ) + def download_repo( self, repo_id: str, diff --git a/src/modelscope_hub/cli/download.py b/src/modelscope_hub/cli/download.py index e041662..74dc750 100644 --- a/src/modelscope_hub/cli/download.py +++ b/src/modelscope_hub/cli/download.py @@ -8,6 +8,7 @@ from ..api import HubApi from ..constants import RepoType +from ..errors import InvalidParameter from .base import CLICommand, SubParsers, add_repo_type_arg, info, make_api, success, warn from .compat import ( PatternAction, @@ -85,6 +86,15 @@ def register(subparsers: SubParsers) -> None: help="Glob to exclude (snapshot mode). Repeatable.", ) p.add_argument("--force", action="store_true", help="Re-download even if cached.") + p.add_argument( + "--continue-on-error", action="store_true", help="Continue with a persistent report; exit 3 for PARTIAL." + ) + p.add_argument( + "--tracker-path", + default=None, + help="Single hidden JSON for recovery hints and summary, outside downloaded contents.", + ) + p.add_argument("--retry-failed", action="store_true", help="Explicitly reset isolated files' recovery budgets.") # Legacy compat (hidden from --help) add_legacy_download_args(p) @@ -95,6 +105,12 @@ def register(subparsers: SubParsers) -> None: def execute(self) -> None: normalize_download_args(self.args) + continuing = getattr(self.args, "continue_on_error", False) + if not continuing and (getattr(self.args, "tracker_path", None) or getattr(self.args, "retry_failed", False)): + raise InvalidParameter("--tracker-path and --retry-failed require --continue-on-error") + if continuing and (self.args.force or self.args.repo_type == "collection"): + raise InvalidParameter("--continue-on-error does not support --force or collection downloads") + # Handle collection download separately if self.args.repo_type == "collection": self._download_collection() @@ -104,6 +120,25 @@ def execute(self) -> None: cache_dir: Path | None = Path(self.args.cache_dir) if self.args.cache_dir else None local_dir: Path | None = Path(self.args.local_dir) if self.args.local_dir else None + if continuing: + report = api.download_repo_report( + self.args.repo_id, + self.args.repo_type, + revision=self.args.revision, + cache_dir=cache_dir, + local_dir=local_dir, + allow_patterns=self.args.files or normalize_patterns(self.args.allow_patterns), + ignore_patterns=normalize_patterns(self.args.ignore_patterns), + max_workers=self.args.max_workers, + tracker_path=self.args.tracker_path, + retry_failed=self.args.retry_failed, + ) + info(f"{report.status}: {report.succeeded}/{report.total} verified; report: {report.report_path}") + if not report.complete: + warn("Partial data is not a complete snapshot. Only VERIFIED files may be consumed.") + raise SystemExit(3 if report.global_error is None else 1) + return + if self.args.files: for file_path in self.args.files: local = api.download_file( diff --git a/src/modelscope_hub/compat/snapshot_download.py b/src/modelscope_hub/compat/snapshot_download.py index b06208d..1cf93ec 100644 --- a/src/modelscope_hub/compat/snapshot_download.py +++ b/src/modelscope_hub/compat/snapshot_download.py @@ -15,7 +15,8 @@ from ..api import HubApi from ..constants import RepoType -from ..errors import AuthenticationError, NotExistError, PermissionDeniedError +from ..errors import AuthenticationError, InvalidParameter, NotExistError, PermissionDeniedError +from ..types import DownloadRetryPolicy from ..utils import append_user_agent from ..utils.patterns import normalize_patterns from .file_download import _resolve_legacy_paths @@ -43,6 +44,10 @@ def snapshot_download( local_files_only: bool = False, user_agent: dict | str | None = None, progress_callbacks: list[type[ProgressCallback]] | None = None, + continue_on_error: bool = False, + tracker_path: str | None = None, + retry_failed: bool = False, + retry_policy: DownloadRetryPolicy | None = None, ) -> str: """Download a repo snapshot (legacy signature). @@ -51,7 +56,9 @@ def snapshot_download( ``allow_file_pattern``/``ignore_file_pattern`` aliases when both are set. ``progress_callbacks`` takes a list of :class:`ProgressCallback` subclasses (not instances); each is instantiated per file to report - download progress. + download progress. continue_on_error opts into durable failed-file isolation + and returns the output path even when the persisted report is PARTIAL. Use + HubApi.download_repo_report for direct access to verified-file results. """ effective_id = repo_id or model_id if not effective_id: @@ -86,6 +93,27 @@ def snapshot_download( local_dir, api, ) + if continue_on_error: + if local_files_only: + raise InvalidParameter("Report-mode downloads cannot use local_files_only") + report = api.download_repo_report( + effective_id, + effective_type, + revision=revision, + cache_dir=effective_cache, + local_dir=effective_local, + allow_patterns=include, + ignore_patterns=exclude, + max_workers=max_workers, + tracker_path=tracker_path, + retry_failed=retry_failed, + retry_policy=retry_policy, + user_agent=user_agent, + progress_callbacks=progress_callbacks, + ) + return report.output_dir + if tracker_path is not None or retry_failed or retry_policy is not None: + raise InvalidParameter("Tracker/recovery options require continue_on_error=True") try: result = api.download_repo( effective_id, @@ -121,8 +149,12 @@ def dataset_snapshot_download( endpoint: str | None = None, local_files_only: bool = False, user_agent: dict | str | None = None, + continue_on_error: bool = False, + tracker_path: str | None = None, + retry_failed: bool = False, + retry_policy: DownloadRetryPolicy | None = None, ) -> str: - """Download a dataset snapshot (legacy signature).""" + """Download a dataset snapshot; report-mode options match snapshot_download.""" if cookies is not None: warnings.warn( "The 'cookies' parameter is deprecated and ignored. Use 'token' instead.", @@ -146,6 +178,26 @@ def dataset_snapshot_download( local_dir, api, ) + if continue_on_error: + if local_files_only: + raise InvalidParameter("Report-mode downloads cannot use local_files_only") + report = api.download_repo_report( + dataset_id, + "dataset", + revision=revision, + cache_dir=effective_cache, + local_dir=effective_local, + allow_patterns=include, + ignore_patterns=exclude, + max_workers=max_workers, + tracker_path=tracker_path, + retry_failed=retry_failed, + retry_policy=retry_policy, + user_agent=user_agent, + ) + return report.output_dir + if tracker_path is not None or retry_failed or retry_policy is not None: + raise InvalidParameter("Tracker/recovery options require continue_on_error=True") try: result = api.download_repo( dataset_id, diff --git a/src/modelscope_hub/types.py b/src/modelscope_hub/types.py index 4cf8316..c6b90ac 100644 --- a/src/modelscope_hub/types.py +++ b/src/modelscope_hub/types.py @@ -7,11 +7,13 @@ from __future__ import annotations -from collections.abc import Mapping +import json +import threading +from collections.abc import Iterator, Mapping from dataclasses import dataclass, field, fields from datetime import datetime, timezone from enum import Enum -from typing import Any, ClassVar, Generic, TypedDict, TypeVar +from typing import Any, ClassVar, Generic, TextIO, TypedDict, TypeVar from .constants import RepoType, Visibility @@ -52,6 +54,129 @@ def from_dict(cls: type[_TDataclass], data: Mapping[str, Any] | None) -> _TDatac return cls(**kwargs) # type: ignore[arg-type] +@dataclass(frozen=True, slots=True) +class DownloadRetryPolicy: + """Short recovery limits for report-mode downloads, shared by all file parts. + + Healthy transfers keep the regular total file timeout. Only failed-request + inactivity and retry waits consume recovery_timeout. Longer Retry-After + values defer the file instead of occupying a worker or retrying early. + """ + + max_retries: int = 2 + recovery_timeout: float = 60 + idle_timeout: float = 15 + low_speed_window: float = 10 + max_retry_delay: float = 5 + + def __post_init__(self) -> None: + if self.max_retries < 0 or any( + value <= 0 + for value in ( + self.recovery_timeout, + self.idle_timeout, + self.low_speed_window, + self.max_retry_delay, + ) + ): + raise ValueError("Download recovery limits must be positive; max_retries may be zero") + + +@dataclass(slots=True) +class DownloadFileResult(_FromDictMixin): + """A file validation result or an auxiliary recovery hint.""" + + path: str = "" + status: str = "PENDING" + size: int | None = None + sha256: str | None = None + retries: int = 0 + recovery_seconds: float = 0 + error: str | None = None + error_type: str | None = None + retryable: bool = False + + +class _DownloadReportResults: + """Own an anonymous result spool without adding it to dataclass serialization.""" + + _file_results: TextIO | None = None + + def _attach_file_results(self, stream: TextIO) -> None: + self._file_results = stream + self._file_results_lock = threading.Lock() + + def close(self) -> None: + """Release this report's temporary per-file results after consumption.""" + if self._file_results is not None: + with self._file_results_lock: + if self._file_results is not None: + self._file_results.close() + self._file_results = None + + def _iter_file_results(self) -> Iterator[dict]: + if self._file_results is None: + raise ValueError( + "Per-file results are available only on an open live DownloadReport; " + "the persisted JSON contains summary and recovery hints, not a verified-file manifest. " + "Rerun download_repo_report to validate actual files." + ) + offset = 0 + while True: + with self._file_results_lock: + if self._file_results is None: + raise ValueError("DownloadReport results have been closed") + self._file_results.seek(offset) + line = self._file_results.readline() + offset = self._file_results.tell() + if not line: + break + yield json.loads(line) + + +@dataclass(slots=True) +class DownloadReport(_DownloadReportResults, _FromDictMixin): + """Report-mode completion is explicit: COMPLETE, PARTIAL, or CANCELLED. + + Counts cover this invocation's selected files, not every previous batch. + Per-file results use an anonymous temporary spool owned by this live object, + preserving earlier calls without permanent manifests or unbounded memory. + One hidden JSON holds the latest summary and sparse recovery hints only. + VERIFIED describes validation during this call, not a permanent guarantee: + downstream users must check changed files. close() releases the spool; + state_warnings reports unavailable or ignored auxiliary recovery hints. + """ + + repo_id: str = "" + revision: str = "" + output_dir: str = "" + tracker_path: str = "" + report_path: str = "" + results_path: str = "" + status: str = "PARTIAL" + total: int = 0 + succeeded: int = 0 + failed: int = 0 + deferred: int = 0 + pending: int = 0 + skipped_failures: int = 0 + listing_complete: bool = True + listing_errors: list[str] = field(default_factory=list) + global_error: str | None = None + state_warnings: list[str] = field(default_factory=list) + + @property + def complete(self) -> bool: + return self.status == "COMPLETE" + + def iter_files(self, status: str | None = None) -> Iterator[DownloadFileResult]: + """Stream this live call's results; persisted summaries cannot prove success.""" + for raw in self._iter_file_results(): + result = DownloadFileResult.from_dict(raw) + if status is None or result.status == status: + yield result + + # --------------------------------------------------------------------------- # User # --------------------------------------------------------------------------- @@ -606,6 +731,9 @@ class TokenSignPayload(TypedDict): __all__ = [ + "DownloadRetryPolicy", + "DownloadFileResult", + "DownloadReport", "AgentIdentity", "AgentIdentitySummary", "AgentIdConfiguration", diff --git a/tests/download_corruption_harness.py b/tests/download_corruption_harness.py new file mode 100644 index 0000000..2e2e08b --- /dev/null +++ b/tests/download_corruption_harness.py @@ -0,0 +1,473 @@ +"""Deterministic HTTP corruption injection using the unmodified SDK download pipeline.""" + +from __future__ import annotations + +import contextlib +import hashlib +import json +import socket +import threading +import time +from collections.abc import Callable, Iterator +from dataclasses import asdict, dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, quote, unquote, urlparse + +from modelscope_hub import DownloadRetryPolicy, HubApi +from modelscope_hub._download import ProgressCallback +from modelscope_hub._download_state import safe_error + +GROUPS = { + 1: "persistent_empty_200", + 2: "persistent_short_object", + 3: "persistent_short_chunked", + 4: "persistent_bit_flips", + 5: "persistent_wrong_object", + 6: "persistent_overlong_chunked", + 7: "transient_empty_200", + 8: "transient_hash_mismatch", + 9: "truncated_connection_then_resume", + 10: "wrong_range_start_then_restart", +} + + +@dataclass(frozen=True) +class FaultCase: + """One unique repository file, never an alias for another corruption case.""" + + path: str + content: bytes + group: int = 0 + lfs: bool = True + + @property + def sha256(self) -> str: + return hashlib.sha256(self.content).hexdigest() + + @property + def cut(self) -> int: + return len(self.content) // 3 + 13 + + @property + def metadata(self) -> dict: + return {"Path": self.path, "Type": "blob", "Size": len(self.content), "Sha256": self.sha256} + + +def synthetic_cases() -> list[FaultCase]: + """Provide ten distinct files per group plus one hundred normal controls.""" + cases = [] + for index in range(100): + group = index // 10 + 1 + path = f"group-{group:02}/file-{index:03}.bin" + content = hashlib.shake_256(path.encode()).digest(32768 + index * 17) + cases.append(FaultCase(path, content, group, lfs=group >= 9 or index % 2 == 0)) + path = f"controls/normal-{index:03}.bin" + content = hashlib.shake_256(path.encode()).digest(32768 + index * 19) + cases.append(FaultCase(path, content, lfs=index % 2 == 0)) + return cases + + +class FaultHTTPServer: + """Serve valid HTTP framing, truncated sockets, redirects and incorrect ranges.""" + + def __init__(self, cases: list[FaultCase]) -> None: + self.cases = {case.path: case for case in cases} + self.lock = threading.Lock() + self.attempts: dict[tuple[str, str], int] = {} + self.wire: list[dict] = [] + self.credential_requests = 0 + state = self + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args) -> None: + pass + + def do_GET(self) -> None: # noqa: N802 - http.server API + if self.headers.get("Authorization") or self.headers.get("Cookie"): + with state.lock: + state.credential_requests += 1 + self._send(403, b"Credentials must not be sent to the local fault server") + return + url = urlparse(self.path) + path = ( + unquote(url.path.removeprefix("/cdn/")) + if url.path.startswith("/cdn/") + else parse_qs(url.query).get("FilePath", [""])[0] + ) + case = state.cases.get(path) + if case is None: + self._send(404, b"Unknown test file") + return + stage = self.headers.get("X-Test-Stage", "first") + with state.lock: + state.wire.append({"stage": stage, "path": path, "cdn": url.path.startswith("/cdn/")}) + if case.lfs and not url.path.startswith("/cdn/"): + self._send(302, b"", {"Location": f"/cdn/{quote(path)}"}) + return + with state.lock: + key = (stage, path) + attempt = state.attempts.get(key, 0) + 1 + state.attempts[key] = attempt + inject = self.headers.get("X-Test-Inject") == "1" + active = inject and (case.group <= 6 or attempt == 1) + body = case.content + group = case.group if active else 0 + if group in (1, 7): + self._send(200, b"") + elif group == 2: + self._send(200, body[: case.cut]) + elif group == 3: + self._chunked(body[: case.cut]) + elif group in (4, 8): + damaged = bytearray(body) + for index in (0, len(body) // 2, len(body) - 1): + damaged[index] ^= 1 + self._send(200, bytes(damaged)) + elif group == 5: + self._send(200, bytes(byte ^ 0xA5 for byte in body)) + elif group == 6: + self._chunked(body + b"EXCESS-CONTENT" * 128) + elif group == 9: + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body[: case.cut]) + self.wfile.flush() + self.close_connection = True + self.connection.shutdown(socket.SHUT_WR) + elif group == 10: + requested = self.headers.get("Range", "bytes=0-") + start = int(requested.removeprefix("bytes=").partition("-")[0]) + 1 + self._send(206, body[start:], {"Content-Range": f"bytes {start}-{len(body) - 1}/{len(body)}"}) + else: + requested = self.headers.get("Range") + if requested and case.lfs: + start_text, _, end_text = requested.removeprefix("bytes=").partition("-") + start = int(start_text) + end = min(int(end_text), len(body) - 1) if end_text else len(body) - 1 + if start >= len(body): + self._send(416, b"", {"Content-Range": f"bytes */{len(body)}"}) + else: + self._send( + 206, body[start : end + 1], {"Content-Range": f"bytes {start}-{end}/{len(body)}"} + ) + elif case.lfs: + self._send(200, body, {"X-Linked-ETag": case.sha256}) + else: + self._chunked(body) + + def _send(self, status: int, body: bytes, headers: dict | None = None) -> None: + self.send_response(status) + for name, value in (headers or {}).items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _chunked(self, body: bytes) -> None: + self.send_response(200) + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + for start in range(0, len(body), 4096): + chunk = body[start : start + 4096] + self.wfile.write(f"{len(chunk):x}\r\n".encode() + chunk + b"\r\n") + self.wfile.write(b"0\r\n\r\n") + + class QuietServer(ThreadingHTTPServer): + daemon_threads = True + + def handle_error(self, request, client_address) -> None: + # Integrity rejection legitimately closes an unread response body. + pass + + self.server = QuietServer(("127.0.0.1", 0), Handler) + self.url = f"http://127.0.0.1:{self.server.server_port}" + self.thread = threading.Thread(target=self.server.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True) + + def __enter__(self) -> FaultHTTPServer: + self.thread.start() + return self + + def __exit__(self, *args) -> None: + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=2) + + +class FaultRouter: + """Inject at the Response boundary, preserving all SDK orchestration and checks.""" + + def __init__(self, server: FaultHTTPServer, *, live: bool = False) -> None: + self.server = server + self.live = live + self.stage = "first" + self.inject = True + self.records: list[dict] = [] + self.lock = threading.Lock() + self.local_api = HubApi(endpoint=server.url, token="") + self.local_api.legacy._read_session.trust_env = False + + def attach(self, api: HubApi) -> None: + original = api.legacy.download_stream + + def stream(repo_id, repo_type, file_path, revision="master", headers=None): + case = self.server.cases[file_path] + use_local = not self.live or self.inject and case.group != 0 + record = { + "stage": self.stage, + "path": file_path, + "range": (headers or {}).get("Range"), + "transport": "fault_http" if use_local else "modelscope.cn", + } + with self.lock: + self.records.append(record) + started = time.monotonic() + try: + if use_local: + control = api.legacy._read_state.control + local_headers = { + **(headers or {}), + "X-Test-Stage": self.stage, + "X-Test-Inject": "1" if self.inject and case.group else "0", + } + # Never forward auth from the real API facade to localhost. + local_headers = { + key: value + for key, value in local_headers.items() + if key.lower() not in ("authorization", "cookie") + } + with self.local_api.legacy.read_context(control): + response = self.local_api.legacy.download_stream( + repo_id, repo_type, file_path, revision, headers=local_headers + ) + else: + response = original(repo_id, repo_type, file_path, revision, headers=headers) + record.update( + status=response.status_code, + redirect_hops=len(response.history), + content_length=response.headers.get("Content-Length"), + content_range=response.headers.get("Content-Range"), + ) + return response + except Exception as exc: + record["error"] = safe_error(exc) + raise + finally: + record["header_seconds"] = round(time.monotonic() - started, 4) + + api.legacy.download_stream = stream + + def for_stage(self, stage: str) -> list[dict]: + return [record for record in self.records if record["stage"] == stage] + + def close(self) -> None: + self.local_api.legacy._read_session.close() + self.local_api.legacy._session.close() + + +def verify_file(target: Path, metadata: dict) -> None: + assert target.is_file(), str(target) + assert target.stat().st_size == int(metadata["Size"]), str(target) + digest = hashlib.sha256() + with target.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + assert digest.hexdigest() == metadata["Sha256"].lower(), str(target) + + +def assert_single_json(root: Path, cases: list[FaultCase]) -> None: + data_paths = {root / "files" / case.path for case in cases} + generated = {path for path in root.rglob("*") if path.is_file() and path not in data_paths} + assert {path for path in generated if path.suffix in (".json", ".jsonl")} == {root / ".ms_download_state.json"} + assert not any(path.name.endswith((".tmp", ".incomplete", ".parallel_tmp")) for path in generated) + + +def exercise_stages( + root: Path, + cases: list[FaultCase], + api_factory: Callable[[], HubApi], + router: FaultRouter, + *, + fresh_manifest: Callable[[], list[dict]] | None = None, + announce: Callable[[str], None] = lambda message: None, +) -> dict: + """Exercise first failure, restart, no-reset recovery, explicit repair and reuse.""" + assert len(cases) == len({case.path for case in cases}) == 200 + assert sum(case.group > 0 for case in cases) == 100 + assert all(sum(case.group == group for case in cases) == 10 for group in GROUPS) + output = root / "files" + tracker = root / ".ms_download_state.json" + assert not tracker.exists() and not output.exists(), "The corruption experiment needs a fresh isolated directory" + persistent = {case.path for case in cases if 1 <= case.group <= 6} + metadata = [case.metadata for case in cases] + for case in cases: + if case.group == 10: + target = output / case.path + target.parent.mkdir(parents=True, exist_ok=True) + target.with_suffix(target.suffix + ".incomplete").write_bytes(case.content[: case.cut]) + policy = DownloadRetryPolicy() + evidence: dict = {"policy": asdict(policy), "fault_files": 100, "control_files": 100, "stages": {}, "cases": []} + report = None + try: + for stage in ("first", "restart", "source_repaired_without_reset", "explicit_repair", "reuse"): + router.stage = stage + router.inject = stage in ("first", "restart") + api = api_factory() + router.attach(api) + started = time.monotonic() + transfer_times: dict[str, list[float]] = {} + timing_lock = threading.Lock() + + class CaseProgress(ProgressCallback): + def __init__(self, filename: str, file_size: int): + super().__init__(filename, file_size) + with timing_lock: + transfer_times.setdefault(filename, [time.monotonic(), time.monotonic()]) + + def end(self) -> None: + with timing_lock: + transfer_times[self.filename][1] = time.monotonic() + + try: + report = api.download_repo_report( + "wangxingjun778/test_upload_large_2g", + "dataset", + local_dir=output, + tracker_path=tracker, + file_list=iter(metadata), + max_workers=4, + retry_policy=policy, + retry_failed=stage == "explicit_repair", + progress_callbacks=[CaseProgress], + ) + rows = {row.path: row for row in report.iter_files()} + requests = router.for_stage(stage) + evidence["stages"][stage] = { + "summary": asdict(report), + "seconds": round(time.monotonic() - started, 4), + "logical_file_requests": len(requests), + "remote_file_requests": sum(request["transport"] == "modelscope.cn" for request in requests), + "redirect_hops": sum(request.get("redirect_hops", 0) for request in requests), + } + announce( + f"{stage}: {report.status}，成功 {report.succeeded}，失败 {report.failed}，请求 {len(requests)}" + ) + assert report.total == 200 and len(rows) == 200 + assert report.global_error is None and report.listing_complete and not report.state_warnings + if stage in ("first", "restart", "source_repaired_without_reset"): + assert (report.status, report.succeeded, report.failed, report.deferred, report.pending) == ( + "PARTIAL", + 140, + 60, + 0, + 0, + ), evidence["stages"][stage] + assert {row.path for row in rows.values() if row.status == "FAILED"} == persistent + assert all(not (output / path).exists() for path in persistent) + else: + assert report.complete and report.succeeded == 200 + hints = json.loads(tracker.read_text(encoding="utf-8")) + assert hints["files"] == {} and hints["pages"] == {} + if stage == "first": + assert len(requests) == 300 + for case in cases: + selected = [request for request in requests if request["path"] == case.path] + row = rows[case.path] + assert len(selected) == (2 if case.group else 1), (case.path, selected) + if not case.group: + continue + assert row.retries == 1, (case.path, row) + if case.group == 9: + assert [request["range"] for request in selected] == [None, f"bytes={case.cut}-"] + assert selected[1]["status"] == 206 + if case.group == 10: + assert [request["range"] for request in selected] == [f"bytes={case.cut}-", None] + assert [request["status"] for request in selected] == [206, 200] + if case.group <= 6: + expected_error = "FileIntegrityError" if case.group in (4, 5) else "FileSizeMismatchError" + assert row.error_type == expected_error, row + group_index = sum(item["group"] == case.group for item in evidence["cases"]) + 1 + evidence["cases"].append( + { + "id": f"G{case.group}-{group_index:02}", + "group": case.group, + "fault": GROUPS[case.group], + "path": case.path, + "size": len(case.content), + "sha256": case.sha256, + "lfs": case.lfs, + "result": asdict(row), + "requests": selected, + "passed": True, + "seconds": round(transfer_times[case.path][1] - transfer_times[case.path][0], 4), + } + ) + assert (output / cases[-1].path).is_file(), "The tail control file must complete" + evidence["tail_control_completed"] = cases[-1].path + for case in cases: + if case.path not in persistent: + verify_file(output / case.path, case.metadata) + elif stage in ("restart", "source_repaired_without_reset"): + assert report.skipped_failures == 60 and requests == [] + elif stage == "explicit_repair": + assert len(requests) == 60 and {request["path"] for request in requests} == persistent + if fresh_manifest is not None: + current = {entry["Path"]: entry for entry in fresh_manifest() if entry.get("Type") != "tree"} + for entry in metadata: + remote = current[entry["Path"]] + assert (int(remote["Size"]), remote["Sha256"].lower()) == (entry["Size"], entry["Sha256"]) + for entry in metadata: + verify_file(output / entry["Path"], entry) + evidence["independent_size_sha256_verified"] = 200 + evidence["verified_bytes"] = sum(entry["Size"] for entry in metadata) + elif stage == "reuse": + assert requests == [] + assert_single_json(root, cases) + assert router.server.credential_requests == 0 + finally: + if report is not None: + report.close() + report = None + api.legacy._read_session.close() + api.legacy._session.close() + evidence["passed"] = True + evidence["passed_fault_cases"] = len(evidence["cases"]) + evidence["group_results"] = { + f"G{group}": { + "fault": name, + "cases": 10, + "passed": 10, + "first_status": "FAILED" if group <= 6 else "VERIFIED", + } + for group, name in GROUPS.items() + } + return evidence + except BaseException as exc: + evidence["passed"] = False + evidence["error"] = safe_error(exc) + evidence["observed_requests"] = router.records + raise + finally: + # Keep both success and failure evidence in the task's only JSON file. + if tracker.exists(): + from tempfile import NamedTemporaryFile + + data = json.loads(tracker.read_text(encoding="utf-8")) + data["corruption_regression"] = evidence + with NamedTemporaryFile(mode="w", encoding="utf-8", dir=root, suffix=".tmp", delete=False) as stream: + json.dump(data, stream, ensure_ascii=False, separators=(",", ":")) + temporary = Path(stream.name) + temporary.replace(tracker) + + +@contextlib.contextmanager +def local_experiment(cases: list[FaultCase]) -> Iterator[tuple[FaultHTTPServer, FaultRouter]]: + with FaultHTTPServer(cases) as server: + router = FaultRouter(server) + try: + yield server, router + finally: + router.close() diff --git a/tests/test_download_corruption.py b/tests/test_download_corruption.py new file mode 100644 index 0000000..6420542 --- /dev/null +++ b/tests/test_download_corruption.py @@ -0,0 +1,38 @@ +"""One hundred corruption cases, real HTTP framing, isolation and explicit repair.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from modelscope_hub import HubApi +from tests.download_corruption_harness import exercise_stages, local_experiment, synthetic_cases + + +def test_hundred_corrupt_files_continue_isolate_repair_and_reuse(tmp_path, monkeypatch): + monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost") + monkeypatch.setenv("MODELSCOPE_DOWNLOAD_INTRA_CLOUD", "false") + monkeypatch.delenv("MODELSCOPE_DOWNLOAD_INTRA_CLOUD_REGION", raising=False) + monkeypatch.delenv("MODELSCOPE_DOWNLOAD_INTER_CLOUD_REGIONS", raising=False) + cases = synthetic_cases() + root = tmp_path / "hundred-corrupt-files" + with local_experiment(cases) as (server, router): + evidence = exercise_stages(root, cases, lambda: HubApi(endpoint=server.url, token=""), router) + assert server.credential_requests == 0 + assert evidence["passed_fault_cases"] == 100 + assert all(group["passed"] == 10 for group in evidence["group_results"].values()) + assert evidence["independent_size_sha256_verified"] == 200 + stages = evidence["stages"] + assert [stage["logical_file_requests"] for stage in stages.values()] == [300, 0, 0, 60, 0] + assert stages["first"]["summary"]["succeeded"] == 140 + assert stages["explicit_repair"]["summary"]["succeeded"] == 200 + ranges = [row for row in evidence["cases"] if row["group"] in (9, 10)] + assert len(ranges) == 20 + assert all(row["result"]["retries"] == 1 for row in evidence["cases"]) + tracker = root / ".ms_download_state.json" + saved = json.loads(tracker.read_text()) + assert saved["files"] == {} and saved["pages"] == {} + assert saved["corruption_regression"]["passed"] + assert len(list(root.rglob("*.json"))) == 1 + assert not list(root.rglob("*.jsonl")) + assert Path(stages["reuse"]["summary"]["report_path"]) == tracker diff --git a/tests/test_download_lock_path.py b/tests/test_download_lock_path.py index 1d70d0e..75664b1 100644 --- a/tests/test_download_lock_path.py +++ b/tests/test_download_lock_path.py @@ -83,6 +83,8 @@ def test_dataset_file_download_accepts_long_file_path(self, tmp_path, monkeypatc been acquired, and write the expected target file directly. """ + acquired_lock_files = [] + def fake_download_with_resume( self, repo_id: str, @@ -92,6 +94,7 @@ def fake_download_with_resume( target: Path, **kwargs, ) -> Path: + acquired_lock_files.extend((cache_dir / ".lock").glob("*.lock")) target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(b"image-bytes") return target @@ -108,6 +111,6 @@ def fake_download_with_resume( ) assert Path(result).read_bytes() == b"image-bytes" - lock_files = list((cache_dir / ".lock").glob("*.lock")) - assert lock_files - assert all(len(path.name.encode("utf-8")) < 255 for path in lock_files) + # New filelock releases remove Unix lock files, so inspect them while held. + assert acquired_lock_files + assert all(len(path.name.encode("utf-8")) < 255 for path in acquired_lock_files) diff --git a/tests/test_download_report.py b/tests/test_download_report.py new file mode 100644 index 0000000..efd31ae --- /dev/null +++ b/tests/test_download_report.py @@ -0,0 +1,526 @@ +"""Persistent isolation and bounded recovery across large download batches.""" + +from __future__ import annotations + +import importlib +import json +import threading +import time +from dataclasses import asdict +from pathlib import Path + +import pytest + +from modelscope_hub import DownloadReport, DownloadRetryPolicy, HubApi +from modelscope_hub.cli.main import run_cmd +from modelscope_hub.errors import InvalidParameter, NetworkError +from tests.test_download_integrity import REPO, _sha256 +from tests.test_download_integrity import api as api +from tests.test_download_integrity import dirs as dirs +from tests.test_download_integrity import hub as hub + + +def manifest(hub): + return [ + {"Path": path, "Size": len(file.content), "Sha256": _sha256(file.content)} for path, file in hub.files.items() + ] + + +def run(api, dirs, **kwargs): + return api.download_repo_report(REPO, "dataset", local_dir=dirs[0], cache_dir=dirs[1], **kwargs) + + +def test_failure_does_not_stop_files_after_the_first_batch(hub, api, dirs): + for i in range(25): + hub.add(f"shards/{i:02}.tar", b"data" * 1024, served=b"" if i == 3 else None) + report = run(api, dirs, max_workers=2) + assert report.status == "PARTIAL" + assert (report.total, report.succeeded, report.failed) == (25, 24, 1) + assert report.listing_complete + assert len(hub.seen("shards/03.tar", "cdn")) == 2 + assert (dirs[0] / "shards/24.tar").exists() + assert not (dirs[0] / "shards/03.tar").exists() + assert len(list(report.iter_files("VERIFIED"))) == 24 + assert json.loads(Path(report.report_path).read_text())["report"]["status"] == "PARTIAL" + + +def test_new_api_instances_and_batches_share_failure_isolation(hub, api, dirs): + hub.add("bad.bin", b"data", served=b"") + hub.add("next.bin", b"good") + first = run(api, dirs, allow_patterns=["bad.bin"]) + again = run(HubApi(endpoint=hub.url, token=""), dirs) + assert first.tracker_path == again.tracker_path + assert again.skipped_failures == 1 + assert len(hub.seen("bad.bin", "cdn")) == 2 + assert (dirs[0] / "next.bin").read_bytes() == b"good" + assert first.total == 1 # The first report remains immutable after later batches. + assert [file.path for file in first.iter_files()] == ["bad.bin"] + + +def test_failed_file_is_only_reopened_explicitly_or_on_metadata_change(hub, api, dirs): + hub.add("bad.bin", b"data", served=b"") + first = run(api, dirs) + hub.files["bad.bin"].served = None + second = run(api, dirs) + assert not second.complete + assert len(hub.seen("bad.bin", "cdn")) == 2 + recovered = run(api, dirs, retry_failed=True) + assert recovered.complete + assert len(hub.seen("bad.bin", "cdn")) == 3 + hub.files["bad.bin"].content = b"new version" + changed = run(api, dirs) + assert changed.complete + assert (dirs[0] / "bad.bin").read_bytes() == b"new version" + assert first.failed == 1 + + +def test_tracker_scope_rejects_another_output_or_revision(hub, api, dirs, tmp_path): + hub.add("x.bin", b"data") + ledger = tmp_path / ".ms_download_state.json" + run(api, dirs, tracker_path=ledger) + with pytest.raises(InvalidParameter): + run(api, (tmp_path / "other", dirs[1]), tracker_path=ledger) + with pytest.raises(InvalidParameter): + run(api, dirs, tracker_path=ledger, revision="v2") + with pytest.raises(InvalidParameter): + run(api, dirs, tracker_path=dirs[0] / ".ms_download_state.json") + + +def test_long_retry_after_defers_without_sleeping_or_occupying_a_worker(hub, api, dirs): + hub.add("busy.bin", b"data", status=429, retry_after="9999") + hub.add("next.bin", b"good") + started = time.monotonic() + report = run(api, dirs, max_workers=1) + assert time.monotonic() - started < 1.5 + assert report.deferred == 1 + assert report.succeeded == 1 + assert len(hub.seen("busy.bin", "cdn")) == 1 + run(api, dirs) + assert len(hub.seen("busy.bin", "cdn")) == 1 + + +def test_transport_and_hash_retries_share_one_file_budget(hub, api, dirs): + hub.add("bad.bin", b"good" * 2048, served=b"evil" * 2048, truncate_first=3000) + report = run(api, dirs, file_list=manifest(hub)) + result = next(report.iter_files()) + assert result.status == "FAILED" + assert result.retries == 2 + assert len(hub.seen("bad.bin", "cdn")) == 3 + assert not (dirs[0] / "bad.bin").exists() + + +def test_complete_but_corrupt_partial_is_rejected_before_network_and_repaired(hub, api, dirs): + content = hub.add("partial.bin", b"correct data" * 4096) + dirs[0].mkdir() + partial = dirs[0] / "partial.bin.incomplete" + partial.write_bytes(b"wrong bytes!" * 4096) + assert partial.stat().st_size == len(content) + report = run(api, dirs, file_list=manifest(hub)) + assert report.complete + assert next(report.iter_files()).retries == 1 + assert (dirs[0] / "partial.bin").read_bytes() == content + assert not partial.exists() + assert len(hub.seen("partial.bin", "cdn")) == 1 + report.close() + + +def test_same_size_corrupt_parallel_parts_are_rejected_and_do_not_stop_next_file(hub, api, dirs, monkeypatch): + from modelscope_hub import _download + + monkeypatch.setattr(_download, "DOWNLOAD_PARALLEL_THRESHOLD", 64 * 1024) + monkeypatch.setattr(_download, "DOWNLOAD_PART_SIZE", 64 * 1024) + monkeypatch.setattr(_download, "DOWNLOAD_PARALLELS", 4) + hub.add("bad.bin", b"good" * 75000, served=b"evil" * 75000) + good = hub.add("next.bin", b"correct tail") + report = run(api, dirs, file_list=manifest(hub), max_workers=1) + assert (report.failed, report.succeeded) == (1, 1) + failed = next(report.iter_files("FAILED")) + assert failed.error_type == "FileIntegrityError" and failed.retries == 1 + assert len(hub.seen("bad.bin", "cdn")) == 10 # Five ranges per attempt, two complete attempts. + assert all(request.range for request in hub.seen("bad.bin", "cdn")) + assert not list(dirs[0].glob("bad.bin*")) + assert (dirs[0] / "next.bin").read_bytes() == good + report.close() + + +def test_stalled_headers_use_short_policy_and_no_nested_retries(hub, api, dirs): + hub.add("stall.bin", b"x", header_delay=2) + hub.add("good.bin", b"good") + policy = DownloadRetryPolicy(max_retries=1, idle_timeout=0.1, recovery_timeout=0.4) + report = run(api, dirs, retry_policy=policy, max_workers=1) + assert report.deferred == 1 + assert report.succeeded == 1 + assert len(hub.seen("stall.bin", "cdn")) == 2 + assert next(report.iter_files("DEFERRED")).retries == 1 + + +def test_invalid_metadata_and_paths_are_isolated_without_network(hub, api, dirs, tmp_path): + hub.add("good.bin", b"data") + entries = [ + {"Path": "missing.bin"}, + {"Path": "negative.bin", "Size": -1}, + {"Path": "../escape.bin", "Size": 1}, + {"Path": "bad-hash.bin", "Sha256": "invalid"}, + *manifest(hub), + ] + report = run(api, dirs, file_list=iter(entries)) + assert (report.failed, report.succeeded) == (4, 1) + assert not (tmp_path / "escape.bin").exists() + assert {request.path for request in hub.requests} == {"good.bin"} + + +def test_pagination_downloads_page_one_before_page_two_finishes(hub, api, dirs, monkeypatch): + hub.add("a.bin", b"a") + hub.add("b.bin", b"b") + entries = manifest(hub) + + def pages(*args, **kwargs): + yield 1, entries[:1], None + limit = time.monotonic() + 2 + while not (dirs[0] / "a.bin").exists() and time.monotonic() < limit: + time.sleep(0.01) + assert (dirs[0] / "a.bin").exists() + yield 2, [], NetworkError("page missing") + yield 3, entries[1:], None + + monkeypatch.setattr(api.legacy, "iter_dataset_file_pages", pages) + report = run(api, dirs) + assert report.succeeded == 2 + assert not report.listing_complete + assert "page=2" in report.listing_errors[0] + + +def test_manifest_iterable_is_bounded_and_not_materialized(hub, api, dirs): + hub.add("x.bin", b"x") + + def entries(): + yield manifest(hub)[0] + limit = time.monotonic() + 2 + while not (dirs[0] / "x.bin").exists() and time.monotonic() < limit: + time.sleep(0.01) + assert (dirs[0] / "x.bin").exists() + + assert run(api, dirs, file_list=entries()).complete + + +def test_cancelled_run_has_durable_pending_state_and_is_resumable(hub, api, dirs, monkeypatch): + import modelscope_hub._resilient_download as module + + hub.add("x.bin", b"x" * 100, header_delay=2) + entered = threading.Event() + original_wait = module.wait + + def interrupt(*args, **kwargs): + if not entered.is_set(): + entered.set() + time.sleep(0.05) + raise KeyboardInterrupt + return original_wait(*args, **kwargs) + + monkeypatch.setattr(module, "wait", interrupt) + with pytest.raises(KeyboardInterrupt): + run(api, dirs) + reports = list((dirs[1] / ".download_state").glob("*.json")) + assert len(reports) == 1 + assert json.loads(reports[0].read_text())["report"]["status"] == "CANCELLED" + hub.files["x.bin"].header_delay = 0 + assert run(api, dirs).complete + + +def test_crashed_recovery_does_not_receive_a_fresh_automatic_budget(hub, api, dirs): + hub.add("x.bin", b"x", served=b"") + first = run(api, dirs) + ledger = Path(first.tracker_path) + data = json.loads(ledger.read_text()) + data["files"]["x.bin"]["status"] = "DOWNLOADING" + ledger.write_text(json.dumps(data)) + again = run(api, dirs) + assert again.skipped_failures == 1 + assert again.deferred == 1 + assert len(hub.seen("x.bin", "cdn")) == 2 + + +def test_cli_partial_exit_three_and_strict_mode_unchanged(hub, dirs, monkeypatch): + monkeypatch.setenv("MODELSCOPE_API_TOKEN", "") + hub.add("bad.bin", b"data", served=b"") + hub.add("good.bin", b"good") + args = [ + "--endpoint", + hub.url, + "download", + REPO, + "--repo-type", + "dataset", + "--local-dir", + str(dirs[0]), + "--cache-dir", + str(dirs[1]), + ] + assert run_cmd([*args, "--continue-on-error"]) == 3 + assert (dirs[0] / "good.bin").exists() + assert run_cmd([*args, "--continue-on-error"]) == 3 + assert len(hub.seen("bad.bin", "cdn")) == 2 + assert run_cmd(args) == 1 + + +@pytest.mark.parametrize("name", ["snapshot_download", "dataset_snapshot_download"]) +def test_compat_continue_mode_is_explicit_and_survives_new_instances(hub, dirs, name): + module = importlib.import_module("modelscope_hub.compat.snapshot_download") + function = getattr(module, name) + hub.add("bad.bin", b"data", served=b"") + hub.add("good.bin", b"good") + kwargs = dict(endpoint=hub.url, token="", local_dir=str(dirs[0]), cache_dir=str(dirs[1]), continue_on_error=True) + if name == "snapshot_download": + kwargs["repo_type"] = "dataset" + assert function(REPO, **kwargs) == str(dirs[0]) + assert function(REPO, **kwargs) == str(dirs[0]) + assert len(hub.seen("bad.bin", "cdn")) == 2 + + +def test_bad_listing_page_is_durably_skipped_and_does_not_hide_later_pages(hub, api, dirs, monkeypatch): + hub.add("a.bin", b"a") + hub.add("b.bin", b"b") + entries = manifest(hub) + calls = [] + + def request(path, params, **kwargs): + page = params["PageNumber"] + calls.append(page) + if page == 2: + raise NetworkError("missing page") + response = type( + "Response", + (), + {"json": lambda self: {"Data": {"Files": [entries[0]] * 200 if page == 1 else [entries[1]]}}}, + )() + return response + + monkeypatch.setattr(api.legacy, "_request_repo_tree", request) + first = run(api, dirs) + assert first.succeeded == 2 + assert not first.listing_complete + assert calls == [1, 2, 3] + calls.clear() + again = run(api, dirs) + assert not again.listing_complete + assert calls == [1, 3] + + +def test_report_header_trickle_is_stopped_without_waiting_for_whole_file_deadline(hub, api, dirs): + hub.add("slow.bin", b"data", header_trickle=True) + hub.add("next.bin", b"next") + started = time.monotonic() + report = run(api, dirs, retry_policy=DownloadRetryPolicy(max_retries=0, idle_timeout=0.15)) + assert time.monotonic() - started < 1 + assert report.deferred == 1 + assert report.succeeded == 1 + + +def test_duplicates_cannot_reset_an_active_files_metadata(hub, api, dirs): + hub.add("x.bin", b"data") + entry = manifest(hub)[0] + report = run(api, dirs, file_list=[entry, dict(entry, Size=999), entry]) + assert report.total == 1 + assert report.succeeded == 1 + assert len(hub.seen("x.bin", "cdn")) == 1 + + +def test_business_listing_failure_is_not_an_empty_complete_dataset(hub, api, dirs, monkeypatch): + from unittest.mock import MagicMock + + response = MagicMock() + response.json.return_value = {"Success": False, "Code": 500} + monkeypatch.setattr(api.legacy, "_request_repo_tree", lambda *args, **kwargs: response) + report = run(api, dirs) + assert not report.complete + assert not report.listing_complete + assert len(report.listing_errors) == 2 + assert report.total == 0 + + +def test_healthy_files_do_not_write_persistent_records_per_file(hub, api, dirs, monkeypatch): + from modelscope_hub._download_state import _DownloadState + + writes = [] + original_save = _DownloadState._save + + def saved(state): + writes.append(1) + original_save(state) + + monkeypatch.setattr(_DownloadState, "_save", saved) + for i in range(30): + hub.add(f"file-{i}.bin", b"good") + report = run(api, dirs) + assert report.complete + assert writes == [1, 1] # One scope header and one final summary, independent of file count. + data = json.loads(Path(report.tracker_path).read_text()) + assert data["files"] == {} + assert Path(report.tracker_path).name.startswith(".") + assert len(list(report.iter_files("VERIFIED"))) == 30 + run(api, dirs) + assert writes == [1, 1, 1] # A rerun only replaces the latest summary. + + +def test_repaired_file_overrides_stale_failure_hint_without_network(hub, api, dirs): + content = hub.add("bad.bin", b"valid data", served=b"") + first = run(api, dirs) + assert first.failed == 1 + (dirs[0] / "bad.bin").write_bytes(content) + repaired = run(api, dirs) + assert repaired.complete + assert repaired.skipped_failures == 0 + assert len(hub.seen("bad.bin", "cdn")) == 2 + assert json.loads(Path(repaired.tracker_path).read_text())["files"] == {} + + +def test_deleted_state_does_not_redownload_valid_files(hub, api, dirs): + hub.add("good.bin", b"valid data") + first = run(api, dirs) + Path(first.tracker_path).unlink() + again = run(api, dirs) + assert again.complete + assert len(hub.seen("good.bin", "cdn")) == 1 + + +@pytest.mark.parametrize("damaged", ['{"version":', '{"version": 1, "identity": null}', "\ufffd"]) +def test_corrupt_state_warns_and_revalidates_actual_files(hub, api, dirs, damaged): + hub.add("good.bin", b"valid data") + first = run(api, dirs) + Path(first.tracker_path).write_text(damaged) + again = run(api, dirs) + assert again.complete + assert again.state_warnings + assert len(hub.seen("good.bin", "cdn")) == 1 + + +def test_success_hint_cannot_hide_same_size_corruption(hub, api, dirs): + hub.add("good.bin", b"good") + first = run(api, dirs) + (dirs[0] / "good.bin").write_bytes(b"evil") + data = json.loads(Path(first.tracker_path).read_text()) + data["files"]["good.bin"] = asdict(next(first.iter_files())) + Path(first.tracker_path).write_text(json.dumps(data)) + again = run(api, dirs) + assert again.complete + assert again.state_warnings # Ignore VERIFIED claims in auxiliary state. + assert (dirs[0] / "good.bin").read_bytes() == b"good" + assert len(hub.seen("good.bin", "cdn")) == 2 + + +def test_cannot_save_hints_does_not_block_downloads(hub, api, dirs, monkeypatch): + hub.add("bad.bin", b"data", served=b"") + hub.add("good.bin", b"good") + original = Path.replace + + def fail_state_replace(path, target): + if str(target).endswith(".json") and ".report." not in str(target): + raise PermissionError("hint file cannot be replaced") + return original(path, target) + + monkeypatch.setattr(Path, "replace", fail_state_replace) + report = run(api, dirs) + assert report.failed == 1 + assert report.succeeded == 1 + assert report.state_warnings + assert (dirs[0] / "good.bin").read_bytes() == b"good" + + +def test_missing_state_does_not_promote_a_partial_file_to_verified(hub, api, dirs): + hub.add("good.bin", b"good") + dirs[0].mkdir() + (dirs[0] / "good.bin").write_bytes(b"go") + report = run(api, dirs) + assert report.complete + assert (dirs[0] / "good.bin").read_bytes() == b"good" + + +def test_import_works_when_embedded_database_modules_are_unavailable(): + import subprocess + import sys + + code = """ +import importlib.abc +import sys +class BlockDatabase(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split(".")[0] in {"sqlite3", "duckdb", "_sqlite3"}: + raise ImportError("Embedded databases are unavailable") +sys.meta_path.insert(0, BlockDatabase()) +import modelscope_hub._download_state +import modelscope_hub._resilient_download +""" + subprocess.run([sys.executable, "-c", code], check=True, capture_output=True, text=True) + + +def test_single_json_is_reused_across_failure_restart_and_repair(hub, api, dirs, tmp_path): + hub.add("bad.bin", b"data", served=b"") + hub.add("good.bin", b"good") + task = tmp_path / "task" + tracker = task / ".ms_download_state.json" + first = run(api, dirs, tracker_path=tracker) + second = run(api, dirs, tracker_path=tracker) + repaired = run(api, dirs, tracker_path=tracker, retry_failed=True) + assert first.failed == second.failed == repaired.failed == 1 + hub.files["bad.bin"].served = None + final = run(api, dirs, tracker_path=tracker, retry_failed=True) + assert final.complete + assert all(report.report_path == report.tracker_path == str(tracker) for report in (first, second, final)) + assert final.results_path == "" + assert list(task.glob("*.json")) == [tracker] + assert not list(task.glob("*.jsonl")) + assert not list(task.glob("*.tmp")) + data = json.loads(tracker.read_text()) + assert data["files"] == {} and data["pages"] == {} + assert data["report"]["succeeded"] == 2 + assert data["report"]["status"] == "COMPLETE" + assert [result.status for result in first.iter_files("FAILED")] == ["FAILED"] + assert len(list(final.iter_files("VERIFIED"))) == 2 + assert "_file_results" not in asdict(final) + + +@pytest.mark.parametrize("reset", [False, True]) +def test_explicit_manifest_recovery_resets_selected_page_hints(hub, api, dirs, reset): + hub.add("good.bin", b"good") + first = run(api, dirs) + tracker = Path(first.tracker_path) + data = json.loads(tracker.read_text()) + data["pages"] = {"/": {"13": "Previous page timeout"}} + tracker.write_text(json.dumps(data)) + recovered = run(api, dirs, file_list=manifest(hub), retry_failed=reset) + assert recovered.complete + assert json.loads(tracker.read_text())["pages"] == ({} if reset else data["pages"]) + assert len(hub.seen("good.bin", "cdn")) == 1 + + +def test_result_iterators_are_independent_and_close_releases_spool(hub, api, dirs): + for i in range(3): + hub.add(f"{i}.bin", b"good") + report = run(api, dirs) + left, right = report.iter_files(), report.iter_files() + assert next(left).path == next(right).path + assert [entry.path for entry in left] == [entry.path for entry in right] + assert len(list(report.iter_files())) == 3 + spool = report._file_results + report.close() + report.close() + assert spool.closed + with pytest.raises(ValueError, match="open live DownloadReport"): + list(report.iter_files()) + + +def test_persisted_summary_cannot_be_used_as_verified_file_evidence(hub, api, dirs): + hub.add("good.bin", b"good") + report = run(api, dirs) + summary = json.loads(Path(report.report_path).read_text())["report"] + restored = DownloadReport.from_dict(summary) + assert restored.complete + with pytest.raises(ValueError, match="not a verified-file manifest"): + list(restored.iter_files("VERIFIED")) + + +def test_report_types_and_policy_are_public(): + assert not DownloadReport().complete + assert asdict(DownloadRetryPolicy())["max_retries"] == 2 + with pytest.raises(ValueError): + DownloadRetryPolicy(idle_timeout=0) From 2535234821cc92b27a64b0375c575ab60c9fe1ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=AD=E6=89=AC?= Date: Mon, 26 Oct 2026 19:42:50 +0800 Subject: [PATCH 19/22] bump version --- src/modelscope_hub/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modelscope_hub/version.py b/src/modelscope_hub/version.py index 8844a57..48b097a 100644 --- a/src/modelscope_hub/version.py +++ b/src/modelscope_hub/version.py @@ -1,3 +1,3 @@ """Version information for modelscope_hub.""" -__version__ = "0.5.0+main" +__version__ = "0.5.1+main" From dd02f613959fecec96fadb15a47c1aed530cc86f Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Thu, 29 Oct 2026 16:21:54 +0800 Subject: [PATCH 20/22] Route model/dataset writes through OpenAPI and add zone metadata support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model and dataset creation, settings updates and file deletion now go through /openapi/v1, keeping the legacy endpoint for inputs OpenAPI cannot express (internal visibility, non-enum licenses, AIGC compat calls). - add create_repo routing with gated mode (auto/manual) and visibility mapping - add get_model/get_dataset detail reads, including zone fields - add set/update model and dataset category plus options endpoints with TTL cache - add delete_files payload validation and a legacy-preserving delete path - align constants, types and README with the new wire protocol Signed-off-by: 班扬 --- README.md | 97 +- src/modelscope_hub/__init__.py | 18 +- src/modelscope_hub/_openapi.py | 299 +- src/modelscope_hub/api.py | 1103 +- src/modelscope_hub/cli/repo.py | 6 +- src/modelscope_hub/compat/hub_api.py | 116 + src/modelscope_hub/constants.py | 209 + src/modelscope_hub/types.py | 278 + tests/data/openapi.json | 16855 +++++++++++++++++-------- tests/test_compat_get_model_files.py | 75 +- tests/test_create_repo_routing.py | 444 + tests/test_dataset_category.py | 191 + tests/test_model_category.py | 261 + tests/test_model_detail_fields.py | 235 + tests/test_model_options.py | 207 + tests/test_openapi_coverage.py | 6 +- tests/test_openapi_delete_files.py | 160 + tests/test_repo_info_visibility.py | 121 + 18 files changed, 15607 insertions(+), 5074 deletions(-) create mode 100644 tests/test_create_repo_routing.py create mode 100644 tests/test_dataset_category.py create mode 100644 tests/test_model_category.py create mode 100644 tests/test_model_detail_fields.py create mode 100644 tests/test_model_options.py create mode 100644 tests/test_openapi_delete_files.py create mode 100644 tests/test_repo_info_visibility.py diff --git a/README.md b/README.md index 7b0922f..bac0b6d 100644 --- a/README.md +++ b/README.md @@ -269,6 +269,62 @@ ms-hub create my-org/my-model --repo-type model --visibility private api.create_repo("my-org/my-model", "model", visibility="private", license="apache-2.0") ``` +### Visibility and gated access + +A repository is `public`, `private`, or `gated`. Gated repositories are +discoverable, but downloading requires an application, approved either +automatically (`"auto"`) or by a maintainer (`"manual"`): + +```python +api.create_repo("my-org/my-data", "dataset", visibility="gated", gated="manual") +api.update_repo_settings("my-org/my-data", "dataset", visibility="gated", gated="auto") + +# Turn gating off by leaving the gated state. +api.update_repo_settings("my-org/my-data", "dataset", visibility="private") +``` + +The older boolean `gated_mode=True` / `gated_mode=False` still works and keeps +its original behaviour exactly, but it cannot express the approval mode. Prefer +`gated`. + +`visibility="internal"` (visible within an organisation) has no equivalent in +the OpenAPI surface, so those requests continue to use the legacy endpoint. + +### Model and dataset zones + +Models belong to the `base`, `aigc`, or `science` zone; datasets to `base` or +`science`. A new repository is always `base`. Upload the files first, then set +the zone — the AIGC endpoint checks that each weight path already exists: + +```python +api.create_repo("my-org/my-lora", "model") +api.upload_folder("my-org/my-lora", "model", "./my-lora-files") + +options = api.list_aigc_model_options() # enum ids, no credentials needed +api.set_model_category( + "my-org/my-lora", + model_category="aigc", + aigc_type="LoRA", + base_model_architecture_type=options["base_architectures"][0]["id"], + base_model_architecture_subtype=options["base_architectures"][0]["subtypes"][0]["id"], + version_name="v1.0", + weight_files=[{"path_in_repo": "model.safetensors"}], + sample_images=[{"url": "https://example.com/sample.png", "prompt": "a cat"}], +) + +# Edit a version's metadata in place, addressed by version name or numeric id. +api.update_model_category("my-org/my-lora", version_name="v1.0", trigger_words=["anime"]) + +# Read the zone fields back; which keys are present depends on the category. +detail = api.get_model("my-org/my-lora") +print(detail["model_category"], [v["version_id"] for v in detail.get("versions", [])]) +``` + +Enum values (`aigc_type`, architectures, subject areas, `task`, `license`) come +from the options endpoints; pass each option's `id`, not its `label`. An +existing version's weights are immutable — changing them means creating a new +version with `set_model_category`. + ### Deploy a Studio ```bash @@ -831,9 +887,19 @@ api = HubApi(token="...", endpoint="https://modelscope.ai") | | `whoami()` | Get current user info | | **Repo** | `create_repo(repo_id, repo_type, ...)` | Create a repository | | | `get_repo(repo_id, repo_type)` | Get repository metadata | +| | `get_model(repo_id)` / `model_info(repo_id)` | Model detail, including zone fields | +| | `get_dataset(repo_id)` / `dataset_info(repo_id)` | Dataset detail, including science fields | | | `list_repos(repo_type, ...)` | Paginated listing (`model`, `dataset`, `studio`, `skill`, `mcp`) | | | `delete_repo(repo_id, repo_type)` | Delete a repository *(deprecated — see note below)* | | | `repo_exists(repo_id, repo_type)` | Check existence | +| **Zones** | `set_model_category(repo_id, model_category=...)` | Move a model into `base` / `aigc` / `science` | +| | `update_model_category(repo_id, ...)` | Edit a model's zone fields in place | +| | `set_dataset_category(repo_id, dataset_category=...)` | Move a dataset into `base` / `science` | +| | `update_dataset_category(repo_id, ...)` | Edit a dataset's science fields in place | +| | `list_aigc_model_options()` | AIGC enums (types, architectures, content categories) | +| | `list_science_model_options()` | Science model subject areas | +| | `list_science_dataset_options()` | Science dataset subject areas | +| | `list_model_options()` | Valid model `task` and `license` values | | **Files** | `upload_file(repo_id, repo_type, local, remote)` | Upload a single file | | | `upload_folder(repo_id, repo_type, folder, ...)` | Upload a directory | | | `download_file(repo_id, repo_type, file, ...)` | Download a single file (with retry, resume, offline mode) | @@ -845,7 +911,7 @@ api = HubApi(token="...", endpoint="https://modelscope.ai") | **Deploy** | `deploy_repo(repo_id, repo_type)` | Deploy Studio or MCP | | | `stop_repo(repo_id, repo_type)` | Stop deployment | | | `get_repo_logs(repo_id, ...)` | Fetch logs | -| | `update_repo_settings(repo_id, repo_type, ...)` | Update settings | +| | `update_repo_settings(repo_id, repo_type, ...)` | Update settings (`model`, `dataset`, `studio`, `skill`) | | **Secrets** | `add_secret(repo_id, key, value)` | Add a secret | | | `list_secrets(repo_id)` | List secret keys (values are never returned) | | | `update_secret(repo_id, key, value)` | Update a secret | @@ -942,6 +1008,7 @@ Token is persisted locally after `ms-hub login` and auto-loaded in subsequent se | `MODELSCOPE_API_TIMEOUT` | `60` | HTTP request timeout (seconds) | | `MODELSCOPE_API_CONNECT_TIMEOUT` | `10` | HTTP connect timeout (seconds) | | `MODELSCOPE_API_MAX_RETRIES` | `5` | Regular API retries; downloads and file listings use separate retry policies | +| `MODELSCOPE_OPTIONS_CACHE_TTL` | `300` | Seconds to cache the model/dataset enum option endpoints; `0` disables caching | | `MODELSCOPE_REPO_TREE_PAGE_TIMEOUT_SECONDS` | `15` | Per-page listing timeout (seconds) | | `MODELSCOPE_REPO_TREE_PAGE_MAX_ATTEMPTS` | `4` | Maximum attempts per listing page, including the initial request | | `MODELSCOPE_REPO_TREE_PAGE_RETRY_MAX_DELAY_SECONDS` | `8` | Maximum listing retry delay (seconds) | @@ -1014,6 +1081,34 @@ from modelscope_hub.compat import LegacyHubApi as HubApi All legacy parameter names (`allow_file_pattern`, `ignore_file_pattern`, `cookies`, etc.) are accepted and mapped to the new implementation. +### Behaviour changes when model/dataset writes moved to OpenAPI + +Model and dataset creation and settings now use the `/openapi/v1` endpoints. +Inputs those endpoints cannot express keep using the legacy ones, so existing +calls continue to work; three observable differences remain: + +| Change | Detail | +|--------|--------| +| Gated repositories in metadata | A gated repository is now reported as `Visibility.PRIVATE` with `gated=True`. Previously the server's `private: false` made it read back as `Visibility.PUBLIC`. The field stays a `Visibility` integer. | +| Default license | When `license` is omitted the Hub applies its own default (`apache-2.0`) instead of the SDK sending `Apache-2.0`. Only affects calls that take the OpenAPI route. | +| `LegacyHubApi.create_dataset(description=...)` | The argument was previously accepted and then dropped; it is now sent. | + +Unchanged on purpose: + +- **`gated_mode=True` / `gated_mode=False`** keep the exact legacy request + (`Visibility=PRIVATE` plus `ProtectedMode=1`/`2`), including the warn-and-ignore + when visibility is public. The boolean cannot name an approval mode and the two + surfaces default differently, so translating it could have turned a + review-gated repository into an auto-approving one. `ms-hub create --gated` + is therefore also byte-for-byte unchanged. +- `delete_files()` still commits deletions directly, so it keeps glob support + and has no 100-path ceiling. +- `visibility="internal"`, licenses outside the OpenAPI enum, a `task` on a + dataset, and extra request fields all still reach the legacy endpoint. +- The AIGC creation path in `modelscope_hub.compat` + (`create_model(aigc_model=...)`, `create_model_tag(aigc_model=...)`) is + untouched. + --- ## Development diff --git a/src/modelscope_hub/__init__.py b/src/modelscope_hub/__init__.py index b6dc9c4..1594e8a 100644 --- a/src/modelscope_hub/__init__.py +++ b/src/modelscope_hub/__init__.py @@ -19,7 +19,18 @@ ) from .api import HubApi from .config import HubConfig, get_default_config, set_default_config -from .constants import License, RepoType, StudioVisibility, TokenScope, Visibility +from .constants import ( + AigcModelType, + DatasetCategory, + GatedMode, + License, + ModelCategory, + RepoType, + RepoVisibility, + StudioVisibility, + TokenScope, + Visibility, +) from .errors import ( APIError, AuthenticationError, @@ -75,8 +86,13 @@ "get_default_config", "set_default_config", # Enums + "AigcModelType", + "DatasetCategory", + "GatedMode", "License", + "ModelCategory", "RepoType", + "RepoVisibility", "StudioVisibility", "TokenScope", "Visibility", diff --git a/src/modelscope_hub/_openapi.py b/src/modelscope_hub/_openapi.py index 7217f09..46bb777 100644 --- a/src/modelscope_hub/_openapi.py +++ b/src/modelscope_hub/_openapi.py @@ -44,7 +44,9 @@ API_TIMEOUT, DOWNLOAD_CHUNK_SIZE, DOWNLOAD_RETRY_MAX_DELAY_SECONDS, + OPENAPI_DELETE_FILES_MAX_PATHS, OPENAPI_PREFIX, + OPTIONS_CACHE_TTL_SECONDS, TokenScope, ) from .errors import ( @@ -60,13 +62,22 @@ ) from .types import ( CreateAgentIdentityPayload, + CreateDatasetPayload, + CreateModelPayload, CreateSkillPayload, CreateStudioPayload, + DeleteFilesPayload, DeployMcpServerPayload, PauseAgentPayload, ResetAgentKeyPairPayload, + SetDatasetCategoryPayload, + SetModelCategoryPayload, TokenSignPayload, UpdateAgentIdentityPayload, + UpdateDatasetCategoryPayload, + UpdateDatasetSettingsPayload, + UpdateModelCategoryPayload, + UpdateModelSettingsPayload, UpdateSkillSettingsPayload, UpdateStudioSettingsPayload, ) @@ -124,6 +135,27 @@ def _validate_choice(name: str, value: str | None, allowed: tuple[str, ...]) -> raise InvalidParameter(f"{name} must be one of {', '.join(allowed)} (got {value!r}).") +def _validated_delete_files(payload: Mapping[str, Any]) -> dict[str, Any]: + """Check a delete-files payload against the limits the endpoint declares. + + The path count is bounded at both ends, so an empty list and an oversized + one are both rejected before the request goes out -- the server answers an + opaque 400 for either. + """ + body = dict(payload) + paths = body.get("file_paths") + if not isinstance(paths, list) or not paths: + raise InvalidParameter("file_paths must be a non-empty list of repository-relative file paths.") + if len(paths) > OPENAPI_DELETE_FILES_MAX_PATHS: + raise InvalidParameter( + f"file_paths accepts at most {OPENAPI_DELETE_FILES_MAX_PATHS} paths per request " + f"(got {len(paths)}). Use HubApi.delete_files() for larger batches." + ) + if len(set(paths)) != len(paths): + raise InvalidParameter("file_paths must not contain duplicate paths.") + return body + + def _as_wire_bool(value: bool | None) -> str | None: """Render a tri-state flag the way query strings expect (``true``/``false``).""" if value is None: @@ -184,6 +216,26 @@ def _as_wire_bool(value: bool | None) -> str | None: "addStudioVariable": ("add_studio_variable", TokenScope.WRITE), "updateStudioVariable": ("update_studio_variable", TokenScope.WRITE), "deleteStudioVariable": ("delete_studio_variable", TokenScope.WRITE), + # -- Models ------------------------------------------------------------- + "listModels": ("list_models", TokenScope.READ), + "createModel": ("create_model", TokenScope.WRITE), + "getModel": ("get_model", TokenScope.READ), + "setModelCategory": ("set_model_category", TokenScope.WRITE), + "updateModelCategory": ("update_model_category", TokenScope.WRITE), + "updateModelSettings": ("update_model_settings", TokenScope.WRITE), + "deleteModelFiles": ("delete_model_files", TokenScope.WRITE), + "getModelOptions": ("get_model_options", TokenScope.READ), + "listAigcModelOptions": ("list_aigc_model_options", TokenScope.READ), + "listScienceModelOptions": ("list_science_model_options", TokenScope.READ), + # -- Datasets ----------------------------------------------------------- + "listDatasets": ("list_datasets", TokenScope.READ), + "createDataset": ("create_dataset", TokenScope.WRITE), + "getDataset": ("get_dataset", TokenScope.READ), + "setDatasetCategory": ("set_dataset_category", TokenScope.WRITE), + "updateDatasetCategory": ("update_dataset_category", TokenScope.WRITE), + "updateDatasetSettings": ("update_dataset_settings", TokenScope.WRITE), + "deleteDatasetFiles": ("delete_dataset_files", TokenScope.WRITE), + "listScienceDatasetOptions": ("list_science_dataset_options", TokenScope.READ), } @@ -241,6 +293,11 @@ def __init__( # Tri-state cache for whether this deployment serves ``GET /mcp/servers``. # ``None`` means "not probed yet"; see :meth:`list_mcp_servers`. self._mcp_list_supports_get: bool | None = None + # ``path -> (monotonic timestamp, payload)`` for the ``*/options`` + # endpoints. Guarded by a lock because one client is shared across the + # worker threads that upload and download. + self._options_cache: dict[str, tuple[float, Any]] = {} + self._options_cache_lock = threading.Lock() self._read_state = threading.local() self._read_session = requests.Session() self._read_session.headers.update(self._session.headers) @@ -714,9 +771,153 @@ def list_models( return self._request("GET", "/models", params=params, require_token=False) def get_model(self, owner: str, repo_name: str) -> JSON: - """``GET /models/{owner}/{repo_name}`` — fetch a model's metadata.""" + """``GET /models/{owner}/{repo_name}`` — fetch a model's metadata. + + The payload is a ``ModelDetail``: the common repository fields plus + ``model_category`` and, for a model in a zone, that zone's fields only + (AIGC architecture and ``versions``, or the science subject areas). + Fields belonging to the other zones are absent rather than null. + """ return self._request("GET", f"/models/{owner}/{repo_name}", require_token=False) + # ------------------------------------------------------------------ + # Model / dataset enum options + # + # These endpoints publish product configuration and take no credentials. + # Their values change on a release cadence, so repeated reads within one + # process are served from a short-lived cache (see + # :data:`OPTIONS_CACHE_TTL_SECONDS`); a caller that sets a category no + # longer re-fetches the whole enum tree on every call. + # ------------------------------------------------------------------ + def _cached_options(self, path: str) -> JSON: + """Fetch an options endpoint, reusing a recent response when allowed.""" + ttl = OPTIONS_CACHE_TTL_SECONDS + if ttl <= 0: + return self._request("GET", path, require_token=False) + now = time.monotonic() + with self._options_cache_lock: + entry = self._options_cache.get(path) + if entry is not None and now - entry[0] < ttl: + return entry[1] + payload = self._request("GET", path, require_token=False) + with self._options_cache_lock: + self._options_cache[path] = (time.monotonic(), payload) + return payload + + def clear_options_cache(self) -> None: + """Drop every cached ``*/options`` response.""" + with self._options_cache_lock: + self._options_cache.clear() + + def list_aigc_model_options(self) -> JSON: + """``GET /model/aigc/options`` — AIGC zone enums. + + Returns ``aigc_types``, the ``base_architectures`` cascade (first-level + architecture plus ``subtypes``), and the ``content_styles`` / + ``content_functions`` / ``content_elements`` triples. Every option is a + ``{type, id, label}`` triple; submit ``id``, display ``label``. + """ + return self._cached_options("/model/aigc/options") + + def list_science_model_options(self) -> JSON: + """``GET /model/science/options`` — science model subject-area tree.""" + return self._cached_options("/model/science/options") + + def get_model_options(self) -> JSON: + """``GET /models/options`` — the task and license option lists. + + ``tasks[].id`` is what ``task`` accepts on create and settings, and what + ``filter.task`` accepts when listing; ``license[].id`` is what + ``license`` accepts. Needs no credentials. + """ + return self._cached_options("/models/options") + + # ------------------------------------------------------------------ + # Model writes + # ------------------------------------------------------------------ + def create_model(self, payload: CreateModelPayload | Mapping[str, Any]) -> JSON: + """``POST /models`` — create a general model repository. + + Accepts general metadata only; the new repository is ``base``. Zone + fields go through :meth:`set_model_category` after the weights are + uploaded. Unknown fields are rejected by the server with a 400. + """ + return self._request("POST", "/models", json_body=dict(payload), required_scope=TokenScope.WRITE) + + def set_model_category( + self, + owner: str, + repo_name: str, + payload: SetModelCategoryPayload | Mapping[str, Any], + ) -> JSON: + """``PUT /models/{owner}/{repo_name}/category`` — set or change the zone. + + ``PUT`` because this replaces the zone wholesale: the required fields + follow from the target ``model_category`` alone, and moving to ``base`` + clears every zone field. Partial edits within a zone use + :meth:`update_model_category`. + """ + return self._request( + "PUT", + f"/models/{owner}/{repo_name}/category", + json_body=dict(payload), + required_scope=TokenScope.WRITE, + ) + + def update_model_category( + self, + owner: str, + repo_name: str, + payload: UpdateModelCategoryPayload | Mapping[str, Any], + ) -> JSON: + """``PATCH /models/{owner}/{repo_name}/category`` — edit zone metadata. + + The server reads the current zone from its own database, so the payload + must not carry ``model_category``. AIGC version fields are addressed by + the numeric ``version_id``. + """ + return self._request( + "PATCH", + f"/models/{owner}/{repo_name}/category", + json_body=dict(payload), + required_scope=TokenScope.WRITE, + ) + + def update_model_settings( + self, + owner: str, + repo_name: str, + settings: UpdateModelSettingsPayload | Mapping[str, Any], + ) -> JSON: + """``PATCH /models/{owner}/{repo_name}/settings`` — general settings. + + Omitted fields are unchanged; explicit nulls clear. Zone fields are not + accepted here. + """ + return self._request( + "PATCH", + f"/models/{owner}/{repo_name}/settings", + json_body=dict(settings), + required_scope=TokenScope.WRITE, + ) + + def delete_model_files( + self, + owner: str, + repo_name: str, + payload: DeleteFilesPayload | Mapping[str, Any], + ) -> JSON: + """``DELETE /models/{owner}/{repo_name}/files`` — one atomic commit. + + See :meth:`delete_dataset_files` for the shared constraints. + """ + return self._request( + "DELETE", + f"/models/{owner}/{repo_name}/files", + json_body=_validated_delete_files(payload), + required_scope=TokenScope.WRITE, + ) + # ================================================================== # Datasets # ================================================================== @@ -746,9 +947,103 @@ def list_datasets( return self._request("GET", "/datasets", params=params, require_token=False) def get_dataset(self, owner: str, repo_name: str) -> JSON: - """``GET /datasets/{owner}/{repo_name}`` — fetch a dataset's metadata.""" + """``GET /datasets/{owner}/{repo_name}`` — fetch a dataset's metadata. + + The payload is a ``DatasetDetail``: the common repository fields plus + ``dataset_category``, ``login_required``, and -- only for a science + dataset -- the subject areas and cover image. + """ return self._request("GET", f"/datasets/{owner}/{repo_name}", require_token=False) + def list_science_dataset_options(self) -> JSON: + """``GET /dataset/science/options`` — science dataset subject-area tree.""" + return self._cached_options("/dataset/science/options") + + # ------------------------------------------------------------------ + # Dataset writes + # ------------------------------------------------------------------ + def create_dataset(self, payload: CreateDatasetPayload | Mapping[str, Any]) -> JSON: + """``POST /datasets`` — create a general dataset repository. + + The new repository is ``base``; science fields go through + :meth:`set_dataset_category`. + """ + return self._request("POST", "/datasets", json_body=dict(payload), required_scope=TokenScope.WRITE) + + def set_dataset_category( + self, + owner: str, + repo_name: str, + payload: SetDatasetCategoryPayload | Mapping[str, Any], + ) -> JSON: + """``PUT /datasets/{owner}/{repo_name}/category`` — set or change the zone.""" + return self._request( + "PUT", + f"/datasets/{owner}/{repo_name}/category", + json_body=dict(payload), + required_scope=TokenScope.WRITE, + ) + + def update_dataset_category( + self, + owner: str, + repo_name: str, + payload: UpdateDatasetCategoryPayload | Mapping[str, Any], + ) -> JSON: + """``PATCH /datasets/{owner}/{repo_name}/category`` — edit science fields. + + The payload must not carry ``dataset_category``; the server reads the + current zone itself and rejects the field. + """ + return self._request( + "PATCH", + f"/datasets/{owner}/{repo_name}/category", + json_body=dict(payload), + required_scope=TokenScope.WRITE, + ) + + def update_dataset_settings( + self, + owner: str, + repo_name: str, + settings: UpdateDatasetSettingsPayload | Mapping[str, Any], + ) -> JSON: + """``PATCH /datasets/{owner}/{repo_name}/settings`` — general settings. + + Omitted fields are unchanged; explicit nulls clear. Zone fields and + ``login_required`` are not accepted here. + """ + return self._request( + "PATCH", + f"/datasets/{owner}/{repo_name}/settings", + json_body=dict(settings), + required_scope=TokenScope.WRITE, + ) + + def delete_dataset_files( + self, + owner: str, + repo_name: str, + payload: DeleteFilesPayload | Mapping[str, Any], + ) -> JSON: + """``DELETE /datasets/{owner}/{repo_name}/files`` — one atomic commit. + + Takes 1 to 100 distinct repo-relative file paths -- no directories, no + traversal -- and ``revision`` must be a branch. The request is + all-or-nothing: one unresolvable path fails every deletion. + + :meth:`HubApi.delete_files` does *not* route here. It commits deletions + directly, which lifts the 100-path ceiling and supports glob patterns; + this method exists for callers that want the endpoint's ``commit_sha`` + and its strict all-or-nothing guarantee. + """ + return self._request( + "DELETE", + f"/datasets/{owner}/{repo_name}/files", + json_body=_validated_delete_files(payload), + required_scope=TokenScope.WRITE, + ) + # ================================================================== # Files # ================================================================== diff --git a/src/modelscope_hub/api.py b/src/modelscope_hub/api.py index 99035b0..097ffb3 100644 --- a/src/modelscope_hub/api.py +++ b/src/modelscope_hub/api.py @@ -27,7 +27,7 @@ import time from collections.abc import Callable, Iterable, Mapping from pathlib import Path -from typing import Any, BinaryIO, TypeAlias +from typing import Any, BinaryIO, TypeAlias, TypeVar from urllib.parse import urlparse from requests.cookies import RequestsCookieJar @@ -42,14 +42,33 @@ from ._upload import UploadManager from .agent_idp import sign_agent_token_request from .config import HubConfig, get_default_config -from .constants import DEFAULT_ENDPOINT, RepoType, StudioVisibility, Visibility +from .constants import ( + DEFAULT_ENDPOINT, + OPENAPI_CREATE_DATASET_FIELDS, + OPENAPI_CREATE_MODEL_FIELDS, + OPENAPI_MODEL_LICENSES, + OPENAPI_VERSION_NAME_MAX_LENGTH, + WEIGHT_FILE_SUFFIXES, + AigcModelType, + DatasetCategory, + GatedMode, + ModelCategory, + RepoType, + RepoVisibility, + StrEnum, + StudioVisibility, + Visibility, +) from .errors import ( + AlreadyExistsError, AuthenticationError, HubError, InvalidParameter, NotExistError, NotSupportedError, PermissionDeniedError, + ServerError, + is_repo_exists_error, ) from .types import ( AgentIdConfiguration, @@ -76,11 +95,19 @@ RepoTypeLike: TypeAlias = "str | RepoType" +# Any of the string enums the category payloads validate against. +_StrEnumT = TypeVar("_StrEnumT", bound=StrEnum) + # Routing tables — declarative dispatch keeps :class:`HubApi` free of long # if/elif chains and makes adding new repo types a one-line change. _CREATABLE_TYPES: frozenset[RepoType] = frozenset({RepoType.MODEL, RepoType.DATASET, RepoType.STUDIO, RepoType.SKILL}) -_OPENAPI_CREATE_TYPES: frozenset[RepoType] = frozenset({RepoType.STUDIO, RepoType.SKILL}) +_OPENAPI_CREATE_TYPES: frozenset[RepoType] = frozenset( + {RepoType.MODEL, RepoType.DATASET, RepoType.STUDIO, RepoType.SKILL} +) +# Repo kinds whose OpenAPI create endpoint can be bypassed for inputs it cannot +# express. Studio and Skill have no legacy create endpoint, so they are absent. +_LEGACY_CREATE_FALLBACK_TYPES: frozenset[RepoType] = frozenset({RepoType.MODEL, RepoType.DATASET}) _OPENAPI_DETAIL_TYPES: frozenset[RepoType] = frozenset( {RepoType.MODEL, RepoType.DATASET, RepoType.STUDIO, RepoType.SKILL} ) @@ -118,6 +145,11 @@ # These MUST NOT be overridden via extra kwargs to avoid silent conflicts. _RESERVED_EXTRA_FIELDS: frozenset[str] = frozenset({"Path", "Owner", "Name"}) +# Settings the model/dataset settings endpoints report but cannot change. +# ``login_required`` appears in a dataset's detail payload, yet no request +# schema accepts it, so it can only be changed through the web UI. +_UNSETTABLE_FIELDS: frozenset[str] = frozenset({"login_required"}) + def _unwrap_list(payload: Any, keys: tuple[str, ...]) -> list[dict]: """Pull a list out of an OpenAPI payload, tolerating either shape. @@ -427,27 +459,42 @@ def _repo_info_from_payload( # gated is orthogonal and does not affect visibility mapping. raw_visibility = normalised.get("visibility") if isinstance(raw_visibility, str): - # Studio payloads report visibility as a string enum. ``public`` and - # ``private`` have integer equivalents; ``protected`` (app public, - # code hidden) does not, so it is preserved verbatim rather than - # forced into a value that would misrepresent it. + # Two string encodings arrive here. Studio payloads use + # ``public``/``protected``/``private``; model and dataset detail + # payloads use ``public``/``private``/``gated``. ``gated`` must not + # fall through as a raw string: ``visibility`` is typed as the + # integer :class:`Visibility`, and leaking a ``str`` would break + # ``.label``, ``int()`` and equality against the enum -- including + # in the compat dict, which the legacy SDK reads as an integer. + # + # A gated repository maps to PRIVATE because that is what it is on + # the legacy wire (``Visibility=PRIVATE`` plus ``ProtectedMode=1``); + # the approval mode itself is carried separately by ``gated`` and + # ``gated_mode``. The server reports ``private: false`` for gated + # repos, so without this branch they would be reported as PUBLIC. + repo_visibility = RepoVisibility.parse(raw_visibility) studio_visibility = StudioVisibility.parse(raw_visibility) - if studio_visibility is StudioVisibility.PUBLIC: + if repo_visibility is RepoVisibility.GATED: + normalised["visibility"] = Visibility.PRIVATE + normalised.setdefault("gated", True) + elif studio_visibility is StudioVisibility.PUBLIC: normalised["visibility"] = Visibility.PUBLIC elif studio_visibility is StudioVisibility.PRIVATE: normalised["visibility"] = Visibility.PRIVATE elif studio_visibility is not None: + # ``protected`` (app public, code hidden) has no integer + # equivalent, so it is preserved verbatim rather than forced + # into a value that would misrepresent it. normalised["visibility"] = studio_visibility.value elif raw_visibility is None: private_flag = normalised.get("private") - if isinstance(private_flag, bool): - if private_flag: - normalised["visibility"] = Visibility.PRIVATE - else: - normalised["visibility"] = Visibility.PUBLIC - elif normalised.get("gated"): - # Rule 4: visibility unknown + gated=True → imply PRIVATE + if normalised.get("gated"): + # Checked before ``private``: the server reports ``private: + # false`` for a gated repo, so trusting that flag first would + # report an application-gated repo as public. normalised["visibility"] = Visibility.PRIVATE + elif isinstance(private_flag, bool): + normalised["visibility"] = Visibility.PRIVATE if private_flag else Visibility.PUBLIC # The OpenAPI list endpoints return ``id`` as "owner/name". # Split it so the computed ``repo_id`` property works. @@ -813,6 +860,10 @@ def create_repo( chinese_name: str | None = None, description: str | None = None, gated_mode: bool | None = None, + gated: bool | str | GatedMode | None = None, + license_name: str | None = None, + license_link: str | None = None, + task: str | None = None, **extra: Any, ) -> RepoInfo: """Create a new repository. @@ -820,7 +871,9 @@ def create_repo( Routing is decided by ``repo_type``: * ``studio`` / ``skill`` → OpenAPI ``POST /studios`` / ``POST /skills`` - * ``model`` / ``dataset`` → legacy ``POST /api/v1/{type}s`` + * ``model`` / ``dataset`` → OpenAPI ``POST /models`` / ``POST /datasets``, + falling back to the legacy endpoint for inputs the OpenAPI schema + cannot express (see :meth:`_legacy_create_reason`) Parameters ---------- @@ -830,18 +883,33 @@ def create_repo( One of ``"model"``, ``"dataset"``, ``"studio"``, ``"skill"``. visibility : int, str or Visibility, optional Visibility level. Accepts the integer wire encoding, a label - (``"public"`` / ``"private"``) or a :class:`Visibility` value. - Defaults to public. + (``"public"`` / ``"private"`` / ``"gated"``) or a + :class:`Visibility` value. Defaults to public. license : str, optional - SPDX-style license identifier (e.g. ``"apache-2.0"``). + SPDX-style license identifier (e.g. ``"apache-2.0"``). Values the + OpenAPI model schema does not list are still accepted, via the + legacy endpoint. chinese_name : str, optional - Chinese display name shown on the Hub UI. + Display name shown on the Hub UI. description : str, optional Short description of the repository. gated_mode : bool, optional - Enable gated (application-based download) mode for private repos. - True = gated, False = normal private. Only effective when - visibility is PRIVATE; ignored otherwise. + Deprecated. ``True`` requests an application-gated repository and + ``False`` an ordinary private one. Prefer ``gated``, which can also + say *how* applications are approved. + gated : bool, "auto", "manual" or GatedMode, optional + Application-gated download policy. ``"auto"`` approves applications + automatically, ``"manual"`` requires review, ``True`` means + ``"auto"``, and ``False`` means not gated. Only meaningful when + visibility is gated or unset; ignored with a warning otherwise. + license_name : str, optional + Custom license name. Required when ``license="other"``. + license_link : str, optional + Custom license URL, honoured when ``license="other"``. When omitted + the Hub creates a blank ``LICENSE.md``. + task : str, optional + Model task identifier, as published by + :meth:`list_model_options`. Models only. **extra : Any Additional fields forwarded verbatim to the underlying client. @@ -853,7 +921,8 @@ def create_repo( Raises ------ InvalidParameter - When ``repo_id`` does not have the ``owner/name`` shape. + When ``repo_id`` does not have the ``owner/name`` shape, or when + ``visibility`` is a value the repo type does not support. AuthenticationError When the token is missing or invalid. @@ -871,9 +940,9 @@ def create_repo( >>> info.repo_id 'alice/llama-7b-finetuned' - Create a private gated dataset: + Create a dataset that requires a reviewed application to download: - >>> api.create_repo("alice/my-data", "dataset", visibility="private", gated_mode=True) + >>> api.create_repo("alice/my-data", "dataset", visibility="gated", gated="manual") Create a public Studio space: @@ -887,48 +956,372 @@ def create_repo( ) owner, name = self._parse_repo_id(repo_id) studio_visibility = StudioVisibility.parse(visibility) if rt is RepoType.STUDIO else None - vis = None if studio_visibility is not None else self._normalize_visibility(visibility) + repo_visibility = RepoVisibility.parse(visibility) if rt in _LEGACY_CREATE_FALLBACK_TYPES else None + vis = ( + None + if studio_visibility is not None or repo_visibility is RepoVisibility.GATED + else self._normalize_visibility(visibility) + ) if license is not None: license = _LICENSE_DISPLAY_TO_SPDX.get(license, license) - if rt in _OPENAPI_CREATE_TYPES: - is_private = vis is not None and vis == int(Visibility.PRIVATE) - if rt is RepoType.STUDIO: - payload: dict[str, Any] = { - "owner": owner, - "repo_name": name, - } - if studio_visibility is not None: - # ``visibility`` is the expressive form (it is the only way to - # ask for ``protected``); ``private`` is sent alongside it for - # servers that predate the tri-state field. The two are - # consistent by construction, so they cannot disagree. - payload["visibility"] = studio_visibility.value - payload["private"] = studio_visibility is StudioVisibility.PRIVATE - elif vis is not None: - payload["private"] = is_private - if chinese_name is not None: - payload["display_name"] = chinese_name + if rt in _LEGACY_CREATE_FALLBACK_TYPES: + return self._create_model_or_dataset( + rt, + owner, + name, + visibility=vis, + repo_visibility=repo_visibility, + license=license, + chinese_name=chinese_name, + description=description, + gated_mode=gated_mode, + gated=gated, + license_name=license_name, + license_link=license_link, + task=task, + extra=extra, + ) + + is_private = vis is not None and vis == int(Visibility.PRIVATE) + if rt is RepoType.STUDIO: + payload: dict[str, Any] = { + "owner": owner, + "repo_name": name, + } + if studio_visibility is not None: + # ``visibility`` is the expressive form (it is the only way to + # ask for ``protected``); ``private`` is sent alongside it for + # servers that predate the tri-state field. The two are + # consistent by construction, so they cannot disagree. + payload["visibility"] = studio_visibility.value + payload["private"] = studio_visibility is StudioVisibility.PRIVATE + elif vis is not None: + payload["private"] = is_private + if chinese_name is not None: + payload["display_name"] = chinese_name + else: + payload = { + "owner": owner, + "skill_name": name, + } + if vis is not None: + payload["private"] = is_private + if chinese_name is not None: + payload["display_name"] = chinese_name + if license is not None: + payload["license"] = license + if description is not None: + payload["description"] = description + # Only Studios get the Studio field normalisation: a Skill has no + # cover image and no visibility field, so rewriting those keys on a + # Skill payload would invent fields its endpoint does not accept. + payload.update(_normalise_studio_fields(extra) if rt is RepoType.STUDIO else extra) + data = self.openapi.create_studio(payload) if rt is RepoType.STUDIO else self.openapi.create_skill(payload) + return self._repo_info_from_payload(data, rt, owner_hint=owner, name_hint=name) + + @staticmethod + def _resolve_gated( + gated: bool | str | GatedMode | None, + gated_mode: bool | None, + ) -> GatedMode | bool | None: + """Collapse the two gated parameters into one value. + + ``gated`` is expressive (it can name the approval mode); ``gated_mode`` + is the older boolean and wins only when ``gated`` is not given. A + ``True`` from either resolves to :data:`GatedMode.AUTO`. + + For ``create_repo`` that resolved mode is only ever used as "gating was + requested": a call that used the boolean is routed to the legacy + endpoint, whose ``ProtectedMode`` field carries no mode at all, so the + historical behaviour is preserved exactly. + """ + value = gated if gated is not None else gated_mode + if value is None or value is False: + return value + if value is True: + return GatedMode.AUTO + mode = GatedMode.parse(value) + if mode is None: + allowed = ", ".join(repr(m.value) for m in GatedMode) + raise InvalidParameter(f"gated must be False, True, or one of {allowed} (got {value!r}).") + return mode + + @staticmethod + def _legacy_create_reason( + rt: RepoType, + *, + visibility: int | None, + license: str | None, + extra: Mapping[str, Any], + gated_mode: bool | None = None, + task: str | None = None, + ) -> str | None: + """Explain why these inputs cannot go through the OpenAPI create endpoint. + + The OpenAPI schemas are closed: ``visibility`` has no ``internal``, + ``POST /models`` restricts ``license`` to a published set, and both + endpoints answer 400 for any field they do not declare. The legacy + endpoint accepts all three, so rather than failing a call that works + today, these inputs keep using it. Returns ``None`` when the OpenAPI + endpoint can serve the request. + """ + if gated_mode is not None: + # The historical boolean has no approval mode, and the two surfaces + # default differently: legacy sends ``ProtectedMode=1`` and lets the + # server decide, while the OpenAPI schema defaults ``gated_mode`` to + # ``auto``. Translating the boolean could therefore silently turn a + # repository that required review into one that auto-approves, so it + # keeps using the wire format it has always used. ``gated`` is the + # parameter that can state the mode, and it does go through OpenAPI. + return "gated_mode is the historical boolean; use gated= to reach the OpenAPI endpoint" + if visibility is not None and visibility == int(Visibility.INTERNAL): + return "visibility=internal is not part of the OpenAPI visibility enum" + if rt is RepoType.MODEL and license is not None and license not in OPENAPI_MODEL_LICENSES: + return f"license={license!r} is not in the OpenAPI model license enum" + if rt is not RepoType.MODEL and task is not None: + # Only the model schema declares ``task``. The legacy endpoint used + # to receive it verbatim through ``**extra``, so route there rather + # than reject a call that used to work. + return f"task is not declared by the OpenAPI {rt.value} create schema" + allowed = OPENAPI_CREATE_MODEL_FIELDS if rt is RepoType.MODEL else OPENAPI_CREATE_DATASET_FIELDS + unknown = sorted(set(extra) - allowed) + if unknown: + return f"field(s) {unknown} are not declared by the OpenAPI create schema" + return None + + def _create_model_or_dataset( + self, + rt: RepoType, + owner: str, + name: str, + *, + visibility: int | None, + repo_visibility: RepoVisibility | None, + license: str | None, + chinese_name: str | None, + description: str | None, + gated_mode: bool | None, + gated: bool | str | GatedMode | None, + license_name: str | None, + license_link: str | None, + task: str | None, + extra: Mapping[str, Any], + ) -> RepoInfo: + """Create a model or dataset, preferring OpenAPI over the legacy endpoint.""" + if repo_visibility is not None and repo_visibility is not RepoVisibility.GATED: + # ``public``/``private`` already round-tripped through the integer + # encoding; only ``gated`` needs the string form carried forward. + repo_visibility = None + resolved_gated = self._resolve_gated(gated, gated_mode) + reason = self._legacy_create_reason( + rt, + visibility=visibility, + license=license, + extra=extra, + gated_mode=gated_mode, + task=task, + ) + if reason is None: + try: + return self._create_via_openapi( + rt, + owner, + name, + visibility=visibility, + repo_visibility=repo_visibility, + license=license, + chinese_name=chinese_name, + description=description, + gated=resolved_gated, + license_name=license_name, + license_link=license_link, + task=task, + extra=extra, + ) + except (NotExistError, ServerError) as exc: + # 404 means this deployment has not published the endpoint yet; + # 501 means it knows the route but will not serve it. Both are + # "not available here", so fall through rather than fail. + if exc.status_code not in (404, 501): + raise + reason = f"the OpenAPI create endpoint answered {exc.status_code}" + logger.debug("Creating %s %s/%s through the legacy endpoint: %s", rt.value, owner, name, reason) + return self._create_via_legacy( + rt, + owner, + name, + visibility=visibility, + repo_visibility=repo_visibility, + license=license, + chinese_name=chinese_name, + description=description, + gated=resolved_gated, + gated_mode=gated_mode, + # ``task`` / ``license_name`` / ``license_link`` are named parameters + # only because the OpenAPI schema declares them. The legacy endpoint + # has always received them through ``**extra`` under exactly these + # keys, so they are restored rather than renamed -- a legacy request + # must not change shape just because the signature grew. + extra={ + **extra, + **({"task": task} if task is not None else {}), + **({"license_name": license_name} if license_name is not None else {}), + **({"license_link": license_link} if license_link is not None else {}), + }, + ) + + def _create_via_openapi( + self, + rt: RepoType, + owner: str, + name: str, + *, + visibility: int | None, + repo_visibility: RepoVisibility | None, + license: str | None, + chinese_name: str | None, + description: str | None, + gated: GatedMode | bool | None, + license_name: str | None, + license_link: str | None, + task: str | None, + extra: Mapping[str, Any], + ) -> RepoInfo: + """Create a model or dataset via ``POST /models`` / ``POST /datasets``.""" + payload: dict[str, Any] = {"owner": owner, "repo_name": name} + wire_visibility = self._openapi_visibility(rt, visibility, repo_visibility, gated) + if wire_visibility is not None: + payload["visibility"] = wire_visibility.value + if isinstance(gated, GatedMode): + if wire_visibility is RepoVisibility.GATED: + payload["gated_mode"] = gated.value else: - payload = { - "owner": owner, - "skill_name": name, - } - if vis is not None: - payload["private"] = is_private - if chinese_name is not None: - payload["display_name"] = chinese_name - if license is not None: - payload["license"] = license - if description is not None: - payload["description"] = description - # Only Studios get the Studio field normalisation: a Skill has no - # cover image and no visibility field, so rewriting those keys on a - # Skill payload would invent fields its endpoint does not accept. - payload.update(_normalise_studio_fields(extra) if rt is RepoType.STUDIO else extra) - data = self.openapi.create_studio(payload) if rt is RepoType.STUDIO else self.openapi.create_skill(payload) - return self._repo_info_from_payload(data, rt, owner_hint=owner, name_hint=name) + # The endpoint rejects ``gated_mode`` outside a gated repo, and + # the legacy path has always warned-and-ignored here. Keep that. + logger.warning( + "gated is only effective when visibility is gated, ignored (visibility=%s).", + wire_visibility.value if wire_visibility else "public", + ) + if chinese_name is not None: + payload["display_name"] = chinese_name + # ``license`` is omitted rather than defaulted: the server's own default + # (``apache-2.0``) is inside the enum, whereas the legacy fallback value + # (``"Apache-2.0"``) is not and would be rejected. + if license is not None: + payload["license"] = license + if license_name is not None: + payload["license_name"] = license_name + if license_link is not None: + payload["license_link"] = license_link + if description is not None: + payload["description"] = description + if task is not None: + if rt is not RepoType.MODEL: + raise InvalidParameter("task is only supported for repo_type='model'.") + payload["task"] = task + payload.update(extra) + + try: + data = self.openapi.create_model(payload) if rt is RepoType.MODEL else self.openapi.create_dataset(payload) + except InvalidParameter as exc: + # ``POST /datasets`` declares no 409, so a duplicate may arrive as a + # 400. Callers catch AlreadyExistsError, so reclassify rather than + # let a rename of an existing repo look like bad input. + if not isinstance(exc, AlreadyExistsError) and is_repo_exists_error(exc): + raise AlreadyExistsError( + str(exc.message or exc), + status_code=exc.status_code, + request_id=exc.request_id, + response_body=exc.response_body, + ) from exc + raise + info = self._repo_info_from_payload(data, rt, owner_hint=owner, name_hint=name) + # The create response carries only identity fields, while the legacy one + # echoed the settings back. Backfill from the request so a caller reading + # ``info.visibility`` after create does not see None where it saw a value. + # Derived from the wire value rather than the raw argument: the argument + # may be absent (gating alone implies a visibility) and must not leak a + # plain ``int`` past the dataclass's enum coercion. + if info.visibility is None and wire_visibility is not None: + info.visibility = Visibility.PUBLIC if wire_visibility is RepoVisibility.PUBLIC else Visibility.PRIVATE + if info.license is None: + info.license = license + if info.description is None: + info.description = description + if info.gated is None: + info.gated = wire_visibility is RepoVisibility.GATED + if info.gated_mode is None and isinstance(gated, GatedMode) and wire_visibility is RepoVisibility.GATED: + info.gated_mode = gated.value + return info + + @staticmethod + def _openapi_visibility( + rt: RepoType, + visibility: int | None, + repo_visibility: RepoVisibility | None, + gated: GatedMode | bool | None, + ) -> RepoVisibility | None: + """Pick the ``visibility`` string the create endpoints expect. + + Returns ``None`` to omit the field entirely, letting the server apply + its own default of ``public``. + + A private repository that also carries an approval mode *is* a gated + repository: the legacy encoding spelled exactly that as + ``Visibility=PRIVATE`` plus ``ProtectedMode=1``. Resolving it to plain + ``private`` would quietly drop the gating, and would disagree with the + legacy route for the same arguments. + """ + if repo_visibility is RepoVisibility.GATED: + return RepoVisibility.GATED + wants_gated = isinstance(gated, GatedMode) + if visibility is not None: + if visibility == int(Visibility.PRIVATE): + return RepoVisibility.GATED if wants_gated else RepoVisibility.PRIVATE + if visibility == int(Visibility.PUBLIC): + # Gating a public repository is contradictory; the caller is + # warned and the gating dropped, as the legacy path has always + # done. + return RepoVisibility.PUBLIC + raise InvalidParameter( + f"visibility={visibility!r} is not supported for repo_type={rt.value!r}. " + f"Expected one of: {', '.join(v.value for v in RepoVisibility)}." + ) + if gated is None: + return None + # Visibility was not stated but the gating intent was. The legacy path + # has always made that imply a non-public repository, so both a request + # for gating and an explicit opt-out land on a private repo. + return RepoVisibility.GATED if wants_gated else RepoVisibility.PRIVATE + + def _create_via_legacy( + self, + rt: RepoType, + owner: str, + name: str, + *, + visibility: int | None, + repo_visibility: RepoVisibility | None, + license: str | None, + chinese_name: str | None, + description: str | None, + gated: GatedMode | bool | None, + gated_mode: bool | None, + extra: Mapping[str, Any], + ) -> RepoInfo: + """Create a model or dataset via the legacy ``POST /api/v1/{type}s``. + The body is built in the order this endpoint has always received it, and + ``extra`` is appended last and verbatim. Callers reach this path because + the OpenAPI schema could not express their input, so the request they + get must be the one they got before the OpenAPI route existed. + """ + vis = visibility + if repo_visibility is RepoVisibility.GATED: + # The legacy encoding has no ``gated`` state: it is PRIVATE plus + # ``ProtectedMode=1``. + vis = int(Visibility.PRIVATE) if rt is RepoType.DATASET: body: dict[str, Any] = { "Owner": owner, @@ -948,16 +1341,17 @@ def create_repo( if description is not None: body["Description"] = description - # gated_mode → ProtectedMode wire field (1=gated, 2=off). - # gated only effective with PRIVATE; when vis=None + gated=True, + # gated -> ProtectedMode wire field (1=gated, 2=off). + # gated only effective with PRIVATE; when vis=None + gated requested, # implicitly set visibility to PRIVATE (user intent: gated repo). - if gated_mode is not None: + wants_gated = isinstance(gated, GatedMode) or repo_visibility is RepoVisibility.GATED + if gated is not None or repo_visibility is RepoVisibility.GATED: if vis is None: vis = int(Visibility.PRIVATE) body["Visibility"] = vis - body["ProtectedMode"] = 1 if gated_mode else 2 + body["ProtectedMode"] = 1 if wants_gated else 2 elif vis == int(Visibility.PRIVATE): - body["ProtectedMode"] = 1 if gated_mode else 2 + body["ProtectedMode"] = 1 if wants_gated else 2 else: logger.warning("gated_mode is only effective when visibility is PRIVATE, ignored.") @@ -1044,6 +1438,470 @@ def get_repo( return self._repo_info_from_payload(data, rt, owner_hint=owner, name_hint=name) + def get_model(self, repo_id: str) -> dict: + """Fetch a model's full detail payload, zone fields included. + + Returns the server's ``ModelDetail`` verbatim (snake_case keys) rather + than a :class:`RepoInfo`. The zone-specific fields -- AIGC architecture + and ``versions``, or the science subject areas -- have no place on the + dataclass, and :meth:`get_repo` would silently drop them. + + Which keys are present depends on ``model_category``: ``base`` carries + no zone fields, and the fields of the zones a model is *not* in are + absent rather than null. Read them with ``.get()``. + + Parameters + ---------- + repo_id : str + Canonical ``owner/name`` identifier. + + Returns + ------- + dict + The ``ModelDetail`` payload. + + Examples + -------- + >>> detail = api.get_model("alice/my-anime-lora") + >>> detail["model_category"] + 'aigc' + >>> [v["version_id"] for v in detail.get("versions", [])] + [1070760] + """ + owner, name = self._parse_repo_id(repo_id) + data = self.openapi.get_model(owner, name) + return dict(data) if isinstance(data, Mapping) else {} + + def model_info(self, repo_id: str) -> dict: + """Alias of :meth:`get_model`.""" + return self.get_model(repo_id) + + def get_dataset(self, repo_id: str) -> dict: + """Fetch a dataset's full detail payload, zone fields included. + + The dataset counterpart of :meth:`get_model`; see there for why the raw + payload is returned instead of a :class:`RepoInfo`. Science datasets + additionally carry the subject areas and cover image. + + Examples + -------- + >>> detail = api.get_dataset("alice/rna-seq") + >>> detail["dataset_category"], detail.get("primary_subject_area") + ('science', 'life-sciences') + """ + owner, name = self._parse_repo_id(repo_id) + data = self.openapi.get_dataset(owner, name) + return dict(data) if isinstance(data, Mapping) else {} + + def dataset_info(self, repo_id: str) -> dict: + """Alias of :meth:`get_dataset`.""" + return self.get_dataset(repo_id) + + # ================================================================== + # Zone configuration enums + # ================================================================== + def list_aigc_model_options(self) -> dict: + """Return the enum values accepted when setting the AIGC category. + + Needs no credentials. Every option is a ``{type, id, label}`` triple: + pass ``id`` to :meth:`set_model_category`, show ``label`` to users. + + Returns + ------- + dict + ``aigc_types``, ``base_architectures`` (each with ``subtypes``), + ``content_styles``, ``content_functions``, ``content_elements``. + + Examples + -------- + >>> options = api.list_aigc_model_options() + >>> for arch in options["base_architectures"]: + ... print(arch["id"], [s["id"] for s in arch["subtypes"]]) + """ + data = self.openapi.list_aigc_model_options() + return dict(data) if isinstance(data, Mapping) else {} + + def list_science_model_options(self) -> dict: + """Return the science-model subject-area tree. Needs no credentials. + + Examples + -------- + >>> tree = api.list_science_model_options() + >>> tree["subject_areas"][0]["id"] + 'life-sciences' + """ + data = self.openapi.list_science_model_options() + return dict(data) if isinstance(data, Mapping) else {} + + def list_science_dataset_options(self) -> dict: + """Return the science-dataset subject-area tree. Needs no credentials.""" + data = self.openapi.list_science_dataset_options() + return dict(data) if isinstance(data, Mapping) else {} + + def list_model_options(self) -> dict: + """Return the valid model ``task`` and ``license`` values. + + Needs no credentials. Each entry is a ``{type, id, label}`` triple; pass + ``id`` to :meth:`create_repo` or :meth:`update_repo_settings`. + + Returns + ------- + dict + ``tasks`` and ``license`` lists. + + Examples + -------- + >>> options = api.list_model_options() + >>> [t["id"] for t in options["tasks"]][:2] + ['text-generation', 'image-captioning'] + """ + data = self.openapi.get_model_options() + return dict(data) if isinstance(data, Mapping) else {} + + # ================================================================== + # Zone membership (category) + # ================================================================== + def set_model_category( + self, + repo_id: str, + repo_type: RepoTypeLike = RepoType.MODEL, + *, + model_category: str | ModelCategory, + **fields: Any, + ) -> dict: + """Set or change which zone a model belongs to. + + Which fields are required follows from the *target* category, not from + the model's current one. Upload the weight files first: the endpoint + validates that each ``path_in_repo`` already exists. + + * ``base`` -- no other field; every zone field is cleared. + * ``aigc`` -- ``aigc_type``, ``base_model_architecture_type``, + ``base_model_architecture_subtype``, ``version_name``, and non-empty + ``weight_files`` and ``sample_images``. Creates a new version. + * ``science`` -- ``primary_subject_area`` and + ``secondary_specialization``; ``cover_image`` optional. + + Enum values come from :meth:`list_aigc_model_options` / + :meth:`list_science_model_options`; pass each option's ``id``. + + Parameters + ---------- + repo_id : str + Canonical ``owner/name`` identifier. + repo_type : str or RepoType, optional + Must be ``"model"``. Accepted for symmetry with the other methods. + model_category : str or ModelCategory + Target zone: ``"base"``, ``"aigc"`` or ``"science"``. + **fields : Any + Zone fields for the target category. + + Returns + ------- + dict + Repository identity (``id``, ``repo_name``, ``owner``, ``url``). + + Raises + ------ + NotSupportedError + When ``repo_type`` is not ``"model"``. + InvalidParameter + When ``model_category`` is unknown, or when a required AIGC field is + missing or malformed. + + Examples + -------- + >>> api.set_model_category( + ... "alice/my-anime-lora", + ... model_category="aigc", + ... aigc_type="LoRA", + ... base_model_architecture_type="Z_IMAGE_TURBO", + ... base_model_architecture_subtype="Z_IMAGE_TURBO", + ... version_name="v1.0", + ... weight_files=[{"path_in_repo": "model.safetensors"}], + ... sample_images=[{"url": "https://example.com/a.png", "prompt": "anime girl"}], + ... ) + + Move it back out of the zone: + + >>> api.set_model_category("alice/my-anime-lora", model_category="base") + """ + rt = self._normalize_repo_type(repo_type) + if rt is not RepoType.MODEL: + raise NotSupportedError(f"set_model_category only supports repo_type='model' (got {rt.value!r}).") + owner, name = self._parse_repo_id(repo_id) + category = self._coerce_enum(model_category, ModelCategory, "model_category") + payload: dict[str, Any] = {"model_category": category.value} + payload.update(fields) + self._validate_set_model_category(category, payload) + data = self.openapi.set_model_category(owner, name, payload) + return dict(data) if isinstance(data, Mapping) else {} + + def update_model_category( + self, + repo_id: str, + repo_type: RepoTypeLike = RepoType.MODEL, + *, + version_id: int | None = None, + version_name: str | None = None, + **fields: Any, + ) -> dict: + """Update a model's zone metadata in place, without changing the zone. + + The server reads the current zone from its own records, so the category + is neither needed nor accepted. Use :meth:`set_model_category` to move + between zones or to replace an existing version's weights. + + AIGC version fields (``trigger_words``, ``version_description``, + ``sample_images``) address a specific version. Give either its numeric + ``version_id`` or its ``version_name``, which is resolved to the id with + one extra detail read. Science updates need neither. + + Omitted fields are left unchanged; passing ``None`` clears a field. + + Parameters + ---------- + repo_id : str + Canonical ``owner/name`` identifier. + repo_type : str or RepoType, optional + Must be ``"model"``. + version_id : int, optional + Numeric version id, as reported by :meth:`get_model`. + version_name : str, optional + Version name to resolve into ``version_id``. Ignored when + ``version_id`` is given. + **fields : Any + Zone fields to update. + + Returns + ------- + dict + Repository identity. + + Raises + ------ + NotSupportedError + When ``repo_type`` is not ``"model"``. + InvalidParameter + When a field belongs to :meth:`set_model_category` instead, or when + ``version_name`` matches no version. + + Examples + -------- + >>> api.update_model_category( + ... "alice/my-anime-lora", + ... version_name="v1.0", + ... trigger_words=["anime style"], + ... ) + >>> api.update_model_category("alice/rna-predictor", secondary_specialization="dna") + """ + rt = self._normalize_repo_type(repo_type) + if rt is not RepoType.MODEL: + raise NotSupportedError(f"update_model_category only supports repo_type='model' (got {rt.value!r}).") + owner, name = self._parse_repo_id(repo_id) + payload = self._strip_category_discriminator(dict(fields), "model_category", "set_model_category") + self._reject_set_only_fields(payload) + if version_id is None and version_name is not None: + version_id = self._resolve_version_id(repo_id, version_name) + if version_id is not None: + payload["version_id"] = version_id + data = self.openapi.update_model_category(owner, name, payload) + return dict(data) if isinstance(data, Mapping) else {} + + def set_dataset_category( + self, + repo_id: str, + repo_type: RepoTypeLike = RepoType.DATASET, + *, + dataset_category: str | DatasetCategory, + **fields: Any, + ) -> dict: + """Set or change which zone a dataset belongs to. + + ``science`` requires a matching ``primary_subject_area`` and + ``secondary_specialization`` (ids from + :meth:`list_science_dataset_options`); ``base`` clears them. Note the + general value is ``base``, not ``basic``. + + Examples + -------- + >>> api.set_dataset_category( + ... "alice/rna-seq", + ... dataset_category="science", + ... primary_subject_area="life-sciences", + ... secondary_specialization="dna", + ... ) + """ + rt = self._normalize_repo_type(repo_type) + if rt is not RepoType.DATASET: + raise NotSupportedError(f"set_dataset_category only supports repo_type='dataset' (got {rt.value!r}).") + owner, name = self._parse_repo_id(repo_id) + category = self._coerce_enum(dataset_category, DatasetCategory, "dataset_category") + payload: dict[str, Any] = {"dataset_category": category.value} + payload.update(fields) + data = self.openapi.set_dataset_category(owner, name, payload) + return dict(data) if isinstance(data, Mapping) else {} + + def update_dataset_category( + self, + repo_id: str, + repo_type: RepoTypeLike = RepoType.DATASET, + **fields: Any, + ) -> dict: + """Update a dataset's science fields in place, without changing the zone. + + As with :meth:`update_model_category`, the category is read by the + server and is not accepted in the payload. Omitted fields are left + unchanged; ``None`` clears a field. + + Examples + -------- + >>> api.update_dataset_category("alice/rna-seq", secondary_specialization="medicine") + >>> api.update_dataset_category("alice/rna-seq", cover_image=None) + """ + rt = self._normalize_repo_type(repo_type) + if rt is not RepoType.DATASET: + raise NotSupportedError(f"update_dataset_category only supports repo_type='dataset' (got {rt.value!r}).") + owner, name = self._parse_repo_id(repo_id) + payload = self._strip_category_discriminator(dict(fields), "dataset_category", "set_dataset_category") + data = self.openapi.update_dataset_category(owner, name, payload) + return dict(data) if isinstance(data, Mapping) else {} + + # ------------------------------------------------------------------ + # Category payload checks + # + # Each rule below mirrors one the server enforces. They run locally so a + # request that is certain to fail reports the fix straight away instead of + # costing a round trip for a generic 400. Anything whose valid values the + # server owns (architectures, subject areas, tasks) is deliberately not + # checked here -- those enums change without an SDK release, and its 400 + # message already names the offending field and the options endpoint. + # ------------------------------------------------------------------ + @staticmethod + def _coerce_enum(value: Any, enum_cls: type[_StrEnumT], field: str) -> _StrEnumT: + """Coerce *value* into *enum_cls*, naming the valid values when it fails.""" + if isinstance(value, enum_cls): + return value + try: + return enum_cls(str(value)) + except ValueError as exc: + allowed = ", ".join(repr(member.value) for member in enum_cls) + raise InvalidParameter(f"{field} must be one of {allowed} (got {value!r}).") from exc + + @staticmethod + def _strip_category_discriminator(payload: dict[str, Any], field: str, setter: str) -> dict[str, Any]: + """Drop the category discriminator from a PATCH payload. + + The endpoint answers 400 if the field is present at all -- it reads the + current category itself. Earlier drafts of this API required the field, + so it is removed with a warning rather than forwarded into a failure. + """ + if field in payload: + payload.pop(field) + logger.warning( + "%s is not accepted when updating zone fields and was dropped; use %s() to change the category.", + field, + setter, + ) + return payload + + @staticmethod + def _reject_set_only_fields(payload: Mapping[str, Any]) -> None: + """Reject fields that only ``set_model_category`` can change. + + An existing version's weights are immutable; changing them means + creating a new version. Saying so here is more useful than relaying the + server's rejection. + + ``version_name`` is absent from this list on purpose: the method takes + it as a named argument and resolves it to a ``version_id``, so it never + reaches the payload. + """ + offenders = sorted({"weight_files", "base_model_id"} & set(payload)) + if offenders: + raise InvalidParameter( + f"{', '.join(offenders)} cannot be changed on an existing version. " + f"Use set_model_category(model_category='aigc', ...) to create a new version instead." + ) + + @classmethod + def _validate_set_model_category(cls, category: ModelCategory, payload: Mapping[str, Any]) -> None: + """Check the required-field rules for a zone transition.""" + if category is not ModelCategory.AIGC: + return + missing = sorted( + field + for field in ( + "aigc_type", + "base_model_architecture_type", + "base_model_architecture_subtype", + "version_name", + "weight_files", + "sample_images", + ) + if not payload.get(field) + ) + if missing: + raise InvalidParameter( + f"model_category='aigc' requires non-empty {', '.join(missing)}. " + f"Valid enum values are published by list_aigc_model_options()." + ) + cls._coerce_enum(payload["aigc_type"], AigcModelType, "aigc_type") + version_name = str(payload["version_name"]) + if len(version_name) > OPENAPI_VERSION_NAME_MAX_LENGTH: + raise InvalidParameter( + f"version_name must be at most {OPENAPI_VERSION_NAME_MAX_LENGTH} characters (got {len(version_name)})." + ) + cls._validate_weight_files(payload["weight_files"]) + cls._validate_sample_images(payload["sample_images"]) + + @staticmethod + def _validate_weight_files(weight_files: Any) -> None: + """Check every weight file reference carries a recognised suffix.""" + for entry in weight_files: + path = entry.get("path_in_repo") if isinstance(entry, Mapping) else None + if not path: + raise InvalidParameter("each weight_files entry needs a non-empty 'path_in_repo'.") + if not str(path).lower().endswith(WEIGHT_FILE_SUFFIXES): + raise InvalidParameter( + f"weight file {path!r} does not have a recognised weight suffix. " + f"Supported suffixes: {', '.join(WEIGHT_FILE_SUFFIXES)}." + ) + + @staticmethod + def _validate_sample_images(sample_images: Any) -> None: + """Check sample images carry a URL and no retired ``text`` key.""" + for entry in sample_images: + if not isinstance(entry, Mapping) or not entry.get("url"): + raise InvalidParameter("each sample_images entry needs a non-empty 'url'.") + if "text" in entry: + raise InvalidParameter( + "sample_images does not accept 'text'; use 'prompt' and 'negative_prompt' instead." + ) + + def _resolve_version_id(self, repo_id: str, version_name: str) -> int: + """Look up a version's numeric id by its name. + + The update endpoint addresses versions by numeric id, but callers think + in version names, so the name is resolved here with one detail read. + """ + detail = self.get_model(repo_id) + versions = detail.get("versions") + if not isinstance(versions, list): + versions = [] + names = [] + for version in versions: + if not isinstance(version, Mapping): + continue + name = version.get("version_name") + names.append(str(name)) + if name == version_name and version.get("version_id") is not None: + return int(version["version_id"]) + available = ", ".join(repr(n) for n in names) or "none" + raise InvalidParameter( + f"version_name={version_name!r} not found on {repo_id!r}. Available versions: {available}." + ) + def list_repos( self, repo_type: RepoTypeLike, @@ -2060,16 +2918,23 @@ def update_repo_settings( repo_type: RepoTypeLike, **settings: Any, ) -> dict: - """Update repo settings (Studio or Skill). + """Update a repository's general settings. + + Supported for ``model``, ``dataset``, ``studio`` and ``skill``. Only the + keys you pass are changed; for models and datasets an explicit ``None`` + clears a field. Zone membership is not a setting -- use + :meth:`set_model_category` / :meth:`set_dataset_category` for that. Parameters ---------- repo_id : str Canonical ``owner/name`` identifier. repo_type : str or RepoType - Either ``"studio"`` or ``"skill"``. + ``"model"``, ``"dataset"``, ``"studio"`` or ``"skill"``. **settings : Any - Setting key/value pairs forwarded to the backend. + Setting key/value pairs forwarded to the backend. For models and + datasets: ``display_name``, ``license``, ``visibility``, ``gated`` / + ``gated_mode``, ``description``, and ``task`` (models only). Returns ------- @@ -2079,7 +2944,8 @@ def update_repo_settings( Raises ------ NotSupportedError - When ``repo_type`` is neither studio nor skill. + When ``repo_type`` has no settings endpoint, or when a setting the + endpoint cannot change is requested (notably ``login_required``). Examples -------- @@ -2089,6 +2955,7 @@ def update_repo_settings( ... visibility="public", ... hardware="GPU-A10", ... ) + >>> api.update_repo_settings("alice/rna-seq", "dataset", visibility="gated", gated="manual") """ rt = self._normalize_repo_type(repo_type) owner, name = self._parse_repo_id(repo_id) @@ -2096,8 +2963,100 @@ def update_repo_settings( return self.openapi.update_studio_settings(owner, name, _normalise_studio_fields(settings)) if rt is RepoType.SKILL: return self.openapi.update_skill_settings(owner, name, settings) + if rt is RepoType.DATASET: + payload = self._repo_settings_payload(rt, settings) + data = self.openapi.update_dataset_settings(owner, name, payload) + return dict(data) if isinstance(data, Mapping) else {} + if rt is RepoType.MODEL: + payload = self._repo_settings_payload(rt, settings) + data = self.openapi.update_model_settings(owner, name, payload) + return dict(data) if isinstance(data, Mapping) else {} raise NotSupportedError(f"update_repo_settings is not supported for repo_type={rt.value!r}.") + @classmethod + def _repo_settings_payload(cls, rt: RepoType, settings: Mapping[str, Any]) -> dict[str, Any]: + """Translate model/dataset settings kwargs into the wire payload. + + Two spellings are accepted and they are kept apart on purpose: + + * ``gated`` is the SDK's convenience argument (``True`` / ``False`` / + ``"auto"`` / ``"manual"``). It is translated into the endpoint's two + fields, including the visibility change that turning gating off + requires. + * ``gated_mode`` is the wire field itself. It is forwarded as given, so + the contract's own rules apply -- notably that an empty string is + valid and means "default to auto", while an explicit ``null`` or a + boolean is rejected. + + Settings the endpoint cannot change are rejected rather than dropped, + since silently dropping them would look like the update had succeeded. + """ + payload = dict(settings) + unsupported = sorted(_UNSETTABLE_FIELDS & set(payload)) + if unsupported: + raise NotSupportedError( + f"{', '.join(unsupported)} cannot be changed through update_repo_settings: " + f"the {rt.value} settings endpoint does not accept it. " + f"It is reported read-only by get_{rt.value}()." + ) + + has_native_mode = "gated_mode" in payload + native_mode = payload.pop("gated_mode", None) + gated = cls._resolve_gated(payload.pop("gated", None), None) + if has_native_mode and gated is not None: + raise InvalidParameter("Pass either gated or gated_mode, not both.") + + if "visibility" in payload: + raw = payload["visibility"] + if raw is None or raw == "": + # An explicit empty value clears the setting server-side. + payload["visibility"] = raw + else: + visibility = RepoVisibility.parse(raw) + if visibility is None: + normalised = cls._normalize_visibility(raw) + if normalised == int(Visibility.PUBLIC): + visibility = RepoVisibility.PUBLIC + elif normalised == int(Visibility.PRIVATE): + visibility = RepoVisibility.PRIVATE + else: + allowed = ", ".join(v.value for v in RepoVisibility) + raise InvalidParameter( + f"visibility={raw!r} is not supported for repo_type={rt.value!r}. " + f"Expected one of: {allowed}." + ) + payload["visibility"] = visibility.value + + if has_native_mode: + # Forwarded verbatim. The endpoint accepts ``auto``/``manual``/``""`` + # and rejects an explicit null or a boolean; the boolean is caught + # here because it is the historical ``gated_mode`` spelling and would + # otherwise reach the server as JSON ``true``/``false``. + if isinstance(native_mode, bool) or native_mode is None: + raise InvalidParameter( + f"gated_mode must be one of {', '.join(repr(m.value) for m in GatedMode)} or '' " + f"(got {native_mode!r}). Use gated=True/False to switch gating on or off." + ) + payload["gated_mode"] = native_mode + elif isinstance(gated, GatedMode): + target = payload.get("visibility") + if target is not None and target != RepoVisibility.GATED.value: + # The endpoint rejects the pair outright; refuse locally so the + # contradiction is reported against the caller's own arguments. + raise InvalidParameter( + f"gated={gated.value!r} conflicts with visibility={target!r}. " + f"Set visibility='gated' to keep an approval mode, or drop gated." + ) + if target is None: + # Naming a mode asks for gating, which is a visibility change. + payload["visibility"] = RepoVisibility.GATED.value + payload["gated_mode"] = gated.value + elif gated is False and "visibility" not in payload: + # Turning gating off is expressed by leaving the gated state, which + # the endpoint models as a visibility change. + payload["visibility"] = RepoVisibility.PRIVATE.value + return payload + # ================================================================== # Secrets # ================================================================== diff --git a/src/modelscope_hub/cli/repo.py b/src/modelscope_hub/cli/repo.py index 7072896..361d5d0 100644 --- a/src/modelscope_hub/cli/repo.py +++ b/src/modelscope_hub/cli/repo.py @@ -8,6 +8,7 @@ import argparse from pathlib import Path +from typing import Any from ..constants import RepoType from ..errors import AlreadyExistsError, is_repo_exists_error @@ -119,7 +120,10 @@ def _add_arguments(p) -> None: def execute(self) -> None: api = make_api(self.args) - extra: dict[str, object] = {} + # ``Any`` rather than ``object``: these come from argparse and are + # forwarded verbatim into ``create_repo``'s keyword parameters, whose + # types vary per repo kind. + extra: dict[str, Any] = {} for key in ("sdk_type", "sdk_version", "base_image", "cover_image", "hardware", "category"): value = getattr(self.args, key, None) if value is not None: diff --git a/src/modelscope_hub/compat/hub_api.py b/src/modelscope_hub/compat/hub_api.py index 5d91634..5d18331 100644 --- a/src/modelscope_hub/compat/hub_api.py +++ b/src/modelscope_hub/compat/hub_api.py @@ -167,6 +167,25 @@ def get_model(self, model_id: str, revision: str | None = None) -> dict: info = self._api.get_repo(model_id, RepoType.MODEL, revision=revision) return _repo_info_to_dict(info) + def get_model_detail( + self, + model_id: str, + *, + endpoint: str | None = None, + token: str | None = None, + ) -> dict: + """Fetch a model's full detail payload, zone fields included. + + Distinct from :meth:`get_model`, which keeps its historical shape: a + ``RepoInfo`` flattened to legacy PascalCase keys. That shape has no room + for the AIGC / science zone fields, so this method returns the server's + snake_case ``ModelDetail`` verbatim instead. + """ + api = self._api + if token or endpoint: + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) + return api.get_model(model_id) + def get_model_files( self, model_id: str, @@ -263,6 +282,82 @@ def create_repo( return None raise + def set_model_category( + self, + model_id: str, + *, + token: str | None = None, + endpoint: str | None = None, + **kwargs: Any, + ) -> dict: + """Set or change a model's zone. Wraps :meth:`HubApi.set_model_category`.""" + return self._api_for_write(token, endpoint).set_model_category(model_id, RepoType.MODEL, **kwargs) + + def update_model_category( + self, + model_id: str, + *, + token: str | None = None, + endpoint: str | None = None, + **kwargs: Any, + ) -> dict: + """Update a model's zone fields. Wraps :meth:`HubApi.update_model_category`.""" + return self._api_for_write(token, endpoint).update_model_category(model_id, RepoType.MODEL, **kwargs) + + def set_dataset_category( + self, + dataset_id: str, + *, + token: str | None = None, + endpoint: str | None = None, + **kwargs: Any, + ) -> dict: + """Set or change a dataset's zone. Wraps :meth:`HubApi.set_dataset_category`.""" + return self._api_for_write(token, endpoint).set_dataset_category(dataset_id, RepoType.DATASET, **kwargs) + + def update_dataset_category( + self, + dataset_id: str, + *, + token: str | None = None, + endpoint: str | None = None, + **kwargs: Any, + ) -> dict: + """Update a dataset's science fields. Wraps :meth:`HubApi.update_dataset_category`.""" + return self._api_for_write(token, endpoint).update_dataset_category(dataset_id, RepoType.DATASET, **kwargs) + + def update_dataset( + self, + dataset_id: str, + *, + token: str | None = None, + endpoint: str | None = None, + **kwargs: Any, + ) -> dict: + """Update a dataset's general settings. Wraps :meth:`HubApi.update_repo_settings`.""" + return self._api_for_write(token, endpoint).update_repo_settings(dataset_id, RepoType.DATASET, **kwargs) + + def update_model( + self, + model_id: str, + *, + token: str | None = None, + endpoint: str | None = None, + **kwargs: Any, + ) -> dict: + """Update a model's general settings. Wraps :meth:`HubApi.update_repo_settings`.""" + return self._api_for_write(token, endpoint).update_repo_settings(model_id, RepoType.MODEL, **kwargs) + + def _api_for_write(self, token: str | None, endpoint: str | None) -> HubApi: + """Return the façade to use, honouring per-call token/endpoint overrides.""" + if token or endpoint: + return HubApi( + endpoint=endpoint or self._endpoint, + token=token, + user_agent={"invoked_by": "compat"}, + ) + return self._api + def create_model(self, model_id: str, **kwargs: Any) -> str: """Create a model repo (legacy signature). @@ -918,6 +1013,9 @@ def create_dataset( visibility=visibility, license=license, chinese_name=chinese_name, + # Forwarded explicitly: this argument used to be accepted and then + # dropped, so a caller's description never reached the Hub. + description=description, ) ep = endpoint or self._endpoint or api._config.endpoint return f"{ep}/datasets/{namespace}/{dataset_name}" @@ -956,6 +1054,24 @@ def get_dataset( api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) return api.get_repo(dataset_id, RepoType.DATASET, revision=revision) + def get_dataset_detail( + self, + dataset_id: str, + *, + endpoint: str | None = None, + token: str | None = None, + ) -> dict: + """Fetch a dataset's full detail payload, science fields included. + + Distinct from :meth:`get_dataset`, which keeps its historical shape (a + ``RepoInfo``) and has no room for the science zone fields. Returns the + server's snake_case ``DatasetDetail`` verbatim. + """ + api = self._api + if token or endpoint: + api = HubApi(endpoint=endpoint or self._endpoint, token=token, user_agent={"invoked_by": "compat"}) + return api.get_dataset(dataset_id) + def delete_model( self, model_id: str, diff --git a/src/modelscope_hub/constants.py b/src/modelscope_hub/constants.py index c4ca763..b494d67 100644 --- a/src/modelscope_hub/constants.py +++ b/src/modelscope_hub/constants.py @@ -139,6 +139,97 @@ def parse(cls, value: object) -> StudioVisibility | None: return None +class RepoVisibility(StrEnum): + """Visibility values the OpenAPI model/dataset endpoints accept. + + Deliberately separate from :class:`Visibility`: the ``/openapi/v1`` model and + dataset endpoints take a string enum whose third state (``gated``) the 1/3/5 + integer encoding cannot express, while that encoding's ``INTERNAL`` has no + string counterpart. Neither enum is a relabelling of the other, so they are + mapped explicitly at the call site rather than merged. + + * ``public`` -- visible to everyone. + * ``private`` -- visible only to the owner and collaborators. + * ``gated`` -- discoverable, but downloads require an approved application. + """ + + PUBLIC = "public" + PRIVATE = "private" + GATED = "gated" + + @classmethod + def parse(cls, value: object) -> RepoVisibility | None: + """Return the matching member, or ``None`` when *value* is not one. + + Mirrors :meth:`StudioVisibility.parse`: returning ``None`` lets callers + fall back to the integer :class:`Visibility` encoding for inputs this + enum does not own. + """ + if isinstance(value, cls): + return value + if isinstance(value, str): + try: + return cls(value.strip().lower()) + except ValueError: + return None + return None + + +class GatedMode(StrEnum): + """How download applications against a gated repository are approved. + + Only meaningful when visibility is ``gated``; the endpoints reject the field + for ``public``/``private`` repositories. + """ + + AUTO = "auto" + MANUAL = "manual" + + @classmethod + def parse(cls, value: object) -> GatedMode | None: + """Return the matching member, or ``None`` when *value* is not one.""" + if isinstance(value, cls): + return value + if isinstance(value, str): + try: + return cls(value.strip().lower()) + except ValueError: + return None + return None + + +class ModelCategory(StrEnum): + """Which curated zone a model belongs to. + + A freshly created model is ``base``; ``PUT /models/{owner}/{name}/category`` + moves it into a zone, and passing ``base`` moves it back out and clears the + zone-specific fields. + """ + + BASE = "base" + AIGC = "aigc" + SCIENCE = "science" + + +class DatasetCategory(StrEnum): + """Which curated zone a dataset belongs to. + + Note the general value is ``base`` -- matching the model enum and the wire + protocol -- not ``basic``. + """ + + BASE = "base" + SCIENCE = "science" + + +class AigcModelType(StrEnum): + """Sub-type of an AIGC model. Values are case-sensitive on the wire.""" + + CHECKPOINT = "Checkpoint" + LORA = "LoRA" + VAE = "VAE" + + class TokenScope(StrEnum): """Permission tiers a ModelScope API token can be issued with. @@ -197,6 +288,109 @@ class License(StrEnum): REPO_TYPE_SUPPORT: list[str] = [REPO_TYPE_MODEL, REPO_TYPE_DATASET, REPO_TYPE_STUDIO] DEFAULT_DATASET_REVISION: str = "master" +# --------------------------------------------------------------------------- +# OpenAPI model/dataset contract limits. +# +# These mirror constraints the ``/openapi/v1`` specification declares. They are +# kept here so a request that the server would certainly reject can be rejected +# locally with a message that names the fix, rather than costing a round trip +# for an opaque 400. +# --------------------------------------------------------------------------- + +# Suffixes accepted for an AIGC version's weight files (matched case-insensitively). +WEIGHT_FILE_SUFFIXES: tuple[str, ...] = ( + ".safetensors", + ".ckpt", + ".pt", + ".pth", + ".bin", + ".gguf", + ".ggml", + ".h5", + ".hdf5", + ".npz", + ".msgpack", + ".pdparams", + ".params", + ".onnx", +) + +# ``DELETE /{models,datasets}/{owner}/{name}/files`` accepts 1..100 paths. +OPENAPI_DELETE_FILES_MAX_PATHS: int = 100 + +# Largest ``page_size`` the OpenAPI list endpoints accept. Advisory: the value is +# forwarded as given so a server-side change does not need an SDK release. +OPENAPI_PAGE_SIZE_MAX: int = 50 + +# Longest ``version_name`` accepted when creating an AIGC version. +OPENAPI_VERSION_NAME_MAX_LENGTH: int = 20 + +# ``POST /models`` constrains ``license`` to a closed set, storing the value +# verbatim and case-sensitively. Anything outside it answers 400, so a caller +# supplying another identifier is routed to the legacy endpoint instead, which +# accepts free-form values. ``POST /datasets`` declares no such restriction. +OPENAPI_MODEL_LICENSES: frozenset[str] = frozenset( + { + "", + "apache-2.0", + "Apache License 2.0", + "gpl-2.0", + "GPL-2.0", + "lgpl-3.0", + "LGPL-3.0", + "mit", + "MIT License", + "lgpl-2.1", + "LGPL-2.1", + "gpl-3.0", + "GPL-3.0", + "afl-3.0", + "AFL-3.0", + "ecl-2.0", + "ECL-2.0", + "cc-by-4.0", + "CC-BY-4.0", + "creativeml-openrail-m", + "cc-by-nc-nd", + "CC-BY-NC-ND", + "agpl-3.0", + "cc-by-nc-4.0", + "CC-BY-NC-4.0", + "cc-by-nc-sa-4.0", + "CC-BY-NC-SA-4.0", + "odc-by-1.0", + "ODC-BY-1.0", + "cc-by-sa-4.0", + "CC-BY-SA-4.0", + "cc0-1.0", + "CC0-1.0", + "bsd-3-clause", + "BSD-3-Clause", + "other", + } +) + +# Every field ``POST /models`` and ``POST /datasets`` accept. The endpoints answer +# 400 for anything else, so a caller passing an extra field is routed to the +# legacy endpoint, which forwards unknown keys verbatim. +OPENAPI_CREATE_MODEL_FIELDS: frozenset[str] = frozenset( + { + "owner", + "repo_name", + "display_name", + "license", + "license_name", + "license_link", + "visibility", + "gated_mode", + "description", + "task", + } +) + +# Same list minus ``task``, which the dataset schema does not declare. +OPENAPI_CREATE_DATASET_FIELDS: frozenset[str] = OPENAPI_CREATE_MODEL_FIELDS - {"task"} + # --------------------------------------------------------------------------- # Legacy modelscope domain / endpoint / filesystem constants. # @@ -605,6 +799,21 @@ def _env_register( use. This must be at least as large as the worker count to avoid that churn. """ +OPTIONS_CACHE_TTL_SECONDS: int = _env_int( + "MODELSCOPE_OPTIONS_CACHE_TTL", + 300, + "Seconds to cache the model/dataset enum option endpoints (0 disables)", + "Network", + allow_zero=True, +) +"""Lifetime of the in-process cache for the ``*/options`` endpoints. + +Those endpoints publish product configuration (AIGC architectures, science +subject areas, tasks, licenses) that changes on a release cadence, not per +request, and they need no credentials. Setting a category therefore does not +have to re-fetch the enum every call. Set to ``0`` to disable caching. +""" + REPO_TREE_PAGE_MAX_ATTEMPTS: int = _env_int( "MODELSCOPE_REPO_TREE_PAGE_MAX_ATTEMPTS", 4, diff --git a/src/modelscope_hub/types.py b/src/modelscope_hub/types.py index c6b90ac..bbdb3ca 100644 --- a/src/modelscope_hub/types.py +++ b/src/modelscope_hub/types.py @@ -417,6 +417,13 @@ class RepoInfo(_FromDictMixin): hardware: str | None = None mcp_support: bool | None = None runtime: dict[str, Any] | None = None + # Zone membership. Reported by the model/dataset detail endpoints; ``None`` + # for repo kinds that have no zones (studio / skill / mcp) and for payloads + # predating the field. + model_category: str | None = None + dataset_category: str | None = None + # Approval mode of a gated repository, returned only when ``gated`` is true. + gated_mode: str | None = None def __post_init__(self) -> None: if isinstance(self.repo_type, str): @@ -451,6 +458,9 @@ def to_dict(self) -> dict: "hardware", "mcp_support", "runtime", + "model_category", + "dataset_category", + "gated_mode", } result = {} for f in fields(self): # type: ignore[arg-type] @@ -730,7 +740,275 @@ class TokenSignPayload(TypedDict): signature: str +# --------------------------------------------------------------------------- +# Model / dataset zone structures +# +# Shapes shared by the category endpoints and the enum option endpoints. Field +# names follow the OpenAPI schemas exactly: the server rejects unknown keys, so +# a convenience alias here would turn into a 400 at the call site. +# --------------------------------------------------------------------------- +class SampleImage(TypedDict, total=False): + """A sample image attached to an AIGC model version. + + ``url`` is required. The schema accepts only these three keys -- notably + there is no ``text`` field. + """ + + url: str + prompt: str + negative_prompt: str + + +class WeightFileReference(TypedDict): + """Reference to a weight file already uploaded to the repository. + + Only the repo-relative path is sent; the server resolves size and digest + itself. The suffix must be one of :data:`WEIGHT_FILE_SUFFIXES`. + """ + + path_in_repo: str + + +class AigcContentCategories(TypedDict, total=False): + """Three-level content classification of an AIGC model.""" + + style: str + function: str + element: str + + +class ModelOptionItem(TypedDict): + """One selectable value returned by an ``*/options`` endpoint. + + Submit ``id``; ``label`` is for display only. ``type`` is self-describing + (``task``, ``license``, ``base_architecture``, ``specialization``, ...). + """ + + type: str + id: str + label: str + + +class BaseArchitectureOption(TypedDict): + """A first-level AIGC base architecture and its second-level subtypes.""" + + type: str + id: str + label: str + subtypes: list[ModelOptionItem] + + +class ScienceSubjectOption(TypedDict): + """A first-level science subject area and its specializations.""" + + type: str + id: str + label: str + specializations: list[ModelOptionItem] + + +# --------------------------------------------------------------------------- +# Model / dataset request payloads +# +# Each mirrors one OpenAPI request schema exactly. Fields the schema omits are +# omitted here too, because the endpoints reject unknown keys with a 400 -- +# notably ``base_model_id`` (the base model is derived from the architecture +# subtype), ``login_required`` (read-only), and the category discriminator on +# the PATCH payloads (the server reads the current category from its own +# database and rejects the field). +# --------------------------------------------------------------------------- +class CreateModelPayload(TypedDict, total=False): + """Payload for ``POST /models``. ``owner`` and ``repo_name`` are required. + + Carries general metadata only; a newly created model is ``base``. Zone + fields are set afterwards via ``PUT /models/{owner}/{repo_name}/category``. + """ + + owner: str + repo_name: str + display_name: str + license: str + license_name: str + license_link: str + visibility: str + gated_mode: str + description: str + # The schema declares ``string | null``: an explicit null means "not set". + task: str | None + + +class CreateDatasetPayload(TypedDict, total=False): + """Payload for ``POST /datasets``. ``owner`` and ``repo_name`` are required. + + Mirrors :class:`CreateModelPayload` without ``task``, which the dataset + schema does not declare. + """ + + owner: str + repo_name: str + display_name: str + license: str + license_name: str + license_link: str + visibility: str + gated_mode: str + description: str + + +class SetModelCategoryPayload(TypedDict, total=False): + """Payload for ``PUT /models/{owner}/{repo_name}/category``. + + ``model_category`` is required and names the *target* zone; which other + fields are required follows from it, not from the model's current zone: + + * ``base`` -- nothing else; every zone field is cleared. + * ``aigc`` -- ``aigc_type``, both architecture levels, ``version_name``, and + non-empty ``weight_files`` and ``sample_images``. Creates a new version. + * ``science`` -- ``primary_subject_area`` and ``secondary_specialization``; + ``cover_image`` is optional. + + Passing a field that does not apply to the target zone returns 400. + """ + + model_category: str + # AIGC + aigc_type: str + base_model_architecture_type: str + base_model_architecture_subtype: str + content_categories: AigcContentCategories + version_name: str + version_description: str + trigger_words: list[str] + sample_images: list[SampleImage] + weight_files: list[WeightFileReference] + # Science + primary_subject_area: str + secondary_specialization: str + cover_image: str + + +class UpdateModelCategoryPayload(TypedDict, total=False): + """Payload for ``PATCH /models/{owner}/{repo_name}/category``. + + Updates zone metadata in place. The server determines the zone itself, so + ``model_category`` is not accepted; use ``PUT`` to change zones. + + For AIGC, ``version_id`` (the numeric id from a detail response's + ``versions[]``) is required to update ``trigger_words``, + ``version_description`` or ``sample_images``. Science updates must not send + it. An existing version's weights are immutable: ``weight_files``, + ``version_name`` and ``base_model_id`` are rejected -- create a new version + with ``PUT`` instead. Omitted fields are unchanged; explicit nulls clear. + """ + + version_id: int + # AIGC + aigc_type: str | None + base_model_architecture_type: str | None + base_model_architecture_subtype: str | None + content_categories: AigcContentCategories | None + trigger_words: list[str] | None + version_description: str | None + sample_images: list[SampleImage] + # Science + primary_subject_area: str | None + secondary_specialization: str | None + cover_image: str | None + + +class SetDatasetCategoryPayload(TypedDict, total=False): + """Payload for ``PUT /datasets/{owner}/{repo_name}/category``. + + ``dataset_category`` is required and is ``base`` or ``science`` -- the + general value is ``base``, not ``basic``. ``science`` requires a matching + subject area and specialization; ``base`` clears the science fields. + """ + + dataset_category: str + primary_subject_area: str | None + secondary_specialization: str | None + cover_image: str | None + + +class UpdateDatasetCategoryPayload(TypedDict, total=False): + """Payload for ``PATCH /datasets/{owner}/{repo_name}/category``. + + ``dataset_category`` is not accepted -- the server reads the current one and + returns 400 if the field is present; use ``PUT`` to change zones. + """ + + primary_subject_area: str | None + secondary_specialization: str | None + cover_image: str | None + + +class UpdateDatasetSettingsPayload(TypedDict, total=False): + """Payload for ``PATCH /datasets/{owner}/{repo_name}/settings``. + + General settings only; zone fields are rejected. ``login_required`` is + deliberately absent -- the schema does not accept it, the field is only ever + reported back by the detail endpoint. + + Omitted fields are unchanged; explicit nulls clear. ``gated_mode`` is valid + only alongside ``visibility="gated"`` (or on a repo that is already gated). + """ + + display_name: str | None + license: str | None + visibility: str | None + gated_mode: str + description: str | None + + +class UpdateModelSettingsPayload(TypedDict, total=False): + """Payload for ``PATCH /models/{owner}/{repo_name}/settings``. + + General settings only; zone fields are rejected. Mirrors + :class:`UpdateDatasetSettingsPayload` plus ``task``. + + Omitted fields are unchanged; explicit nulls clear. ``gated_mode`` is valid + only alongside ``visibility="gated"`` (or on a repo that is already gated). + """ + + task: str | None + display_name: str | None + license: str | None + visibility: str | None + gated_mode: str + description: str | None + + +class DeleteFilesPayload(TypedDict, total=False): + """Payload for ``DELETE /{models,datasets}/{owner}/{repo_name}/files``. + + ``file_paths`` is required: 1 to 100 distinct repo-relative file paths. No + directories and no traversal. The whole request is one commit and is + all-or-nothing -- a single unresolvable path fails the lot. + + ``revision`` must name a branch, not a tag. + """ + + file_paths: list[str] + revision: str + commit_message: str + + __all__ = [ + "AigcContentCategories", + "BaseArchitectureOption", + "CreateDatasetPayload", + "CreateModelPayload", + "DeleteFilesPayload", + "ModelOptionItem", + "SampleImage", + "ScienceSubjectOption", + "SetDatasetCategoryPayload", + "SetModelCategoryPayload", + "UpdateDatasetCategoryPayload", + "UpdateDatasetSettingsPayload", + "UpdateModelCategoryPayload", + "UpdateModelSettingsPayload", + "WeightFileReference", "DownloadRetryPolicy", "DownloadFileResult", "DownloadReport", diff --git a/tests/data/openapi.json b/tests/data/openapi.json index aaed4a3..4ee1a83 100644 --- a/tests/data/openapi.json +++ b/tests/data/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "ModelScope OpenAPI", "description": "魔搭社区 OpenAPI 文档", - "version": "1.1.0+master.20260813T030041Z", + "version": "1.1.0+master.20261022T105704Z", "termsOfService": "https://www.modelscope.cn/protocol/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC", "contact": { "name": "modelscope", @@ -50,6 +50,10 @@ "name": "Skills", "description": "技能相关接口" }, + { + "name": "AgentHub", + "description": "AgentHub 相关接口" + }, { "name": "Files", "description": "文件上传相关接口" @@ -69,6 +73,10 @@ { "name": "Galleries", "description": "Gallery 相关接口" + }, + { + "name": "Agent Playground", + "description": "实验场（Agent Playground）相关接口（beta版本）" } ], "externalDocs": { @@ -294,13 +302,63 @@ "bearerAuth": [] } ] + }, + "post": { + "operationId": "createModel", + "summary": "创建通用模型仓库", + "description": "创建一个新的通用模型仓库。默认 `model_category` 为 `base`，即未进入任何专区的普通模型。\n\n如需创建 AIGC 模型或科学模型，请先调用本接口创建仓库，上传权重等文件后，再调用 `PUT /models/{owner}/{repo_name}/category` 设置模型类型及专区业务字段。\n\n使用用户访问令牌（Bearer Token）认证。只能在自己的命名空间或拥有写入/管理员权限的组织下创建。当前 OAuth 仅读取仓库的授权不支持创建。\n\n省略 license 时默认 apache-2.0；license=\"other\" 时必须提供 license_name，省略 license_link 时自动创建空白 LICENSE.md。创建时仅 visibility=\"gated\" 可传 gated_mode：auto 自动审批，manual 人工审批；省略或空字符串默认 auto。null、布尔值、数字及其他非法值返回 400。visibility 为 public/private（含省略后默认 public）时禁止传 gated_mode。请求仅接受文档列出的字段，未知字段返回 400。\n\n重复仓库返回 HTTP 409（DuplicateEntity）；权限或私有仓库配额不足返回 HTTP 403。创建成功返回 HTTP 201，所有 JSON 响应包含 request_id。\n\n本接口仅接收通用元数据字段，不接受 `aigc_type`、`base_model_architecture_type`、`primary_subject_area` 等专区字段。", + "tags": [ + "Models" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateModelRequest" + }, + "example": { + "owner": "alice", + "repo_name": "my-model", + "visibility": "gated", + "gated_mode": "manual", + "task": "text-generation" + } + } + } + }, + "responses": { + "201": { + "$ref": "#/components/responses/create-model-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "409": { + "$ref": "#/components/responses/conflict" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] } }, "/models/{owner}/{repo_name}": { "get": { "operationId": "getModel", "summary": "获取模型详情", - "description": "根据 owner/repo_name 获取模型详情", + "description": "根据 owner/repo_name 获取模型详情。model_category=base 时无专区字段；science 返回学科与封面；aigc 返回架构、内容类别及 versions。每个版本含数值 version_id、固定 Git tag 的 weight_files 元数据和 sample_images；AIGC 不返回 cover_image。所有响应均有 request_id。", "tags": [ "Models" ], @@ -336,6 +394,228 @@ ] } }, + "/models/{owner}/{repo_name}/category": { + "put": { + "operationId": "setModelCategory", + "summary": "设置/变更模型类型", + "description": "设置或变更模型的专区类型（`base` / `aigc` / `science`）。\n\n本接口是 PUT 类型转换；PATCH /category 仅局部更新当前类型，不能转换类型。必填项由目标 model_category 决定，与来源类型无关。\n\n| 目标类型 | 必填字段（除 model_category 外） | 可选字段 | 不适用字段 |\n| --- | --- | --- | --- |\n| base | 无 | 无 | AIGC 和 Science 专区字段 |\n| aigc | aigc_type、base_model_architecture_type、base_model_architecture_subtype、version_name、非空 weight_files、非空 sample_images | version_description、trigger_words、content_categories | version_id、cover_image、Science 学科字段 |\n| science | primary_subject_area、secondary_specialization | cover_image | version_id、version_name、weight_files、sample_images 等 AIGC 字段 |\n\n- 类型转换不修改 README.md 内容或 YAML 头；模型类型和专区元数据保存在数据库，AIGC 权重配置仍通过 configuration.json 管理\n- 转换为 AIGC 时创建新版本，传 version_name，不传用于更新已有版本的 version_id\n- 转换为 Science 时不创建 AIGC 版本/tag；无需 version_id，详情不返回 versions\n- 传入 `base` 时，只需提供 `model_category`，所有专区业务字段将被清除（回退为通用模型）\n- 目标为 aigc（包括创建新 tag）时 weight_files 和 sample_images 均必填且非空\n- 基础模型由架构子类型确定\n- AIGC 图片使用 sample_images（url 必填、prompt 和 negative_prompt 选填），cover_image 仅适用于科学专区\n- 需要仓库维护者权限；仅有读取权限的 OAuth scope 不允许修改\n- 科学分类和 AIGC 架构的实际枚举以 options 接口返回的产品数据为准\n- 传入非目标类型的字段将返回 400\n- 必填字段缺失时返回 400，`message` 一次性列出全部缺失字段名，并指引查询对应 options 接口", + "tags": [ + "Models" + ], + "parameters": [ + { + "$ref": "#/components/parameters/owner" + }, + { + "$ref": "#/components/parameters/repo_name" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SetModelCategoryRequest" + }, + "examples": { + "toBase": { + "summary": "转换为普通模型", + "value": { + "model_category": "base" + } + }, + "toAigc": { + "summary": "转换为 AIGC 并创建新版本", + "value": { + "model_category": "aigc", + "aigc_type": "Checkpoint", + "base_model_architecture_type": "Z_IMAGE_TURBO", + "base_model_architecture_subtype": "Z_IMAGE_TURBO", + "version_name": "v1.0", + "weight_files": [ + { + "path_in_repo": "model.safetensors" + } + ], + "sample_images": [ + { + "url": "https://modelscope.oss-cn-beijing.aliyuncs.com/resource/modelscope_logo.png", + "prompt": "清晰的山间风景", + "negative_prompt": "模糊、低清晰度" + } + ] + } + }, + "toScience": { + "summary": "转换为 Science，学科必填、封面选填", + "value": { + "model_category": "science", + "primary_subject_area": "life-sciences", + "secondary_specialization": "medicine" + } + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/set-model-category-success" + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "patch": { + "operationId": "updateModelCategory", + "summary": "更新模型专区与已有 tag 元数据", + "description": "Science 更新 primary_subject_area、secondary_specialization、cover_image 不需要 version_id，且不应传 AIGC 版本字段；Science 详情不返回 versions。仅更新当前模型专区或已有 AIGC tag 的元数据；类型由服务端从数据库读取当前仓库确定；不接受 model_category，传入返回 400，类型转换使用 PUT /category。更新 tag 描述、触发词或样例图时，必须提供 GET 详情 versions[].version_id 的数值 ID。省略不变，显式空值清空，但必填字段和非空样例图约束仍需满足。已有 tag 的权重不可修改：weight_files、version_name 和 base_model_id 不接受，权重变化请 PUT /category 创建新 tag。AIGC 不接受 cover_image，样例图使用 sample_images。不适用于数据库当前类型的字段返回 400，message 列出当前类型及所有不适用字段；显式 null 也按传入字段校验。所有响应均有 request_id。此接口不修改 README.md 或 YAML 头。", + "tags": [ + "Models" + ], + "responses": { + "200": { + "description": "成功；所有响应含 request_id。", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SetModelCategoryResponse" + } + } + } + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/owner" + }, + { + "$ref": "#/components/parameters/repo_name" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateModelCategoryRequest" + }, + "examples": { + "aigcVersion": { + "summary": "更新 AIGC 版本字段，需要 version_id", + "value": { + "version_id": 1070760, + "version_description": null, + "trigger_words": [] + } + }, + "scienceCover": { + "summary": "更新 Science 封面，无需 version_id", + "value": { + "cover_image": "https://modelscope.oss-cn-beijing.aliyuncs.com/resource/modelscope_banner.png" + } + }, + "scienceSubjects": { + "summary": "更新 Science 学科，无需 version_id", + "value": { + "primary_subject_area": "life-sciences", + "secondary_specialization": "medicine" + } + } + } + } + } + } + } + }, + "/model/aigc/options": { + "get": { + "operationId": "listAigcModelOptions", + "summary": "查询 AIGC 模型配置项枚举", + "description": "返回设置 AIGC 模型类型时可用的配置项枚举：aigc_types（模型子类型）、base_architectures（基础模型架构一级/二级级联）、content_styles / content_functions / content_elements（内容类别三级联动）。设置或更新 AIGC 模型的枚举字段取值以本接口返回的 id 为准。 所有层级的选项使用 label 作为展示名称，提交参数使用 id。 基础架构涵盖 IMAGE、VIDEO、AUDIO，base_architectures 为一级架构，subtypes 为二级架构；每项均包含 type、id、label。", + "tags": [ + "Models" + ], + "responses": { + "200": { + "$ref": "#/components/responses/aigc-model-options-success" + }, + "429": { + "$ref": "#/components/responses/too-many-requests" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [] + } + }, + "/model/science/options": { + "get": { + "operationId": "listScienceModelOptions", + "summary": "查询科学模型配置项枚举", + "description": "返回设置科学模型类型时可用的学科分类树：subject_areas（一级学科）及其下属 specializations（二级专业方向）。primary_subject_area / secondary_specialization 的取值以本接口返回的 id 为准。 所有层级的选项使用 label 作为展示名称，提交参数使用 id。", + "tags": [ + "Models" + ], + "responses": { + "200": { + "$ref": "#/components/responses/science-model-options-success" + }, + "429": { + "$ref": "#/components/responses/too-many-requests" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + }, + "503": { + "$ref": "#/components/responses/service-unavailable" + } + }, + "security": [] + } + }, "/datasets": { "get": { "operationId": "listDatasets", @@ -468,13 +748,70 @@ "bearerAuth": [] } ] + }, + "post": { + "operationId": "createDataset", + "summary": "创建通用数据集仓库", + "description": "创建通用数据集仓库，默认 dataset_category=base。只接受通用元数据；科学字段通过 PUT /category 设置。license=other 时需要 license_name；省略 license_link 自动创建 LICENSE.md。创建时仅 visibility=\"gated\" 可传 gated_mode：auto 自动审批，manual 人工审批；省略或空字符串默认 auto。null、布尔值、数字及其他非法值返回 400。visibility 为 public/private（含省略后默认 public）时禁止传 gated_mode。请求仅接受文档列出的字段，未知字段返回 400。成功返回 201，所有响应均有 request_id。", + "tags": [ + "Datasets" + ], + "responses": { + "201": { + "description": "成功；所有响应含 request_id。", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DatasetWriteResponse" + } + } + } + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateDatasetRequest" + }, + "example": { + "owner": "alice", + "repo_name": "my-dataset", + "visibility": "gated", + "gated_mode": "manual", + "license": "apache-2.0" + } + } + } + } } }, "/datasets/{owner}/{repo_name}": { "get": { "operationId": "getDataset", "summary": "获取数据集详情", - "description": "根据 owner/repo_name 获取数据集详情", + "description": "根据 owner/repo_name 获取数据集详情。dataset_category=base 时无科学字段；science 返回 primary_subject_area、secondary_specialization 和 cover_image。通用字段包含 visibility 和 login_required。所有响应均有 request_id。", "tags": [ "Datasets" ], @@ -1184,7 +1521,7 @@ "get": { "operationId": "listStudioSecrets", "summary": "获取创空间密文变量列表", - "description": "获取指定 Studio 的所有密文变量 key 列表，不返回 value", + "description": "获取指定 Studio 的所有密文变量 key 列表，不返回 value；创建或更新时填写的 description 会一并返回", "tags": [ "Studios" ], @@ -1232,7 +1569,7 @@ "post": { "operationId": "addStudioSecret", "summary": "添加创空间密文变量", - "description": "为指定的 Studio 添加一个密文变量，value 不会公开展示", + "description": "为指定的 Studio 添加一个密文变量，value 不会公开展示；可通过可选的 description 记录该变量的用途（最长 512 字符），描述含敏感内容时返回 400", "tags": [ "Studios" ], @@ -1287,7 +1624,7 @@ "put": { "operationId": "updateStudioSecret", "summary": "更新创空间密文变量", - "description": "更新指定 Studio 的一个密文变量值", + "description": "更新指定 Studio 的一个密文变量值，并可同时更新可选的 description（最长 512 字符）；未提供 description 时保留原有描述，描述含敏感内容时返回 400", "tags": [ "Studios" ], @@ -1393,7 +1730,7 @@ "get": { "operationId": "listStudioVariables", "summary": "获取创空间明文变量列表", - "description": "获取指定 Studio 的所有明文变量的 key 和 value（与密文变量不同，明文变量的值公开可见）", + "description": "获取指定 Studio 的所有明文变量的 key、value 和可选的 description（与密文变量不同，明文变量的值公开可见）", "tags": [ "Studios" ], @@ -1431,7 +1768,7 @@ "post": { "operationId": "addStudioVariable", "summary": "添加创空间明文变量", - "description": "为指定的 Studio 添加一个明文变量（key 和 value 都公开可见，敏感信息请使用密文变量接口）", + "description": "为指定的 Studio 添加一个明文变量（key 和 value 都公开可见，敏感信息请使用密文变量接口）；可通过可选的 description 记录该变量的用途（最长 512 字符），描述含敏感内容时返回 400", "tags": [ "Studios" ], @@ -1485,7 +1822,7 @@ "put": { "operationId": "updateStudioVariable", "summary": "更新创空间明文变量", - "description": "更新指定 Studio 的一个明文变量值", + "description": "更新指定 Studio 的一个明文变量值，并可同时更新可选的 description（最长 512 字符）；未提供 description 时保留原有描述，描述含敏感内容时返回 400", "tags": [ "Studios" ], @@ -2031,11 +2368,242 @@ ] } }, - "/files/upload": { - "post": { - "operationId": "uploadFile", - "summary": "上传文件", - "description": "通用文件上传接口（最大 5MB），返回文件 ID。该 ID 可作为入参传给其他接口，用于引用已上传的文件。", + "/agents": { + "get": { + "operationId": "listAgents", + "summary": "获取 Agent 列表", + "description": "返回 ModelScope Agent 广场的 Agent 列表，支持关键词搜索与多维度筛选。\n\n默认只返回已发布/预发布的 Agent；携带 Token 访问时，同时返回当前用户及其所属组织下的私有 Agent。\n\n分页总量（`page_number` × `page_size`）不得超过 2400，超出返回 `QuotaLimitExceed`。\n", + "tags": [ + "AgentHub" + ], + "parameters": [ + { + "name": "search", + "in": "query", + "description": "关键词搜索，支持 Agent 名称、描述等", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "filter.framework", + "in": "query", + "description": "按框架类型筛选", + "required": false, + "schema": { + "type": "string" + }, + "examples": { + "ms-agent": { + "value": "ms-agent", + "summary": "ModelScope Agent 框架" + }, + "qwenpaw": { + "value": "qwenpaw", + "summary": "QwenPaw 框架" + } + } + }, + { + "name": "filter.category", + "in": "query", + "description": "按分类筛选（对应 Agent 的 categories）", + "required": false, + "schema": { + "type": "string" + }, + "examples": { + "tools": { + "value": "tools", + "summary": "工具类" + } + } + }, + { + "name": "filter.license", + "in": "query", + "description": "按许可证筛选", + "required": false, + "schema": { + "type": "string" + }, + "examples": { + "apache": { + "value": "Apache License 2.0", + "summary": "Apache 2.0 许可证" + } + } + }, + { + "name": "filter.custom_tag", + "in": "query", + "description": "按自定义标签筛选", + "required": false, + "schema": { + "type": "string" + }, + "examples": { + "weather": { + "value": "weather", + "summary": "天气" + } + } + }, + { + "name": "filter.owner", + "in": "query", + "description": "按所有者筛选（用户名或组织名）", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "sort", + "in": "query", + "description": "排序方式：default（综合）/ created_at（创建时间）/ last_modified（最近更新）/ views（浏览量）/ downloads（下载量）/ likes（喜欢数）", + "required": false, + "schema": { + "type": "string", + "enum": [ + "default", + "created_at", + "last_modified", + "views", + "downloads", + "likes" + ], + "default": "default", + "examples": [ + "default", + "created_at", + "last_modified", + "views", + "downloads", + "likes" + ] + } + }, + { + "name": "page_number", + "in": "query", + "description": "页码，从 1 开始", + "required": false, + "schema": { + "type": "integer", + "default": 1 + } + }, + { + "name": "page_size", + "in": "query", + "description": "每页条数，取值 1~100", + "required": false, + "schema": { + "type": "integer", + "default": 10 + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-agents-success" + }, + "400": { + "$ref": "#/components/responses/agent-bad-request" + }, + "401": { + "$ref": "#/components/responses/agent-unauthorized" + }, + "403": { + "$ref": "#/components/responses/agent-quota-limit-exceeded" + }, + "404": { + "$ref": "#/components/responses/agent-list-failed" + } + } + }, + "post": { + "operationId": "createAgent", + "summary": "创建 Agent", + "description": "创建一个新的 Agent。\n\n使用 `owner` 和 `agent_name` 指定 Agent 所有者与唯一名称。\n\n支持设置框架类型（`framework`）、是否私有（`private`）与自定义标签（`custom_tags`）等配置。\n\n请求字段与查看 Agent 详情接口保持同名（`logo_url` / `private` / `custom_tags` 等）。\n", + "tags": [ + "AgentHub" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateAgentRequest" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/create-agent-success" + }, + "400": { + "$ref": "#/components/responses/agent-bad-request" + }, + "401": { + "$ref": "#/components/responses/agent-unauthorized" + }, + "500": { + "$ref": "#/components/responses/agent-internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/agents/{id}": { + "get": { + "operationId": "getAgent", + "summary": "查看 Agent 详情", + "description": "根据 Agent ID 返回指定 Agent 的详细信息，包括关联的模型、技能、MCP 配置等", + "tags": [ + "AgentHub" + ], + "parameters": [ + { + "$ref": "#/components/parameters/agent_id" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/get-agent-success" + }, + "400": { + "$ref": "#/components/responses/agent-bad-request" + }, + "403": { + "$ref": "#/components/responses/agent-forbidden" + }, + "404": { + "$ref": "#/components/responses/agent-not-found" + }, + "500": { + "$ref": "#/components/responses/agent-internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/files/upload": { + "post": { + "operationId": "uploadFile", + "summary": "上传文件", + "description": "通用文件上传接口（最大 5MB），返回文件 ID。该 ID 可作为入参传给其他接口，用于引用已上传的文件。", "tags": [ "Files" ], @@ -3286,1247 +3854,1283 @@ } ] } - } - }, - "components": { - "schemas": { - "User": { - "allOf": [ - { - "$ref": "#/components/schemas/UserPublic" + }, + "/models/{owner}/{repo_name}/settings": { + "patch": { + "operationId": "updateModelSettings", + "summary": "更新模型通用设置", + "description": "局部更新模型通用设置。省略不变，显式空值清空。gated_mode 取 auto 自动审批、manual 人工审批；显式 visibility=\"gated\" 时，gated_mode 省略或空字符串默认 auto；两字段均省略保持原设置；无论 visibility 取值，显式 null、布尔值、数字及其他非法值均返回 400；visibility 为 public/private 时禁止传 gated_mode（含空串和 null），否则返回 400；PATCH 省略 visibility 时，仅当前仓库已是申请制才可用 gated_mode 修改审批方式。关闭申请制请将 visibility 改为 public 或 private。通用设置不包含专区字段。所有响应均有 request_id。", + "tags": [ + "Models" + ], + "responses": { + "200": { + "description": "成功；所有响应含 request_id。", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SetModelCategoryResponse" + } + } + } + }, + "400": { + "$ref": "#/components/responses/bad-request" + }, + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" }, + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ { - "$ref": "#/components/schemas/UserPrivate" + "bearerAuth": [] } ], - "description": "用户完整信息(包含私有信息)" - }, - "UserResponse": { - "type": "object", - "description": "用户信息响应", - "properties": { - "success": { - "type": "boolean", - "description": "成功响应时始终为 true", - "examples": [ - true - ] - }, - "data": { - "$ref": "#/components/schemas/User", - "description": "用户信息" + "parameters": [ + { + "$ref": "#/components/parameters/owner" }, - "request_id": { - "type": "string", - "description": "请求ID", - "examples": [ - "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" - ] + { + "$ref": "#/components/parameters/repo_name" } - }, - "required": [ - "success", - "request_id" - ] - }, - "ErrorResponse": { - "type": "object", - "description": "错误响应格式", - "properties": { - "success": { - "type": "boolean", - "description": "错误响应时始终为 false", - "examples": [ - false - ] + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ModelSettingsRequest" + }, + "example": { + "description": null, + "visibility": "gated", + "gated_mode": "manual", + "task": "text-generation" + } + } + } + } + } + }, + "/datasets/{owner}/{repo_name}/category": { + "put": { + "operationId": "setDatasetCategory", + "summary": "设置数据集类型", + "description": "设置或切换数据集类型。science 必须提供匹配的 primary_subject_area 和 secondary_specialization，枚举查询 GET /dataset/science/options；base 清除科学专区字段。不支持 AIGC 字段。需要仓库维护权限，所有响应均有 request_id。", + "tags": [ + "Datasets" + ], + "responses": { + "200": { + "description": "成功；所有响应含 request_id。", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DatasetWriteResponse" + } + } + } }, - "code": { - "type": "string", - "description": "错误码", - "examples": [ - "InputParameterError" - ] + "400": { + "$ref": "#/components/responses/bad-request" }, - "message": { - "type": "string", - "description": "错误消息", - "examples": [ - "Unauthorized access" - ] + "401": { + "$ref": "#/components/responses/unauthorized" }, - "request_id": { - "type": "string", - "description": "请求ID", - "examples": [ - "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" - ] + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" } }, - "required": [ - "success", - "code", - "message", - "request_id" - ] + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/owner" + }, + { + "$ref": "#/components/parameters/repo_name" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DatasetCategoryRequest" + }, + "example": { + "dataset_category": "science", + "primary_subject_area": "life-sciences", + "secondary_specialization": "dna" + } + } + } + } }, - "Model": { - "type": "object", - "description": "模型摘要信息", - "properties": { - "id": { - "type": "string", - "description": "模型仓库名（repo_id，格式为 owner/repo_name）", - "examples": [ - "iic/QwenLong-L1.5-30B-A3B", - "ZhipuAI/AutoGLM-Phone-9B" - ] + "patch": { + "operationId": "updateDatasetCategory", + "summary": "局部更新科学数据集字段", + "description": "仅更新当前数据集专区字段；类型由服务端从数据库读取当前仓库确定，不接受 dataset_category（传入返回 400），改变类型请用 PUT /category。省略不变，显式空值清空，学科必填约束仍需满足；cover_image 可用 null 清空。不适用于数据库当前类型的字段返回 400，message 列出当前类型及所有不适用字段；显式 null 也按传入字段校验。所有响应均有 request_id。", + "tags": [ + "Datasets" + ], + "responses": { + "200": { + "description": "成功；所有响应含 request_id。", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DatasetWriteResponse" + } + } + } }, - "display_name": { - "type": [ - "string", - "null" - ], - "description": "模型展示名称", - "examples": [ - "QwenLong-L1.5-30B-A3B", - "AutoGLM-Phone-9B" - ] + "400": { + "$ref": "#/components/responses/bad-request" }, - "description": { - "type": [ - "string", - "null" - ], - "description": "模型简介", - "examples": [ - "" - ] + "401": { + "$ref": "#/components/responses/unauthorized" }, - "downloads": { - "type": "integer", - "description": "下载量", - "examples": [ - 1466, - 28525 - ] + "403": { + "$ref": "#/components/responses/forbidden" }, - "likes": { - "type": "integer", - "description": "喜欢量", - "examples": [ - 25, - 265 - ] + "404": { + "$ref": "#/components/responses/not-found" }, - "license": { - "type": [ - "string", - "null" - ], - "description": "许可证", - "examples": [ - "apache-2.0", - "MIT License" - ] + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/owner" }, - "tasks": { - "type": "array", - "description": "任务列表", - "items": { - "type": "string" - }, - "examples": [ - [ - "text-generation" - ], - [ - "image-captioning" - ] - ] + { + "$ref": "#/components/parameters/repo_name" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateDatasetCategoryRequest" + }, + "example": { + "cover_image": null + } + } + } + } + } + }, + "/datasets/{owner}/{repo_name}/settings": { + "patch": { + "operationId": "updateDatasetSettings", + "summary": "更新数据集通用设置", + "description": "局部更新数据集通用设置。省略不变，显式空值清空。gated_mode 取 auto 自动审批、manual 人工审批；显式 visibility=\"gated\" 时，gated_mode 省略或空字符串默认 auto；两字段均省略保持原设置；无论 visibility 取值，显式 null、布尔值、数字及其他非法值均返回 400；visibility 为 public/private 时禁止传 gated_mode（含空串和 null），否则返回 400；PATCH 省略 visibility 时，仅当前仓库已是申请制才可用 gated_mode 修改审批方式。关闭申请制请将 visibility 改为 public 或 private。所有响应均有 request_id。", + "tags": [ + "Datasets" + ], + "responses": { + "200": { + "description": "成功；所有响应含 request_id。", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DatasetWriteResponse" + } + } + } }, - "created_at": { - "type": "string", - "format": "date-time", - "description": "创建时间（ISO 8601 UTC）", - "examples": [ - "2025-12-14T12:26:15Z" - ] + "400": { + "$ref": "#/components/responses/bad-request" }, - "last_modified": { - "type": "string", - "format": "date-time", - "description": "Git 文件或设置的最后修改时间（ISO 8601 UTC）", - "examples": [ - "2025-12-16T03:34:26Z" - ] + "401": { + "$ref": "#/components/responses/unauthorized" }, - "file_size": { - "type": "integer", - "description": "文件大小（字节）", - "examples": [ - 61081671123 - ] + "403": { + "$ref": "#/components/responses/forbidden" }, - "params": { - "type": "integer", - "description": "参数量", - "examples": [ - 30532122624 - ] + "404": { + "$ref": "#/components/responses/not-found" }, - "tags": { - "type": "array", - "description": "标签列表", - "items": { - "type": "string" - }, - "examples": [ - [ - "license:apache-2.0", - "model_type:qwen3_moe", - "library:transformer", - "library:safetensors", - "library:pytorch", - "task:text-generation", - "deploy:swingdeploy" - ] - ] - }, - "private": { - "type": "boolean", - "description": "是否非公开模型", - "examples": [ - false - ] - }, - "gated": { - "type": "boolean", - "description": "是否申请制模型，通过申请后可访问", - "examples": [ - false - ] + "500": { + "$ref": "#/components/responses/internal-server-error" } }, - "required": [ - "id", - "downloads", - "likes", - "tasks", - "created_at", - "last_modified", - "file_size", - "params", - "private", - "gated" - ] - }, - "ModelList": { - "type": "object", - "description": "模型列表与分页信息", - "properties": { - "models": { - "type": "array", - "description": "结果模型列表", - "items": { - "$ref": "#/components/schemas/Model" + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/owner" + }, + { + "$ref": "#/components/parameters/repo_name" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DatasetSettingsRequest" + }, + "example": { + "description": null, + "visibility": "gated", + "gated_mode": "manual" + } + } + } + } + } + }, + "/models/{owner}/{repo_name}/files": { + "delete": { + "operationId": "deleteModelFiles", + "summary": "单次 commit 原子批量删除文件", + "description": "批量删除仓库文件，整个请求只产生一次 commit；任一文件不存在或校验失败时全部失败，不允许部分成功。file_paths 为 1–100 个不重复的仓库相对文件路径，不接受目录或越界路径。revision 默认为 master，只能指定分支，不能指定 tag。成功返回 commit_sha 和完整 deleted_files，failed_files 为空；所有响应均有 request_id。", + "tags": [ + "Models" + ], + "responses": { + "200": { + "description": "成功；所有响应含 request_id。", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteFilesResponse" + } + } } }, - "total_count": { - "type": "integer", - "description": "符合筛选条件的模型总数", - "examples": [ - 43127 - ] + "400": { + "$ref": "#/components/responses/bad-request" }, - "page_number": { - "type": "integer", - "description": "当前页码", - "examples": [ - 1 - ] + "401": { + "$ref": "#/components/responses/unauthorized" }, - "page_size": { - "type": "integer", - "description": "每页大小", - "examples": [ - 10 - ] - } - }, - "required": [ - "models", - "total_count", - "page_number", - "page_size" - ] - }, - "ModelListResponse": { - "type": "object", - "description": "列出模型的标准响应包裹格式", - "properties": { - "success": { - "type": "boolean", - "description": "成功时始终为 true", - "examples": [ - true - ] + "403": { + "$ref": "#/components/responses/forbidden" }, - "data": { - "$ref": "#/components/schemas/ModelList", - "description": "模型列表数据" + "404": { + "$ref": "#/components/responses/not-found" }, - "request_id": { - "type": "string", - "description": "请求 ID", - "examples": [ - "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" - ] + "500": { + "$ref": "#/components/responses/internal-server-error" } }, - "required": [ - "success", - "data", - "request_id" - ] - }, - "ModelDetail": { - "allOf": [ + "security": [ { - "$ref": "#/components/schemas/Model" + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/owner" }, { - "type": "object", - "description": "模型详情信息", - "properties": { - "readme": { - "type": [ - "string", - "null" + "$ref": "#/components/parameters/repo_name" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteFilesRequest" + }, + "example": { + "file_paths": [ + "obsolete/a.txt", + "obsolete/b.txt" ], - "description": "README 内容", - "examples": [ - "hello" - ] + "revision": "master", + "commit_message": "Remove obsolete files" } } } - ] - }, - "ModelDetailResponse": { - "type": "object", - "description": "获取模型详情的标准响应包裹格式", - "properties": { - "success": { - "type": "boolean", - "description": "成功时始终为 true", - "examples": [ - true - ] + } + } + }, + "/datasets/{owner}/{repo_name}/files": { + "delete": { + "operationId": "deleteDatasetFiles", + "summary": "单次 commit 原子批量删除文件", + "description": "批量删除仓库文件，整个请求只产生一次 commit；任一文件不存在或校验失败时全部失败，不允许部分成功。file_paths 为 1–100 个不重复的仓库相对文件路径，不接受目录或越界路径。revision 默认为 master，只能指定分支，不能指定 tag。成功返回 commit_sha 和完整 deleted_files，failed_files 为空；所有响应均有 request_id。", + "tags": [ + "Datasets" + ], + "responses": { + "200": { + "description": "成功；所有响应含 request_id。", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteFilesResponse" + } + } + } }, - "data": { - "$ref": "#/components/schemas/ModelDetail", - "description": "模型详情数据" + "400": { + "$ref": "#/components/responses/bad-request" }, - "request_id": { - "type": "string", - "description": "请求 ID", - "examples": [ - "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" - ] + "401": { + "$ref": "#/components/responses/unauthorized" + }, + "403": { + "$ref": "#/components/responses/forbidden" + }, + "404": { + "$ref": "#/components/responses/not-found" + }, + "500": { + "$ref": "#/components/responses/internal-server-error" } }, - "required": [ - "success", - "data", - "request_id" - ] - }, - "Dataset": { - "type": "object", - "description": "数据集摘要信息", - "properties": { - "id": { - "type": "string", - "description": "数据集仓库名（repo_id，格式为 owner/repo_name）", - "examples": [ - "DAMO_NLP/jd", - "iic/nlp_domain_classification_chinese_testset" - ] + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/owner" }, - "display_name": { - "type": [ - "string", - "null" - ], - "description": "数据集展示名称", - "examples": [ - "商品评论情感预测", - "中文文本领域分类测试集" - ] - }, - "description": { - "type": [ - "string", - "null" - ], - "description": "数据集简介", - "examples": [ - "", - "..." - ] - }, - "file_size": { - "type": "integer", - "description": "文件大小（字节）", - "examples": [ - 0 - ] + { + "$ref": "#/components/parameters/repo_name" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteFilesRequest" + }, + "example": { + "file_paths": [ + "obsolete/a.txt", + "obsolete/b.txt" + ], + "revision": "master", + "commit_message": "Remove obsolete files" + } + } + } + } + } + }, + "/dataset/science/options": { + "get": { + "operationId": "listScienceDatasetOptions", + "summary": "获取科学数据集学科枚举", + "description": "获取科学数据集学科枚举。 所有层级的选项使用 label 作为展示名称，一级学科和二级专业方向均使用 id 作为提交值，分别用于 primary_subject_area 和 secondary_specialization。", + "tags": [ + "Datasets" + ], + "responses": { + "200": { + "description": "成功；所有响应含 request_id。", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScienceDatasetOptionsResponse" + }, + "examples": { + "science-dataset-options-response": { + "$ref": "#/components/examples/science-dataset-options-response" + } + } + } + } }, - "downloads": { - "type": "integer", - "description": "下载量", - "examples": [ - 53996, - 2401 - ] + "400": { + "$ref": "#/components/responses/bad-request" }, - "likes": { - "type": "integer", - "description": "喜欢量", - "examples": [ - 102, - 23 - ] + "401": { + "$ref": "#/components/responses/unauthorized" }, - "license": { - "type": [ - "string", - "null" - ], - "description": "许可证", - "examples": [ - "Apache License 2.0", - "cc-by-4.0" - ] + "403": { + "$ref": "#/components/responses/forbidden" }, - "tasks": { - "type": "array", - "description": "任务列表", - "items": { - "type": "string" - }, - "examples": [ - [ - "text-classification" - ], - [] - ] + "404": { + "$ref": "#/components/responses/not-found" }, - "created_at": { - "type": "string", - "format": "date-time", - "description": "创建时间（ISO 8601 UTC）", - "examples": [ - "2022-09-28T06:30:31Z", - "2025-11-25T14:30:22Z" - ] + "500": { + "$ref": "#/components/responses/internal-server-error" + } + }, + "security": [] + } + }, + "/models/options": { + "get": { + "operationId": "getModelOptions", + "summary": "获取模型任务与许可证选项", + "description": "返回 tasks 和 license 两个列表，每项仅包含 type、id、label。tasks[].id 用于创建模型、更新模型设置的 task 或列表筛选 filter.task；license[].id 用于创建模型的 license，label 是可用于各处直接展示的名称。", + "tags": [ + "Models" + ], + "responses": { + "200": { + "description": "Success. Every response includes request_id.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ModelOptionsResponse" + } + } + } }, - "last_modified": { - "type": "string", - "format": "date-time", - "description": "Git 文件或设置的最后修改时间（ISO 8601 UTC）", - "examples": [ - "2022-10-30T03:36:39Z", - "2026-01-01T00:31:42Z" - ] + "400": { + "$ref": "#/components/responses/bad-request" }, - "tags": { - "type": "array", - "description": "标签列表", - "items": { - "type": "string" - }, - "examples": [ - [ - "license:Apache License 2.0", - "task:text-classification" - ], - [ - "license:cc-by-4.0", - "custom_tag:common-crawl" - ] - ] + "401": { + "$ref": "#/components/responses/unauthorized" }, - "private": { - "type": "boolean", - "description": "是否非公开数据集", - "examples": [ - false - ] + "403": { + "$ref": "#/components/responses/forbidden" }, - "gated": { - "type": "boolean", - "description": "是否申请制数据集，通过申请后可访问", - "examples": [ - false - ] + "404": { + "$ref": "#/components/responses/not-found" }, - "login_required": { - "type": "boolean", - "description": "是否登录后才可下载", - "examples": [ - false - ] + "500": { + "$ref": "#/components/responses/internal-server-error" } }, - "required": [ - "id", - "downloads", - "likes", - "private", - "gated", - "login_required", - "created_at", - "last_modified" - ] - }, - "DatasetList": { - "type": "object", - "description": "数据集列表与分页信息", - "properties": { - "datasets": { - "type": "array", - "description": "结果数据集列表", - "items": { - "$ref": "#/components/schemas/Dataset" + "security": [] + } + }, + "/agent/playground/instances": { + "post": { + "operationId": "createAgentPlaygroundInstance", + "summary": "创建实验场实例", + "description": "创建一个实验场（Agent Playground）实例。\n\n异步创建：响应中 `status` 为 `starting`，就绪后转为 `running`。\n\n可选的 `models` / `mcps` / `skills` / `message` 会在实例就绪后按「模型 → MCP → Skill → 对话」的顺序\n作用于实例内 `default` Agent。引用在创建时同步校验，不合法直接返回 400；应用阶段失败的项会被跳过\n（实例照常转为 `running`），错误经一次性通知（`init_error`）返回。\n\n不传 `models` 时会自动配置平台默认模型。\n\n实例数超出账号配额时返回 400。\n", + "tags": [ + "Agent Playground" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateAgentPlaygroundInstanceRequest" + }, + "examples": { + "create-agent-playground-instance-request": { + "$ref": "#/components/examples/create-agent-playground-instance-request" + } + } } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/create-agent-playground-instance-success" }, - "total_count": { - "type": "integer", - "description": "符合筛选条件的数据集总数", - "examples": [ - 60 - ] + "400": { + "$ref": "#/components/responses/agent-playground-bad-request" }, - "page_number": { - "type": "integer", - "description": "当前页码", - "examples": [ - 1 - ] + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" }, - "page_size": { - "type": "integer", - "description": "每页大小", - "examples": [ - 10 - ] + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" + }, + "409": { + "$ref": "#/components/responses/agent-playground-conflict" + }, + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" } }, - "required": [ - "datasets", - "total_count", - "page_number", - "page_size" + "security": [ + { + "bearerAuth": [] + } ] }, - "DatasetListResponse": { - "type": "object", - "description": "列出数据集的标准响应包裹格式", - "properties": { - "success": { - "type": "boolean", - "description": "成功时始终为 true", - "examples": [ - true - ] + "get": { + "operationId": "listAgentPlaygroundInstances", + "summary": "获取实验场实例列表", + "description": "分页返回调用者名下的实验场实例。\n", + "tags": [ + "Agent Playground" + ], + "parameters": [ + { + "name": "owner", + "in": "query", + "description": "实例归属账号，缺省为调用者；只能是调用者本人，否则返回 403", + "required": false, + "schema": { + "type": "string", + "examples": [ + "username" + ] + } }, - "data": { - "$ref": "#/components/schemas/DatasetList", - "description": "数据集列表数据" + { + "name": "page_number", + "in": "query", + "description": "页码，需大于 0；非数字或小于等于 0 返回 400", + "required": false, + "schema": { + "type": "integer", + "default": 1, + "minimum": 1 + } }, - "request_id": { - "type": "string", - "description": "请求 ID", - "examples": [ - "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" - ] - } - }, - "required": [ - "success", - "data", - "request_id" - ] - }, - "DatasetDetail": { - "allOf": [ { - "$ref": "#/components/schemas/Dataset" + "name": "page_size", + "in": "query", + "description": "每页条数，取值 1~100；越界或非数字返回 400", + "required": false, + "schema": { + "type": "integer", + "default": 10, + "minimum": 1, + "maximum": 100 + } }, { - "type": "object", - "description": "数据集详情信息", - "properties": { - "readme": { - "type": [ - "string", - "null" - ], - "description": "README 内容", - "examples": [ - "hello" - ] - } + "name": "sort_by", + "in": "query", + "description": "排序字段，大小写不敏感；`user_modified_at` 为用户最近操作时间（相同时按创建时间）", + "required": false, + "schema": { + "type": "string", + "enum": [ + "user_modified_at", + "gmt_created" + ], + "default": "user_modified_at" + } + }, + { + "name": "order", + "in": "query", + "description": "排序方向", + "required": false, + "schema": { + "type": "string", + "enum": [ + "desc", + "asc" + ], + "default": "desc" } } - ] - }, - "DatasetDetailResponse": { - "type": "object", - "description": "获取数据集详情的标准响应包裹格式", - "properties": { - "success": { - "type": "boolean", - "description": "成功时始终为 true", - "examples": [ - true - ] + ], + "responses": { + "200": { + "$ref": "#/components/responses/list-agent-playground-instances-success" }, - "data": { - "$ref": "#/components/schemas/DatasetDetail", - "description": "数据集详情数据" + "400": { + "$ref": "#/components/responses/agent-playground-bad-request" }, - "request_id": { - "type": "string", - "description": "请求 ID", - "examples": [ - "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" - ] + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" + }, + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" + }, + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" } }, - "required": [ - "success", - "data", - "request_id" + "security": [ + { + "bearerAuth": [] + } ] - }, - "McpServerLocale": { - "type": "object", - "description": "MCP服务在对应语言环境下的内容", - "properties": { - "name": { - "type": "string", - "description": "MCP服务在对应语言环境下的服务名称", - "examples": [ - "Fetch网页内容抓取" - ] + } + }, + "/agent/playground/instances/from-agent-hub": { + "post": { + "operationId": "createAgentPlaygroundInstanceFromAgentHub", + "summary": "从 Agent Hub 创建实验场实例", + "description": "创建实例，并把 Agent Hub 上指定 agent 的内容同步到实例内 `default` Agent（覆盖写入）。\n\n与普通创建的区别：\n\n- 内容同步目前只支持 `qwenpaw` / `ms-agent`。\n- **不会**自动配置默认模型：未传 `models` 时实例内模型取决于源 agent 的内容，可能为空。\n- 同步完成后产生一次性通知（`import_result`；带 `message` 时为 `conversation`），payload 里标明来源与去向。\n\n其余字段的校验与普通创建一致。\n", + "tags": [ + "Agent Playground" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateAgentPlaygroundInstanceFromAgentHubRequest" + }, + "examples": { + "create-agent-playground-instance-from-agent-hub-request": { + "$ref": "#/components/examples/create-agent-playground-instance-from-agent-hub-request" + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/create-agent-playground-instance-from-agent-hub-success" }, - "description": { - "type": "string", - "description": "MCP服务在对应语言环境下的服务简介", - "examples": [ - "该服务器使大型语言模型能够检索和处理网页内容..." - ] + "400": { + "$ref": "#/components/responses/agent-playground-bad-request" }, - "readme": { - "type": "string", - "description": "MCP服务介绍（仅详情接口返回）", - "examples": [ - "# Fetch MCP Server..." - ] - } - } - }, - "McpServerLocales": { - "type": "object", - "description": "不同语言环境下的内容展示", - "properties": { - "zh": { - "$ref": "#/components/schemas/McpServerLocale", - "description": "中文环境" + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" }, - "en": { - "$ref": "#/components/schemas/McpServerLocale", - "description": "英文环境" - } - } - }, - "McpOperationalUrl": { - "type": "object", - "description": "MCP服务托管链接信息", - "properties": { - "id": { - "type": "string", - "description": "MCP服务链接命名标识", - "examples": [ - "platform-pool" - ] + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" }, - "url": { - "type": "string", - "description": "MCP服务远程连接地址", - "examples": [ - "https://mcp.api-inference.modelscope.net/xxxx/mcp" - ] + "404": { + "$ref": "#/components/responses/agent-playground-not-found" }, - "transport_type": { - "type": "string", - "description": "MCP服务链接传输方式", - "examples": [ - "streamable_http", - "sse" - ] + "409": { + "$ref": "#/components/responses/agent-playground-conflict" }, - "auth_required": { - "type": "boolean", - "description": "远程 url 连接时是否需要提供魔搭访问令牌鉴权", - "examples": [ - false - ] + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/agent/playground/instances/{uuid}": { + "get": { + "operationId": "getAgentPlaygroundInstance", + "summary": "获取实验场实例详情", + "description": "按 `uuid` 返回实例详情。\n", + "tags": [ + "Agent Playground" + ], + "parameters": [ + { + "$ref": "#/components/parameters/agent_playground_instance_uuid" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/get-agent-playground-instance-success" }, - "expiration": { - "type": "string", - "description": "MCP服务链接有效期", - "examples": [ - "2025-10-01 21:00:00" - ] + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" }, - "accessible": { - "type": "boolean", - "description": "是否有权限访问该MCP服务详情。当所有者设置为非公开或MCP服务被删除时，无权访问。", - "examples": [ - true - ] + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" + }, + "404": { + "$ref": "#/components/responses/agent-playground-not-found" + }, + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" } - } + }, + "security": [ + { + "bearerAuth": [] + } + ] }, - "McpServerSummary": { - "type": "object", - "description": "MCP服务摘要信息", - "properties": { - "id": { - "type": "string", - "description": "服务ID。由平台后台收录的MCP，ID 默认为@author/server_name，当被社区用户认领后将会变更为user_name/server_name；由社区用户自发提交的MCP，ID 为user_name/server_name。", - "examples": [ - "@modelcontextprotocol/fetch" - ] - }, - "name": { - "type": "string", - "description": "MCP服务在魔搭社区的名称", - "examples": [ - "Fetch网页内容抓取" - ] - }, - "chinese_name": { - "type": "string", - "description": "MCP服务在魔搭社区的中文名称", - "examples": [ - "Fetch网页内容抓取" - ] - }, - "description": { - "type": "string", - "description": "MCP服务简介", - "examples": [ - "该服务器使大型语言模型能够检索和处理网页内容..." - ] + "put": { + "operationId": "updateAgentPlaygroundInstance", + "summary": "更新实验场实例元信息", + "description": "只允许修改 `instance_name`、`display_name`、`description`、`logo`，其余字段不可改。\n\n覆盖语义：`display_name`、`description` 缺省或传空串都会清空；`logo` 缺省保持原值，显式传入则覆盖。\n", + "tags": [ + "Agent Playground" + ], + "parameters": [ + { + "$ref": "#/components/parameters/agent_playground_instance_uuid" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateAgentPlaygroundInstanceRequest" + }, + "examples": { + "update-agent-playground-instance-request": { + "$ref": "#/components/examples/update-agent-playground-instance-request" + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/update-agent-playground-instance-success" }, - "logo_url": { - "type": "string", - "description": "logo图片URL链接", - "examples": [ - "https://resources.modelscope.cn/..." - ] + "400": { + "$ref": "#/components/responses/agent-playground-bad-request" }, - "publisher": { - "type": "string", - "description": "发布时原始ID", - "examples": [ - "@modelcontextprotocol/fetch" - ] + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" }, - "categories": { - "type": "array", - "description": "该MCP服务所属分类", - "items": { - "type": "string" - }, - "examples": [ - [ - "browser-automation" - ] - ] + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" }, - "tags": { - "type": "array", - "description": "该MCP服务的标签列表", - "items": { - "type": "string" - }, - "examples": [ - [ - "browser-automation" - ] - ] + "404": { + "$ref": "#/components/responses/agent-playground-not-found" }, - "view_count": { - "type": "integer", - "description": "该MCP服务的累计访问量", - "examples": [ - 30667 - ] + "409": { + "$ref": "#/components/responses/agent-playground-conflict" }, - "locales": { - "$ref": "#/components/schemas/McpServerLocales" + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" } }, - "required": [ - "id" - ] - }, - "McpServerDetail": { - "type": "object", - "description": "MCP服务详细信息", - "allOf": [ - { - "$ref": "#/components/schemas/McpServerSummary" - }, + "security": [ { - "type": "object", - "properties": { - "author": { - "type": "string", - "description": "MCP服务开发者", - "examples": [ - "modelcontextprotocol" - ] - }, - "owner": { - "type": "string", - "description": "MCP服务归属的魔搭用户/组织ID", - "examples": [ - "" - ] - }, - "readme": { - "type": "string", - "description": "MCP服务介绍", - "examples": [ - "# Fetch MCP Server..." - ] - }, - "source_url": { - "type": "string", - "description": "MCP服务来源项目主页地址", - "examples": [ - "https://github.com/modelcontextprotocol/..." - ] - }, - "github_stars": { - "type": "integer", - "description": "该 MCP 服务对应 github 原始仓库的 stars 数量", - "examples": [ - 1234 - ] - }, - "is_hosted": { - "type": "boolean", - "description": "该MCP服务是否支持托管部署", - "examples": [ - true - ] - }, - "is_verified": { - "type": "boolean", - "description": "托管MCP服务是否经过平台验证测试", - "examples": [ - true - ] - }, - "env_schema": { - "type": "object", - "description": "该服务连接部署的环境变量配置", - "additionalProperties": true - }, - "server_config": { - "type": "array", - "description": "MCP服务配置信息", - "items": {} - }, - "operational_urls": { - "type": "array", - "description": "当get_operational_url=True且当前用户已连接该 MCP 服务时则返回，否则为空", - "items": { - "$ref": "#/components/schemas/McpOperationalUrl" - } - } - } + "bearerAuth": [] } ] }, - "McpServerOperational": { - "type": "object", - "description": "用户托管的MCP服务信息", - "allOf": [ + "delete": { + "operationId": "deleteAgentPlaygroundInstance", + "summary": "删除实验场实例", + "description": "删除实例，同时清除该实例的通知。\n\n幂等：对不存在或删除中的实例同样返回 200 且 `deleted: true`。\n", + "tags": [ + "Agent Playground" + ], + "parameters": [ { - "$ref": "#/components/schemas/McpServerSummary" + "$ref": "#/components/parameters/agent_playground_instance_uuid" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/delete-agent-playground-instance-success" + }, + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" + }, + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" }, + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" + } + }, + "security": [ { - "type": "object", - "properties": { - "operational_urls": { - "type": "array", - "description": "MCP服务托管链接列表", - "items": { - "$ref": "#/components/schemas/McpOperationalUrl" - } - } - } + "bearerAuth": [] } ] - }, - "McpServerListResponse": { - "type": "object", - "description": "MCP服务列表响应", - "properties": { - "success": { - "type": "boolean", - "description": "成功时始终为 true", - "examples": [ - true - ] + } + }, + "/agent/playground/instances/{uuid}/pause": { + "post": { + "operationId": "pauseAgentPlaygroundInstance", + "summary": "暂停实验场实例", + "description": "暂停实例，工作区内容保留。\n\n任意状态都可调用且幂等，响应 `status` 立即为 `sleeping`。\n\n实例空闲一段时间后也会被系统自动休眠，自动休眠不更新 `user_modified_at`。\n", + "tags": [ + "Agent Playground" + ], + "parameters": [ + { + "$ref": "#/components/parameters/agent_playground_instance_uuid" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/pause-agent-playground-instance-success" }, - "data": { - "$ref": "#/components/schemas/McpServerList", - "description": "MCP服务列表数据" + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" }, - "message": { - "type": "string", - "description": "响应消息", - "examples": [ - "success" - ] + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" }, - "request_id": { - "type": "string", - "description": "请求 ID", - "examples": [ - "c55709d2-4e09-4bd8-be84-d8518bc46363" - ] + "404": { + "$ref": "#/components/responses/agent-playground-not-found" + }, + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" } }, - "required": [ - "success", - "request_id" + "security": [ + { + "bearerAuth": [] + } ] - }, - "McpServerOperationalListResponse": { - "type": "object", - "description": "用户托管MCP服务列表响应", - "properties": { - "success": { - "type": "boolean", - "description": "成功时始终为 true", - "examples": [ - true - ] + } + }, + "/agent/playground/instances/{uuid}/resume": { + "post": { + "operationId": "resumeAgentPlaygroundInstance", + "summary": "恢复实验场实例", + "description": "恢复实例运行。任意状态都可调用（幂等）。\n\n响应 `status` 按调用前的状态给出：\n\n- `sleeping`：返回 `running`。唤醒通常秒级完成，期间查询详情可能短暂返回 `starting`。\n- `starting` / `failed`：返回 `starting`。\n- `running`：返回 `running`。\n\n`failed` 实例恢复不会触发重建，请改用重启接口。\n", + "tags": [ + "Agent Playground" + ], + "parameters": [ + { + "$ref": "#/components/parameters/agent_playground_instance_uuid" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/resume-agent-playground-instance-success" }, - "data": { - "$ref": "#/components/schemas/McpServerOperationalList", - "description": "用户托管MCP服务列表数据" + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" }, - "message": { - "type": "string", - "description": "响应消息", - "examples": [ - "success" - ] + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" }, - "request_id": { - "type": "string", - "description": "请求 ID", - "examples": [ - "c55709d2-4e09-4bd8-be84-d8518bc46363" - ] + "404": { + "$ref": "#/components/responses/agent-playground-not-found" + }, + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" } }, - "required": [ - "success", - "request_id" + "security": [ + { + "bearerAuth": [] + } ] - }, - "McpServerDetailResponse": { - "type": "object", - "description": "MCP服务详情响应", - "properties": { - "success": { - "type": "boolean", - "description": "成功时始终为 true", - "examples": [ - true - ] + } + }, + "/agent/playground/instances/{uuid}/restart": { + "post": { + "operationId": "restartAgentPlaygroundInstance", + "summary": "冷重启实验场实例", + "description": "冷重启：实例内进程与内存状态重置，工作区内容保留。\n\n任意状态都可调用，`sleeping` / `failed` 实例会被拉起。异步操作，响应 `status` 为 `starting`。\n", + "tags": [ + "Agent Playground" + ], + "parameters": [ + { + "$ref": "#/components/parameters/agent_playground_instance_uuid" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/restart-agent-playground-instance-success" }, - "data": { - "$ref": "#/components/schemas/McpServerDetail", - "description": "MCP服务详情数据" + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" }, - "message": { - "type": "string", - "description": "响应消息", - "examples": [ - "success" - ] + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" }, - "request_id": { - "type": "string", - "description": "请求 ID", - "examples": [ - "c55709d2-4e09-4bd8-be84-d8518bc46363" - ] + "404": { + "$ref": "#/components/responses/agent-playground-not-found" + }, + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" } }, - "required": [ - "success", - "request_id" + "security": [ + { + "bearerAuth": [] + } ] - }, - "McpDeployResponse": { - "type": "object", - "description": "MCP服务部署响应", - "properties": { - "success": { - "type": "boolean", - "description": "成功时始终为 true", - "examples": [ - true - ] + } + }, + "/agent/playground/instances/{uuid}/agents": { + "get": { + "operationId": "getAgentPlaygroundInstanceAgents", + "summary": "获取实例内 Agent 信息", + "description": "返回实例内所有 Agent 的概要（`summary`）与详情（`detail`）。用于确认下发结果，以及拿到下发接口需要的 `agent_id`。\n\n休眠实例返回最近一次已知的数据，不会被唤醒；实例还在启动时返回空列表而不是报错，可以直接轮询；\n实例内 Agent 服务尚未就绪时返回 503，稍后重试。\n\n框架差异：qwenpaw 可有多个 Agent，persona 为工作区中的 Markdown 文件，`mcp_clients.tools` 仅对启用的客户端返回；\nms-agent 恒为 1 个 `default` Agent，persona 固定 `AGENTS.md`、`PROFILE.md`，`mcp_clients.tools` 恒为空数组。\n", + "tags": [ + "Agent Playground" + ], + "parameters": [ + { + "$ref": "#/components/parameters/agent_playground_instance_uuid" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/get-agent-playground-instance-agents-success" }, - "data": { - "$ref": "#/components/schemas/McpOperationalUrl", - "description": "部署后的MCP服务链接信息" + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" }, - "message": { - "type": "string", - "description": "响应消息", - "examples": [ - "success" - ] + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" }, - "request_id": { - "type": "string", - "description": "请求 ID", - "examples": [ - "c55709d2-4e09-4bd8-be84-d8518bc46363" - ] + "404": { + "$ref": "#/components/responses/agent-playground-not-found" + }, + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" + }, + "502": { + "$ref": "#/components/responses/agent-playground-bad-gateway" + }, + "503": { + "$ref": "#/components/responses/agent-playground-service-unavailable" } }, - "required": [ - "success", - "request_id" + "security": [ + { + "bearerAuth": [] + } ] - }, - "McpUndeployResponse": { - "type": "object", - "description": "MCP服务解除部署响应", - "properties": { - "success": { - "type": "boolean", - "description": "成功时始终为 true", - "examples": [ - true - ] + } + }, + "/agent/playground/instances/{uuid}/setup/model": { + "post": { + "operationId": "setupAgentPlaygroundModel", + "summary": "向实例下发模型配置", + "description": "为目标 Agent 配置模型：按模型引用解析后下发给运行中的实例，并**设为该 Agent 的生效模型**。\n重复下发同一个模型是幂等的。\n\n下发类接口的公共约定：\n\n- 调用者须是实例 owner，且实例 `status` 为 `running`，否则返回 400（休眠实例请先恢复）。\n- 同步下发，最长等待 50 秒，超时返回 504（操作可能仍在实例内继续完成）。\n- 实例内返回的错误会透传，其余实例内失败返回 503。\n- 成功返回最新的实例对象，下发不改变实例 `status`。\n", + "tags": [ + "Agent Playground" + ], + "parameters": [ + { + "$ref": "#/components/parameters/agent_playground_instance_uuid" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SetupAgentPlaygroundModelRequest" + }, + "examples": { + "setup-agent-playground-model-request": { + "$ref": "#/components/examples/setup-agent-playground-model-request" + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/setup-agent-playground-instance-success" }, - "message": { - "type": "string", - "description": "响应消息", - "examples": [ - "success" - ] + "400": { + "$ref": "#/components/responses/agent-playground-bad-request" }, - "request_id": { - "type": "string", - "description": "请求 ID", - "examples": [ - "c55709d2-4e09-4bd8-be84-d8518bc46363" - ] + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" + }, + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" + }, + "404": { + "$ref": "#/components/responses/agent-playground-not-found" + }, + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" + }, + "502": { + "$ref": "#/components/responses/agent-playground-bad-gateway" + }, + "503": { + "$ref": "#/components/responses/agent-playground-service-unavailable" + }, + "504": { + "$ref": "#/components/responses/agent-playground-request-timeout" } }, - "required": [ - "success", - "request_id" - ] - }, - "StudioHardware": { - "type": "string", - "description": "硬件配置，修改后需重新部署才能生效。\n可用值不固定，请通过 `GET /studios/hardware` 接口查询当前可用的硬件配置列表。\n免费资源常见格式为 `platform/...`、`xgpu/...`、`amd/...`；付费资源格式为 `paid/`，例如 `paid/ecs.gn7i-c8g1.2xlarge`。\n", - "examples": [ - "platform/2v-cpu-16g-mem", - "paid/ecs.gn7i-c8g1.2xlarge" - ] - }, - "StudioSDKType": { - "type": "string", - "description": "SDK 类型，修改后需重新部署才能生效", - "enum": [ - "gradio", - "streamlit", - "docker", - "static" - ] - }, - "StudioSDKVersion": { - "type": "string", - "description": "SDK 版本，仅对 Gradio 类型生效，默认最新版，建议选用最新版本，修改后需重新部署才能生效。\n可用版本会随平台更新，请通过 `GET /studios/sdk-versions?sdk_type=gradio` 查询当前可用的 Gradio 版本列表。\n", - "examples": [ - "6.2.0" - ] - }, - "StudioBaseImage": { - "type": "string", - "description": "基础镜像，仅 Docker 类型不支持，建议选用最新版本，修改后需重新部署才能生效。\n可用值不固定，请通过 `GET /studios/base-images` 接口查询当前可用的基础镜像列表。\n", - "examples": [ - "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0" + "security": [ + { + "bearerAuth": [] + } ] - }, - "CreateStudioRequest": { - "type": "object", - "description": "创建 Studio 请求参数", - "properties": { - "repo_name": { - "type": "string", - "description": "仓库名称", - "maxLength": 64, - "examples": [ - "my-demo-app" - ] + } + }, + "/agent/playground/instances/{uuid}/setup/mcp": { + "post": { + "operationId": "setupAgentPlaygroundMcp", + "summary": "向实例下发 MCP 配置", + "description": "为目标 Agent 添加一个 MCP 连接，`server_config` 校验通过后原样下发。\n\n**仅新增**：目标 Agent 下已有同名 client key 时返回 409，不会覆盖。qwenpaw 会把 key 中的斜杠与反斜杠\n替换为 `-`，实例内实际 key 以实例内 Agent 信息接口为准。\n\n连接方式：带 `command` 的配置按 stdio 处理；带 `url` 且未写 `type` 时按 SSE 处理，\nStreamable HTTP 服务必须显式写 `\"type\": \"streamable_http\"`。\n\n前置条件、超时与错误透传规则同下发模型接口。\n", + "tags": [ + "Agent Playground" + ], + "parameters": [ + { + "$ref": "#/components/parameters/agent_playground_instance_uuid" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SetupAgentPlaygroundMcpRequest" + }, + "examples": { + "setup-agent-playground-mcp-request": { + "$ref": "#/components/examples/setup-agent-playground-mcp-request" + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/setup-agent-playground-instance-success" }, - "owner": { - "type": "string", - "description": "所有者（用户名或组织名）", - "examples": [ - "username" - ] + "400": { + "$ref": "#/components/responses/agent-playground-bad-request" }, - "display_name": { - "type": "string", - "description": "显示名称", - "maxLength": 128, - "examples": [ - "我的演示应用" - ] + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" }, - "license": { - "type": "string", - "description": "许可证，默认 apache-2.0", - "default": "apache-2.0", - "examples": [ - "apache-2.0" - ] + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" }, - "visibility": { - "$ref": "#/components/schemas/StudioVisibility" + "404": { + "$ref": "#/components/responses/agent-playground-not-found" }, - "private": { - "type": "boolean", - "description": "Deprecated: 请使用 visibility 字段", - "deprecated": true, - "default": false + "409": { + "$ref": "#/components/responses/agent-playground-conflict" }, - "description": { - "type": "string", - "description": "描述", - "maxLength": 2000, - "examples": [ - "这是一个演示应用" - ] + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" }, - "cover_image": { - "type": "string", - "description": "封面图 URL，为空时使用平台默认图" + "502": { + "$ref": "#/components/responses/agent-playground-bad-gateway" }, - "sdk_type": { - "allOf": [ - { - "$ref": "#/components/schemas/StudioSDKType" + "503": { + "$ref": "#/components/responses/agent-playground-service-unavailable" + }, + "504": { + "$ref": "#/components/responses/agent-playground-request-timeout" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/agent/playground/instances/{uuid}/setup/skill": { + "post": { + "operationId": "setupAgentPlaygroundSkill", + "summary": "向实例下发 Skill", + "description": "按 Skill Hub 引用把 Skill 安装到运行中实例的目标 Agent。\n\n- 引用格式不合法返回 400（`invalid skill ref, expected path/name`）；Skill Hub 中不存在返回 400（`skill not found`）。\n- 私有 Skill 只有有读权限的账号能下发（本人、组织成员、`@` 命名空间）；没有权限时与「不存在」返回同一个错误。\n- **仅新增**：实例内已有同名 Skill 返回 409，不会覆盖。\n- 安装失败返回 503，超时返回 504。\n\n前置条件与错误透传规则同下发模型接口。\n", + "tags": [ + "Agent Playground" + ], + "parameters": [ + { + "$ref": "#/components/parameters/agent_playground_instance_uuid" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SetupAgentPlaygroundSkillRequest" + }, + "examples": { + "setup-agent-playground-skill-request": { + "$ref": "#/components/examples/setup-agent-playground-skill-request" + } } - ], - "default": "gradio" + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/setup-agent-playground-instance-success" }, - "sdk_version": { - "$ref": "#/components/schemas/StudioSDKVersion" + "400": { + "$ref": "#/components/responses/agent-playground-bad-request" }, - "base_image": { - "$ref": "#/components/schemas/StudioBaseImage" + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" }, - "hardware": { - "allOf": [ - { - "$ref": "#/components/schemas/StudioHardware" + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" + }, + "404": { + "$ref": "#/components/responses/agent-playground-not-found" + }, + "409": { + "$ref": "#/components/responses/agent-playground-conflict" + }, + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" + }, + "502": { + "$ref": "#/components/responses/agent-playground-bad-gateway" + }, + "503": { + "$ref": "#/components/responses/agent-playground-service-unavailable" + }, + "504": { + "$ref": "#/components/responses/agent-playground-request-timeout" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/agent/playground/instances/{uuid}/import": { + "post": { + "operationId": "importToAgentPlaygroundInstance", + "summary": "向实例导入 Model / MCP / Skill", + "description": "往已有实例追加导入 model / mcp / skill，可附带一句话 `message` 开启对话。**休眠实例会被自动唤醒**。\n\n应用顺序与创建时一致；实例内已存在的同名 MCP / Skill 视为已完成，失败的项会被跳过并经 `init_error` 通知返回。\n\n- 四个内容字段至少传一个，否则返回 400。导入内容恒作用于实例内 `default` Agent（本接口不支持 `agent_id`）。\n- 引用在受理时同步校验，不合法返回 400，`message` 指明是哪一项。\n- 成功响应 `status` 为 `starting`，完成后回到 `running`；结果经一次性通知返回（`import_result` / `conversation`）。\n", + "tags": [ + "Agent Playground" + ], + "parameters": [ + { + "$ref": "#/components/parameters/agent_playground_instance_uuid" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ImportToAgentPlaygroundInstanceRequest" + }, + "examples": { + "import-to-agent-playground-instance-request": { + "$ref": "#/components/examples/import-to-agent-playground-instance-request" + } } - ], - "default": "platform/2v-cpu-16g-mem", - "description": "硬件配置。未传时使用默认免费硬件配置" + } } }, - "required": [ - "repo_name", - "owner" + "responses": { + "200": { + "$ref": "#/components/responses/import-to-agent-playground-instance-success" + }, + "400": { + "$ref": "#/components/responses/agent-playground-bad-request" + }, + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" + }, + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" + }, + "404": { + "$ref": "#/components/responses/agent-playground-not-found" + }, + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "/agent/playground/instances/{uuid}/notifications": { + "get": { + "operationId": "getAgentPlaygroundInstanceNotifications", + "summary": "拉取实例一次性通知", + "description": "返回并**消费**实例的全部未读通知（按产生顺序）：同一条通知只会返回一次，适合轮询后一次性展示。\n通知保留 7 天，超期未拉取的也会被清理；实例删除时一并清除。\n\n通知类型：\n\n- `import_result`：创建或导入的内容应用完成（未附带 `message`）。从 Agent Hub 创建时另有 `agent_repo`。\n- `conversation`：内容应用完成且附带了一句话，payload 含会话信息。\n- `agent_import`：从 Agent Hub 导入 Agent 完成，payload 含新建 Agent 的 `agent_id`。\n- `init_error`：某一项内容被跳过（payload 含 `item` / `error` / `code`），实例仍会 `running`。\n- `framework_update_error`：框架版本更新失败（payload 含 `target` / `error` / `code`）。\n\n`payload` 为原样 JSON、键名不做转换；其中 `conversation` / `agent` 部分不同框架结构不同，\n用 `framework` 字段区分。\n", + "tags": [ + "Agent Playground" + ], + "parameters": [ + { + "$ref": "#/components/parameters/agent_playground_instance_uuid" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/get-agent-playground-instance-notifications-success" + }, + "401": { + "$ref": "#/components/responses/agent-playground-unauthorized" + }, + "403": { + "$ref": "#/components/responses/agent-playground-forbidden" + }, + "404": { + "$ref": "#/components/responses/agent-playground-not-found" + }, + "500": { + "$ref": "#/components/responses/agent-playground-internal-server-error" + } + }, + "security": [ + { + "bearerAuth": [] + } ] + } + } + }, + "components": { + "schemas": { + "User": { + "allOf": [ + { + "$ref": "#/components/schemas/UserPublic" + }, + { + "$ref": "#/components/schemas/UserPrivate" + } + ], + "description": "用户完整信息(包含私有信息)" }, - "CreateStudioResponse": { + "UserResponse": { "type": "object", - "description": "创建 Studio 响应", + "description": "用户信息响应", "properties": { "success": { "type": "boolean", + "description": "成功响应时始终为 true", "examples": [ true ] }, "data": { - "type": "object", - "properties": { - "id": { - "type": "string", - "description": "Studio ID (owner/repo_name)", - "examples": [ - "username/my-demo-app" - ] - }, - "repo_name": { - "type": "string", - "description": "仓库名称", - "examples": [ - "my-demo-app" - ] - }, - "display_name": { - "type": "string", - "description": "显示名称", - "examples": [ - "我的演示应用" - ] - }, - "owner": { - "type": "string", - "description": "所有者", - "examples": [ - "username" - ] - }, - "url": { - "type": "string", - "description": "Studio URL", - "examples": [ - "https://modelscope.cn/studios/username/my-demo-app" - ] - } - } + "$ref": "#/components/schemas/User", + "description": "用户信息" }, "request_id": { - "type": "string" + "type": "string", + "description": "请求ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" + ] } }, "required": [ @@ -4534,1151 +5138,1154 @@ "request_id" ] }, - "StudioConfig": { - "type": "object", - "description": "Studio 运行时配置", - "properties": { - "hardware": { - "$ref": "#/components/schemas/StudioHardware" - }, - "base_image": { - "allOf": [ - { - "$ref": "#/components/schemas/StudioBaseImage" - } - ], - "description": "基础镜像（Docker 类型不返回）" - }, - "sdk_type": { - "$ref": "#/components/schemas/StudioSDKType" - }, - "sdk_version": { - "type": "string", - "description": "SDK 版本（仅 gradio 类型返回）" - } - } - }, - "StudioRuntime": { + "ErrorResponse": { "type": "object", - "description": "Studio 运行时信息", + "description": "错误响应格式", "properties": { - "status": { - "type": "string", - "description": "运行状态", - "enum": [ - "Initialized", - "Building", - "BuildFailed", - "Deploying", - "DeployFailed", - "Running", - "Stopping", - "Stopped", - "Duplicating", - "Sleeping" + "success": { + "type": "boolean", + "description": "错误响应时始终为 false", + "examples": [ + false ] }, - "active_config": { - "description": "当前实际运行中的配置", - "$ref": "#/components/schemas/StudioConfig" - }, - "created_at": { + "code": { "type": "string", - "description": "部署时间" + "description": "错误码", + "examples": [ + "InputParameterError" + ] }, - "error_message": { + "message": { "type": "string", - "description": "失败信息（仅在错误状态时返回）" - } - } - }, - "GetStudioResponse": { - "type": "object", - "description": "获取 Studio 详情响应", - "properties": { - "success": { - "type": "boolean" - }, - "data": { - "type": "object", - "properties": { - "id": { - "type": "string", - "description": "Studio ID (owner/repo_name)" - }, - "repo_name": { - "type": "string", - "description": "仓库名称" - }, - "display_name": { - "type": "string", - "description": "显示名称" - }, - "owner": { - "type": "string", - "description": "所有者" - }, - "description": { - "type": "string", - "description": "描述" - }, - "cover_image": { - "type": "string", - "description": "封面图 URL" - }, - "likes": { - "type": "integer", - "description": "喜欢数" - }, - "view_count": { - "type": "integer", - "description": "访问量" - }, - "tags": { - "type": "array", - "items": { - "type": "string" - } - }, - "visibility": { - "$ref": "#/components/schemas/StudioVisibility" - }, - "private": { - "type": "boolean", - "description": "Deprecated: 请使用 visibility 字段", - "deprecated": true - }, - "last_modified": { - "type": "string", - "description": "最后修改时间" - }, - "sdk_type": { - "$ref": "#/components/schemas/StudioSDKType" - }, - "sdk_version": { - "$ref": "#/components/schemas/StudioSDKVersion" - }, - "hardware": { - "allOf": [ - { - "$ref": "#/components/schemas/StudioHardware" - } - ], - "description": "期望的硬件配置（数据库中保存的值）" - }, - "base_image": { - "allOf": [ - { - "$ref": "#/components/schemas/StudioBaseImage" - } - ], - "description": "期望的基础镜像（Docker 类型不返回）" - }, - "license": { - "type": "string", - "description": "许可证" - }, - "host": { - "type": "string", - "description": "访问地址" - }, - "mcp_support": { - "type": "boolean", - "description": "是否支持 MCP" - }, - "runtime": { - "$ref": "#/components/schemas/StudioRuntime" - } - } + "description": "错误消息", + "examples": [ + "Unauthorized access" + ] }, "request_id": { - "type": "string" + "type": "string", + "description": "请求ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" + ] } }, "required": [ "success", + "code", + "message", "request_id" ] }, - "UpdateStudioSettingsRequest": { + "Model": { "type": "object", - "description": "更新 Studio 设置请求参数（传入哪个字段修改哪个字段）", + "description": "模型摘要信息", "properties": { - "display_name": { - "type": "string", - "description": "显示名称", - "maxLength": 128 - }, - "license": { + "id": { "type": "string", - "description": "许可证" - }, - "visibility": { - "$ref": "#/components/schemas/StudioVisibility" + "description": "模型仓库名（repo_id，格式为 owner/repo_name）", + "examples": [ + "iic/QwenLong-L1.5-30B-A3B", + "ZhipuAI/AutoGLM-Phone-9B" + ] }, - "private": { - "type": "boolean", - "description": "Deprecated: 请使用 visibility 字段", - "deprecated": true + "display_name": { + "type": [ + "string", + "null" + ], + "description": "模型展示名称", + "examples": [ + "QwenLong-L1.5-30B-A3B", + "AutoGLM-Phone-9B" + ] }, "description": { - "type": "string", - "description": "描述", - "maxLength": 2000 + "type": [ + "string", + "null" + ], + "description": "模型简介", + "examples": [ + "" + ] }, - "cover_image": { - "type": "string", - "description": "封面图 URL" + "downloads": { + "type": "integer", + "description": "下载量", + "examples": [ + 1466, + 28525 + ] }, - "sdk_type": { - "$ref": "#/components/schemas/StudioSDKType" + "likes": { + "type": "integer", + "description": "喜欢量", + "examples": [ + 25, + 265 + ] }, - "sdk_version": { - "$ref": "#/components/schemas/StudioSDKVersion" + "license": { + "type": [ + "string", + "null" + ], + "description": "许可证", + "examples": [ + "apache-2.0", + "MIT License" + ] }, - "base_image": { - "$ref": "#/components/schemas/StudioBaseImage" + "tasks": { + "type": "array", + "description": "任务列表", + "items": { + "type": "string" + }, + "examples": [ + [ + "text-generation" + ], + [ + "image-captioning" + ] + ] }, - "hardware": { - "allOf": [ - { - "$ref": "#/components/schemas/StudioHardware" - } - ], - "description": "硬件配置，修改后需重新部署才能生效" - } - } - }, - "AddStudioSecretRequest": { - "type": "object", - "description": "添加 Studio 密文变量请求参数", - "properties": { - "key": { + "created_at": { "type": "string", - "description": "密文变量名称", - "maxLength": 128, + "format": "date-time", + "description": "创建时间（ISO 8601 UTC）", "examples": [ - "API_KEY" + "2025-12-14T12:26:15Z" ] }, - "value": { + "last_modified": { "type": "string", - "description": "密文变量值（添加后不会公开展示）", - "maxLength": 4096, + "format": "date-time", + "description": "Git 文件或设置的最后修改时间（ISO 8601 UTC）", "examples": [ - "sk-xxxxxxxxxxxx" + "2025-12-16T03:34:26Z" ] - } - }, - "required": [ - "key", - "value" - ] - }, - "UpdateStudioSecretRequest": { - "type": "object", - "description": "更新 Studio 密文变量请求参数", - "properties": { - "key": { - "type": "string", - "description": "密文变量名称", - "maxLength": 128, + }, + "file_size": { + "type": "integer", + "description": "文件大小（字节）", "examples": [ - "API_KEY" + 61081671123 ] }, - "value": { - "type": "string", - "description": "密文变量新值", - "maxLength": 4096, + "params": { + "type": "integer", + "description": "参数量", "examples": [ - "sk-xxxxxxxxxxxx" + 30532122624 ] - } - }, - "required": [ - "key", - "value" - ] - }, - "StudioSecretItem": { - "type": "object", - "description": "密文变量列表项（仅返回 key，不返回 value）", - "properties": { - "key": { - "type": "string", - "description": "密文变量名称" - } - }, - "required": [ - "key" - ] - }, - "ListStudioSecretsResponse": { - "type": "object", - "description": "获取 Studio 密文变量列表响应", - "properties": { - "success": { - "type": "boolean" }, - "data": { - "type": "object", - "properties": { - "secrets": { - "type": "array", - "items": { - "$ref": "#/components/schemas/StudioSecretItem" - } - } - } + "tags": { + "type": "array", + "description": "标签列表", + "items": { + "type": "string" + }, + "examples": [ + [ + "license:apache-2.0", + "model_type:qwen3_moe", + "library:transformer", + "library:safetensors", + "library:pytorch", + "task:text-generation", + "deploy:swingdeploy" + ] + ] }, - "request_id": { - "type": "string" + "private": { + "type": "boolean", + "description": "是否普通私有模型；申请制模型返回 false，请结合 gated 和 visibility 判断访问模式", + "examples": [ + false + ] + }, + "gated": { + "type": "boolean", + "description": "是否申请制模型，通过申请后可访问", + "examples": [ + false + ] } }, "required": [ - "success", - "data", - "request_id" + "id", + "downloads", + "likes", + "tasks", + "created_at", + "last_modified", + "file_size", + "params", + "private", + "gated" ] }, - "DeleteStudioSecretRequest": { + "ModelList": { "type": "object", - "description": "删除 Studio 密文变量请求参数", + "description": "模型列表与分页信息", "properties": { - "key": { - "type": "string", - "description": "密文变量名称" + "models": { + "type": "array", + "description": "结果模型列表", + "items": { + "$ref": "#/components/schemas/Model" + } + }, + "total_count": { + "type": "integer", + "description": "符合筛选条件的模型总数", + "examples": [ + 43127 + ] + }, + "page_number": { + "type": "integer", + "description": "当前页码", + "examples": [ + 1 + ] + }, + "page_size": { + "type": "integer", + "description": "每页大小", + "examples": [ + 10 + ] } }, "required": [ - "key" + "models", + "total_count", + "page_number", + "page_size" ] }, - "StudioRuntimeResponse": { + "ModelListResponse": { "type": "object", - "description": "Studio 运行时操作响应", + "description": "列出模型的标准响应包裹格式", "properties": { "success": { - "type": "boolean" + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] }, "data": { - "$ref": "#/components/schemas/StudioRuntime" + "$ref": "#/components/schemas/ModelList", + "description": "模型列表数据" }, "request_id": { - "type": "string" + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] } }, "required": [ "success", + "data", "request_id" ] }, - "StudioLogsResponse": { - "type": "object", - "description": "获取 Studio 日志响应", - "properties": { - "success": { - "type": "boolean" + "ModelDetail": { + "allOf": [ + { + "$ref": "#/components/schemas/Model" }, - "data": { + { "type": "object", + "description": "模型详情，包含 README 和当前模型类型", "properties": { - "logs": { - "type": "array", - "items": { - "type": "string" - } + "readme": { + "type": [ + "string", + "null" + ], + "description": "README 内容", + "examples": [ + "hello" + ] }, - "page_num": { - "type": "integer" + "model_category": { + "$ref": "#/components/schemas/ModelCategory" }, - "page_size": { - "type": "integer" + "owner": { + "type": "string" }, - "total_count": { - "type": "integer" + "repo_name": { + "type": "string" }, - "total_page_num": { - "type": "integer" + "visibility": { + "type": "string", + "enum": [ + "public", + "private", + "gated" + ] + }, + "gated_mode": { + "type": "string", + "enum": [ + "auto", + "manual" + ], + "description": "申请制审批方式。仅 gated=true 时返回；auto 表示申请后自动通过，manual 表示需要人工审批。" + }, + "aigc_type": { + "allOf": [ + { + "$ref": "#/components/schemas/AigcModelType" + } + ], + "description": "【AIGC 必填】模型子类型，可选值为 Checkpoint、LoRA、VAE（区分大小写）；其他值返回 HTTP 400。" + }, + "base_model_architecture_type": { + "type": "string", + "description": "【AIGC 必填】基础模型架构类型（一级选择）。可选值查询 GET /model/aigc/options", + "examples": [ + "Z_IMAGE_TURBO", + "FLUX_2" + ] + }, + "base_model_architecture_subtype": { + "type": "string", + "description": "【AIGC 必填】基础模型架构子类型（二级选择），需与 base_model_architecture_type 匹配。可选值查询 GET /model/aigc/options", + "examples": [ + "Z_IMAGE_TURBO", + "FLUX2_DEV" + ] + }, + "content_categories": { + "allOf": [ + { + "$ref": "#/components/schemas/AigcContentCategories" + } + ], + "description": "【AIGC 可选】内容类别三级联动" + }, + "primary_subject_area": { + "allOf": [ + { + "type": "string" + } + ], + "description": "【科学 必填】一级学科分类，实际值以 GET /model/science/options 为准，例如 life-sciences" + }, + "secondary_specialization": { + "allOf": [ + { + "type": "string" + } + ], + "description": "【科学 必填】二级专业分类，应与 primary_subject_area 匹配；实际值以 GET /model/science/options 为准，例如 dna" + }, + "cover_image": { + "type": "string", + "description": "【Science 可选】模型封面图；可省略或传空字符串，非空时须为 HTTP(S) URL。", + "format": "uri" + }, + "versions": { + "type": "array", + "description": "仅 AIGC 返回；其他类型不返回专区字段。", + "items": { + "$ref": "#/components/schemas/ModelVersion" + } } } - }, - "request_id": { - "type": "string" } - }, - "required": [ - "success", - "request_id" ] }, - "SuccessResponse": { + "ModelDetailResponse": { "type": "object", - "description": "通用成功响应", + "description": "获取模型详情的标准响应包裹格式", "properties": { "success": { - "type": "boolean" + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] }, - "message": { - "type": "string" + "data": { + "$ref": "#/components/schemas/ModelDetail", + "description": "模型详情数据" }, "request_id": { - "type": "string" + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] } }, "required": [ "success", + "data", "request_id" ] }, - "StudioVisibility": { - "type": "string", - "description": "创空间可见性：\n- public: 代码和体验都公开\n- protected: 体验公开，代码仓库不可见\n- private: 都不公开\n", - "enum": [ - "public", - "protected", - "private" - ] - }, - "HardwareItem": { + "Dataset": { "type": "object", - "description": "硬件配置项。免费资源仅返回基础标识字段；付费资源会返回 ECS 规格、资源明细和价格字段", + "description": "数据集摘要信息", "properties": { - "name": { + "id": { "type": "string", - "description": "OpenAPI hardware 参数值。付费资源格式为 `paid/`", + "description": "数据集仓库名（repo_id，格式为 owner/repo_name）", "examples": [ - "platform/2v-cpu-16g-mem", - "paid/ecs.gn7i-c8g1.2xlarge" + "DAMO_NLP/jd", + "iic/nlp_domain_classification_chinese_testset" ] }, - "resource_type": { - "type": "string", - "description": "资源类型，free 表示平台免费资源，paid 表示使用用户自己的云账号付费部署", - "enum": [ - "free", - "paid" + "display_name": { + "type": [ + "string", + "null" + ], + "description": "数据集展示名称", + "examples": [ + "商品评论情感预测", + "中文文本领域分类测试集" ] }, - "instance_type": { - "type": "string", - "description": "ECS 规格名，仅付费资源返回", + "description": { + "type": [ + "string", + "null" + ], + "description": "数据集简介", "examples": [ - "ecs.gn7i-c8g1.2xlarge" + "", + "..." ] }, - "cpu": { + "file_size": { "type": "integer", - "description": "CPU 核数，仅付费资源返回" + "description": "文件大小（字节）", + "examples": [ + 0 + ] }, - "gpu": { + "downloads": { "type": "integer", - "description": "GPU 数量，仅付费资源返回" + "description": "下载量", + "examples": [ + 53996, + 2401 + ] }, - "memory": { + "likes": { "type": "integer", - "description": "内存（GB），仅付费资源返回" - }, - "gpu_type": { - "type": "string", - "description": "GPU 类型，仅付费资源返回" + "description": "喜欢量", + "examples": [ + 102, + 23 + ] }, - "gpu_memory": { - "type": "integer", - "description": "显存（GB），仅付费资源返回" + "license": { + "type": [ + "string", + "null" + ], + "description": "许可证", + "examples": [ + "Apache License 2.0", + "cc-by-4.0" + ] }, - "supported_sdk_types": { + "tasks": { "type": "array", + "description": "任务列表", "items": { "type": "string" }, - "description": "支持的 SDK 类型" - }, - "has_stock": { - "type": "boolean", - "description": "是否有库存，仅付费资源返回" - }, - "stock": { - "type": "integer", - "description": "库存数量" - }, - "cost_after_discount": { - "type": "number", - "format": "float", - "description": "折后价格，仅付费资源返回" - }, - "original_cost": { - "type": "number", - "format": "float", - "description": "原价，仅付费资源返回" - } - } - }, - "ListHardwareResponse": { - "type": "object", - "description": "查询可用硬件配置响应", - "properties": { - "success": { - "type": "boolean" - }, - "data": { - "type": "object", - "properties": { - "hardware": { - "type": "array", - "items": { - "$ref": "#/components/schemas/HardwareItem" - } - } - } - }, - "request_id": { - "type": "string" - } - }, - "required": [ - "success", - "request_id" - ] - }, - "SDKVersionItem": { - "type": "object", - "description": "SDK 版本项", - "properties": { - "sdk_type": { - "type": "string", - "description": "SDK 类型", "examples": [ - "gradio" + [ + "text-classification" + ], + [] ] }, - "tag": { + "created_at": { "type": "string", - "description": "版本标签", + "format": "date-time", + "description": "创建时间（ISO 8601 UTC）", "examples": [ - "latest" + "2022-09-28T06:30:31Z", + "2025-11-25T14:30:22Z" ] }, - "version": { + "last_modified": { "type": "string", - "description": "SDK 版本", + "format": "date-time", + "description": "Git 文件或设置的最后修改时间（ISO 8601 UTC）", "examples": [ - "6.2.0" + "2022-10-30T03:36:39Z", + "2026-01-01T00:31:42Z" ] - } - }, - "required": [ - "sdk_type", - "tag", - "version" - ] - }, - "ListSDKVersionsResponse": { - "type": "object", - "description": "查询可用 SDK 版本响应", - "properties": { - "success": { - "type": "boolean" }, - "data": { - "type": "object", - "properties": { - "sdk_versions": { - "type": "array", - "description": "SDK 版本列表。仅 `sdk_type=gradio` 时返回 Gradio 版本列表；其他 SDK 类型返回空数组", - "items": { - "$ref": "#/components/schemas/SDKVersionItem" - } - }, - "total_count": { - "type": "integer", - "format": "int64", - "description": "SDK 版本总数" - } - } + "tags": { + "type": "array", + "description": "标签列表", + "items": { + "type": "string" + }, + "examples": [ + [ + "license:Apache License 2.0", + "task:text-classification" + ], + [ + "license:cc-by-4.0", + "custom_tag:common-crawl" + ] + ] }, - "request_id": { - "type": "string" + "private": { + "type": "boolean", + "description": "是否普通私有数据集；申请制数据集返回 false，请结合 gated 和 visibility 判断访问模式", + "examples": [ + false + ] + }, + "gated": { + "type": "boolean", + "description": "是否申请制数据集，通过申请后可访问", + "examples": [ + false + ] + }, + "login_required": { + "type": "boolean", + "description": "是否登录后才可下载", + "examples": [ + false + ] } }, "required": [ - "success", - "request_id" + "id", + "downloads", + "likes", + "private", + "gated", + "login_required", + "created_at", + "last_modified" ] }, - "BaseImageItem": { + "DatasetList": { "type": "object", - "description": "基础镜像项", + "description": "数据集列表与分页信息", "properties": { - "name": { - "type": "string", - "description": "镜像名称", + "datasets": { + "type": "array", + "description": "结果数据集列表", + "items": { + "$ref": "#/components/schemas/Dataset" + } + }, + "total_count": { + "type": "integer", + "description": "符合筛选条件的数据集总数", "examples": [ - "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0" + 60 ] }, - "tag": { - "type": "string", - "description": "标签", + "page_number": { + "type": "integer", + "description": "当前页码", "examples": [ - "latest" + 1 + ] + }, + "page_size": { + "type": "integer", + "description": "每页大小", + "examples": [ + 10 ] } - } + }, + "required": [ + "datasets", + "total_count", + "page_number", + "page_size" + ] }, - "ListBaseImagesResponse": { + "DatasetListResponse": { "type": "object", - "description": "查询可用基础镜像响应", + "description": "列出数据集的标准响应包裹格式", "properties": { "success": { - "type": "boolean" + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] }, "data": { - "type": "object", - "properties": { - "base_images": { - "type": "array", - "items": { - "$ref": "#/components/schemas/BaseImageItem" - } - } - } + "$ref": "#/components/schemas/DatasetList", + "description": "数据集列表数据" }, "request_id": { - "type": "string" + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] } }, "required": [ "success", + "data", "request_id" ] }, - "AddStudioVariableRequest": { + "DatasetDetail": { + "allOf": [ + { + "$ref": "#/components/schemas/Dataset" + }, + { + "type": "object", + "description": "数据集详情信息", + "properties": { + "readme": { + "type": [ + "string", + "null" + ], + "description": "README 内容", + "examples": [ + "hello" + ] + }, + "owner": { + "type": "string" + }, + "repo_name": { + "type": "string" + }, + "dataset_category": { + "type": "string", + "enum": [ + "base", + "science" + ] + }, + "visibility": { + "type": "string", + "enum": [ + "public", + "private", + "gated" + ] + }, + "gated_mode": { + "type": "string", + "enum": [ + "auto", + "manual" + ], + "description": "申请制审批方式。仅 gated=true 时返回；auto 表示申请后自动通过，manual 表示需要人工审批。" + }, + "login_required": { + "type": "boolean" + }, + "primary_subject_area": { + "type": "string" + }, + "secondary_specialization": { + "type": "string" + }, + "cover_image": { + "type": "string" + } + } + } + ] + }, + "DatasetDetailResponse": { "type": "object", - "description": "添加 Studio 明文变量请求参数（key 和 value 都公开可见，敏感信息请使用密文变量接口）", + "description": "获取数据集详情的标准响应包裹格式", "properties": { - "key": { - "type": "string", - "description": "明文变量名称", - "maxLength": 128, + "success": { + "type": "boolean", + "description": "成功时始终为 true", "examples": [ - "GRADIO_TEMP_DIR" + true ] }, - "value": { + "data": { + "$ref": "#/components/schemas/DatasetDetail", + "description": "数据集详情数据" + }, + "request_id": { "type": "string", - "description": "明文变量值", - "maxLength": 4096, + "description": "请求 ID", "examples": [ - "/tmp/gradio" + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" ] } }, "required": [ - "key", - "value" + "success", + "data", + "request_id" ] }, - "UpdateStudioVariableRequest": { + "McpServerLocale": { "type": "object", - "description": "更新 Studio 明文变量请求参数", + "description": "MCP服务在对应语言环境下的内容", "properties": { - "key": { + "name": { "type": "string", - "description": "明文变量名称", - "maxLength": 128, + "description": "MCP服务在对应语言环境下的服务名称", "examples": [ - "GRADIO_TEMP_DIR" + "Fetch网页内容抓取" ] }, - "value": { + "description": { "type": "string", - "description": "明文变量新值", - "maxLength": 4096, + "description": "MCP服务在对应语言环境下的服务简介", "examples": [ - "/new/path" + "该服务器使大型语言模型能够检索和处理网页内容..." + ] + }, + "readme": { + "type": "string", + "description": "MCP服务介绍（仅详情接口返回）", + "examples": [ + "# Fetch MCP Server..." ] } - }, - "required": [ - "key", - "value" - ] + } }, - "DeleteStudioVariableRequest": { + "McpServerLocales": { "type": "object", - "description": "删除 Studio 明文变量请求参数", + "description": "不同语言环境下的内容展示", "properties": { - "key": { - "type": "string", - "description": "明文变量名称" + "zh": { + "$ref": "#/components/schemas/McpServerLocale", + "description": "中文环境" + }, + "en": { + "$ref": "#/components/schemas/McpServerLocale", + "description": "英文环境" } - }, - "required": [ - "key" - ] + } }, - "StudioVariableItem": { + "McpOperationalUrl": { "type": "object", - "description": "明文变量列表项（返回 key 和 value）", + "description": "MCP服务托管链接信息", "properties": { - "key": { + "id": { "type": "string", - "description": "明文变量名称" + "description": "MCP服务链接命名标识", + "examples": [ + "platform-pool" + ] }, - "value": { + "url": { "type": "string", - "description": "明文变量值" - } - }, - "required": [ - "key", - "value" - ] - }, - "ListStudioVariablesResponse": { - "type": "object", - "description": "获取 Studio 明文变量列表响应", - "properties": { - "success": { - "type": "boolean" - }, - "data": { - "type": "object", - "properties": { - "variables": { - "type": "array", - "items": { - "$ref": "#/components/schemas/StudioVariableItem" - } - } - } + "description": "MCP服务远程连接地址", + "examples": [ + "https://mcp.api-inference.modelscope.net/xxxx/mcp" + ] }, - "request_id": { - "type": "string" - } - }, - "required": [ - "success", - "data", - "request_id" - ] - }, - "SkillLocale": { - "type": "object", - "description": "技能的单语言本地化信息", - "properties": { - "description": { + "transport_type": { "type": "string", - "description": "技能描述的本地化翻译" + "description": "MCP服务链接传输方式", + "examples": [ + "streamable_http", + "sse" + ] }, - "category": { + "auth_required": { + "type": "boolean", + "description": "远程 url 连接时是否需要提供魔搭访问令牌鉴权", + "examples": [ + false + ] + }, + "expiration": { "type": "string", - "description": "技能分类的本地化翻译" - } - } - }, - "SkillLocales": { - "type": "object", - "description": "技能的多语言本地化信息", - "properties": { - "en": { - "$ref": "#/components/schemas/SkillLocale" + "description": "MCP服务链接有效期", + "examples": [ + "2025-10-01 21:00:00" + ] }, - "zh": { - "$ref": "#/components/schemas/SkillLocale" + "accessible": { + "type": "boolean", + "description": "是否有权限访问该MCP服务详情。当所有者设置为非公开或MCP服务被删除时，无权访问。", + "examples": [ + true + ] } } }, - "SkillSummary": { + "McpServerSummary": { "type": "object", - "description": "技能摘要信息", + "description": "MCP服务摘要信息", "properties": { - "_id": { + "id": { "type": "string", - "description": "技能内部唯一标识", + "description": "服务ID。由平台后台收录的MCP，ID 默认为@author/server_name，当被社区用户认领后将会变更为user_name/server_name；由社区用户自发提交的MCP，ID 为user_name/server_name。", "examples": [ - "62+FGAr3b7f4Bblezwx8hQ==" + "@modelcontextprotocol/fetch" ] }, - "id": { + "name": { "type": "string", - "description": "技能 ID，格式为 @author/skill_name 或 owner/skill_name", + "description": "MCP服务在魔搭社区的名称", "examples": [ - "@AMap-Web/amap-lbs-skill" + "Fetch网页内容抓取" ] }, - "display_name": { + "chinese_name": { "type": "string", - "description": "技能展示名称", + "description": "MCP服务在魔搭社区的中文名称", "examples": [ - "高德地图综合服务Skill" + "Fetch网页内容抓取" ] }, "description": { "type": "string", - "description": "技能描述" - }, - "owner": { - "type": "string", - "description": "技能所有者（用户名或组织名）", + "description": "MCP服务简介", "examples": [ - "guoyanlian01" + "该服务器使大型语言模型能够检索和处理网页内容..." ] }, - "license": { + "logo_url": { "type": "string", - "description": "许可证", + "description": "logo图片URL链接", "examples": [ - "MIT License" + "https://resources.modelscope.cn/..." ] }, - "developer": { + "publisher": { "type": "string", - "description": "开发者", + "description": "发布时原始ID", "examples": [ - "AMap-Web" + "@modelcontextprotocol/fetch" ] }, - "source_url": { - "type": "string", - "description": "源码地址" - }, - "category": { - "type": "string", - "description": "技能分类", + "categories": { + "type": "array", + "description": "该MCP服务所属分类", + "items": { + "type": "string" + }, "examples": [ - "developer-tools" + [ + "browser-automation" + ] ] }, "tags": { - "type": [ - "array", - "null" - ], - "description": "标签列表，格式为 `key:value`，key 包括 category / license / developer / custom_tag", + "type": "array", + "description": "该MCP服务的标签列表", "items": { "type": "string" }, "examples": [ [ - "category:developer-tools", - "license:MIT License", - "developer:AMap-Web", - "custom_tag:api-design" + "browser-automation" ] ] }, - "logo_url": { - "type": "string", - "description": "logo 图片 URL" - }, "view_count": { "type": "integer", - "description": "浏览量", - "examples": [ - 13460 - ] - }, - "downloads": { - "type": "integer", - "description": "下载量", + "description": "该MCP服务的累计访问量", "examples": [ - 1203 + 30667 ] }, "locales": { - "$ref": "#/components/schemas/SkillLocales" - }, - "private": { - "type": "boolean", - "description": "是否为私有技能", - "examples": [ - false - ] - }, - "custom_tag": { - "type": [ - "array", - "null" - ], - "description": "自定义标签列表（无标签时为 null）", - "items": { - "type": "string" - }, - "examples": [ - [ - "api-design", - "general-tools" - ], - null - ] + "$ref": "#/components/schemas/McpServerLocales" } - } + }, + "required": [ + "id" + ] }, - "SkillDetail": { + "McpServerDetail": { "type": "object", - "description": "技能详细信息，包含安装命令", + "description": "MCP服务详细信息", "allOf": [ { - "$ref": "#/components/schemas/SkillSummary" + "$ref": "#/components/schemas/McpServerSummary" }, { "type": "object", "properties": { - "install_command": { - "type": "array", - "description": "安装命令列表", - "items": { - "type": "string" - }, + "author": { + "type": "string", + "description": "MCP服务开发者", "examples": [ - [ - "npx skills add https://modelscope.cn/skills/@AMap-Web/amap-lbs-skill", - "curl -fsSL https://modelscope.cn/skills/install.sh | bash -s -- @AMap-Web/amap-lbs-skill" - ] + "modelcontextprotocol" ] - } - } - } - ] - }, - "SkillListResponse": { - "type": "object", - "description": "技能列表响应", - "properties": { - "success": { - "type": "boolean", - "examples": [ - true - ] - }, - "request_id": { - "type": "string", - "examples": [ - "c5bca202-aa4f-478b-b7e2-c568160583cb" - ] - }, - "data": { - "type": "object", - "properties": { - "skills": { - "type": "array", - "description": "技能列表", - "items": { - "$ref": "#/components/schemas/SkillSummary" - } }, - "total": { - "type": "integer", - "format": "int64", - "description": "总记录数", + "owner": { + "type": "string", + "description": "MCP服务归属的魔搭用户/组织ID", "examples": [ - 20 + "" ] }, - "page_number": { - "type": "integer", - "description": "当前页码", + "readme": { + "type": "string", + "description": "MCP服务介绍", "examples": [ - 1 + "# Fetch MCP Server..." ] }, - "page_size": { + "source_url": { + "type": "string", + "description": "MCP服务来源项目主页地址", + "examples": [ + "https://github.com/modelcontextprotocol/..." + ] + }, + "github_stars": { "type": "integer", - "description": "每页条数", + "description": "该 MCP 服务对应 github 原始仓库的 stars 数量", "examples": [ - 20 + 1234 + ] + }, + "is_hosted": { + "type": "boolean", + "description": "该MCP服务是否支持托管部署", + "examples": [ + true + ] + }, + "is_verified": { + "type": "boolean", + "description": "托管MCP服务是否经过平台验证测试", + "examples": [ + true ] + }, + "env_schema": { + "type": "object", + "description": "该服务连接部署的环境变量配置", + "additionalProperties": true + }, + "server_config": { + "type": "array", + "description": "MCP服务配置信息", + "items": {} + }, + "operational_urls": { + "type": "array", + "description": "当get_operational_url=True且当前用户已连接该 MCP 服务时则返回，否则为空", + "items": { + "$ref": "#/components/schemas/McpOperationalUrl" + } } } } - } + ] }, - "SkillDetailResponse": { + "McpServerOperational": { "type": "object", - "description": "技能详情响应", + "description": "用户托管的MCP服务信息", + "allOf": [ + { + "$ref": "#/components/schemas/McpServerSummary" + }, + { + "type": "object", + "properties": { + "operational_urls": { + "type": "array", + "description": "MCP服务托管链接列表", + "items": { + "$ref": "#/components/schemas/McpOperationalUrl" + } + } + } + } + ] + }, + "McpServerListResponse": { + "type": "object", + "description": "MCP服务列表响应", "properties": { "success": { "type": "boolean", + "description": "成功时始终为 true", "examples": [ true ] }, - "request_id": { + "data": { + "$ref": "#/components/schemas/McpServerList", + "description": "MCP服务列表数据" + }, + "message": { "type": "string", + "description": "响应消息", "examples": [ - "105c898f-fd6d-437d-aa0b-9006036782da" + "success" ] }, - "data": { - "$ref": "#/components/schemas/SkillDetail" + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "c55709d2-4e09-4bd8-be84-d8518bc46363" + ] } - } + }, + "required": [ + "success", + "request_id" + ] }, - "CreateSkillRequest": { + "McpServerOperationalListResponse": { "type": "object", - "description": "创建 Skill 请求参数", + "description": "用户托管MCP服务列表响应", "properties": { - "skill_name": { - "type": "string", - "description": "英文名称，创建后不可更改，仅允许小写字母、数字和连字符", - "pattern": "^[a-z0-9-]+$", - "maxLength": 64, + "success": { + "type": "boolean", + "description": "成功时始终为 true", "examples": [ - "my-awesome-skill" + true ] }, - "owner": { - "type": "string", - "description": "所有者（用户名或组织名）", - "examples": [ - "iSolver" - ] + "data": { + "$ref": "#/components/schemas/McpServerOperationalList", + "description": "用户托管MCP服务列表数据" }, - "display_name": { + "message": { "type": "string", - "description": "展示名称", - "maxLength": 128, + "description": "响应消息", "examples": [ - "我的技能" + "success" ] }, - "source_url": { + "request_id": { "type": "string", - "format": "uri", - "description": "来源地址", + "description": "请求 ID", "examples": [ - "https://github.com/example/my-skill" + "c55709d2-4e09-4bd8-be84-d8518bc46363" ] - }, - "private": { + } + }, + "required": [ + "success", + "request_id" + ] + }, + "McpServerDetailResponse": { + "type": "object", + "description": "MCP服务详情响应", + "properties": { + "success": { "type": "boolean", - "description": "是否私有，默认 false（公开）", - "default": false, + "description": "成功时始终为 true", "examples": [ - false + true ] }, - "description": { + "data": { + "$ref": "#/components/schemas/McpServerDetail", + "description": "MCP服务详情数据" + }, + "message": { "type": "string", - "description": "描述", - "maxLength": 2000, + "description": "响应消息", "examples": [ - "这是一个示例技能" + "success" ] }, - "license": { + "request_id": { "type": "string", - "description": "开源协议证书，不填默认 Apache-2.0", - "default": "Apache-2.0", + "description": "请求 ID", "examples": [ - "Apache-2.0" + "c55709d2-4e09-4bd8-be84-d8518bc46363" ] - }, - "category": { - "allOf": [ - { - "$ref": "#/components/schemas/SkillCategory" - } - ], - "description": "Skill 类型/分类", + } + }, + "required": [ + "success", + "request_id" + ] + }, + "McpDeployResponse": { + "type": "object", + "description": "MCP服务部署响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", "examples": [ - "developer-tools" + true ] }, - "tags": { - "type": "array", - "description": "自定义标签", - "items": { - "type": "string" - }, - "examples": [ - [ - "Skill Creation" - ] - ] + "data": { + "$ref": "#/components/schemas/McpOperationalUrl", + "description": "部署后的MCP服务链接信息" }, - "logo_url": { + "message": { "type": "string", - "format": "uri", - "description": "Skill 图标 URL", + "description": "响应消息", "examples": [ - "https://example.com/logo.png" + "success" ] }, - "skill_file": { + "request_id": { "type": "string", - "description": "Skill 项目文件 ID（通过 POST /files/upload 上传 zip 包后获取）。\nzip 包要求：\n- 最大 5MB\n- zip 根目录下必须包含且仅包含 1 个 SKILL.md 文件\n- 可包含子目录及其他辅助文件（如 prompts/、examples/ 等）\n- SKILL.md 必须包含 YAML front-matter（name、version、description）\n", + "description": "请求 ID", "examples": [ - "8c378570-8991-431b-a82c-96f3d0b4f0f4" + "c55709d2-4e09-4bd8-be84-d8518bc46363" ] } }, "required": [ - "skill_name", - "owner", - "license", - "category", - "skill_file" + "success", + "request_id" ] }, - "CreateSkillResponse": { + "McpUndeployResponse": { "type": "object", - "description": "创建 Skill 响应", + "description": "MCP服务解除部署响应", "properties": { "success": { "type": "boolean", + "description": "成功时始终为 true", "examples": [ true ] }, - "request_id": { + "message": { "type": "string", + "description": "响应消息", "examples": [ - "c55709d2-4e09-4bd8-be84-d8518bc46363" + "success" ] }, - "data": { - "type": "object", - "properties": { - "id": { - "type": "string", - "description": "Skill ID (owner/skill_name)", - "examples": [ - "iSolver/my-awesome-skill" - ] - }, - "name": { - "type": "string", - "description": "Skill 英文名称", - "examples": [ - "my-awesome-skill" - ] - }, - "display_name": { - "type": "string", - "description": "展示名称", - "examples": [ - "我的技能" - ] - }, - "owner": { - "type": "string", - "description": "所有者", - "examples": [ - "iSolver" - ] - }, - "url": { - "type": "string", - "description": "Skill 页面 URL", - "examples": [ - "https://modelscope.cn/skills/iSolver/my-awesome-skill" - ] - } - } + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "c55709d2-4e09-4bd8-be84-d8518bc46363" + ] } }, "required": [ @@ -5686,110 +6293,126 @@ "request_id" ] }, - "UpdateSkillSettingsRequest": { + "StudioHardware": { + "type": "string", + "description": "硬件配置，修改后需重新部署才能生效。\n可用值不固定，请通过 `GET /studios/hardware` 接口查询当前可用的硬件配置列表。\n免费资源常见格式为 `platform/...`、`xgpu/...`、`amd/...`；付费资源格式为 `paid/`，例如 `paid/ecs.gn7i-c8g1.2xlarge`。\n", + "examples": [ + "platform/2v-cpu-16g-mem", + "paid/ecs.gn7i-c8g1.2xlarge" + ] + }, + "StudioSDKType": { + "type": "string", + "description": "SDK 类型，修改后需重新部署才能生效", + "enum": [ + "gradio", + "streamlit", + "docker", + "static" + ] + }, + "StudioSDKVersion": { + "type": "string", + "description": "SDK 版本，仅对 Gradio 类型生效，默认最新版，建议选用最新版本，修改后需重新部署才能生效。\n可用版本会随平台更新，请通过 `GET /studios/sdk-versions?sdk_type=gradio` 查询当前可用的 Gradio 版本列表。\n", + "examples": [ + "6.2.0" + ] + }, + "StudioBaseImage": { + "type": "string", + "description": "基础镜像，仅 Docker 类型不支持，建议选用最新版本，修改后需重新部署才能生效。\n可用值不固定，请通过 `GET /studios/base-images` 接口查询当前可用的基础镜像列表。\n", + "examples": [ + "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0" + ] + }, + "CreateStudioRequest": { "type": "object", - "description": "更新 Skill 设置请求参数（传入哪个字段修改哪个字段，不传的字段不修改）", + "description": "创建 Studio 请求参数", "properties": { + "repo_name": { + "type": "string", + "description": "仓库名称", + "maxLength": 64, + "examples": [ + "my-demo-app" + ] + }, + "owner": { + "type": "string", + "description": "所有者（用户名或组织名）", + "examples": [ + "username" + ] + }, "display_name": { "type": "string", - "description": "展示名称", + "description": "显示名称", "maxLength": 128, "examples": [ - "新的展示名称" + "我的演示应用" ] }, - "source_url": { + "license": { "type": "string", - "format": "uri", - "description": "来源地址", + "description": "许可证，默认 apache-2.0", + "default": "apache-2.0", "examples": [ - "https://github.com/example/my-skill" + "apache-2.0" ] }, + "visibility": { + "$ref": "#/components/schemas/StudioVisibility" + }, "private": { "type": "boolean", - "description": "是否私有", - "examples": [ - true - ] + "description": "Deprecated: 请使用 visibility 字段", + "deprecated": true, + "default": false }, "description": { "type": "string", "description": "描述", "maxLength": 2000, "examples": [ - "更新后的技能描述" + "这是一个演示应用" ] }, - "license": { + "cover_image": { "type": "string", - "description": "开源协议证书", - "examples": [ - "Apache-2.0" - ] + "description": "封面图 URL，为空时使用平台默认图" }, - "category": { + "sdk_type": { "allOf": [ { - "$ref": "#/components/schemas/SkillCategory" + "$ref": "#/components/schemas/StudioSDKType" } ], - "description": "Skill 类型/分类", - "examples": [ - "developer-tools" - ] + "default": "gradio" }, - "tags": { - "type": "array", - "description": "自定义标签（传入时整体覆盖原标签列表）", - "items": { - "type": "string" - }, - "examples": [ - [ - "Image/Video Gen", - "Media Processing" - ] - ] + "sdk_version": { + "$ref": "#/components/schemas/StudioSDKVersion" }, - "logo_url": { - "type": "string", - "format": "uri", - "description": "Skill 图标 URL（传空字符串恢复为平台默认图标）", - "examples": [ - "https://example.com/new-logo.png" - ] + "base_image": { + "$ref": "#/components/schemas/StudioBaseImage" }, - "skill_file": { - "type": "string", - "description": "Skill 项目文件 ID（通过 POST /files/upload 上传 zip 包后获取，传入时整体覆盖原文件，不传则保留原文件不变）。\nzip 包要求：\n- 最大 5MB\n- zip 根目录下必须包含且仅包含 1 个 SKILL.md 文件\n- 可包含子目录及其他辅助文件（如 prompts/、examples/ 等）\n- SKILL.md 必须包含 YAML front-matter（name、version、description）\n", - "examples": [ - "new-file-id-xxx" - ] + "hardware": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioHardware" + } + ], + "default": "platform/2v-cpu-16g-mem", + "description": "硬件配置。未传时使用默认免费硬件配置" } - } - }, - "SkillCategory": { - "type": "string", - "description": "Skill 类型/分类枚举", - "enum": [ - "skill-management", - "developer-tools", - "marketing-seo", - "frontend-development", - "ai-media", - "code-quality-testing", - "mobile-development", - "cloud-devops", - "other" - ], - "examples": [ - "developer-tools" + }, + "required": [ + "repo_name", + "owner" ] }, - "GetBalanceResponse": { + "CreateStudioResponse": { "type": "object", - "description": "余额查询响应", + "description": "创建 Studio 响应", "properties": { "success": { "type": "boolean", @@ -5800,34 +6423,45 @@ "data": { "type": "object", "properties": { - "total_balance": { - "type": "number", - "description": "总额度（可用额度 + 预扣额度）", + "id": { + "type": "string", + "description": "Studio ID (owner/repo_name)", "examples": [ - 1500 + "username/my-demo-app" ] }, - "available_balance": { - "type": "number", - "description": "可用额度", + "repo_name": { + "type": "string", + "description": "仓库名称", "examples": [ - 1300 + "my-demo-app" ] }, - "frozen_amount": { - "type": "number", - "description": "预扣额度（进行中任务未返回结果时预扣减）", + "display_name": { + "type": "string", + "description": "显示名称", "examples": [ - 200 + "我的演示应用" + ] + }, + "owner": { + "type": "string", + "description": "所有者", + "examples": [ + "username" + ] + }, + "url": { + "type": "string", + "description": "Studio URL", + "examples": [ + "https://modelscope.cn/studios/username/my-demo-app" ] } } }, "request_id": { - "type": "string", - "examples": [ - "c55709d2-4e09-4bd8-be84-d8518bc46363" - ] + "type": "string" } }, "required": [ @@ -5835,1145 +6469,990 @@ "request_id" ] }, - "UploadFileRequest": { + "StudioConfig": { "type": "object", - "description": "上传文件请求参数（multipart/form-data）", + "description": "Studio 运行时配置", "properties": { - "file": { - "type": "string", - "format": "binary", - "description": "上传的文件，最大 5MB" + "hardware": { + "$ref": "#/components/schemas/StudioHardware" }, - "filename": { - "type": [ - "string", - "null" + "base_image": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioBaseImage" + } ], - "description": "自定义文件名（选填），支持多级目录路径（如 images/sub/logo.png）；不提供则使用上传文件的原始文件名。不允许绝对路径或目录穿越（..）。", - "examples": [ - "images/sub/logo.png" - ] + "description": "基础镜像（Docker 类型不返回）" + }, + "sdk_type": { + "$ref": "#/components/schemas/StudioSDKType" + }, + "sdk_version": { + "type": "string", + "description": "SDK 版本（仅 gradio 类型返回）" } - }, - "required": [ - "file" - ] + } }, - "UploadFileResponse": { + "StudioRuntime": { "type": "object", - "description": "上传文件响应", + "description": "Studio 运行时信息", "properties": { - "success": { - "type": "boolean", - "examples": [ - true + "status": { + "type": "string", + "description": "运行状态", + "enum": [ + "Initialized", + "Building", + "BuildFailed", + "Deploying", + "DeployFailed", + "Running", + "Stopping", + "Stopped", + "Duplicating", + "Sleeping" ] }, - "request_id": { + "active_config": { + "description": "当前实际运行中的配置", + "$ref": "#/components/schemas/StudioConfig" + }, + "created_at": { "type": "string", - "examples": [ - "99ee2083-63e0-46dc-b393-623c22645078" - ] + "description": "部署时间" + }, + "error_message": { + "type": "string", + "description": "失败信息（仅在错误状态时返回）" + } + } + }, + "GetStudioResponse": { + "type": "object", + "description": "获取 Studio 详情响应", + "properties": { + "success": { + "type": "boolean" }, "data": { "type": "object", "properties": { "id": { "type": "string", - "description": "文件 ID，可在其他接口中作为入参引用该文件", - "examples": [ - "8c378570-8991-431b-a82c-96f3d0b4f0f4" - ] + "description": "Studio ID (owner/repo_name)" + }, + "repo_name": { + "type": "string", + "description": "仓库名称" + }, + "display_name": { + "type": "string", + "description": "显示名称" + }, + "owner": { + "type": "string", + "description": "所有者" + }, + "description": { + "type": "string", + "description": "描述" + }, + "cover_image": { + "type": "string", + "description": "封面图 URL" + }, + "likes": { + "type": "integer", + "description": "喜欢数" + }, + "view_count": { + "type": "integer", + "description": "访问量" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "visibility": { + "$ref": "#/components/schemas/StudioVisibility" + }, + "private": { + "type": "boolean", + "description": "Deprecated: 请使用 visibility 字段", + "deprecated": true + }, + "last_modified": { + "type": "string", + "description": "最后修改时间" + }, + "sdk_type": { + "$ref": "#/components/schemas/StudioSDKType" + }, + "sdk_version": { + "$ref": "#/components/schemas/StudioSDKVersion" + }, + "hardware": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioHardware" + } + ], + "description": "期望的硬件配置（数据库中保存的值）" + }, + "base_image": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioBaseImage" + } + ], + "description": "期望的基础镜像（Docker 类型不返回）" + }, + "license": { + "type": "string", + "description": "许可证" + }, + "host": { + "type": "string", + "description": "访问地址" + }, + "mcp_support": { + "type": "boolean", + "description": "是否支持 MCP" + }, + "runtime": { + "$ref": "#/components/schemas/StudioRuntime" } - }, - "required": [ - "id" - ] + } + }, + "request_id": { + "type": "string" } }, "required": [ "success", - "request_id", - "data" + "request_id" ] }, - "JWK": { + "UpdateStudioSettingsRequest": { "type": "object", - "description": "JSON Web Key（仅用于 Ed25519 公钥）", - "required": [ - "kty", - "crv", - "x", - "kid" - ], + "description": "更新 Studio 设置请求参数（传入哪个字段修改哪个字段）", "properties": { - "kty": { + "display_name": { "type": "string", - "description": "密钥类型，固定 OKP", - "enum": [ - "OKP" - ], - "examples": [ - "OKP" - ] + "description": "显示名称", + "maxLength": 128 }, - "crv": { + "license": { "type": "string", - "description": "椭圆曲线类型，固定 Ed25519", - "enum": [ - "Ed25519" - ], - "examples": [ - "Ed25519" - ] + "description": "许可证" }, - "x": { - "type": "string", - "description": "公钥 base64url 编码（无 padding）", - "examples": [ - "f7QtzyEJhqFL8Jk5OYOq7010Ry9PipDGQkTzl7XtiEs" - ] + "visibility": { + "$ref": "#/components/schemas/StudioVisibility" }, - "kid": { - "type": "string", - "description": "密钥 ID", - "examples": [ - "agent-key-001" - ] + "private": { + "type": "boolean", + "description": "Deprecated: 请使用 visibility 字段", + "deprecated": true }, - "alg": { + "description": { "type": "string", - "description": "签名算法", - "examples": [ - "EdDSA" - ] + "description": "描述", + "maxLength": 2000 }, - "use": { + "cover_image": { "type": "string", - "description": "用途", - "examples": [ - "sig" - ] + "description": "封面图 URL" + }, + "sdk_type": { + "$ref": "#/components/schemas/StudioSDKType" + }, + "sdk_version": { + "$ref": "#/components/schemas/StudioSDKVersion" + }, + "base_image": { + "$ref": "#/components/schemas/StudioBaseImage" + }, + "hardware": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioHardware" + } + ], + "description": "硬件配置，修改后需重新部署才能生效" } } }, - "Principal": { + "AddStudioSecretRequest": { "type": "object", - "description": "Agent 归属主体", + "description": "添加 Studio 密文变量请求参数", "properties": { - "type": { + "key": { "type": "string", - "description": "主体类型", - "enum": [ - "user", - "org" - ], + "description": "密文变量名称", + "maxLength": 128, "examples": [ - "user" + "API_KEY" ] }, - "id": { + "value": { "type": "string", - "description": "主体 ID", + "description": "密文变量值（添加后不会公开展示）", + "maxLength": 4096, "examples": [ - "alice" + "sk-xxxxxxxxxxxx" + ] + }, + "description": { + "type": "string", + "description": "密文变量描述（用途说明），非必填", + "maxLength": 512, + "examples": [ + "推理服务使用的访问密钥" ] } - } - }, - "AgentIdentityStatus": { - "type": "string", - "description": "Agent 状态", - "enum": [ - "active", - "paused" - ], - "examples": [ - "active" + }, + "required": [ + "key", + "value" ] }, - "CreateAgentIdentityRequest": { + "UpdateStudioSecretRequest": { "type": "object", - "description": "创建 Agent 身份请求", - "required": [ - "agent_name", - "public_key" - ], + "description": "更新 Studio 密文变量请求参数", "properties": { - "agent_name": { + "key": { "type": "string", - "description": "Agent 名称（最长 128 字符）", + "description": "密文变量名称", "maxLength": 128, "examples": [ - "my-agent" + "API_KEY" ] }, - "description": { - "type": "string", - "description": "Agent 描述（最长 2000 字符）", - "maxLength": 2000 - }, - "public_key": { - "$ref": "#/components/schemas/JWK" - }, - "key_alg_type": { + "value": { "type": "string", - "description": "密钥算法类型，固定 Ed25519", - "enum": [ - "Ed25519" - ], + "description": "密文变量新值", + "maxLength": 4096, "examples": [ - "Ed25519" + "sk-xxxxxxxxxxxx" ] }, - "token_expire_time": { - "type": "integer", - "description": "Token 过期时间（秒），可选值 300/600/1800/3600", - "enum": [ - 300, - 600, - 1800, - 3600 - ], + "description": { + "type": "string", + "description": "密文变量描述（用途说明），非必填；更新时不传则保留原有描述", + "maxLength": 512, "examples": [ - 3600 + "推理服务使用的访问密钥" ] } - } + }, + "required": [ + "key", + "value" + ] }, - "UpdateAgentIdentityRequest": { + "StudioSecretItem": { "type": "object", - "description": "更新 Agent 身份设置请求", + "description": "密文变量列表项（仅返回 key，不返回 value）", "properties": { - "agent_name": { + "key": { "type": "string", - "maxLength": 128 + "description": "密文变量名称" }, "description": { "type": "string", - "maxLength": 2000 - }, - "token_expire_time": { - "type": "integer", - "enum": [ - 300, - 600, - 1800, - 3600 - ] + "description": "密文变量描述（用途说明），未填写时不返回", + "maxLength": 512 } - } + }, + "required": [ + "key" + ] }, - "ResetKeyPairRequest": { + "ListStudioSecretsResponse": { "type": "object", - "description": "重置密钥对请求（客户端本地生成新密钥后上传公钥）", - "required": [ - "public_key" - ], + "description": "获取 Studio 密文变量列表响应", "properties": { - "public_key": { - "$ref": "#/components/schemas/JWK" + "success": { + "type": "boolean" }, - "key_alg_type": { - "type": "string", - "enum": [ - "Ed25519" - ], - "examples": [ - "Ed25519" - ] + "data": { + "type": "object", + "properties": { + "secrets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/StudioSecretItem" + } + } + } + }, + "request_id": { + "type": "string" } - } - }, - "PauseAgentRequest": { - "type": "object", - "description": "启用/停用 Agent 请求", + }, "required": [ - "paused" - ], - "properties": { - "paused": { - "type": "boolean", - "description": "true 表示停用，false 表示启用", - "examples": [ - true - ] - } - } + "success", + "data", + "request_id" + ] }, - "AgentIdentity": { + "DeleteStudioSecretRequest": { "type": "object", - "description": "Agent 身份详情", + "description": "删除 Studio 密文变量请求参数", "properties": { - "agent_id": { - "type": "string", - "description": "Agent 唯一 ID", - "examples": [ - "agent_id:modelscope:agent_1234567890ab" - ] - }, - "agent_name": { - "type": "string", - "examples": [ - "my-agent" - ] - }, - "description": { - "type": "string" - }, - "token_expire_time": { - "type": "integer", - "examples": [ - 3600 - ] - }, - "principal": { - "$ref": "#/components/schemas/Principal" - }, - "kid": { - "type": "string", - "description": "当前密钥 ID", - "examples": [ - "agent-key-001" - ] - }, - "public_key": { - "$ref": "#/components/schemas/JWK" - }, - "status": { - "$ref": "#/components/schemas/AgentIdentityStatus" - }, - "create_time": { - "type": "string", - "description": "创建时间 (ISO 8601)", - "examples": [ - "2026-06-03T10:00:00Z" - ] - }, - "update_time": { + "key": { "type": "string", - "description": "更新时间 (ISO 8601)", - "examples": [ - "2026-06-03T10:30:00Z" - ] + "description": "密文变量名称" } - } + }, + "required": [ + "key" + ] }, - "AgentIdentityListItem": { + "StudioRuntimeResponse": { "type": "object", - "description": "Agent 身份列表项", + "description": "Studio 运行时操作响应", "properties": { - "agent_id": { - "type": "string" - }, - "agent_name": { - "type": "string" - }, - "kid": { - "type": "string" - }, - "status": { - "$ref": "#/components/schemas/AgentIdentityStatus" + "success": { + "type": "boolean" }, - "token_expire_time": { - "type": "integer" + "data": { + "$ref": "#/components/schemas/StudioRuntime" }, - "create_time": { + "request_id": { "type": "string" } - } + }, + "required": [ + "success", + "request_id" + ] }, - "ListAgentIdentitiesData": { + "StudioLogsResponse": { "type": "object", - "description": "Agent 身份列表数据", + "description": "获取 Studio 日志响应", "properties": { - "agent_identities": { - "type": "array", - "items": { - "$ref": "#/components/schemas/AgentIdentityListItem" - } - }, - "total_count": { - "type": "integer", - "format": "int64" + "success": { + "type": "boolean" }, - "page_number": { - "type": "integer" + "data": { + "type": "object", + "properties": { + "logs": { + "type": "array", + "items": { + "type": "string" + } + }, + "page_num": { + "type": "integer" + }, + "page_size": { + "type": "integer" + }, + "total_count": { + "type": "integer" + }, + "total_page_num": { + "type": "integer" + } + } }, - "page_size": { - "type": "integer" + "request_id": { + "type": "string" } - } + }, + "required": [ + "success", + "request_id" + ] }, - "ResetKeyPairData": { + "SuccessResponse": { "type": "object", - "description": "重置密钥对返回数据", + "description": "通用成功响应", "properties": { - "agent_id": { - "type": "string" + "success": { + "type": "boolean" }, - "kid": { + "message": { "type": "string" }, - "public_key": { - "$ref": "#/components/schemas/JWK" + "request_id": { + "type": "string" } - } + }, + "required": [ + "success", + "request_id" + ] }, - "AgentIdentityResponse": { + "StudioVisibility": { + "type": "string", + "description": "创空间可见性：\n- public: 代码和体验都公开\n- protected: 体验公开，代码仓库不可见\n- private: 都不公开\n", + "enum": [ + "public", + "protected", + "private" + ] + }, + "HardwareItem": { "type": "object", - "description": "Agent 身份详情响应", + "description": "硬件配置项。免费资源仅返回基础标识字段；付费资源会返回 ECS 规格、资源明细和价格字段", "properties": { - "success": { - "type": "boolean", + "name": { + "type": "string", + "description": "OpenAPI hardware 参数值。付费资源格式为 `paid/`", "examples": [ - true + "platform/2v-cpu-16g-mem", + "paid/ecs.gn7i-c8g1.2xlarge" ] }, - "data": { - "$ref": "#/components/schemas/AgentIdentity" + "resource_type": { + "type": "string", + "description": "资源类型，free 表示平台免费资源，paid 表示使用用户自己的云账号付费部署", + "enum": [ + "free", + "paid" + ] }, - "message": { + "instance_type": { "type": "string", + "description": "ECS 规格名，仅付费资源返回", "examples": [ - "success" + "ecs.gn7i-c8g1.2xlarge" ] }, - "request_id": { - "type": "string" - } - } - }, - "ListAgentIdentitiesResponse": { - "type": "object", - "description": "Agent 身份列表响应", - "properties": { - "success": { - "type": "boolean" + "cpu": { + "type": "integer", + "description": "CPU 核数，仅付费资源返回" }, - "data": { - "$ref": "#/components/schemas/ListAgentIdentitiesData" + "gpu": { + "type": "integer", + "description": "GPU 数量，仅付费资源返回" }, - "message": { - "type": "string" + "memory": { + "type": "integer", + "description": "内存（GB），仅付费资源返回" }, - "request_id": { - "type": "string" - } - } - }, - "ResetKeyPairResponse": { - "type": "object", - "description": "重置密钥对响应", - "properties": { - "success": { - "type": "boolean" + "gpu_type": { + "type": "string", + "description": "GPU 类型，仅付费资源返回" }, - "data": { - "$ref": "#/components/schemas/ResetKeyPairData" + "gpu_memory": { + "type": "integer", + "description": "显存（GB），仅付费资源返回" }, - "message": { - "type": "string" + "supported_sdk_types": { + "type": "array", + "items": { + "type": "string" + }, + "description": "支持的 SDK 类型" }, - "request_id": { - "type": "string" - } - } - }, - "DeleteAgentIdentityResponse": { - "type": "object", - "description": "删除 Agent 身份响应", - "properties": { - "success": { - "type": "boolean" + "has_stock": { + "type": "boolean", + "description": "是否有库存，仅付费资源返回" }, - "message": { - "type": "string" + "stock": { + "type": "integer", + "description": "库存数量" }, - "request_id": { - "type": "string" + "cost_after_discount": { + "type": "number", + "format": "float", + "description": "折后价格，仅付费资源返回" + }, + "original_cost": { + "type": "number", + "format": "float", + "description": "原价，仅付费资源返回" } } }, - "PauseAgentResponse": { + "ListHardwareResponse": { "type": "object", - "description": "启用/停用 Agent 响应", + "description": "查询可用硬件配置响应", "properties": { "success": { "type": "boolean" }, - "message": { - "type": "string" + "data": { + "type": "object", + "properties": { + "hardware": { + "type": "array", + "items": { + "$ref": "#/components/schemas/HardwareItem" + } + } + } }, "request_id": { "type": "string" } - } + }, + "required": [ + "success", + "request_id" + ] }, - "TokenSignRequest": { + "SDKVersionItem": { "type": "object", - "description": "Token 签发请求（Agent 用私钥签名 message=`agent_id|kid|audience|timestamp` 后提交）", - "required": [ - "agent_id", - "kid", - "audience", - "timestamp", - "signature" - ], + "description": "SDK 版本项", "properties": { - "agent_id": { - "type": "string", - "examples": [ - "agent_id:modelscope:agent_1234567890ab" - ] - }, - "kid": { + "sdk_type": { "type": "string", + "description": "SDK 类型", "examples": [ - "agent-key-001" + "gradio" ] }, - "audience": { + "tag": { "type": "string", - "description": "目标 Hub 应用的 client_id", - "examples": [ - "hub-app-abcdef" - ] - }, - "timestamp": { - "type": "integer", - "format": "int64", - "description": "签名时间戳（Unix 秒），需在服务器时间 ±5 分钟之内", + "description": "版本标签", "examples": [ - 1717410000 + "latest" ] }, - "signature": { + "version": { "type": "string", - "description": "Ed25519 签名 base64url 编码（无 padding）", + "description": "SDK 版本", "examples": [ - "MEUCIQDxxx..." + "6.2.0" ] } - } + }, + "required": [ + "sdk_type", + "tag", + "version" + ] }, - "TokenSignData": { + "ListSDKVersionsResponse": { "type": "object", - "description": "Token 签发返回数据", - "properties": { - "access_token": { - "type": "string", - "description": "签发的 JWT", - "examples": [ - "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCIsImtpZCI6ImlkcC1rZXktMDAxIn0..." - ] - }, - "token_type": { - "type": "string", - "enum": [ - "Bearer" - ], - "examples": [ - "Bearer" - ] - }, - "expire_at": { - "type": "integer", - "format": "int64", - "description": "Token 失效时间（Unix 时间戳，单位秒）", - "examples": [ - 1717413600 - ] - }, - "jti": { - "type": "string", - "description": "JWT ID", - "examples": [ - "jti-1234567890" - ] - } - } - }, - "TokenSignResponse": { - "type": "object", - "description": "Token 签发响应", + "description": "查询可用 SDK 版本响应", "properties": { "success": { "type": "boolean" }, "data": { - "$ref": "#/components/schemas/TokenSignData" - }, - "message": { - "type": "string" + "type": "object", + "properties": { + "sdk_versions": { + "type": "array", + "description": "SDK 版本列表。仅 `sdk_type=gradio` 时返回 Gradio 版本列表；其他 SDK 类型返回空数组", + "items": { + "$ref": "#/components/schemas/SDKVersionItem" + } + }, + "total_count": { + "type": "integer", + "format": "int64", + "description": "SDK 版本总数" + } + } }, "request_id": { "type": "string" } - } + }, + "required": [ + "success", + "request_id" + ] }, - "TokenRecordItem": { + "BaseImageItem": { "type": "object", - "description": "Token 签发记录项", + "description": "基础镜像项", "properties": { - "token_id": { - "type": "string" - }, - "audience": { - "type": "string", - "description": "目标 Hub 的 client_id" - }, - "issued_at": { - "type": "string", - "description": "签发时间 (ISO 8601)" - }, - "expire_at": { - "type": "string", - "description": "过期时间 (ISO 8601)" - }, - "status": { + "name": { "type": "string", - "enum": [ - "active", - "expired", - "revoked" + "description": "镜像名称", + "examples": [ + "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0" ] }, - "jwt": { + "tag": { "type": "string", - "description": "JWT 字符串" - } - } - }, - "ListTokenRecordsData": { - "type": "object", - "description": "Token 签发记录列表数据", - "properties": { - "token_records": { - "type": "array", - "items": { - "$ref": "#/components/schemas/TokenRecordItem" - } - }, - "total_count": { - "type": "integer", - "format": "int64" - }, - "page_number": { - "type": "integer" - }, - "page_size": { - "type": "integer" + "description": "标签", + "examples": [ + "latest" + ] } } }, - "ListTokenRecordsResponse": { + "ListBaseImagesResponse": { "type": "object", - "description": "Token 签发记录列表响应", + "description": "查询可用基础镜像响应", "properties": { "success": { "type": "boolean" }, "data": { - "$ref": "#/components/schemas/ListTokenRecordsData" - }, - "message": { - "type": "string" + "type": "object", + "properties": { + "base_images": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BaseImageItem" + } + } + } }, "request_id": { "type": "string" } - } + }, + "required": [ + "success", + "request_id" + ] }, - "AgentIdConfiguration": { + "AddStudioVariableRequest": { "type": "object", - "description": "Agent IDP OIDC Discovery 配置", + "description": "添加 Studio 明文变量请求参数（key 和 value 都公开可见，敏感信息请使用密文变量接口）", "properties": { - "issuer": { + "key": { "type": "string", + "description": "明文变量名称", + "maxLength": 128, "examples": [ - "https://modelscope.cn" + "GRADIO_TEMP_DIR" ] }, - "token_endpoint": { + "value": { "type": "string", + "description": "明文变量值", + "maxLength": 4096, "examples": [ - "https://modelscope.cn/openapi/v1/agent_id/token" + "/tmp/gradio" ] }, - "jwks_uri": { + "description": { "type": "string", + "description": "明文变量描述（用途说明），非必填", + "maxLength": 512, "examples": [ - "https://modelscope.cn/openapi/v1/agent_id/.well-known/agentid-jwks" + "Gradio 运行时临时目录" ] - }, - "registration_endpoint": { + } + }, + "required": [ + "key", + "value" + ] + }, + "UpdateStudioVariableRequest": { + "type": "object", + "description": "更新 Studio 明文变量请求参数", + "properties": { + "key": { "type": "string", + "description": "明文变量名称", + "maxLength": 128, "examples": [ - "https://modelscope.cn/openapi/v1/agent_ids" + "GRADIO_TEMP_DIR" ] }, - "activity_endpoint": { - "type": "string" + "value": { + "type": "string", + "description": "明文变量新值", + "maxLength": 4096, + "examples": [ + "/new/path" + ] }, - "id_token_signing_alg_values_supported": { + "description": { "type": "string", + "description": "明文变量描述（用途说明），非必填；更新时不传则保留原有描述", + "maxLength": 512, "examples": [ - "EdDSA" + "Gradio 运行时临时目录" ] } - } + }, + "required": [ + "key", + "value" + ] }, - "JWKS": { + "DeleteStudioVariableRequest": { "type": "object", - "description": "JSON Web Key Set", + "description": "删除 Studio 明文变量请求参数", "properties": { - "keys": { - "type": "array", - "items": { - "$ref": "#/components/schemas/JWK" - } + "key": { + "type": "string", + "description": "明文变量名称" } - } + }, + "required": [ + "key" + ] }, - "Collection": { + "StudioVariableItem": { "type": "object", - "description": "Collection 详情对象", + "description": "明文变量列表项（返回 key 和 value）", "properties": { - "slug": { + "key": { "type": "string", - "description": "Collection 唯一标识，格式 {owner}/{slug}。新建为 {owner}/{title_slug}；历史为 {owner}/{title_slug}-{short_id}", - "examples": [ - "iSolver/best-nlp-models-6a3b2c" - ] + "description": "明文变量名称" }, - "title": { + "value": { "type": "string", - "description": "Collection 标题", - "examples": [ - "最佳 NLP 模型合集" - ] + "description": "明文变量值" }, "description": { - "type": [ - "string", - "null" - ], - "description": "Collection 描述（Markdown）", - "examples": [ - "精选的自然语言处理模型..." - ] - }, - "owner": { - "type": "string", - "description": "所有者（用户名或组织名）", - "examples": [ - "iSolver" - ] - }, - "visibility": { "type": "string", - "description": "可见性：public（公开）/ private（私有）", - "enum": [ - "public", - "private" - ], - "examples": [ - "public" - ] + "description": "明文变量描述（用途说明），未填写时不返回", + "maxLength": 512 + } + }, + "required": [ + "key", + "value" + ] + }, + "ListStudioVariablesResponse": { + "type": "object", + "description": "获取 Studio 明文变量列表响应", + "properties": { + "success": { + "type": "boolean" }, - "theme": { - "type": [ - "string", - "null" - ], - "description": "主题标签，枚举值：Blue/Pink/Purple/Cyan", + "data": { + "type": "object", + "properties": { + "variables": { + "type": "array", + "items": { + "$ref": "#/components/schemas/StudioVariableItem" + } + } + } + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "success", + "data", + "request_id" + ] + }, + "SkillLocale": { + "type": "object", + "description": "技能的单语言本地化信息", + "properties": { + "description": { + "type": "string", + "description": "技能描述的本地化翻译" + }, + "category": { + "type": "string", + "description": "技能分类的本地化翻译" + } + } + }, + "SkillLocales": { + "type": "object", + "description": "技能的多语言本地化信息", + "properties": { + "en": { + "$ref": "#/components/schemas/SkillLocale" + }, + "zh": { + "$ref": "#/components/schemas/SkillLocale" + } + } + }, + "SkillSummary": { + "type": "object", + "description": "技能摘要信息", + "properties": { + "_id": { + "type": "string", + "description": "技能内部唯一标识", "examples": [ - "Blue" + "62+FGAr3b7f4Bblezwx8hQ==" ] }, - "item_count": { - "type": "integer", - "description": "Collection 中条目数量", + "id": { + "type": "string", + "description": "技能 ID，格式为 @author/skill_name 或 owner/skill_name", "examples": [ - 12 + "@AMap-Web/amap-lbs-skill" ] }, - "likes": { - "type": "integer", - "description": "喜欢数", + "display_name": { + "type": "string", + "description": "技能展示名称", "examples": [ - 128 + "高德地图综合服务Skill" ] }, - "view_count": { - "type": "integer", - "description": "浏览量", + "description": { + "type": "string", + "description": "技能描述" + }, + "owner": { + "type": "string", + "description": "技能所有者（用户名或组织名）", "examples": [ - 1024 + "guoyanlian01" ] }, - "items": { - "type": "array", - "description": "Collection 中的条目列表", - "items": { - "$ref": "#/components/schemas/CollectionItem" - } - }, - "created_at": { + "license": { "type": "string", - "format": "date-time", - "description": "创建时间，ISO 8601 UTC", + "description": "许可证", "examples": [ - "2025-01-15T08:00:00Z" + "MIT License" ] }, - "updated_at": { + "developer": { "type": "string", - "format": "date-time", - "description": "更新时间，ISO 8601 UTC", + "description": "开发者", "examples": [ - "2025-04-20T10:30:00Z" + "AMap-Web" ] }, - "url": { + "source_url": { "type": "string", - "description": "Collection 页面 URL", + "description": "源码地址" + }, + "category": { + "type": "string", + "description": "技能分类", "examples": [ - "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + "developer-tools" ] }, - "install_command": { + "tags": { "type": [ "array", "null" ], - "description": "当 items 包含 skill 时，返回下载命令数组", + "description": "标签列表，格式为 `key:value`，key 包括 category / license / developer / custom_tag", "items": { "type": "string" }, "examples": [ [ - "modelscope skill install @iSolver/my-skill" + "category:developer-tools", + "license:MIT License", + "developer:AMap-Web", + "custom_tag:api-design" ] ] - } - }, - "required": [ - "slug", - "title", - "owner", - "visibility", - "item_count", - "likes", - "view_count", - "items", - "created_at", - "updated_at", - "url" - ] - }, - "CollectionItem": { - "type": "object", - "description": "Collection 条目对象。条目通过 item_type + item_object_id 唯一标识，不维护独立 item_id", - "properties": { - "item_type": { - "type": "string", - "description": "资源类型：model / dataset / studio / paper / skill / mcp", - "enum": [ - "model", - "dataset", - "studio", - "paper", - "skill", - "mcp" - ], - "examples": [ - "model" - ] }, - "item_object_id": { + "logo_url": { "type": "string", - "description": "资源标识，如 damo/nlp_bert_base 或论文 ID", - "examples": [ - "damo/nlp_bert_base" - ] - }, - "note": { - "type": [ - "string", - "null" - ], - "description": "用户对该条目的备注说明（Markdown）", - "examples": [ - "这个模型在中文 NLP 任务上表现优异" - ] + "description": "logo 图片 URL" }, - "position": { + "view_count": { "type": "integer", - "description": "排序位置（从 1 开始）", + "description": "浏览量", "examples": [ - 1 + 13460 ] }, - "created_at": { - "type": "string", - "format": "date-time", - "description": "添加时间，ISO 8601 UTC", + "downloads": { + "type": "integer", + "description": "下载量", "examples": [ - "2025-01-15T08:00:00Z" + 1203 ] }, - "visibility": { - "type": "string", - "description": "可见性：public / protected / private。protected 对应“仅公开体验”语义", - "enum": [ - "public", - "protected", - "private" - ], - "examples": [ - "public" - ] + "locales": { + "$ref": "#/components/schemas/SkillLocales" }, - "gated": { + "private": { "type": "boolean", - "description": "visibility 为 public 时，是否需先申请并经同意后才可访问", + "description": "是否为私有技能", "examples": [ false ] }, - "protected_mode": { - "type": "integer", - "description": "protected（仅公开体验）模式标识；用于表达条目/资源的仅公开体验语义，具体取值含义以产品后端实现为准（如 0=无、2=仅公开体验）", + "custom_tag": { + "type": [ + "array", + "null" + ], + "description": "自定义标签列表（无标签时为 null）", + "items": { + "type": "string" + }, "examples": [ - 0 - ] - } - }, - "required": [ - "item_type", - "item_object_id", - "position", - "created_at", - "visibility", - "gated", - "protected_mode" - ] - }, - "FailedCollectionItem": { - "type": "object", - "description": "批量操作中失败的条目及原因", - "properties": { - "item_type": { - "type": "string", - "description": "资源类型", - "examples": [ - "model" - ] - }, - "item_object_id": { - "type": "string", - "description": "资源标识", - "examples": [ - "not/exist-model" - ] - }, - "reason": { - "type": "string", - "description": "失败原因错误码，如 ResourceNotFound、ItemAlreadyInCollection、ItemNotInCollection、PermissionDenied", - "examples": [ - "ResourceNotFound" - ] - } - }, - "required": [ - "item_type", - "item_object_id", - "reason" - ] - }, - "CreateCollectionRequest": { - "type": "object", - "description": "创建 Collection 请求", - "properties": { - "title": { - "type": "string", - "maxLength": 128, - "description": "Collection 标题（最长 128 字符）", - "examples": [ - "最佳 NLP 模型合集" - ] - }, - "owner": { - "type": "string", - "description": "所有者(用户名或组织名)", - "examples": [ - "iSolver" - ] - }, - "description": { - "type": [ - "string", - "null" - ], - "description": "Collection 描述（Markdown）", - "examples": [ - "精选的自然语言处理模型..." - ] - }, - "visibility": { - "type": "string", - "description": "可见性：public / private，默认 public", - "enum": [ - "public", - "private" - ], - "default": "public", - "examples": [ - "public" - ] - }, - "theme": { - "type": [ - "string", - "null" - ], - "description": "主题标签，枚举值：Blue/Pink/Purple/Cyan", - "examples": [ - "Blue" - ] - } - }, - "required": [ - "title", - "owner" - ] - }, - "UpdateCollectionRequest": { - "type": "object", - "description": "更新 Collection 请求。传入哪个字段即更新哪个字段，未传字段保持原值", - "properties": { - "title": { - "type": [ - "string", - "null" - ], - "maxLength": 128, - "description": "Collection 标题（最长 128 字符）", - "examples": [ - "最佳 NLP 模型合集（已更名）" - ] - }, - "description": { - "type": [ - "string", - "null" - ], - "description": "Collection 描述（Markdown）", - "examples": [ - "更新后的描述..." - ] - }, - "owner": { - "type": [ - "string", - "null" - ], - "description": "所有者", - "examples": [ - "iSolver" - ] - }, - "visibility": { - "type": [ - "string", - "null" - ], - "description": "可见性：public / private", - "enum": [ - "public", - "private", - null - ], - "examples": [ - "private" - ] - }, - "theme": { - "type": [ - "string", - "null" - ], - "description": "主题标签，枚举值：Blue/Pink/Purple/Cyan", - "examples": [ - "Purple" + [ + "api-design", + "general-tools" + ], + null ] } } }, - "CollectionResponse": { + "SkillDetail": { "type": "object", - "description": "获取 Collection 详情的标准响应", - "properties": { - "success": { - "type": "boolean", - "description": "成功时始终为 true", - "examples": [ - true - ] - }, - "request_id": { - "type": "string", - "description": "请求 ID", - "examples": [ - "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" - ] + "description": "技能详细信息，包含安装命令", + "allOf": [ + { + "$ref": "#/components/schemas/SkillSummary" }, - "data": { - "$ref": "#/components/schemas/Collection" + { + "type": "object", + "properties": { + "install_command": { + "type": "array", + "description": "安装命令列表", + "items": { + "type": "string" + }, + "examples": [ + [ + "npx skills add https://modelscope.cn/skills/@AMap-Web/amap-lbs-skill", + "curl -fsSL https://modelscope.cn/skills/install.sh | bash -s -- @AMap-Web/amap-lbs-skill" + ] + ] + } + } } - }, - "required": [ - "success", - "request_id", - "data" ] }, - "CollectionListResponse": { + "SkillListResponse": { "type": "object", - "description": "获取 Collection 列表的标准响应", + "description": "技能列表响应", "properties": { "success": { "type": "boolean", - "description": "成功时始终为 true", "examples": [ true ] }, "request_id": { "type": "string", - "description": "请求 ID", "examples": [ - "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + "c5bca202-aa4f-478b-b7e2-c568160583cb" ] }, "data": { "type": "object", - "description": "列表数据", "properties": { - "collection_list": { + "skills": { "type": "array", - "description": "Collection 列表，返回完整 Collection 对象", + "description": "技能列表", "items": { - "$ref": "#/components/schemas/Collection" + "$ref": "#/components/schemas/SkillSummary" } }, - "total_count": { + "total": { "type": "integer", - "description": "符合筛选条件的合集总数", + "format": "int64", + "description": "总记录数", "examples": [ - 128 + 20 ] }, "page_number": { @@ -6985,851 +7464,1068 @@ }, "page_size": { "type": "integer", - "description": "每页大小", + "description": "每页条数", "examples": [ - 10 + 20 ] } - }, - "required": [ - "collection_list", - "total_count", - "page_number", - "page_size" - ] + } } - }, - "required": [ - "success", - "request_id", - "data" - ] + } }, - "CollectionItemListResponse": { + "SkillDetailResponse": { "type": "object", - "description": "获取 Collection 条目列表的标准响应", + "description": "技能详情响应", "properties": { "success": { "type": "boolean", - "description": "成功时始终为 true", "examples": [ true ] }, "request_id": { "type": "string", - "description": "请求 ID", "examples": [ - "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + "105c898f-fd6d-437d-aa0b-9006036782da" ] }, "data": { - "type": "object", - "description": "条目列表数据", - "properties": { - "items": { - "type": "array", - "description": "条目列表", - "items": { - "$ref": "#/components/schemas/CollectionItem" - } - }, - "total_count": { - "type": "integer", - "description": "条目总数", - "examples": [ - 50 - ] - }, - "page_number": { - "type": "integer", - "description": "当前页码", - "examples": [ - 1 - ] - }, - "page_size": { - "type": "integer", - "description": "每页大小", - "examples": [ - 10 - ] - } - }, - "required": [ - "items", - "total_count", - "page_number", - "page_size" - ] + "$ref": "#/components/schemas/SkillDetail" } - }, - "required": [ - "success", - "request_id", - "data" - ] + } }, - "CollectionItemRequest": { + "CreateSkillRequest": { "type": "object", - "description": "待添加条目", + "description": "创建 Skill 请求参数", "properties": { - "item_type": { + "skill_name": { "type": "string", - "description": "资源类型：model / dataset / studio / paper / skill / mcp", - "enum": [ - "model", - "dataset", - "studio", - "paper", - "skill", - "mcp" - ], + "description": "英文名称，创建后不可更改，仅允许小写字母、数字和连字符", + "pattern": "^[a-z0-9-]+$", + "maxLength": 64, "examples": [ - "model" + "my-awesome-skill" ] }, - "item_object_id": { + "owner": { "type": "string", - "description": "资源标识，如 damo/nlp_bert_base_chinese", + "description": "所有者（用户名或组织名）", "examples": [ - "damo/nlp_bert_base_chinese" + "iSolver" ] }, - "note": { - "type": [ - "string", - "null" - ], - "description": "备注（Markdown）", + "display_name": { + "type": "string", + "description": "展示名称", + "maxLength": 128, "examples": [ - "推荐用于中文文本分类" + "我的技能" ] }, - "position": { - "type": [ - "integer", - "null" + "source_url": { + "type": "string", + "format": "uri", + "description": "来源地址", + "examples": [ + "https://github.com/example/my-skill" + ] + }, + "private": { + "type": "boolean", + "description": "是否私有，默认 false（公开）", + "default": false, + "examples": [ + false + ] + }, + "description": { + "type": "string", + "description": "描述", + "maxLength": 2000, + "examples": [ + "这是一个示例技能" + ] + }, + "license": { + "type": "string", + "description": "开源协议证书，不填默认 Apache-2.0", + "default": "Apache-2.0", + "examples": [ + "Apache-2.0" + ] + }, + "category": { + "allOf": [ + { + "$ref": "#/components/schemas/SkillCategory" + } ], - "description": "排序位置（从 1 开始）", + "description": "Skill 类型/分类", "examples": [ - 1 + "developer-tools" ] - } - }, - "required": [ - "item_type", - "item_object_id" - ] - }, - "AddCollectionItemsRequest": { - "type": "object", - "description": "批量添加条目请求", - "properties": { - "items": { + }, + "tags": { "type": "array", - "description": "待添加条目列表（最少 1 个元素）", - "minItems": 1, + "description": "自定义标签", "items": { - "$ref": "#/components/schemas/CollectionItemRequest" - } + "type": "string" + }, + "examples": [ + [ + "Skill Creation" + ] + ] + }, + "logo_url": { + "type": "string", + "format": "uri", + "description": "Skill 图标 URL", + "examples": [ + "https://example.com/logo.png" + ] + }, + "skill_file": { + "type": "string", + "description": "Skill 项目文件 ID（通过 POST /files/upload 上传 zip 包后获取）。\nzip 包要求：\n- 最大 5MB\n- zip 根目录下必须包含且仅包含 1 个 SKILL.md 文件\n- 可包含子目录及其他辅助文件（如 prompts/、examples/ 等）\n- SKILL.md 必须包含 YAML front-matter（name、version、description）\n", + "examples": [ + "8c378570-8991-431b-a82c-96f3d0b4f0f4" + ] } }, "required": [ - "items" + "skill_name", + "owner", + "license", + "category", + "skill_file" ] }, - "AddCollectionItemsResponse": { + "CreateSkillResponse": { "type": "object", - "description": "批量添加条目的标准响应", + "description": "创建 Skill 响应", "properties": { "success": { "type": "boolean", - "description": "成功时始终为 true（部分失败仍为 true，失败项见 failed_items）", "examples": [ true ] }, "request_id": { "type": "string", - "description": "请求 ID", "examples": [ - "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + "c55709d2-4e09-4bd8-be84-d8518bc46363" ] }, "data": { "type": "object", - "description": "添加结果", "properties": { - "added_count": { - "type": "integer", - "description": "成功添加的条目数量", + "id": { + "type": "string", + "description": "Skill ID (owner/skill_name)", "examples": [ - 2 + "iSolver/my-awesome-skill" ] }, - "url": { + "name": { "type": "string", - "description": "Collection 页面 URL", + "description": "Skill 英文名称", "examples": [ - "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + "my-awesome-skill" ] }, - "failed_items": { - "type": "array", - "description": "添加失败的条目及原因", - "items": { - "$ref": "#/components/schemas/FailedCollectionItem" - } + "display_name": { + "type": "string", + "description": "展示名称", + "examples": [ + "我的技能" + ] + }, + "owner": { + "type": "string", + "description": "所有者", + "examples": [ + "iSolver" + ] + }, + "url": { + "type": "string", + "description": "Skill 页面 URL", + "examples": [ + "https://modelscope.cn/skills/iSolver/my-awesome-skill" + ] } - }, - "required": [ - "added_count", - "url", - "failed_items" - ] + } } }, "required": [ "success", - "request_id", - "data" + "request_id" ] }, - "UpdateCollectionItemRequest": { + "UpdateSkillSettingsRequest": { "type": "object", - "description": "更新单个条目请求。传入哪个字段即更新哪个字段", + "description": "更新 Skill 设置请求参数（传入哪个字段修改哪个字段，不传的字段不修改）", "properties": { - "note": { - "type": [ - "string", - "null" - ], - "description": "备注（Markdown）", + "display_name": { + "type": "string", + "description": "展示名称", + "maxLength": 128, "examples": [ - "更新后的备注" + "新的展示名称" ] }, - "position": { - "type": [ - "integer", - "null" - ], - "description": "排序位置（从 1 开始）", + "source_url": { + "type": "string", + "format": "uri", + "description": "来源地址", "examples": [ - 2 + "https://github.com/example/my-skill" ] - } - } - }, - "UpdateCollectionItemResponse": { - "type": "object", - "description": "更新单个条目的标准响应", - "properties": { - "success": { + }, + "private": { "type": "boolean", - "description": "成功时始终为 true", + "description": "是否私有", "examples": [ true ] }, - "request_id": { + "description": { "type": "string", - "description": "请求 ID", + "description": "描述", + "maxLength": 2000, "examples": [ - "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + "更新后的技能描述" ] }, - "data": { - "$ref": "#/components/schemas/CollectionItem" - } - }, - "required": [ - "success", - "request_id", - "data" - ] - }, - "CreateGalleryRequest": { - "type": "object", - "description": "创建 Gallery 请求", - "properties": { - "name": { + "license": { "type": "string", - "maxLength": 128, - "pattern": "^[0-9a-zA-Z_.-]+$", - "description": "Gallery 名称，最长 128 字符，仅允许字母、数字、下划线、连字符、点", + "description": "开源协议证书", "examples": [ - "my-gallery" + "Apache-2.0" ] }, - "owner": { - "type": "string", - "maxLength": 128, - "description": "所属用户或组织名，最长 128 字符", + "category": { + "allOf": [ + { + "$ref": "#/components/schemas/SkillCategory" + } + ], + "description": "Skill 类型/分类", "examples": [ - "my-org" + "developer-tools" ] }, - "files": { + "tags": { "type": "array", - "maxItems": 50, - "description": "文件 ID 列表（文件上传接口返回的 ID），最多 50 个，单个元素不能为空", + "description": "自定义标签（传入时整体覆盖原标签列表）", "items": { - "type": "string", - "minLength": 1 + "type": "string" }, "examples": [ [ - "8c378570-8991-431b-a82c-96f3d0b4f0f4" + "Image/Video Gen", + "Media Processing" ] ] }, - "entry_file": { + "logo_url": { "type": "string", - "minLength": 1, - "description": "入口文件的相对路径，对应 files 中某个已上传文件；后缀须与 category 匹配（后缀不区分大小写）：notebook→.ipynb，website→.html，pdf→.pdf（file 类别不限后缀）", - "examples": [ - "notebook/demo.ipynb" - ] - }, - "private": { - "type": "boolean", - "default": false, - "description": "是否私有，默认 false（公开）", + "format": "uri", + "description": "Skill 图标 URL（传空字符串恢复为平台默认图标）", "examples": [ - false + "https://example.com/new-logo.png" ] }, - "category": { + "skill_file": { "type": "string", - "description": "类型（必传），可选值：notebook / website / pdf / file；除 file 外须与 entry_file 后缀匹配（后缀不区分大小写）：notebook→.ipynb，website→.html，pdf→.pdf", - "enum": [ - "notebook", - "website", - "pdf", - "file" - ], + "description": "Skill 项目文件 ID（通过 POST /files/upload 上传 zip 包后获取，传入时整体覆盖原文件，不传则保留原文件不变）。\nzip 包要求：\n- 最大 5MB\n- zip 根目录下必须包含且仅包含 1 个 SKILL.md 文件\n- 可包含子目录及其他辅助文件（如 prompts/、examples/ 等）\n- SKILL.md 必须包含 YAML front-matter（name、version、description）\n", "examples": [ - "notebook" + "new-file-id-xxx" ] - }, - "labels": { - "type": [ - "array", - "null" - ], - "maxItems": 10, - "description": "标签列表，最多 10 个，单个标签最长 20 字符", - "items": { - "type": "string", - "maxLength": 20 - }, + } + } + }, + "SkillCategory": { + "type": "string", + "description": "Skill 类型/分类枚举", + "enum": [ + "skill-management", + "developer-tools", + "marketing-seo", + "frontend-development", + "ai-media", + "code-quality-testing", + "mobile-development", + "cloud-devops", + "other" + ], + "examples": [ + "developer-tools" + ] + }, + "AgentSystemFileConfig": { + "type": "object", + "description": "Agent 系统文件配置", + "properties": { + "name": { + "type": "string", + "description": "文件名", "examples": [ - [ - "demo", - "tutorial" - ] + "agent.json" ] }, - "path": { - "type": [ - "string", - "null" - ], - "maxLength": 128, - "pattern": "^[0-9a-zA-Z_-]+$", - "description": "自定义 URL 尾缀（选填），仅允许字母、数字、下划线、连字符，最长 128 字符；不填则不生成，填写后灵感流对应 URL：https://modelscope.cn/gallery/{owner}/{path}", + "content": { + "type": "string", + "description": "文件内容", "examples": [ - "my-custom-path" + "{...}" ] } - }, - "required": [ - "name", - "owner", - "files", - "entry_file", - "category" - ] + } }, - "CreateGalleryResponse": { + "AgentModelsConfig": { "type": "object", - "description": "创建 Gallery 的标准响应", + "description": "Agent 关联模型配置", "properties": { - "success": { - "type": "boolean", - "description": "成功时始终为 true", + "model_name": { + "type": "string", + "description": "模型名称", "examples": [ - true + "qwen-max" ] }, - "request_id": { + "path": { "type": "string", - "description": "请求 ID", + "description": "模型路径" + }, + "provider_id": { + "type": "string", + "description": "模型提供商 ID", "examples": [ - "req-xxxxxx" + "dashscope" ] }, - "data": { - "type": "object", - "description": "创建结果", - "properties": { - "id": { - "type": "string", - "description": "Gallery 唯一标识（UUID）", - "examples": [ - "a1b2c3d4-e5f6-7890-abcd-ef1234567890" - ] - }, - "url": { - "type": "string", - "description": "Gallery 访问链接", - "examples": [ - "https://modelscope.cn/gallery/my-org/a1b2c3d4-e5f6-7890-abcd-ef1234567890" - ] - } - }, - "required": [ - "id", - "url" + "base_url": { + "type": "string", + "description": "模型访问 Base URL", + "examples": [ + "https://dashscope.aliyuncs.com" + ] + }, + "protocol": { + "type": "string", + "description": "通信协议", + "examples": [ + "openai" + ] + }, + "provider": { + "type": "string", + "description": "模型提供商名称", + "examples": [ + "DashScope" ] } - }, - "required": [ - "success", - "request_id", - "data" - ] + } }, - "BatchUpdateCollectionItem": { + "AgentMcpToolsConfig": { "type": "object", - "description": "批量更新条目。通过 item_type + item_object_id 定位，传入哪个字段就修改哪个字段", + "description": "MCP 工具配置", "properties": { - "item_type": { + "name": { "type": "string", - "description": "资源类型，用于定位", - "enum": [ - "model", - "dataset", - "studio", - "paper", - "skill", - "mcp" - ], + "description": "工具名称", "examples": [ - "model" + "get_weather" ] }, - "item_object_id": { + "description": { "type": "string", - "description": "资源标识，用于定位", + "description": "工具描述", "examples": [ - "damo/nlp_bert_base_chinese" + "获取指定城市天气" + ] + } + } + }, + "AgentMcpConfig": { + "type": "object", + "description": "Agent 关联 MCP 配置", + "properties": { + "name": { + "type": "string", + "description": "MCP 服务名称", + "examples": [ + "weather-mcp" ] }, - "note": { - "type": [ - "string", - "null" - ], - "description": "备注（Markdown），不传则不修改", + "description": { + "type": "string", + "description": "MCP 服务描述", "examples": [ - "更新后的备注" + "天气查询 MCP" ] }, - "position": { + "tools": { "type": [ - "integer", + "array", "null" ], - "description": "排序位置（从 1 开始），不传则不修改", - "examples": [ - 1 - ] + "description": "工具列表，可能为 null", + "items": { + "$ref": "#/components/schemas/AgentMcpToolsConfig" + } } - }, - "required": [ - "item_type", - "item_object_id" - ] + } }, - "UpdateCollectionItemsRequest": { + "AgentSkillConfig": { "type": "object", - "description": "批量更新条目请求", + "description": "Agent 关联技能配置", "properties": { - "items": { - "type": "array", - "description": "待更新条目列表（最少 1 个元素）", - "minItems": 1, - "items": { - "$ref": "#/components/schemas/BatchUpdateCollectionItem" - } - } - }, - "required": [ - "items" - ] - }, - "UpdateCollectionItemsResponse": { - "type": "object", - "description": "批量更新条目的标准响应", - "properties": { - "success": { - "type": "boolean", - "description": "成功时始终为 true（部分失败仍为 true，失败项见 failed_items）", + "name": { + "type": "string", + "description": "技能名称", "examples": [ - true + "search-skill" ] }, - "request_id": { + "description": { "type": "string", - "description": "请求 ID", + "description": "技能描述", "examples": [ - "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + "搜索技能" ] }, - "data": { - "type": "object", - "description": "批量更新结果", - "properties": { - "updated_count": { - "type": "integer", - "description": "成功更新的条目数量", - "examples": [ - 2 - ] - }, - "failed_items": { - "type": "array", - "description": "更新失败的条目及原因", - "items": { - "$ref": "#/components/schemas/FailedCollectionItem" - } - } - }, - "required": [ - "updated_count", - "failed_items" + "content": { + "type": "string", + "description": "技能内容（README）", + "examples": [ + "# Search Skill\n..." ] } - }, - "required": [ - "success", - "request_id", - "data" - ] + } }, - "UserPublic": { + "CreateAgentRequest": { "type": "object", - "description": "用户公开信息", + "description": "创建 Agent 请求参数", "properties": { - "username": { + "owner": { "type": "string", - "description": "用户名", + "description": "所有者（用户名或组织名）", "examples": [ - "tester" + "my_org" ] }, - "nickname": { + "agent_name": { "type": "string", - "description": "昵称", + "description": "Agent 唯一名称", "examples": [ - "测试用户" + "weather-agent" + ] + }, + "display_name": { + "type": "string", + "description": "显示名称", + "examples": [ + "天气助手" ] }, "description": { "type": "string", - "description": "描述", + "description": "描述信息", "examples": [ - "这是一个测试用户" + "一个可以查询天气的 Agent" ] }, - "avatar_url": { + "logo_url": { "type": "string", - "format": "uri", - "description": "头像图片URL地址", + "description": "Logo URL", "examples": [ - "https://resouces.modelscope.cn/avatar/516aa51f-5fb7-49eb-930c-e90500ae13ca.png" + "https://example.com/logo.png" ] - } - }, - "required": [ - "username" - ] - }, - "UserPrivate": { - "type": "object", - "description": "用户私有信息(仅自己可见)", - "properties": { - "email": { + }, + "framework": { "type": "string", - "format": "email", - "description": "邮箱地址", + "description": "Agent 框架类型", "examples": [ - "test@modelscope.cn" + "ms-agent", + "qwenpaw" ] - } - } - }, - "McpServerList": { - "type": "object", - "description": "MCP服务列表与分页信息", - "properties": { - "mcp_server_list": { + }, + "license": { + "type": "string", + "description": "许可证类型", + "examples": [ + "Apache License 2.0" + ] + }, + "private": { + "type": "boolean", + "description": "是否为私有 Agent，默认 false（公开）", + "default": false, + "examples": [ + false + ] + }, + "custom_tags": { "type": "array", - "description": "MCP服务列表", + "description": "自定义标签列表", "items": { - "$ref": "#/components/schemas/McpServerSummary" - } - }, - "total_count": { - "type": "integer", - "description": "搜索符合条件的mcp server总个数", + "type": "string" + }, "examples": [ - 100 + [ + "weather", + "demo" + ] ] } }, "required": [ - "mcp_server_list", - "total_count" + "owner", + "agent_name", + "framework" ] }, - "McpServerOperationalList": { + "CreateAgentResponse": { "type": "object", - "description": "用户托管MCP服务列表", + "description": "创建 Agent 响应", "properties": { - "mcp_server_list": { - "type": "array", - "description": "用户托管MCP服务列表", - "items": { - "$ref": "#/components/schemas/McpServerOperational" - } + "success": { + "type": "boolean", + "examples": [ + true + ] }, - "total_count": { - "type": "integer", - "description": "服务链接总数", + "request_id": { + "type": "string", "examples": [ - 5 + "2f1c8a3e-9b7d-4f21-a0c6-1e2d3f4a5b6c" ] + }, + "data": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Agent 全局 ID，格式为 owner/agent_name", + "examples": [ + "my_org/weather-agent" + ] + }, + "name": { + "type": "string", + "description": "Agent 名称", + "examples": [ + "weather-agent" + ] + }, + "display_name": { + "type": "string", + "description": "显示名称", + "examples": [ + "天气助手" + ] + }, + "owner": { + "type": "string", + "description": "所有者", + "examples": [ + "my_org" + ] + }, + "url": { + "type": "string", + "description": "Agent 访问 URL", + "examples": [ + "https://modelscope.cn/agents/my_org/weather-agent" + ] + } + } } }, "required": [ - "mcp_server_list", - "total_count" + "success", + "request_id" ] }, - "data": { + "AgentDetail": { "type": "object", + "description": "Agent 详细信息", "properties": { "id": { "type": "string", - "description": "Studio ID (owner/repo_name)" - }, - "repo_name": { - "type": "string", - "description": "仓库名称" + "description": "Agent ID，格式为 path/name", + "examples": [ + "my_org/weather-agent" + ] }, "display_name": { "type": "string", - "description": "显示名称" + "description": "显示名称", + "examples": [ + "天气助手" + ] }, - "owner": { + "description": { "type": "string", - "description": "所有者" + "description": "描述信息", + "examples": [ + "一个可以查询天气的 Agent" + ] }, - "description": { + "framework": { "type": "string", - "description": "描述" + "description": "框架类型", + "examples": [ + "ms-agent" + ] }, - "cover_image": { + "logo_url": { "type": "string", - "description": "封面图 URL" + "description": "Logo URL", + "examples": [ + "https://example.com/logo.png" + ] }, - "likes": { - "type": "integer", - "description": "喜欢数" + "license": { + "type": "string", + "description": "许可证类型", + "examples": [ + "Apache License 2.0" + ] }, - "view_count": { - "type": "integer", - "description": "访问量" + "private": { + "type": "boolean", + "description": "是否为私有 Agent", + "examples": [ + false + ] }, - "tags": { - "type": "array", + "custom_tags": { + "type": [ + "array", + "null" + ], + "description": "自定义标签列表（无标签时为 null）", "items": { "type": "string" - } - }, - "visibility": { - "$ref": "#/components/schemas/StudioVisibility" - }, - "private": { - "type": "boolean", - "description": "Deprecated: 请使用 visibility 字段", - "deprecated": true + }, + "examples": [ + [ + "weather", + "demo" + ] + ] }, - "last_modified": { - "type": "string", - "description": "最后修改时间" + "categories": { + "type": [ + "array", + "null" + ], + "description": "分类标签列表（无分类时为 null）", + "items": { + "type": "string" + }, + "examples": [ + [ + "tools" + ] + ] }, - "sdk_type": { - "$ref": "#/components/schemas/StudioSDKType" + "last_modified": { + "type": "string", + "format": "date-time", + "description": "最后修改时间（UTC，RFC3339）", + "examples": [ + "2026-07-16T10:30:00Z" + ] }, - "sdk_version": { - "$ref": "#/components/schemas/StudioSDKVersion" + "downloads": { + "type": "integer", + "format": "int64", + "description": "下载量", + "examples": [ + 15 + ] }, - "hardware": { - "allOf": [ - { - "$ref": "#/components/schemas/StudioHardware" - } - ], - "description": "期望的硬件配置（数据库中保存的值）" + "likes": { + "type": "integer", + "format": "int64", + "description": "喜欢数", + "examples": [ + 8 + ] }, - "base_image": { - "allOf": [ - { - "$ref": "#/components/schemas/StudioBaseImage" - } - ], - "description": "期望的基础镜像（Docker 类型不返回）" + "mcps": { + "type": "array", + "description": "关联 MCP 配置列表", + "items": { + "$ref": "#/components/schemas/AgentMcpConfig" + } }, - "license": { - "type": "string", - "description": "许可证" + "skills": { + "type": "array", + "description": "关联技能配置列表", + "items": { + "$ref": "#/components/schemas/AgentSkillConfig" + } }, - "host": { - "type": "string", - "description": "访问地址" + "system_files": { + "type": "array", + "description": "系统文件列表", + "items": { + "$ref": "#/components/schemas/AgentSystemFileConfig" + } }, - "mcp_support": { + "models": { + "type": "array", + "description": "关联模型配置列表", + "items": { + "$ref": "#/components/schemas/AgentModelsConfig" + } + }, + "readme": { + "type": "string", + "description": "README 内容", + "examples": [ + "# Weather Agent\n使用说明..." + ] + } + } + }, + "GetAgentResponse": { + "type": "object", + "description": "Agent 详情响应", + "properties": { + "success": { "type": "boolean", - "description": "是否支持 MCP" + "examples": [ + true + ] }, - "runtime": { - "$ref": "#/components/schemas/StudioRuntime" + "request_id": { + "type": "string", + "examples": [ + "7a2b1c3d-4e5f-6a7b-8c9d-0e1f2a3b4c5d" + ] + }, + "data": { + "$ref": "#/components/schemas/AgentDetail" } } }, - "ListStudiosResponse": { + "AgentSummary": { "type": "object", - "description": "获取创空间列表响应", + "description": "Agent 摘要信息", "properties": { - "studios": { + "id": { + "type": "string", + "description": "Agent ID，格式为 path/name", + "examples": [ + "my_org/weather-agent" + ] + }, + "name": { + "type": "string", + "description": "Agent 名称", + "examples": [ + "weather-agent" + ] + }, + "owner": { + "type": "string", + "description": "所有者（用户名或组织名）", + "examples": [ + "my_org" + ] + }, + "display_name": { + "type": "string", + "description": "展示名称", + "examples": [ + "天气助手" + ] + }, + "description": { + "type": "string", + "description": "描述信息" + }, + "framework": { + "type": "string", + "description": "框架类型", + "examples": [ + "ms-agent" + ] + }, + "logo_url": { + "type": "string", + "description": "Logo URL" + }, + "license": { + "type": "string", + "description": "许可证类型", + "examples": [ + "Apache License 2.0" + ] + }, + "private": { + "type": "boolean", + "description": "是否为私有 Agent", + "examples": [ + false + ] + }, + "categories": { + "type": [ + "array", + "null" + ], + "description": "分类标签列表（无分类时为 null）", + "items": { + "type": "string" + }, + "examples": [ + [ + "tools" + ] + ] + }, + "custom_tags": { + "type": [ + "array", + "null" + ], + "description": "自定义标签列表（无标签时为 null）", + "items": { + "type": "string" + }, + "examples": [ + [ + "weather", + "demo" + ] + ] + }, + "tags": { "type": "array", - "description": "创空间列表", + "description": "标签列表，格式为 `key:value`，key 包括 category / license / framework / custom_tag", "items": { - "$ref": "#/components/schemas/data" - } + "type": "string" + }, + "examples": [ + [ + "category:tools", + "license:Apache License 2.0", + "framework:ms-agent", + "custom_tag:weather" + ] + ] }, - "total_count": { + "views": { "type": "integer", - "description": "符合筛选条件的创空间总数", + "format": "int64", + "description": "浏览量", "examples": [ - 120 + 319 ] }, - "page_number": { + "downloads": { "type": "integer", - "description": "当前页码", + "format": "int64", + "description": "下载量", "examples": [ - 1 + 15 ] }, - "page_size": { + "likes": { "type": "integer", - "description": "每页大小", + "format": "int64", + "description": "喜欢数", "examples": [ - 10 + 8 + ] + }, + "last_modified": { + "type": "string", + "format": "date-time", + "description": "最后修改时间（UTC，RFC3339）", + "examples": [ + "2026-07-16T10:30:00Z" + ] + }, + "created_at": { + "type": "string", + "format": "date-time", + "description": "创建时间（UTC，RFC3339）", + "examples": [ + "2026-07-01T08:00:00Z" ] } - }, - "required": [ - "studios", - "total_count", - "page_number", - "page_size" - ] + } }, - "CollectionMutationResponse": { + "AgentListResponse": { "type": "object", - "description": "创建/更新 Collection 的标准响应", + "description": "Agent 列表响应", "properties": { "success": { "type": "boolean", - "description": "成功时始终为 true", "examples": [ true ] }, "request_id": { "type": "string", - "description": "请求 ID", "examples": [ - "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + "c5bca202-aa4f-478b-b7e2-c568160583cb" ] }, "data": { "type": "object", - "description": "变更结果", "properties": { - "slug": { - "type": "string", - "description": "Collection slug（重命名时可能变化）", + "agents": { + "type": [ + "array", + "null" + ], + "description": "Agent 列表（无结果时为 null）", + "items": { + "$ref": "#/components/schemas/AgentSummary" + } + }, + "total": { + "type": "integer", + "format": "int64", + "description": "总记录数", "examples": [ - "iSolver/best-nlp-models-6a3b2c" + 20 ] }, - "title": { - "type": "string", - "description": "Collection 标题", + "page_number": { + "type": "integer", + "description": "当前页码", "examples": [ - "最佳 NLP 模型合集" + 1 ] }, - "url": { - "type": "string", - "description": "Collection 页面 URL", + "page_size": { + "type": "integer", + "description": "每页条数", "examples": [ - "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + 10 ] } - }, - "required": [ - "slug", - "title", - "url" - ] + } } }, "required": [ "success", - "request_id", - "data" + "request_id" ] }, - "EmptyDataResponse": { + "GetBalanceResponse": { "type": "object", - "description": "删除/移除类操作的标准响应，结果以 success 字段为准，data 为空对象", + "description": "余额查询响应", "properties": { "success": { "type": "boolean", - "description": "成功时始终为 true", "examples": [ true ] }, + "data": { + "type": "object", + "properties": { + "total_balance": { + "type": "number", + "description": "总额度（可用额度 + 预扣额度）", + "examples": [ + 1500 + ] + }, + "available_balance": { + "type": "number", + "description": "可用额度", + "examples": [ + 1300 + ] + }, + "frozen_amount": { + "type": "number", + "description": "预扣额度（进行中任务未返回结果时预扣减）", + "examples": [ + 200 + ] + } + } + }, "request_id": { "type": "string", - "description": "请求 ID", "examples": [ - "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + "c55709d2-4e09-4bd8-be84-d8518bc46363" + ] + } + }, + "required": [ + "success", + "request_id" + ] + }, + "UploadFileRequest": { + "type": "object", + "description": "上传文件请求参数（multipart/form-data）", + "properties": { + "file": { + "type": "string", + "format": "binary", + "description": "上传的文件，最大 5MB" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "自定义文件名（选填），支持多级目录路径（如 images/sub/logo.png）；不提供则使用上传文件的原始文件名。不允许绝对路径或目录穿越（..）。", + "examples": [ + "images/sub/logo.png" + ] + } + }, + "required": [ + "file" + ] + }, + "UploadFileResponse": { + "type": "object", + "description": "上传文件响应", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "examples": [ + "99ee2083-63e0-46dc-b393-623c22645078" ] }, "data": { "type": "object", - "description": "空对象" + "properties": { + "id": { + "type": "string", + "description": "文件 ID，可在其他接口中作为入参引用该文件", + "examples": [ + "8c378570-8991-431b-a82c-96f3d0b4f0f4" + ] + } + }, + "required": [ + "id" + ] } }, "required": [ @@ -7837,2214 +8533,8132 @@ "request_id", "data" ] - } - }, - "responses": { - "bad-request": { - "description": "请求参数错误", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "examples": { - "input-parameter-error": { - "$ref": "#/components/examples/input-parameter-error" - } - } - } - } }, - "unauthorized": { - "description": "认证失败（token 无效、过期或未提供）", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "examples": { - "invalid-authentication": { - "$ref": "#/components/examples/invalid-authentication" - } - } + "JWK": { + "type": "object", + "description": "JSON Web Key（仅用于 Ed25519 公钥）", + "required": [ + "kty", + "crv", + "x", + "kid" + ], + "properties": { + "kty": { + "type": "string", + "description": "密钥类型，固定 OKP", + "enum": [ + "OKP" + ], + "examples": [ + "OKP" + ] + }, + "crv": { + "type": "string", + "description": "椭圆曲线类型，固定 Ed25519", + "enum": [ + "Ed25519" + ], + "examples": [ + "Ed25519" + ] + }, + "x": { + "type": "string", + "description": "公钥 base64url 编码（无 padding）", + "examples": [ + "f7QtzyEJhqFL8Jk5OYOq7010Ry9PipDGQkTzl7XtiEs" + ] + }, + "kid": { + "type": "string", + "description": "密钥 ID", + "examples": [ + "agent-key-001" + ] + }, + "alg": { + "type": "string", + "description": "签名算法", + "examples": [ + "EdDSA" + ] + }, + "use": { + "type": "string", + "description": "用途", + "examples": [ + "sig" + ] } } }, - "forbidden": { - "description": "权限不足（token 有效但无权执行该操作，或配额超限）", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "examples": { - "operation-not-allowed": { - "$ref": "#/components/examples/operation-not-allowed" - }, - "quota-limit-exceed": { - "$ref": "#/components/examples/quota-limit-exceed" - } - } + "Principal": { + "type": "object", + "description": "Agent 归属主体", + "properties": { + "type": { + "type": "string", + "description": "主体类型", + "enum": [ + "user", + "org" + ], + "examples": [ + "user" + ] + }, + "id": { + "type": "string", + "description": "主体 ID", + "examples": [ + "alice" + ] } } }, - "not-found": { - "description": "资源未找到", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "examples": { - "resource-not-found": { - "$ref": "#/components/examples/resource-not-found" - } - } - } - } + "AgentIdentityStatus": { + "type": "string", + "description": "Agent 状态", + "enum": [ + "active", + "paused" + ], + "examples": [ + "active" + ] }, - "internal-server-error": { - "description": "内部服务错误", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "examples": { - "internal-server-error": { - "$ref": "#/components/examples/internal-server-error" - } - } - } - } - }, - "service-unavailable": { - "description": "服务不可用", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "examples": { - "service-unavailable": { - "$ref": "#/components/examples/service-unavailable" - } - } + "CreateAgentIdentityRequest": { + "type": "object", + "description": "创建 Agent 身份请求", + "required": [ + "agent_name", + "public_key" + ], + "properties": { + "agent_name": { + "type": "string", + "description": "Agent 名称（最长 128 字符）", + "maxLength": 128, + "examples": [ + "my-agent" + ] + }, + "description": { + "type": "string", + "description": "Agent 描述（最长 2000 字符）", + "maxLength": 2000 + }, + "public_key": { + "$ref": "#/components/schemas/JWK" + }, + "key_alg_type": { + "type": "string", + "description": "密钥算法类型，固定 Ed25519", + "enum": [ + "Ed25519" + ], + "examples": [ + "Ed25519" + ] + }, + "token_expire_time": { + "type": "integer", + "description": "Token 过期时间（秒），可选值 300/600/1800/3600", + "enum": [ + 300, + 600, + 1800, + 3600 + ], + "examples": [ + 3600 + ] } } }, - "get-current-user-success": { - "description": "成功获取当前用户信息", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/UserResponse" - }, - "examples": { - "user-all-response": { - "$ref": "#/components/examples/user-all-response" - } - } + "UpdateAgentIdentityRequest": { + "type": "object", + "description": "更新 Agent 身份设置请求", + "properties": { + "agent_name": { + "type": "string", + "maxLength": 128 + }, + "description": { + "type": "string", + "maxLength": 2000 + }, + "token_expire_time": { + "type": "integer", + "enum": [ + 300, + 600, + 1800, + 3600 + ] } } }, - "list-models-success": { - "description": "成功获取模型列表", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ModelListResponse" - }, - "examples": { - "list-models-response": { - "$ref": "#/components/examples/list-models-response" - } - } + "ResetKeyPairRequest": { + "type": "object", + "description": "重置密钥对请求（客户端本地生成新密钥后上传公钥）", + "required": [ + "public_key" + ], + "properties": { + "public_key": { + "$ref": "#/components/schemas/JWK" + }, + "key_alg_type": { + "type": "string", + "enum": [ + "Ed25519" + ], + "examples": [ + "Ed25519" + ] } } }, - "get-model-success": { - "description": "成功获取模型详情", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ModelDetailResponse" - }, - "examples": { - "get-model-response": { - "$ref": "#/components/examples/get-model-response" - } - } + "PauseAgentRequest": { + "type": "object", + "description": "启用/停用 Agent 请求", + "required": [ + "paused" + ], + "properties": { + "paused": { + "type": "boolean", + "description": "true 表示停用，false 表示启用", + "examples": [ + true + ] } } }, - "list-datasets-success": { - "description": "成功获取数据集列表", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DatasetListResponse" - }, - "examples": { - "list-datasets-response": { - "$ref": "#/components/examples/list-datasets-response" - } - } + "AgentIdentity": { + "type": "object", + "description": "Agent 身份详情", + "properties": { + "agent_id": { + "type": "string", + "description": "Agent 唯一 ID", + "examples": [ + "agent_id:modelscope:agent_1234567890ab" + ] + }, + "agent_name": { + "type": "string", + "examples": [ + "my-agent" + ] + }, + "description": { + "type": "string" + }, + "token_expire_time": { + "type": "integer", + "examples": [ + 3600 + ] + }, + "principal": { + "$ref": "#/components/schemas/Principal" + }, + "kid": { + "type": "string", + "description": "当前密钥 ID", + "examples": [ + "agent-key-001" + ] + }, + "public_key": { + "$ref": "#/components/schemas/JWK" + }, + "status": { + "$ref": "#/components/schemas/AgentIdentityStatus" + }, + "create_time": { + "type": "string", + "description": "创建时间 (ISO 8601)", + "examples": [ + "2026-06-03T10:00:00Z" + ] + }, + "update_time": { + "type": "string", + "description": "更新时间 (ISO 8601)", + "examples": [ + "2026-06-03T10:30:00Z" + ] } } }, - "get-dataset-success": { - "description": "成功获取数据集详情", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DatasetDetailResponse" - }, - "examples": { - "get-dataset-response": { - "$ref": "#/components/examples/get-dataset-response" - } - } + "AgentIdentityListItem": { + "type": "object", + "description": "Agent 身份列表项", + "properties": { + "agent_id": { + "type": "string" + }, + "agent_name": { + "type": "string" + }, + "kid": { + "type": "string" + }, + "status": { + "$ref": "#/components/schemas/AgentIdentityStatus" + }, + "token_expire_time": { + "type": "integer" + }, + "create_time": { + "type": "string" } } }, - "list-mcp-servers-success": { - "description": "成功获取MCP服务列表", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/McpServerListResponse" - }, - "examples": { - "list-mcp-servers-response": { - "$ref": "#/components/examples/list-mcp-servers-response" - } - } - } - } - }, - "list-operational-mcp-servers-success": { - "description": "成功获取用户托管MCP服务列表", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/McpServerOperationalListResponse" - }, - "examples": { - "list-operational-mcp-servers-response": { - "$ref": "#/components/examples/list-operational-mcp-servers-response" - } + "ListAgentIdentitiesData": { + "type": "object", + "description": "Agent 身份列表数据", + "properties": { + "agent_identities": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AgentIdentityListItem" } + }, + "total_count": { + "type": "integer", + "format": "int64" + }, + "page_number": { + "type": "integer" + }, + "page_size": { + "type": "integer" } } }, - "get-mcp-server-success": { - "description": "成功获取MCP服务详情", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/McpServerDetailResponse" - }, - "examples": { - "get-mcp-server-response": { - "$ref": "#/components/examples/get-mcp-server-response" - } - } + "ResetKeyPairData": { + "type": "object", + "description": "重置密钥对返回数据", + "properties": { + "agent_id": { + "type": "string" + }, + "kid": { + "type": "string" + }, + "public_key": { + "$ref": "#/components/schemas/JWK" } } }, - "deploy-mcp-server-success": { - "description": "成功部署MCP服务", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/McpDeployResponse" - }, - "examples": { - "deploy-mcp-server-response": { - "$ref": "#/components/examples/deploy-mcp-server-response" - } - } + "AgentIdentityResponse": { + "type": "object", + "description": "Agent 身份详情响应", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/AgentIdentity" + }, + "message": { + "type": "string", + "examples": [ + "success" + ] + }, + "request_id": { + "type": "string" } } }, - "undeploy-mcp-server-success": { - "description": "成功解除MCP服务部署", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/McpUndeployResponse" - }, - "examples": { - "undeploy-mcp-server-response": { - "$ref": "#/components/examples/undeploy-mcp-server-response" - } - } + "ListAgentIdentitiesResponse": { + "type": "object", + "description": "Agent 身份列表响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "$ref": "#/components/schemas/ListAgentIdentitiesData" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" } } }, - "create-studio-success": { - "description": "成功创建 Studio", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateStudioResponse" - }, - "examples": { - "create-studio-response": { - "$ref": "#/components/examples/create-studio-response" - } - } + "ResetKeyPairResponse": { + "type": "object", + "description": "重置密钥对响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "$ref": "#/components/schemas/ResetKeyPairData" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" } } }, - "get-studio-success": { - "description": "成功获取 Studio 详情", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/GetStudioResponse" - }, - "examples": { - "get-studio-response": { - "$ref": "#/components/examples/get-studio-response" - } - } + "DeleteAgentIdentityResponse": { + "type": "object", + "description": "删除 Agent 身份响应", + "properties": { + "success": { + "type": "boolean" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" } } }, - "update-studio-settings-success": { - "description": "成功更新 Studio 设置", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/GetStudioResponse" - }, - "examples": { - "update-studio-settings-response": { - "$ref": "#/components/examples/get-studio-response" - } - } + "PauseAgentResponse": { + "type": "object", + "description": "启用/停用 Agent 响应", + "properties": { + "success": { + "type": "boolean" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" } } }, - "studio-operation-success": { - "description": "Studio 操作成功", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SuccessResponse" - } + "TokenSignRequest": { + "type": "object", + "description": "Token 签发请求（Agent 用私钥签名 message=`agent_id|kid|audience|timestamp` 后提交）", + "required": [ + "agent_id", + "kid", + "audience", + "timestamp", + "signature" + ], + "properties": { + "agent_id": { + "type": "string", + "examples": [ + "agent_id:modelscope:agent_1234567890ab" + ] + }, + "kid": { + "type": "string", + "examples": [ + "agent-key-001" + ] + }, + "audience": { + "type": "string", + "description": "目标 Hub 应用的 client_id", + "examples": [ + "hub-app-abcdef" + ] + }, + "timestamp": { + "type": "integer", + "format": "int64", + "description": "签名时间戳（Unix 秒），需在服务器时间 ±5 分钟之内", + "examples": [ + 1717410000 + ] + }, + "signature": { + "type": "string", + "description": "Ed25519 签名 base64url 编码（无 padding）", + "examples": [ + "MEUCIQDxxx..." + ] } } }, - "studio-runtime-success": { - "description": "Studio 运行时操作成功", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/StudioRuntimeResponse" - }, - "examples": { - "studio-runtime-response": { - "$ref": "#/components/examples/studio-runtime-response" - } - } - } - } - }, - "get-studio-logs-success": { - "description": "成功获取 Studio 日志", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/StudioLogsResponse" - }, - "examples": { - "get-studio-logs-response": { - "$ref": "#/components/examples/get-studio-logs-response" - } - } - } - } - }, - "list-hardware-success": { - "description": "成功获取可用硬件配置列表", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ListHardwareResponse" - } + "TokenSignData": { + "type": "object", + "description": "Token 签发返回数据", + "properties": { + "access_token": { + "type": "string", + "description": "签发的 JWT", + "examples": [ + "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCIsImtpZCI6ImlkcC1rZXktMDAxIn0..." + ] + }, + "token_type": { + "type": "string", + "enum": [ + "Bearer" + ], + "examples": [ + "Bearer" + ] + }, + "expire_at": { + "type": "integer", + "format": "int64", + "description": "Token 失效时间（Unix 时间戳，单位秒）", + "examples": [ + 1717413600 + ] + }, + "jti": { + "type": "string", + "description": "JWT ID", + "examples": [ + "jti-1234567890" + ] } } }, - "list-sdk-versions-success": { - "description": "成功获取可用 SDK 版本列表", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ListSDKVersionsResponse" - }, - "examples": { - "list-sdk-versions-response": { - "$ref": "#/components/examples/list-sdk-versions-response" - } - } + "TokenSignResponse": { + "type": "object", + "description": "Token 签发响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "$ref": "#/components/schemas/TokenSignData" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" } } }, - "list-base-images-success": { - "description": "成功获取可用基础镜像列表", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ListBaseImagesResponse" - } + "TokenRecordItem": { + "type": "object", + "description": "Token 签发记录项", + "properties": { + "token_id": { + "type": "string" + }, + "audience": { + "type": "string", + "description": "目标 Hub 的 client_id" + }, + "issued_at": { + "type": "string", + "description": "签发时间 (ISO 8601)" + }, + "expire_at": { + "type": "string", + "description": "过期时间 (ISO 8601)" + }, + "status": { + "type": "string", + "enum": [ + "active", + "expired", + "revoked" + ] + }, + "jwt": { + "type": "string", + "description": "JWT 字符串" } } }, - "list-studio-variables-success": { - "description": "成功获取明文变量列表", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ListStudioVariablesResponse" + "ListTokenRecordsData": { + "type": "object", + "description": "Token 签发记录列表数据", + "properties": { + "token_records": { + "type": "array", + "items": { + "$ref": "#/components/schemas/TokenRecordItem" } + }, + "total_count": { + "type": "integer", + "format": "int64" + }, + "page_number": { + "type": "integer" + }, + "page_size": { + "type": "integer" } } }, - "list-skills-success": { - "description": "成功获取技能列表", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SkillListResponse" - }, - "examples": { - "list-skills-response": { - "$ref": "#/components/examples/list-skills-response" - } - } + "ListTokenRecordsResponse": { + "type": "object", + "description": "Token 签发记录列表响应", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "$ref": "#/components/schemas/ListTokenRecordsData" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" } } }, - "get-skill-success": { - "description": "成功获取技能详情", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SkillDetailResponse" - }, - "examples": { - "get-skill-response": { - "$ref": "#/components/examples/get-skill-response" - } - } + "AgentIdConfiguration": { + "type": "object", + "description": "Agent IDP OIDC Discovery 配置", + "properties": { + "issuer": { + "type": "string", + "examples": [ + "https://modelscope.cn" + ] + }, + "token_endpoint": { + "type": "string", + "examples": [ + "https://modelscope.cn/openapi/v1/agent_id/token" + ] + }, + "jwks_uri": { + "type": "string", + "examples": [ + "https://modelscope.cn/openapi/v1/agent_id/.well-known/agentid-jwks" + ] + }, + "registration_endpoint": { + "type": "string", + "examples": [ + "https://modelscope.cn/openapi/v1/agent_ids" + ] + }, + "activity_endpoint": { + "type": "string" + }, + "id_token_signing_alg_values_supported": { + "type": "string", + "examples": [ + "EdDSA" + ] } } }, - "create-skill-success": { - "description": "成功创建技能", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateSkillResponse" - }, - "examples": { - "create-skill-response": { - "$ref": "#/components/examples/create-skill-response" - } + "JWKS": { + "type": "object", + "description": "JSON Web Key Set", + "properties": { + "keys": { + "type": "array", + "items": { + "$ref": "#/components/schemas/JWK" } } } }, - "update-skill-settings-success": { - "description": "成功更新技能设置", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SuccessResponse" - }, - "examples": { - "update-skill-settings-response": { - "$ref": "#/components/examples/update-skill-settings-response" - } - } - } - } + "Collection": { + "type": "object", + "description": "Collection 详情对象", + "properties": { + "slug": { + "type": "string", + "description": "Collection 唯一标识，格式 {owner}/{slug}。新建为 {owner}/{title_slug}；历史为 {owner}/{title_slug}-{short_id}", + "examples": [ + "iSolver/best-nlp-models-6a3b2c" + ] + }, + "title": { + "type": "string", + "description": "Collection 标题", + "examples": [ + "最佳 NLP 模型合集" + ] + }, + "description": { + "type": [ + "string", + "null" + ], + "description": "Collection 描述（Markdown）", + "examples": [ + "精选的自然语言处理模型..." + ] + }, + "owner": { + "type": "string", + "description": "所有者（用户名或组织名）", + "examples": [ + "iSolver" + ] + }, + "visibility": { + "type": "string", + "description": "可见性：public（公开）/ private（私有）", + "enum": [ + "public", + "private" + ], + "examples": [ + "public" + ] + }, + "theme": { + "type": [ + "string", + "null" + ], + "description": "主题标签，枚举值：Blue/Pink/Purple/Cyan", + "examples": [ + "Blue" + ] + }, + "item_count": { + "type": "integer", + "description": "Collection 中条目数量", + "examples": [ + 12 + ] + }, + "likes": { + "type": "integer", + "description": "喜欢数", + "examples": [ + 128 + ] + }, + "view_count": { + "type": "integer", + "description": "浏览量", + "examples": [ + 1024 + ] + }, + "items": { + "type": "array", + "description": "Collection 中的条目列表", + "items": { + "$ref": "#/components/schemas/CollectionItem" + } + }, + "created_at": { + "type": "string", + "format": "date-time", + "description": "创建时间，ISO 8601 UTC", + "examples": [ + "2025-01-15T08:00:00Z" + ] + }, + "updated_at": { + "type": "string", + "format": "date-time", + "description": "更新时间，ISO 8601 UTC", + "examples": [ + "2025-04-20T10:30:00Z" + ] + }, + "url": { + "type": "string", + "description": "Collection 页面 URL", + "examples": [ + "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + ] + }, + "install_command": { + "type": [ + "array", + "null" + ], + "description": "当 items 包含 skill 时，返回下载命令数组", + "items": { + "type": "string" + }, + "examples": [ + [ + "modelscope skill install @iSolver/my-skill" + ] + ] + } + }, + "required": [ + "slug", + "title", + "owner", + "visibility", + "item_count", + "likes", + "view_count", + "items", + "created_at", + "updated_at", + "url" + ] + }, + "CollectionItem": { + "type": "object", + "description": "Collection 条目对象。条目通过 item_type + item_object_id 唯一标识，不维护独立 item_id", + "properties": { + "item_type": { + "type": "string", + "description": "资源类型：model / dataset / studio / paper / skill / mcp", + "enum": [ + "model", + "dataset", + "studio", + "paper", + "skill", + "mcp" + ], + "examples": [ + "model" + ] + }, + "item_object_id": { + "type": "string", + "description": "资源标识，如 damo/nlp_bert_base 或论文 ID", + "examples": [ + "damo/nlp_bert_base" + ] + }, + "note": { + "type": [ + "string", + "null" + ], + "description": "用户对该条目的备注说明（Markdown）", + "examples": [ + "这个模型在中文 NLP 任务上表现优异" + ] + }, + "position": { + "type": "integer", + "description": "排序位置（从 1 开始）", + "examples": [ + 1 + ] + }, + "created_at": { + "type": "string", + "format": "date-time", + "description": "添加时间，ISO 8601 UTC", + "examples": [ + "2025-01-15T08:00:00Z" + ] + }, + "visibility": { + "type": "string", + "description": "可见性：public / protected / private。protected 对应“仅公开体验”语义", + "enum": [ + "public", + "protected", + "private" + ], + "examples": [ + "public" + ] + }, + "gated": { + "type": "boolean", + "description": "visibility 为 public 时，是否需先申请并经同意后才可访问", + "examples": [ + false + ] + }, + "protected_mode": { + "type": "integer", + "description": "protected（仅公开体验）模式标识；用于表达条目/资源的仅公开体验语义，具体取值含义以产品后端实现为准（如 0=无、2=仅公开体验）", + "examples": [ + 0 + ] + } + }, + "required": [ + "item_type", + "item_object_id", + "position", + "created_at", + "visibility", + "gated", + "protected_mode" + ] + }, + "FailedCollectionItem": { + "type": "object", + "description": "批量操作中失败的条目及原因", + "properties": { + "item_type": { + "type": "string", + "description": "资源类型", + "examples": [ + "model" + ] + }, + "item_object_id": { + "type": "string", + "description": "资源标识", + "examples": [ + "not/exist-model" + ] + }, + "reason": { + "type": "string", + "description": "失败原因错误码，如 ResourceNotFound、ItemAlreadyInCollection、ItemNotInCollection、PermissionDenied", + "examples": [ + "ResourceNotFound" + ] + } + }, + "required": [ + "item_type", + "item_object_id", + "reason" + ] + }, + "CreateCollectionRequest": { + "type": "object", + "description": "创建 Collection 请求", + "properties": { + "title": { + "type": "string", + "maxLength": 128, + "description": "Collection 标题（最长 128 字符）", + "examples": [ + "最佳 NLP 模型合集" + ] + }, + "owner": { + "type": "string", + "description": "所有者(用户名或组织名)", + "examples": [ + "iSolver" + ] + }, + "description": { + "type": [ + "string", + "null" + ], + "description": "Collection 描述（Markdown）", + "examples": [ + "精选的自然语言处理模型..." + ] + }, + "visibility": { + "type": "string", + "description": "可见性：public / private，默认 public", + "enum": [ + "public", + "private" + ], + "default": "public", + "examples": [ + "public" + ] + }, + "theme": { + "type": [ + "string", + "null" + ], + "description": "主题标签，枚举值：Blue/Pink/Purple/Cyan", + "examples": [ + "Blue" + ] + } + }, + "required": [ + "title", + "owner" + ] + }, + "UpdateCollectionRequest": { + "type": "object", + "description": "更新 Collection 请求。传入哪个字段即更新哪个字段，未传字段保持原值", + "properties": { + "title": { + "type": [ + "string", + "null" + ], + "maxLength": 128, + "description": "Collection 标题（最长 128 字符）", + "examples": [ + "最佳 NLP 模型合集（已更名）" + ] + }, + "description": { + "type": [ + "string", + "null" + ], + "description": "Collection 描述（Markdown）", + "examples": [ + "更新后的描述..." + ] + }, + "owner": { + "type": [ + "string", + "null" + ], + "description": "所有者", + "examples": [ + "iSolver" + ] + }, + "visibility": { + "type": [ + "string", + "null" + ], + "description": "可见性：public / private", + "enum": [ + "public", + "private", + null + ], + "examples": [ + "private" + ] + }, + "theme": { + "type": [ + "string", + "null" + ], + "description": "主题标签，枚举值：Blue/Pink/Purple/Cyan", + "examples": [ + "Purple" + ] + } + } + }, + "CollectionResponse": { + "type": "object", + "description": "获取 Collection 详情的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "$ref": "#/components/schemas/Collection" + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "CollectionListResponse": { + "type": "object", + "description": "获取 Collection 列表的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "type": "object", + "description": "列表数据", + "properties": { + "collection_list": { + "type": "array", + "description": "Collection 列表，返回完整 Collection 对象", + "items": { + "$ref": "#/components/schemas/Collection" + } + }, + "total_count": { + "type": "integer", + "description": "符合筛选条件的合集总数", + "examples": [ + 128 + ] + }, + "page_number": { + "type": "integer", + "description": "当前页码", + "examples": [ + 1 + ] + }, + "page_size": { + "type": "integer", + "description": "每页大小", + "examples": [ + 10 + ] + } + }, + "required": [ + "collection_list", + "total_count", + "page_number", + "page_size" + ] + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "CollectionItemListResponse": { + "type": "object", + "description": "获取 Collection 条目列表的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "type": "object", + "description": "条目列表数据", + "properties": { + "items": { + "type": "array", + "description": "条目列表", + "items": { + "$ref": "#/components/schemas/CollectionItem" + } + }, + "total_count": { + "type": "integer", + "description": "条目总数", + "examples": [ + 50 + ] + }, + "page_number": { + "type": "integer", + "description": "当前页码", + "examples": [ + 1 + ] + }, + "page_size": { + "type": "integer", + "description": "每页大小", + "examples": [ + 10 + ] + } + }, + "required": [ + "items", + "total_count", + "page_number", + "page_size" + ] + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "CollectionItemRequest": { + "type": "object", + "description": "待添加条目", + "properties": { + "item_type": { + "type": "string", + "description": "资源类型：model / dataset / studio / paper / skill / mcp", + "enum": [ + "model", + "dataset", + "studio", + "paper", + "skill", + "mcp" + ], + "examples": [ + "model" + ] + }, + "item_object_id": { + "type": "string", + "description": "资源标识，如 damo/nlp_bert_base_chinese", + "examples": [ + "damo/nlp_bert_base_chinese" + ] + }, + "note": { + "type": [ + "string", + "null" + ], + "description": "备注（Markdown）", + "examples": [ + "推荐用于中文文本分类" + ] + }, + "position": { + "type": [ + "integer", + "null" + ], + "description": "排序位置（从 1 开始）", + "examples": [ + 1 + ] + } + }, + "required": [ + "item_type", + "item_object_id" + ] + }, + "AddCollectionItemsRequest": { + "type": "object", + "description": "批量添加条目请求", + "properties": { + "items": { + "type": "array", + "description": "待添加条目列表（最少 1 个元素）", + "minItems": 1, + "items": { + "$ref": "#/components/schemas/CollectionItemRequest" + } + } + }, + "required": [ + "items" + ] + }, + "AddCollectionItemsResponse": { + "type": "object", + "description": "批量添加条目的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true（部分失败仍为 true，失败项见 failed_items）", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "type": "object", + "description": "添加结果", + "properties": { + "added_count": { + "type": "integer", + "description": "成功添加的条目数量", + "examples": [ + 2 + ] + }, + "url": { + "type": "string", + "description": "Collection 页面 URL", + "examples": [ + "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + ] + }, + "failed_items": { + "type": "array", + "description": "添加失败的条目及原因", + "items": { + "$ref": "#/components/schemas/FailedCollectionItem" + } + } + }, + "required": [ + "added_count", + "url", + "failed_items" + ] + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "UpdateCollectionItemRequest": { + "type": "object", + "description": "更新单个条目请求。传入哪个字段即更新哪个字段", + "properties": { + "note": { + "type": [ + "string", + "null" + ], + "description": "备注（Markdown）", + "examples": [ + "更新后的备注" + ] + }, + "position": { + "type": [ + "integer", + "null" + ], + "description": "排序位置（从 1 开始）", + "examples": [ + 2 + ] + } + } + }, + "UpdateCollectionItemResponse": { + "type": "object", + "description": "更新单个条目的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "$ref": "#/components/schemas/CollectionItem" + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "CreateGalleryRequest": { + "type": "object", + "description": "创建 Gallery 请求", + "properties": { + "name": { + "type": "string", + "maxLength": 128, + "pattern": "^[0-9a-zA-Z_.-]+$", + "description": "Gallery 名称，最长 128 字符，仅允许字母、数字、下划线、连字符、点", + "examples": [ + "my-gallery" + ] + }, + "owner": { + "type": "string", + "maxLength": 128, + "description": "所属用户或组织名，最长 128 字符", + "examples": [ + "my-org" + ] + }, + "files": { + "type": "array", + "maxItems": 50, + "description": "文件 ID 列表（文件上传接口返回的 ID），最多 50 个，单个元素不能为空", + "items": { + "type": "string", + "minLength": 1 + }, + "examples": [ + [ + "8c378570-8991-431b-a82c-96f3d0b4f0f4" + ] + ] + }, + "entry_file": { + "type": "string", + "minLength": 1, + "description": "入口文件的相对路径，对应 files 中某个已上传文件；后缀须与 category 匹配（后缀不区分大小写）：notebook→.ipynb，website→.html，pdf→.pdf（file 类别不限后缀）", + "examples": [ + "notebook/demo.ipynb" + ] + }, + "private": { + "type": "boolean", + "default": false, + "description": "是否私有，默认 false（公开）", + "examples": [ + false + ] + }, + "category": { + "type": "string", + "description": "类型（必传），可选值：notebook / website / pdf / file；除 file 外须与 entry_file 后缀匹配（后缀不区分大小写）：notebook→.ipynb，website→.html，pdf→.pdf", + "enum": [ + "notebook", + "website", + "pdf", + "file" + ], + "examples": [ + "notebook" + ] + }, + "labels": { + "type": [ + "array", + "null" + ], + "maxItems": 10, + "description": "标签列表，最多 10 个，单个标签最长 20 字符", + "items": { + "type": "string", + "maxLength": 20 + }, + "examples": [ + [ + "demo", + "tutorial" + ] + ] + }, + "path": { + "type": [ + "string", + "null" + ], + "maxLength": 128, + "pattern": "^[0-9a-zA-Z_-]+$", + "description": "自定义 URL 尾缀（选填），仅允许字母、数字、下划线、连字符，最长 128 字符；不填则不生成，填写后灵感流对应 URL：https://modelscope.cn/gallery/{owner}/{path}", + "examples": [ + "my-custom-path" + ] + } + }, + "required": [ + "name", + "owner", + "files", + "entry_file", + "category" + ] + }, + "CreateGalleryResponse": { + "type": "object", + "description": "创建 Gallery 的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "req-xxxxxx" + ] + }, + "data": { + "type": "object", + "description": "创建结果", + "properties": { + "id": { + "type": "string", + "description": "Gallery 唯一标识（UUID）", + "examples": [ + "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + ] + }, + "url": { + "type": "string", + "description": "Gallery 访问链接", + "examples": [ + "https://modelscope.cn/gallery/my-org/a1b2c3d4-e5f6-7890-abcd-ef1234567890" + ] + } + }, + "required": [ + "id", + "url" + ] + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "BatchUpdateCollectionItem": { + "type": "object", + "description": "批量更新条目。通过 item_type + item_object_id 定位，传入哪个字段就修改哪个字段", + "properties": { + "item_type": { + "type": "string", + "description": "资源类型，用于定位", + "enum": [ + "model", + "dataset", + "studio", + "paper", + "skill", + "mcp" + ], + "examples": [ + "model" + ] + }, + "item_object_id": { + "type": "string", + "description": "资源标识，用于定位", + "examples": [ + "damo/nlp_bert_base_chinese" + ] + }, + "note": { + "type": [ + "string", + "null" + ], + "description": "备注（Markdown），不传则不修改", + "examples": [ + "更新后的备注" + ] + }, + "position": { + "type": [ + "integer", + "null" + ], + "description": "排序位置（从 1 开始），不传则不修改", + "examples": [ + 1 + ] + } + }, + "required": [ + "item_type", + "item_object_id" + ] + }, + "UpdateCollectionItemsRequest": { + "type": "object", + "description": "批量更新条目请求", + "properties": { + "items": { + "type": "array", + "description": "待更新条目列表（最少 1 个元素）", + "minItems": 1, + "items": { + "$ref": "#/components/schemas/BatchUpdateCollectionItem" + } + } + }, + "required": [ + "items" + ] + }, + "UpdateCollectionItemsResponse": { + "type": "object", + "description": "批量更新条目的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true（部分失败仍为 true，失败项见 failed_items）", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "type": "object", + "description": "批量更新结果", + "properties": { + "updated_count": { + "type": "integer", + "description": "成功更新的条目数量", + "examples": [ + 2 + ] + }, + "failed_items": { + "type": "array", + "description": "更新失败的条目及原因", + "items": { + "$ref": "#/components/schemas/FailedCollectionItem" + } + } + }, + "required": [ + "updated_count", + "failed_items" + ] + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "AgentPlaygroundModelSetupRef": { + "type": "object", + "description": "模型引用。创建实例的 `models`、导入的 `models` 与下发模型接口共用。\n\n三种引用形式，按优先级解析：\n\n- 按名称（推荐）：`provider_name` + `model_name`\n- 按供应商 ID：`provider_id` + `model_name`\n- 按模型 ID：`model_id`\n\n供应商只在「调用者本人创建的供应商 + 平台内置供应商」范围内查找，引用他人资源返回 403。\n平台内置供应商目前只有 `ModelScope API-Inference`，使用时 `model_name` 直接作为\nAPI-Inference 的模型 ID。\n", + "properties": { + "provider_name": { + "type": "string", + "description": "模型供应商名称；传了它就按名称解析，此时 `model_name` 必填", + "examples": [ + "ModelScope API-Inference" + ] + }, + "model_name": { + "type": "string", + "description": "模型名称，也可写作 `model`", + "examples": [ + "Qwen/Qwen3.5-27B" + ] + }, + "provider_id": { + "type": "string", + "description": "模型供应商 UUID（自定义供应商记录的 uuid），与实例内 Agent 信息里的 `provider_id` 不是同一个东西", + "examples": [ + "4d32e63f-6f7b-11f1-b963-00163e27a6e0" + ] + }, + "model_id": { + "type": "string", + "description": "模型记录 UUID，也可写作 `uuid`", + "examples": [ + "0a1f5c7e-3b2d-4a68-9f0c-5d7e1b2a3c4d" + ] + } + } + }, + "AgentPlaygroundMcpServerConfig": { + "type": "object", + "description": "MCP 连接配置，校验通过后原样下发给实例。\n\n- 可以是裸配置（`{\"type\": \"sse\", \"url\": \"...\"}`、`{\"command\": \"npx\", \"args\": [...]}`），\n 也可以是 `{\"mcpServers\": {\"\": {...}}}` 包装形；包装形必须恰好包含 1 个条目。\n- `url` 与 `command` 必须有且只有一个非空；`url` 必须公网可达，指向内网或保留地址会被拒绝。\n- 带 `command` 的配置按 stdio 处理；带 `url` 且未写 `type` 时按 SSE 处理，\n Streamable HTTP 服务必须显式写 `\"type\": \"streamable_http\"`。\n", + "additionalProperties": true, + "examples": [ + { + "mcpServers": { + "amap-maps": { + "type": "sse", + "url": "https://mcp.example.com/sse" + } + } + } + ] + }, + "AgentPlaygroundMcpSetupRef": { + "type": "object", + "description": "MCP 引用。创建实例的 `mcps`、导入的 `mcps` 与下发 MCP 接口共用。", + "properties": { + "name": { + "type": "string", + "description": "实例内 MCP client key", + "examples": [ + "amap-maps" + ] + }, + "source": { + "type": "string", + "description": "来源标记，只作记录，大小写不敏感；传其他值返回 400", + "enum": [ + "ms-platform", + "ms-deployed", + "custom" + ], + "examples": [ + "custom" + ] + }, + "server_config": { + "$ref": "#/components/schemas/AgentPlaygroundMcpServerConfig" + } + }, + "required": [ + "name", + "server_config" + ] + }, + "AgentPlaygroundInstance": { + "type": "object", + "description": "实验场实例。\n\n`agent_count` / `model_count` / `mcp_count` / `skill_count` 为统计值，可能落后于最近一次下发或导入。\n", + "properties": { + "uuid": { + "type": "string", + "description": "实例 ID", + "examples": [ + "51a7d9d2-a521-451a-baf5-367cb115a29a" + ] + }, + "owner": { + "type": "string", + "description": "实例归属账号，即调用者本人；暂不支持组织", + "examples": [ + "username" + ] + }, + "instance_name": { + "type": "string", + "description": "实例名称，在本人名下唯一", + "examples": [ + "my-agent" + ] + }, + "display_name": { + "type": "string", + "description": "展示名称，可为空串", + "examples": [ + "我的 Agent" + ] + }, + "description": { + "type": "string", + "description": "实例描述，可为空串", + "examples": [ + "用于测试的 Agent Playground 实例" + ] + }, + "logo": { + "type": "string", + "description": "头像图片 URL，未设置时为空串", + "examples": [ + "https://resources.modelscope.cn/avatar/my-agent.png" + ] + }, + "status": { + "type": "string", + "description": "实例状态：\n\n- `starting`：创建、唤醒、重启或导入内容应用中\n- `running`：运行中\n- `sleeping`：已休眠\n- `failed`：运行异常。不是终态，平台会继续尝试恢复，恢复后自动回到 `running`；\n 也可以直接调用重启 / 暂停 / 删除\n", + "enum": [ + "starting", + "running", + "sleeping", + "failed" + ], + "examples": [ + "running" + ] + }, + "framework_type": { + "type": "string", + "description": "框架类型，取值以平台配置为准，线上为 `qwenpaw` / `ms-agent`", + "examples": [ + "qwenpaw" + ] + }, + "source": { + "type": "string", + "description": "创建来源：`playground`（普通创建）/ `agent_hub`（从 Agent Hub 创建）", + "examples": [ + "playground" + ] + }, + "framework_version": { + "type": "string", + "description": "实例当前的框架版本", + "examples": [ + "2.2.0" + ] + }, + "agent_count": { + "type": "integer", + "description": "实例内 Agent 数", + "examples": [ + 2 + ] + }, + "model_count": { + "type": "integer", + "description": "模型数，跨 Agent 去重", + "examples": [ + 1 + ] + }, + "mcp_count": { + "type": "integer", + "description": "MCP 数，跨 Agent 去重", + "examples": [ + 3 + ] + }, + "skill_count": { + "type": "integer", + "description": "Skill 数，跨 Agent 去重", + "examples": [ + 17 + ] + }, + "gmt_created": { + "type": "string", + "format": "date-time", + "description": "创建时间，RFC 3339、UTC+8、秒级", + "examples": [ + "2026-09-17T15:47:19+08:00" + ] + }, + "gmt_modified": { + "type": "string", + "format": "date-time", + "description": "最后更新时间，由系统维护，不代表用户操作", + "examples": [ + "2026-09-18T17:06:19+08:00" + ] + }, + "user_modified_at": { + "type": "string", + "format": "date-time", + "description": "用户最近一次操作时间（创建 / 改名 / 暂停 / 恢复 / 重启 / 导入），实例列表默认按它倒序；自动休眠不会更新它", + "examples": [ + "2026-09-17T15:47:19+08:00" + ] + } + } + }, + "AgentPlaygroundAgentModel": { + "type": "object", + "description": "实例内 Agent 可用的模型", + "properties": { + "provider_id": { + "type": "string", + "description": "实例内的供应商标识，与模型引用里的供应商名称 / UUID 不通用，不能回填给下发接口", + "examples": [ + "playground-openai-3f2a9c1d" + ] + }, + "provider": { + "type": "string", + "description": "供应商展示名", + "examples": [ + "Playground Openai (3f2a9c1d)" + ] + }, + "model": { + "type": "string", + "description": "模型 ID", + "examples": [ + "Qwen/Qwen3.5-27B" + ] + }, + "type": { + "type": "string", + "description": "协议类型", + "enum": [ + "openai", + "anthropic" + ], + "examples": [ + "openai" + ] + }, + "base_url": { + "type": "string", + "description": "模型服务地址", + "examples": [ + "https://api-inference.modelscope.cn/v1" + ] + }, + "enabled": { + "type": "boolean", + "description": "是否为当前生效模型；其余为已配置可切换的模型", + "examples": [ + true + ] + } + } + }, + "AgentPlaygroundPersonaFile": { + "type": "object", + "description": "persona 文件内容", + "properties": { + "content": { + "type": "string", + "description": "文件内容" + }, + "enabled": { + "type": "boolean", + "description": "该文件是否进入系统提示词", + "examples": [ + true + ] + } + } + }, + "AgentPlaygroundAgentSkill": { + "type": "object", + "description": "实例内已安装的 Skill", + "properties": { + "name": { + "type": "string", + "description": "Skill 名称", + "examples": [ + "web-search" + ] + }, + "description": { + "type": "string", + "description": "Skill 描述" + }, + "enabled": { + "type": "boolean", + "description": "是否启用", + "examples": [ + true + ] + }, + "source": { + "type": "string", + "description": "来源标记", + "examples": [ + "builtin" + ] + } + } + }, + "AgentPlaygroundMcpTool": { + "type": "object", + "description": "MCP 客户端暴露的工具", + "properties": { + "name": { + "type": "string", + "description": "工具名称", + "examples": [ + "maps_geo" + ] + }, + "description": { + "type": "string", + "description": "工具描述" + } + } + }, + "AgentPlaygroundMcpClient": { + "type": "object", + "description": "实例内已配置的 MCP 客户端", + "properties": { + "key": { + "type": "string", + "description": "实例内 MCP client key", + "examples": [ + "amap-maps" + ] + }, + "name": { + "type": "string", + "description": "MCP 名称", + "examples": [ + "amap-maps" + ] + }, + "enabled": { + "type": "boolean", + "description": "是否启用", + "examples": [ + true + ] + }, + "transport": { + "type": "string", + "description": "连接方式：`sse` / `streamable_http` / `stdio`", + "examples": [ + "sse" + ] + }, + "url": { + "type": "string", + "description": "连接地址；stdio 类型为启动命令", + "examples": [ + "https://mcp.example.com/sse" + ] + }, + "tools": { + "type": [ + "array", + "null" + ], + "description": "工具列表；仅对启用的客户端返回，ms-agent 恒为空数组", + "items": { + "$ref": "#/components/schemas/AgentPlaygroundMcpTool" + } + } + } + }, + "AgentPlaygroundAgentSummary": { + "type": "object", + "description": "实例内 Agent 概要", + "properties": { + "id": { + "type": "string", + "description": "实例内 Agent 的 id，下发接口用它区分不同 Agent", + "examples": [ + "default" + ] + }, + "name": { + "type": "string", + "description": "Agent 名称", + "examples": [ + "Default Agent" + ] + }, + "description": { + "type": "string", + "description": "Agent 描述" + }, + "enabled": { + "type": "boolean", + "description": "Agent 是否启用", + "examples": [ + true + ] + }, + "models": { + "type": [ + "array", + "null" + ], + "description": "该 Agent 可用的模型", + "items": { + "$ref": "#/components/schemas/AgentPlaygroundAgentModel" + } + } + } + }, + "AgentPlaygroundAgentDetail": { + "type": "object", + "description": "实例内 Agent 详情；qwenpaw 下某个 Agent 的详情拉取失败时为 null", + "properties": { + "id": { + "type": "string", + "description": "实例内 Agent 的 id", + "examples": [ + "default" + ] + }, + "name": { + "type": "string", + "description": "Agent 名称", + "examples": [ + "Default Agent" + ] + }, + "description": { + "type": "string", + "description": "Agent 描述" + }, + "models": { + "type": [ + "array", + "null" + ], + "description": "该 Agent 可用的模型", + "items": { + "$ref": "#/components/schemas/AgentPlaygroundAgentModel" + } + }, + "persona": { + "type": "object", + "description": "persona 文件映射：key 为文件名（如 `AGENTS.md`，大小写敏感），value 为 `{content, enabled}`。\n", + "additionalProperties": { + "$ref": "#/components/schemas/AgentPlaygroundPersonaFile" + } + }, + "skills": { + "type": [ + "array", + "null" + ], + "description": "已安装的 Skill", + "items": { + "$ref": "#/components/schemas/AgentPlaygroundAgentSkill" + } + }, + "mcp_clients": { + "type": [ + "array", + "null" + ], + "description": "已配置的 MCP 客户端", + "items": { + "$ref": "#/components/schemas/AgentPlaygroundMcpClient" + } + } + } + }, + "AgentPlaygroundAgentItem": { + "type": "object", + "description": "实例内单个 Agent 的概要与详情", + "properties": { + "summary": { + "$ref": "#/components/schemas/AgentPlaygroundAgentSummary" + }, + "detail": { + "oneOf": [ + { + "$ref": "#/components/schemas/AgentPlaygroundAgentDetail" + }, + { + "type": "null" + } + ], + "description": "Agent 详情；详情拉取失败时为 null（summary 仍返回）" + } + } + }, + "AgentPlaygroundNotification": { + "type": "object", + "description": "实例一次性通知。拉取即消费，同一条通知只返回一次；保留 7 天。", + "properties": { + "id": { + "type": "integer", + "format": "int64", + "description": "通知 ID", + "examples": [ + 7 + ] + }, + "kind": { + "type": "string", + "description": "通知类型：\n\n- `import_result`：创建或导入的内容应用完成（未附带 message）\n- `conversation`：内容应用完成且附带了一句话，payload 含会话信息\n- `agent_import`：从 Agent Hub 导入 Agent 完成\n- `init_error`：某一项内容被跳过，实例仍会 running\n- `framework_update_error`：框架版本更新失败\n", + "enum": [ + "import_result", + "conversation", + "agent_import", + "init_error", + "framework_update_error" + ], + "examples": [ + "conversation" + ] + }, + "payload": { + "type": "object", + "description": "原样 JSON，键名不做转换。其中 `conversation` / `agent` 部分不同框架结构不同，\n用 `framework` 字段区分。\n", + "additionalProperties": true + }, + "gmt_created": { + "type": "string", + "format": "date-time", + "description": "通知产生时间，RFC 3339、UTC+8、秒级", + "examples": [ + "2026-07-12T15:32:47+08:00" + ] + } + } + }, + "CreateAgentPlaygroundInstanceRequest": { + "type": "object", + "description": "创建实例请求", + "properties": { + "instance_name": { + "type": "string", + "maxLength": 128, + "description": "实例名称，去掉首尾空白后非空，本人名下唯一（冲突返回 409）", + "examples": [ + "my-agent" + ] + }, + "display_name": { + "type": "string", + "maxLength": 256, + "description": "展示名称，缺省为空串", + "examples": [ + "我的 Agent" + ] + }, + "description": { + "type": "string", + "maxLength": 1024, + "description": "实例描述", + "examples": [ + "用于测试的 Agent Playground 实例" + ] + }, + "logo": { + "type": "string", + "maxLength": 512, + "description": "头像图片 URL，超长返回 400", + "examples": [ + "https://resources.modelscope.cn/avatar/my-agent.png" + ] + }, + "framework_type": { + "type": "string", + "description": "框架类型，须为平台当前配置的框架，线上为 `qwenpaw` / `ms-agent`", + "examples": [ + "qwenpaw" + ] + }, + "owner": { + "type": "string", + "description": "实例归属账号，缺省为调用者；只能填调用者本人，填其他账号或组织返回 403", + "examples": [ + "username" + ] + }, + "models": { + "type": "array", + "description": "就绪后自动下发的模型，依次设为生效模型，最后一个生效。\n\n**不传时会自动配置平台默认模型。**\n", + "items": { + "$ref": "#/components/schemas/AgentPlaygroundModelSetupRef" + } + }, + "mcps": { + "type": "array", + "description": "就绪后自动下发的 MCP", + "items": { + "$ref": "#/components/schemas/AgentPlaygroundMcpSetupRef" + } + }, + "skills": { + "type": "array", + "description": "就绪后自动下发的 Skill Hub 引用，格式 `path/name`；私有 Skill 需调用者有读权限", + "items": { + "type": "string" + }, + "examples": [ + [ + "@modelscope/web-search" + ] + ] + }, + "message": { + "type": "string", + "description": "附带的一句话：内容全部就位后用它对 `default` Agent 发起对话，会话信息经一次性通知返回（kind=`conversation`）", + "examples": [ + "帮我查一下明天的天气" + ] + } + }, + "required": [ + "instance_name", + "framework_type" + ] + }, + "CreateAgentPlaygroundInstanceFromAgentHubRequest": { + "type": "object", + "description": "从 Agent Hub 创建实例请求", + "properties": { + "instance_name": { + "type": "string", + "maxLength": 128, + "description": "实例名称，本人名下唯一", + "examples": [ + "playground-instance" + ] + }, + "display_name": { + "type": "string", + "maxLength": 256, + "description": "展示名称", + "examples": [ + "Playground Instance" + ] + }, + "description": { + "type": "string", + "maxLength": 1024, + "description": "实例描述", + "examples": [ + "从 Agent Hub 导入的实例" + ] + }, + "logo": { + "type": "string", + "maxLength": 512, + "description": "头像图片 URL" + }, + "framework_type": { + "type": "string", + "description": "框架类型；内容同步目前只支持 `qwenpaw` / `ms-agent`", + "examples": [ + "qwenpaw" + ] + }, + "owner": { + "type": "string", + "description": "实例归属账号，缺省为调用者" + }, + "agent_owner": { + "type": "string", + "description": "Agent Hub 源 agent 的归属账号", + "examples": [ + "jakexiao" + ] + }, + "agent_name": { + "type": "string", + "description": "Agent Hub 源 agent 的名称", + "examples": [ + "FirstAgent" + ] + }, + "models": { + "type": "array", + "description": "内容同步完成后下发的模型。\n\n本接口**不会**自动配置默认模型：不传时实例内模型取决于源 agent 的内容，可能为空。\n", + "items": { + "$ref": "#/components/schemas/AgentPlaygroundModelSetupRef" + } + }, + "mcps": { + "type": "array", + "description": "内容同步完成后下发的 MCP", + "items": { + "$ref": "#/components/schemas/AgentPlaygroundMcpSetupRef" + } + }, + "skills": { + "type": "array", + "description": "内容同步完成后下发的 Skill Hub 引用，格式 `path/name`", + "items": { + "type": "string" + } + }, + "message": { + "type": "string", + "description": "附带的一句话，内容就位后用它对 `default` Agent 发起对话" + } + }, + "required": [ + "instance_name", + "framework_type", + "agent_owner", + "agent_name" + ] + }, + "UpdateAgentPlaygroundInstanceRequest": { + "type": "object", + "description": "更新实例元信息请求。只允许修改这四个字段，`framework_type`、`owner` 与 init 内容都不能通过本接口修改。\n", + "properties": { + "instance_name": { + "type": "string", + "maxLength": 128, + "description": "实例名称，非空，本人名下唯一（冲突返回 409）", + "examples": [ + "my-agent-renamed" + ] + }, + "display_name": { + "type": "string", + "maxLength": 256, + "description": "展示名称；整体覆盖语义，缺省或传空串都会清空", + "examples": [ + "My Agent Renamed" + ] + }, + "description": { + "type": "string", + "maxLength": 1024, + "description": "实例描述；缺省或传空串都会清空", + "examples": [ + "Updated description" + ] + }, + "logo": { + "type": "string", + "maxLength": 512, + "description": "头像图片 URL；与其他字段不同，缺省保持原值，显式传入则覆盖（空串清空）", + "examples": [ + "https://resources.modelscope.cn/avatar/my-agent.png" + ] + } + }, + "required": [ + "instance_name" + ] + }, + "ImportToAgentPlaygroundInstanceRequest": { + "type": "object", + "description": "向已有实例追加导入 model / mcp / skill 的请求。四者至少传一个，否则返回 400。\n导入内容恒作用于实例内 `default` Agent。\n", + "properties": { + "models": { + "type": "array", + "description": "追加下发的模型", + "items": { + "$ref": "#/components/schemas/AgentPlaygroundModelSetupRef" + } + }, + "mcps": { + "type": "array", + "description": "追加下发的 MCP", + "items": { + "$ref": "#/components/schemas/AgentPlaygroundMcpSetupRef" + } + }, + "skills": { + "type": "array", + "description": "追加下发的 Skill Hub 引用，格式 `path/name`；私有 Skill 需调用者有读权限", + "items": { + "type": "string" + }, + "examples": [ + [ + "@modelscope/web-search" + ] + ] + }, + "message": { + "type": "string", + "description": "本批内容就位后用它对 `default` Agent 发起对话", + "examples": [ + "帮我查一下明天的天气" + ] + } + } + }, + "SetupAgentPlaygroundModelRequest": { + "type": "object", + "description": "下发模型配置请求。在模型引用（`provider_name` + `model_name` 等）之外额外支持 `agent_id`。\n\n下发后该模型即目标 Agent 的生效模型；重复下发同一个模型是幂等的。\n", + "allOf": [ + { + "$ref": "#/components/schemas/AgentPlaygroundModelSetupRef" + }, + { + "type": "object", + "properties": { + "agent_id": { + "type": "string", + "description": "实例内目标 Agent 的 id，取自实例内 Agent 信息接口的 `detail.id`，缺省 `default`", + "examples": [ + "default" + ] + } + } + } + ] + }, + "SetupAgentPlaygroundMcpRequest": { + "type": "object", + "description": "下发 MCP 配置请求。仅新增：目标 Agent 下已有同名 client key 时返回 409，不会覆盖。\n", + "allOf": [ + { + "$ref": "#/components/schemas/AgentPlaygroundMcpSetupRef" + }, + { + "type": "object", + "properties": { + "agent_id": { + "type": "string", + "description": "实例内目标 Agent 的 id，缺省 `default`", + "examples": [ + "default" + ] + } + } + } + ] + }, + "SetupAgentPlaygroundSkillRequest": { + "type": "object", + "description": "下发 Skill 配置请求。按 Skill Hub 引用把 Skill 安装到目标 Agent。\n\n仅新增：实例内已有同名 Skill 返回 409，不会覆盖。\n", + "properties": { + "agent_id": { + "type": "string", + "description": "实例内目标 Agent 的 id，缺省 `default`", + "examples": [ + "default" + ] + }, + "skill_ref": { + "type": "string", + "description": "Skill Hub 引用，格式 `path/name`（按最后一个 `/` 拆分，path 可带 `@` 前缀）", + "examples": [ + "@modelscope/web-search" + ] + } + }, + "required": [ + "skill_ref" + ] + }, + "AgentPlaygroundInstanceResponse": { + "type": "object", + "description": "单个实例响应", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "examples": [ + "7d3f8f8a-8c1f-4d0b-8f1a-7b6a0d8a1a11" + ] + }, + "data": { + "$ref": "#/components/schemas/AgentPlaygroundInstance" + } + }, + "required": [ + "success", + "request_id" + ] + }, + "AgentPlaygroundInstanceListResponse": { + "type": "object", + "description": "实例列表响应", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "examples": [ + "bb971b6e-36f0-4c4c-a50e-ee3fb6d3ceba" + ] + }, + "data": { + "type": "object", + "properties": { + "instances": { + "type": [ + "array", + "null" + ], + "description": "实例列表", + "items": { + "$ref": "#/components/schemas/AgentPlaygroundInstance" + } + }, + "total_count": { + "type": "integer", + "format": "int64", + "description": "总记录数", + "examples": [ + 5 + ] + }, + "page_number": { + "type": "integer", + "description": "当前页码", + "examples": [ + 1 + ] + }, + "page_size": { + "type": "integer", + "description": "每页条数", + "examples": [ + 10 + ] + } + } + } + }, + "required": [ + "success", + "request_id" + ] + }, + "DeleteAgentPlaygroundInstanceResponse": { + "type": "object", + "description": "删除实例响应", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "examples": [ + "0d2b7a54-1f6c-4a2b-9e33-2c7f9c1d8b40" + ] + }, + "data": { + "type": "object", + "properties": { + "deleted": { + "type": "boolean", + "description": "恒为 true（幂等：对不存在或删除中的实例同样返回）", + "examples": [ + true + ] + } + } + } + }, + "required": [ + "success", + "request_id" + ] + }, + "AgentPlaygroundAgentListResponse": { + "type": "object", + "description": "实例内 Agent 信息响应", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "examples": [ + "3c9a1b2d-4e5f-6a7b-8c9d-0e1f2a3b4c5d" + ] + }, + "data": { + "type": "object", + "properties": { + "agents": { + "type": [ + "array", + "null" + ], + "description": "实例内 Agent 列表；实例未就绪且无可用数据时为空列表", + "items": { + "$ref": "#/components/schemas/AgentPlaygroundAgentItem" + } + }, + "refreshed_at": { + "type": "integer", + "format": "int64", + "description": "这批数据的更新时间（Unix 秒），`0` 表示还没有取到过", + "examples": [ + 1789722407 + ] + } + } + } + }, + "required": [ + "success", + "request_id" + ] + }, + "AgentPlaygroundNotificationListResponse": { + "type": "object", + "description": "实例一次性通知响应", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "examples": [ + "f1e2d3c4-b5a6-4978-8a9b-0c1d2e3f4a5b" + ] + }, + "data": { + "type": "object", + "properties": { + "notifications": { + "type": [ + "array", + "null" + ], + "description": "未读通知列表，按产生顺序返回；返回后即消费", + "items": { + "$ref": "#/components/schemas/AgentPlaygroundNotification" + } + } + } + } + }, + "required": [ + "success", + "request_id" + ] + }, + "UserPublic": { + "type": "object", + "description": "用户公开信息", + "properties": { + "username": { + "type": "string", + "description": "用户名", + "examples": [ + "tester" + ] + }, + "nickname": { + "type": "string", + "description": "昵称", + "examples": [ + "测试用户" + ] + }, + "description": { + "type": "string", + "description": "描述", + "examples": [ + "这是一个测试用户" + ] + }, + "avatar_url": { + "type": "string", + "format": "uri", + "description": "头像图片URL地址", + "examples": [ + "https://resouces.modelscope.cn/avatar/516aa51f-5fb7-49eb-930c-e90500ae13ca.png" + ] + } + }, + "required": [ + "username" + ] + }, + "UserPrivate": { + "type": "object", + "description": "用户私有信息(仅自己可见)", + "properties": { + "email": { + "type": "string", + "format": "email", + "description": "邮箱地址", + "examples": [ + "test@modelscope.cn" + ] + } + } + }, + "CreateModelRequest": { + "type": "object", + "description": "创建通用模型仓库请求参数。本接口仅接收通用元数据字段，创建成功后 `model_category` 默认为 `base`。专区业务字段请通过 `PUT /models/{owner}/{repo_name}/category` 设置。", + "additionalProperties": false, + "properties": { + "task": { + "type": [ + "string", + "null" + ], + "description": "模型任务，取 GET /models/options 的 tasks[].id，例如 text-generation。省略、null 或空字符串表示不设置；未知任务返回 HTTP 400。", + "examples": [ + "text-generation" + ] + }, + "owner": { + "type": "string", + "description": "所有者（用户名或组织名）", + "examples": [ + "alice", + "my-org" + ] + }, + "repo_name": { + "type": "string", + "description": "仓库英文 ID，仅允许字母、数字、点、连字符和下划线，创建后不可修改", + "maxLength": 128, + "pattern": "^[a-zA-Z0-9._-]+$", + "examples": [ + "my-model", + "qwen-7b-finetuned" + ] + }, + "display_name": { + "type": "string", + "description": "模型展示名称（中英文均可），用于 UI 展示", + "maxLength": 128, + "examples": [ + "我的大语言模型" + ] + }, + "license": { + "type": "string", + "description": "创建模型支持 /api/v1/licenses 返回的 19 个许可证名称及下列对应标识符（均原样保存、区分大小写）；省略或空字符串默认 apache-2.0。未列出的值返回 HTTP 400，other 必须提供 license_name。", + "enum": [ + "", + "apache-2.0", + "Apache License 2.0", + "gpl-2.0", + "GPL-2.0", + "lgpl-3.0", + "LGPL-3.0", + "mit", + "MIT License", + "lgpl-2.1", + "LGPL-2.1", + "gpl-3.0", + "GPL-3.0", + "afl-3.0", + "AFL-3.0", + "ecl-2.0", + "ECL-2.0", + "cc-by-4.0", + "CC-BY-4.0", + "creativeml-openrail-m", + "cc-by-nc-nd", + "CC-BY-NC-ND", + "agpl-3.0", + "cc-by-nc-4.0", + "CC-BY-NC-4.0", + "cc-by-nc-sa-4.0", + "CC-BY-NC-SA-4.0", + "odc-by-1.0", + "ODC-BY-1.0", + "cc-by-sa-4.0", + "CC-BY-SA-4.0", + "cc0-1.0", + "CC0-1.0", + "bsd-3-clause", + "BSD-3-Clause", + "other" + ], + "default": "apache-2.0", + "examples": [ + "apache-2.0", + "mit", + "other" + ] + }, + "license_name": { + "type": "string", + "description": "自定义许可证名称。license=\"other\" 时必填，其他许可证下忽略" + }, + "license_link": { + "type": "string", + "description": "【可选】自定义许可证链接，仅当 license=\"other\" 时生效。留空时仓库将自动在根目录创建 LICENSE.md 空白文件", + "format": "uri" + }, + "visibility": { + "type": "string", + "description": "仓库可见性，默认 public。protected 仅适用于 Studio，创建模型时不接受该值", + "enum": [ + "public", + "private", + "gated" + ], + "default": "public" + }, + "gated_mode": { + "description": "创建时仅 visibility=\"gated\" 可传 gated_mode：auto 自动审批，manual 人工审批；省略或空字符串默认 auto。null、布尔值、数字及其他非法值返回 400。visibility 为 public/private（含省略后默认 public）时禁止传 gated_mode。请求仅接受文档列出的字段，未知字段返回 400。", + "type": "string", + "enum": [ + "auto", + "manual", + "" + ] + }, + "description": { + "type": "string", + "description": "模型简介", + "maxLength": 2000, + "examples": [ + "基于 Qwen 微调的对话模型" + ] + } + }, + "required": [ + "owner", + "repo_name" + ] + }, + "ModelCreated": { + "type": "object", + "description": "创建成功后返回的模型仓库核心信息", + "properties": { + "id": { + "type": "string", + "description": "模型仓库唯一标识（格式：owner/repo_name）", + "examples": [ + "alice/my-model" + ] + }, + "repo_name": { + "type": "string", + "description": "仓库英文 ID", + "examples": [ + "my-model" + ] + }, + "display_name": { + "type": [ + "string", + "null" + ], + "description": "模型展示名称", + "examples": [ + "我的大语言模型" + ] + }, + "owner": { + "type": "string", + "description": "所有者（用户名或组织名）", + "examples": [ + "alice" + ] + }, + "url": { + "type": "string", + "format": "uri", + "description": "模型平台访问地址", + "examples": [ + "https://modelscope.cn/models/alice/my-model" + ] + } + }, + "required": [ + "id", + "repo_name", + "owner", + "url" + ] + }, + "CreateModelResponse": { + "type": "object", + "description": "创建模型标准响应包裹格式", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/ModelCreated" + }, + "request_id": { + "type": "string", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" + ] + } + }, + "required": [ + "success", + "data", + "request_id" + ] + }, + "ModelCategory": { + "type": "string", + "description": "模型大类。POST /models 创建后默认为 base；设置/变更专区类型请调用 PUT /models/{owner}/{repo_name}/category。\n- base: 通用模型，无专区业务字段；传入 base 可回退为通用模型\n- aigc: AIGC 专区模型，需填写 aigc_type / base_model_architecture_type / base_model_architecture_subtype / version_name 等\n- science: 科学模型专区，需填写 primary_subject_area / secondary_specialization", + "enum": [ + "base", + "aigc", + "science" + ], + "examples": [ + "base", + "aigc", + "science" + ] + }, + "AigcModelType": { + "type": "string", + "description": "AIGC 模型子类型，可选值为 Checkpoint、LoRA、VAE（区分大小写）；其他值返回 HTTP 400。仅适用于 AIGC 模型。", + "enum": [ + "Checkpoint", + "LoRA", + "VAE" + ], + "examples": [ + "Checkpoint", + "LoRA", + "VAE" + ] + }, + "AigcContentCategories": { + "type": "object", + "description": "AIGC 模型内容类别（三级联动，仅 model_category=aigc 时有效）", + "properties": { + "style": { + "type": "string", + "description": "设计风格", + "examples": [ + "设计风格-插画", + "设计风格-写实" + ] + }, + "function": { + "type": "string", + "description": "模型功能", + "examples": [ + "模型功能-文生图", + "模型功能-图生图" + ] + }, + "element": { + "type": "string", + "description": "内容元素", + "examples": [ + "内容元素-人物", + "内容元素-风景" + ] + } + } + }, + "SampleImage": { + "type": "object", + "additionalProperties": false, + "description": "AIGC 样例图片。请求和响应仅使用 url、prompt、negative_prompt；url 必填，prompt 和 negative_prompt 选填；不接受 text 字段。", + "properties": { + "url": { + "type": "string", + "format": "uri", + "description": "HTTP(S) 图片 URL" + }, + "prompt": { + "type": "string", + "maxLength": 2000, + "description": "正向提示词" + }, + "negative_prompt": { + "type": "string", + "maxLength": 1000, + "description": "负向提示词" + } + }, + "required": [ + "url" + ] + }, + "ModelVersion": { + "type": "object", + "required": [ + "version_id", + "version_name", + "weight_files", + "sample_images", + "created_at" + ], + "properties": { + "version_id": { + "type": "integer", + "format": "int64" + }, + "version_name": { + "type": "string" + }, + "weight_files": { + "type": "array", + "items": { + "type": "object", + "required": [ + "path_in_repo", + "size", + "sha" + ], + "properties": { + "path_in_repo": { + "type": "string" + }, + "size": { + "type": "integer", + "format": "int64" + }, + "sha": { + "type": "string" + } + } + } + }, + "sample_images": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SampleImage" + } + }, + "trigger_words": { + "type": "array", + "items": { + "type": "string" + } + }, + "version_description": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time" + } + } + }, + "WeightFileReference": { + "type": "object", + "description": "AIGC 模型版本权重文件引用。文件需已上传至仓库，通过 path_in_repo 指定相对路径。", + "properties": { + "path_in_repo": { + "type": "string", + "description": "权重文件在仓库内的相对路径。支持的后缀（不区分大小写）：.safetensors、.ckpt、.pt、.pth、.bin、.gguf、.ggml、.h5、.hdf5、.npz、.msgpack、.pdparams、.params、.onnx。每个文件均须符合，非权重后缀返回 400；英文 message 会指出无效字段，引导用户指定正确权重路径、先通过 Git 或网页上传权重，并列出全部支持格式。文件须已存在；后缀检查不代表已验证文件内容或推理兼容性。", + "examples": [ + "model.safetensors", + "checkpoints/model.ckpt", + "vae/diffusion_pytorch_model.bin" + ] + } + }, + "required": [ + "path_in_repo" + ] + }, + "SetModelCategoryRequest": { + "type": "object", + "description": "设置类型；目标为 AIGC（包括创建新 tag）时必须提供架构、version_name、非空 weight_files 和 sample_images。已有 tag 权重不可修改。目标为 Science 时 primary_subject_area、secondary_specialization 必填，cover_image 选填；base 只需 model_category。基础模型由架构子类型确定。", + "additionalProperties": false, + "properties": { + "model_category": { + "$ref": "#/components/schemas/ModelCategory" + }, + "aigc_type": { + "allOf": [ + { + "$ref": "#/components/schemas/AigcModelType" + } + ], + "description": "【AIGC 必填】模型子类型，可选值为 Checkpoint、LoRA、VAE（区分大小写）；其他值返回 HTTP 400。" + }, + "base_model_architecture_type": { + "type": "string", + "description": "【AIGC 必填】基础模型架构类型（一级选择）。可选值查询 GET /model/aigc/options", + "examples": [ + "Z_IMAGE_TURBO", + "FLUX_2" + ] + }, + "base_model_architecture_subtype": { + "type": "string", + "description": "【AIGC 必填】基础模型架构子类型（二级选择），需与 base_model_architecture_type 匹配。可选值查询 GET /model/aigc/options", + "examples": [ + "Z_IMAGE_TURBO", + "FLUX2_DEV" + ] + }, + "content_categories": { + "allOf": [ + { + "$ref": "#/components/schemas/AigcContentCategories" + } + ], + "description": "【AIGC 可选】内容类别三级联动" + }, + "version_name": { + "type": "string", + "description": "【AIGC 必填】版本名称，建议使用语义化版本，如 v1.0", + "maxLength": 20, + "examples": [ + "v1.0" + ] + }, + "trigger_words": { + "type": "array", + "description": "【AIGC 可选】触发词列表", + "items": { + "type": "string" + }, + "examples": [ + [ + "anime style" + ] + ] + }, + "sample_images": { + "type": "array", + "description": "AIGC tag 必填且非空；tag 样例图片。", + "items": { + "$ref": "#/components/schemas/SampleImage" + }, + "minItems": 1 + }, + "weight_files": { + "type": "array", + "description": "AIGC tag 必填且非空；仓库中已存在的权重文件路径。", + "items": { + "$ref": "#/components/schemas/WeightFileReference" + }, + "minItems": 1 + }, + "cover_image": { + "type": "string", + "description": "【Science 可选】模型封面图；可省略或传空字符串，非空时须为 HTTP(S) URL。", + "anyOf": [ + { + "format": "uri", + "pattern": "^https?://" + }, + { + "const": "" + } + ] + }, + "primary_subject_area": { + "allOf": [ + { + "type": "string" + } + ], + "description": "【科学 必填】一级学科分类，实际值以 GET /model/science/options 为准，例如 life-sciences" + }, + "secondary_specialization": { + "allOf": [ + { + "type": "string" + } + ], + "description": "【科学 必填】二级专业分类，应与 primary_subject_area 匹配；实际值以 GET /model/science/options 为准，例如 dna" + }, + "version_description": { + "type": "string", + "description": "AIGC tag 版本描述" + } + }, + "required": [ + "model_category" + ], + "allOf": [ + { + "if": { + "properties": { + "model_category": { + "const": "aigc" + } + }, + "required": [ + "model_category" + ] + }, + "then": { + "required": [ + "aigc_type", + "base_model_architecture_type", + "base_model_architecture_subtype", + "version_name", + "weight_files", + "sample_images" + ], + "properties": { + "model_category": {}, + "content_categories": {}, + "trigger_words": {}, + "version_description": {}, + "aigc_type": {}, + "base_model_architecture_type": {}, + "base_model_architecture_subtype": {}, + "version_name": {}, + "weight_files": {}, + "sample_images": {} + } + } + }, + { + "if": { + "properties": { + "model_category": { + "const": "science" + } + }, + "required": [ + "model_category" + ] + }, + "then": { + "required": [ + "primary_subject_area", + "secondary_specialization" + ], + "properties": { + "model_category": {}, + "cover_image": {}, + "primary_subject_area": {}, + "secondary_specialization": {} + } + } + } + ] + }, + "SetModelCategoryResponse": { + "type": "object", + "description": "设置/变更模型类型标准响应包裹格式", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/ModelCreated" + }, + "request_id": { + "type": "string", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" + ] + } + }, + "required": [ + "success", + "data", + "request_id" + ] + }, + "UpdateModelCategoryRequest": { + "type": "object", + "additionalProperties": false, + "description": "类型由服务器读取当前仓库确定，不接受 model_category（传入返回 400），类型转换使用 PUT /category；tag 字段用 version_id 定位；已有 tag 的权重不可修改，请 PUT /category 创建新 tag。省略不变，显式空值清空；必填约束仍需满足。", + "properties": { + "version_id": { + "type": "integer", + "format": "int64", + "minimum": 1, + "description": "仅适用于 AIGC：更新 trigger_words、version_description 或 sample_images 时必填，取自 GET 详情 versions[].version_id。Science 更新学科或 cover_image 不需要且不应传此字段；Science 详情不返回 versions。" + }, + "aigc_type": { + "anyOf": [ + { + "allOf": [ + { + "$ref": "#/components/schemas/AigcModelType" + } + ], + "description": "【AIGC 字段；省略保持不变，不能清空】模型子类型，可选值为 Checkpoint、LoRA、VAE（区分大小写）；其他值返回 HTTP 400。" + }, + { + "type": "null" + } + ] + }, + "base_model_architecture_type": { + "anyOf": [ + { + "type": "string", + "description": "【AIGC 字段；省略保持不变，不能清空】基础模型架构类型（一级选择）。可选值查询 GET /model/aigc/options", + "examples": [ + "Z_IMAGE_TURBO", + "FLUX_2" + ] + }, + { + "type": "null" + } + ] + }, + "base_model_architecture_subtype": { + "anyOf": [ + { + "type": "string", + "description": "【AIGC 字段；省略保持不变，不能清空】基础模型架构子类型（二级选择），需与 base_model_architecture_type 匹配。可选值查询 GET /model/aigc/options", + "examples": [ + "Z_IMAGE_TURBO", + "FLUX2_DEV" + ] + }, + { + "type": "null" + } + ] + }, + "content_categories": { + "anyOf": [ + { + "allOf": [ + { + "$ref": "#/components/schemas/AigcContentCategories" + } + ], + "description": "【AIGC 可选】内容类别三级联动" + }, + { + "type": "null" + } + ] + }, + "trigger_words": { + "anyOf": [ + { + "type": "array", + "description": "【AIGC 可选】触发词列表", + "items": { + "type": "string" + }, + "examples": [ + [ + "anime style" + ] + ] + }, + { + "type": "null" + } + ] + }, + "sample_images": { + "type": "array", + "description": "可省略以保持原样；传入时替换指定 version_id 的全部样例图，必须非空。图片由现有媒体审核流程处理。", + "items": { + "$ref": "#/components/schemas/SampleImage" + }, + "minItems": 1 + }, + "version_description": { + "anyOf": [ + { + "type": "string", + "description": "AIGC tag 版本描述" + }, + { + "type": "null" + } + ] + }, + "primary_subject_area": { + "anyOf": [ + { + "allOf": [ + { + "type": "string" + } + ], + "description": "【科学 字段；省略保持不变，不能清空】一级学科分类，实际值以 GET /model/science/options 为准，例如 life-sciences" + }, + { + "type": "null" + } + ] + }, + "secondary_specialization": { + "anyOf": [ + { + "allOf": [ + { + "type": "string" + } + ], + "description": "【科学 字段；省略保持不变，不能清空】二级专业分类，应与 primary_subject_area 匹配；实际值以 GET /model/science/options 为准，例如 dna" + }, + { + "type": "null" + } + ] + }, + "cover_image": { + "anyOf": [ + { + "type": "string", + "description": "【仅科学 可选】模型封面图 URL", + "format": "uri" + }, + { + "type": "null" + } + ] + } + } + }, + "ModelOptionItem": { + "type": "object", + "description": "单个可选项，包含 type（自描述分类）、id 和 label；label 可用于各处直接展示，提交参数使用 id", + "properties": { + "type": { + "type": "string", + "description": "选项所属分类（自描述）", + "examples": [ + "aigc_type", + "specialization" + ] + }, + "id": { + "type": "string", + "description": "实际传入 API 的值", + "examples": [ + "LoRA" + ] + }, + "label": { + "type": "string", + "description": "可用于各处直接展示的名称，例如选项列表、筛选项和标签。提交参数请使用 id，不要使用 label。", + "examples": [ + "LoRA" + ] + } + }, + "required": [ + "type", + "id", + "label" + ] + }, + "BaseArchitectureOption": { + "type": "object", + "description": "AIGC 基础模型架构（一级）及其可用子类型（二级）", + "properties": { + "type": { + "type": "string", + "description": "选项所属分类（自描述），固定为 \"base_architecture\"", + "examples": [ + "base_architecture" + ] + }, + "id": { + "type": "string", + "description": "架构类型标识（一级）", + "examples": [ + "Z_IMAGE_TURBO" + ] + }, + "label": { + "type": "string", + "description": "可用于各处直接展示的名称，例如选项列表、筛选项和标签。提交参数请使用 id，不要使用 label。", + "examples": [ + "Z_IMAGE_TURBO" + ] + }, + "subtypes": { + "type": "array", + "description": "该架构下可用的子类型列表（二级）", + "items": { + "$ref": "#/components/schemas/ModelOptionItem" + } + } + }, + "required": [ + "type", + "id", + "label", + "subtypes" + ] + }, + "AigcModelOptions": { + "type": "object", + "description": "AIGC 专区配置项枚举", + "properties": { + "aigc_types": { + "type": "array", + "description": "AIGC 模型子类型选项", + "items": { + "$ref": "#/components/schemas/ModelOptionItem" + } + }, + "base_architectures": { + "type": "array", + "description": "AIGC 基础模型架构级联树（一级 type + 二级 subtype）", + "items": { + "$ref": "#/components/schemas/BaseArchitectureOption" + } + }, + "content_styles": { + "type": "array", + "description": "AIGC 内容类别 - 设计风格选项", + "items": { + "$ref": "#/components/schemas/ModelOptionItem" + } + }, + "content_functions": { + "type": "array", + "description": "AIGC 内容类别 - 模型功能选项", + "items": { + "$ref": "#/components/schemas/ModelOptionItem" + } + }, + "content_elements": { + "type": "array", + "description": "AIGC 内容类别 - 内容元素选项", + "items": { + "$ref": "#/components/schemas/ModelOptionItem" + } + } + }, + "required": [ + "aigc_types", + "base_architectures", + "content_styles", + "content_functions", + "content_elements" + ] + }, + "AigcModelOptionsResponse": { + "type": "object", + "description": "AIGC 专区配置项枚举标准响应包裹格式", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/AigcModelOptions" + }, + "request_id": { + "type": "string", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" + ] + } + }, + "required": [ + "success", + "data", + "request_id" + ] + }, + "ScienceOptionItem": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "specialization" + ] + }, + "id": { + "type": "string", + "description": "二级专业方向标识，用于 secondary_specialization。", + "examples": [ + "medicine" + ] + }, + "label": { + "type": "string", + "description": "可用于各处直接展示的名称；提交参数使用 id。", + "examples": [ + "医学" + ] + } + }, + "required": [ + "type", + "id", + "label" + ] + }, + "ScienceSubjectOption": { + "type": "object", + "description": "科学模型一级学科及其下属二级专业方向", + "properties": { + "type": { + "type": "string", + "description": "选项所属分类（自描述），固定为 \"subject_area\"", + "examples": [ + "subject_area" + ] + }, + "id": { + "type": "string", + "description": "一级学科标识", + "examples": [ + "life-sciences" + ] + }, + "label": { + "type": "string", + "description": "可用于各处直接展示的名称，例如选项列表、筛选项和标签。提交参数请使用 id，不要使用 label。", + "examples": [ + "生命科学" + ] + }, + "specializations": { + "type": "array", + "description": "该学科下的二级专业方向列表", + "items": { + "$ref": "#/components/schemas/ScienceOptionItem" + } + } + }, + "required": [ + "type", + "id", + "label", + "specializations" + ] + }, + "ScienceModelOptions": { + "type": "object", + "description": "科学模型专区配置项枚举", + "properties": { + "subject_areas": { + "type": "array", + "description": "学科分类树（一级学科 + 二级专业方向）", + "items": { + "$ref": "#/components/schemas/ScienceSubjectOption" + } + } + }, + "required": [ + "subject_areas" + ] + }, + "ScienceModelOptionsResponse": { + "type": "object", + "description": "科学模型专区配置项枚举标准响应包裹格式", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/ScienceModelOptions" + }, + "request_id": { + "type": "string", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" + ] + } + }, + "required": [ + "success", + "data", + "request_id" + ] + }, + "CreateDatasetRequest": { + "type": "object", + "description": "创建通用数据集仓库；成功后 dataset_category 为 base。科学专区字段通过 PUT /datasets/{owner}/{repo_name}/category 设置。", + "additionalProperties": false, + "properties": { + "owner": { + "type": "string", + "description": "所有者（用户名或组织名）", + "examples": [ + "alice", + "my-org" + ] + }, + "repo_name": { + "type": "string", + "description": "仓库英文 ID，仅允许字母、数字、点、连字符和下划线，创建后不可修改", + "maxLength": 128, + "pattern": "^[a-zA-Z0-9._-]+$", + "examples": [ + "my-dataset", + "multilingual-dialogues" + ] + }, + "display_name": { + "type": "string", + "description": "数据集展示名称", + "maxLength": 128, + "examples": [ + "我的对话数据集" + ] + }, + "license": { + "type": "string", + "description": "许可证（SPDX 标识符），默认 apache-2.0。当传入 \"other\" 时可通过 license_name 和 license_link 自定义许可证", + "default": "apache-2.0", + "examples": [ + "apache-2.0", + "mit", + "other" + ] + }, + "license_name": { + "type": "string", + "description": "自定义许可证名称。license=\"other\" 时必填，其他许可证下忽略" + }, + "license_link": { + "type": "string", + "description": "【可选】自定义许可证链接，仅当 license=\"other\" 时生效。留空时仓库将自动在根目录创建 LICENSE.md 空白文件", + "format": "uri" + }, + "visibility": { + "type": "string", + "description": "仓库可见性，默认 public。protected 仅适用于 Studio，创建数据集时不接受该值", + "enum": [ + "public", + "private", + "gated" + ], + "default": "public" + }, + "gated_mode": { + "description": "创建时仅 visibility=\"gated\" 可传 gated_mode：auto 自动审批，manual 人工审批；省略或空字符串默认 auto。null、布尔值、数字及其他非法值返回 400。visibility 为 public/private（含省略后默认 public）时禁止传 gated_mode。请求仅接受文档列出的字段，未知字段返回 400。", + "type": "string", + "enum": [ + "auto", + "manual", + "" + ] + }, + "description": { + "type": "string", + "description": "数据集简介", + "maxLength": 2000, + "examples": [ + "用于多语言对话训练的数据集" + ] + } + }, + "required": [ + "owner", + "repo_name" + ] + }, + "DatasetCreated": { + "type": "object", + "description": "创建或更新后的数据集仓库标识", + "properties": { + "id": { + "type": "string", + "description": "数据集仓库唯一标识（格式：owner/repo_name）", + "examples": [ + "alice/my-dataset" + ] + }, + "repo_name": { + "type": "string", + "description": "仓库英文 ID", + "examples": [ + "my-dataset" + ] + }, + "display_name": { + "type": [ + "string", + "null" + ], + "description": "数据集展示名称", + "examples": [ + "我的数据集" + ] + }, + "owner": { + "type": "string", + "description": "所有者（用户名或组织名）", + "examples": [ + "alice" + ] + }, + "url": { + "type": "string", + "format": "uri", + "description": "数据集平台访问地址", + "examples": [ + "https://modelscope.cn/datasets/alice/my-dataset" + ] + } + }, + "required": [ + "id", + "repo_name", + "owner", + "url" + ] + }, + "DatasetWriteResponse": { + "type": "object", + "description": "创建或更新数据集的标准响应；所有成功和错误响应均有 request_id。", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/DatasetCreated" + }, + "request_id": { + "type": "string", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" + ] + } + }, + "required": [ + "success", + "data", + "request_id" + ] + }, + "McpServerList": { + "type": "object", + "description": "MCP服务列表与分页信息", + "properties": { + "mcp_server_list": { + "type": "array", + "description": "MCP服务列表", + "items": { + "$ref": "#/components/schemas/McpServerSummary" + } + }, + "total_count": { + "type": "integer", + "description": "搜索符合条件的mcp server总个数", + "examples": [ + 100 + ] + } + }, + "required": [ + "mcp_server_list", + "total_count" + ] + }, + "McpServerOperationalList": { + "type": "object", + "description": "用户托管MCP服务列表", + "properties": { + "mcp_server_list": { + "type": "array", + "description": "用户托管MCP服务列表", + "items": { + "$ref": "#/components/schemas/McpServerOperational" + } + }, + "total_count": { + "type": "integer", + "description": "服务链接总数", + "examples": [ + 5 + ] + } + }, + "required": [ + "mcp_server_list", + "total_count" + ] + }, + "data": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Studio ID (owner/repo_name)" + }, + "repo_name": { + "type": "string", + "description": "仓库名称" + }, + "display_name": { + "type": "string", + "description": "显示名称" + }, + "owner": { + "type": "string", + "description": "所有者" + }, + "description": { + "type": "string", + "description": "描述" + }, + "cover_image": { + "type": "string", + "description": "封面图 URL" + }, + "likes": { + "type": "integer", + "description": "喜欢数" + }, + "view_count": { + "type": "integer", + "description": "访问量" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "visibility": { + "$ref": "#/components/schemas/StudioVisibility" + }, + "private": { + "type": "boolean", + "description": "Deprecated: 请使用 visibility 字段", + "deprecated": true + }, + "last_modified": { + "type": "string", + "description": "最后修改时间" + }, + "sdk_type": { + "$ref": "#/components/schemas/StudioSDKType" + }, + "sdk_version": { + "$ref": "#/components/schemas/StudioSDKVersion" + }, + "hardware": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioHardware" + } + ], + "description": "期望的硬件配置（数据库中保存的值）" + }, + "base_image": { + "allOf": [ + { + "$ref": "#/components/schemas/StudioBaseImage" + } + ], + "description": "期望的基础镜像（Docker 类型不返回）" + }, + "license": { + "type": "string", + "description": "许可证" + }, + "host": { + "type": "string", + "description": "访问地址" + }, + "mcp_support": { + "type": "boolean", + "description": "是否支持 MCP" + }, + "runtime": { + "$ref": "#/components/schemas/StudioRuntime" + } + } + }, + "ListStudiosResponse": { + "type": "object", + "description": "获取创空间列表响应", + "properties": { + "studios": { + "type": "array", + "description": "创空间列表", + "items": { + "$ref": "#/components/schemas/data" + } + }, + "total_count": { + "type": "integer", + "description": "符合筛选条件的创空间总数", + "examples": [ + 120 + ] + }, + "page_number": { + "type": "integer", + "description": "当前页码", + "examples": [ + 1 + ] + }, + "page_size": { + "type": "integer", + "description": "每页大小", + "examples": [ + 10 + ] + } + }, + "required": [ + "studios", + "total_count", + "page_number", + "page_size" + ] + }, + "CollectionMutationResponse": { + "type": "object", + "description": "创建/更新 Collection 的标准响应", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "type": "object", + "description": "变更结果", + "properties": { + "slug": { + "type": "string", + "description": "Collection slug（重命名时可能变化）", + "examples": [ + "iSolver/best-nlp-models-6a3b2c" + ] + }, + "title": { + "type": "string", + "description": "Collection 标题", + "examples": [ + "最佳 NLP 模型合集" + ] + }, + "url": { + "type": "string", + "description": "Collection 页面 URL", + "examples": [ + "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + ] + } + }, + "required": [ + "slug", + "title", + "url" + ] + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "EmptyDataResponse": { + "type": "object", + "description": "删除/移除类操作的标准响应，结果以 success 字段为准，data 为空对象", + "properties": { + "success": { + "type": "boolean", + "description": "成功时始终为 true", + "examples": [ + true + ] + }, + "request_id": { + "type": "string", + "description": "请求 ID", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd3" + ] + }, + "data": { + "type": "object", + "description": "空对象" + } + }, + "required": [ + "success", + "request_id", + "data" + ] + }, + "ModelSettingsRequest": { + "type": "object", + "additionalProperties": false, + "description": "PATCH：省略不变，显式空值清空；visibility 决定访问模式，gated_mode 用于申请制仓库的审批方式；可同时传 visibility=gated 开启申请制。", + "properties": { + "task": { + "type": [ + "string", + "null" + ], + "description": "模型任务，取 GET /models/options 的 tasks[].id。省略保持不变；null 或空字符串清空。未知任务返回 HTTP 400。", + "examples": [ + "text-generation" + ] + }, + "display_name": { + "type": [ + "string", + "null" + ], + "description": "模型展示名称（中英文均可），用于 UI 展示", + "maxLength": 128, + "examples": [ + "我的大语言模型" + ] + }, + "license": { + "type": [ + "string", + "null" + ], + "description": "许可证标识；省略保持不变，显式 null 或空串清空。", + "examples": [ + "apache-2.0", + "mit", + "other" + ] + }, + "visibility": { + "type": [ + "string", + "null" + ], + "enum": [ + "public", + "private", + "gated", + "", + null + ], + "description": "仓库访问模式；public/private 关闭申请制，gated 开启申请制；省略保持不变。" + }, + "gated_mode": { + "type": "string", + "enum": [ + "auto", + "manual", + "" + ], + "description": "gated_mode 取 auto 自动审批、manual 人工审批；显式 visibility=\"gated\" 时，gated_mode 省略或空字符串默认 auto；两字段均省略保持原设置；无论 visibility 取值，显式 null、布尔值、数字及其他非法值均返回 400；visibility 为 public/private 时禁止传 gated_mode（含空串和 null），否则返回 400；PATCH 省略 visibility 时，仅当前仓库已是申请制才可用 gated_mode 修改审批方式。关闭申请制请将 visibility 改为 public 或 private。" + }, + "description": { + "type": [ + "string", + "null" + ], + "description": "模型简介", + "maxLength": 2000, + "examples": [ + "基于 Qwen 微调的对话模型" + ] + } + } + }, + "DatasetCategoryRequest": { + "type": "object", + "additionalProperties": false, + "required": [ + "dataset_category" + ], + "properties": { + "dataset_category": { + "type": "string", + "enum": [ + "base", + "science" + ] + }, + "primary_subject_area": { + "type": [ + "string", + "null" + ] + }, + "secondary_specialization": { + "type": [ + "string", + "null" + ] + }, + "cover_image": { + "type": [ + "string", + "null" + ], + "description": "科学数据集封面；显式空值清空。" + } + } + }, + "UpdateDatasetCategoryRequest": { + "type": "object", + "additionalProperties": false, + "description": "类型由服务器读取当前仓库确定，不接受 dataset_category（传入返回 400），类型转换使用 PUT /category。", + "properties": { + "primary_subject_area": { + "type": [ + "string", + "null" + ] + }, + "secondary_specialization": { + "type": [ + "string", + "null" + ] + }, + "cover_image": { + "type": [ + "string", + "null" + ], + "description": "科学数据集封面；显式空值清空。" + } + } + }, + "DatasetSettingsRequest": { + "type": "object", + "additionalProperties": false, + "description": "PATCH：省略不变，显式空值清空；visibility 决定访问模式，gated_mode 用于申请制仓库的审批方式；可同时传 visibility=gated 开启申请制。", + "properties": { + "display_name": { + "type": [ + "string", + "null" + ], + "description": "数据集展示名称（中英文均可），用于 UI 展示", + "maxLength": 128, + "examples": [ + "我的数据集" + ] + }, + "license": { + "type": [ + "string", + "null" + ], + "description": "许可证标识；省略保持不变，显式 null 或空串清空。", + "examples": [ + "apache-2.0", + "mit", + "other" + ] + }, + "visibility": { + "type": [ + "string", + "null" + ], + "enum": [ + "public", + "private", + "gated", + "", + null + ], + "description": "仓库访问模式；public/private 关闭申请制，gated 开启申请制；省略保持不变。" + }, + "gated_mode": { + "type": "string", + "enum": [ + "auto", + "manual", + "" + ], + "description": "gated_mode 取 auto 自动审批、manual 人工审批；显式 visibility=\"gated\" 时，gated_mode 省略或空字符串默认 auto；两字段均省略保持原设置；无论 visibility 取值，显式 null、布尔值、数字及其他非法值均返回 400；visibility 为 public/private 时禁止传 gated_mode（含空串和 null），否则返回 400；PATCH 省略 visibility 时，仅当前仓库已是申请制才可用 gated_mode 修改审批方式。关闭申请制请将 visibility 改为 public 或 private。" + }, + "description": { + "type": [ + "string", + "null" + ], + "description": "数据集简介", + "maxLength": 2000, + "examples": [ + "用于文本分类的数据集" + ] + } + } + }, + "DeleteFilesRequest": { + "type": "object", + "additionalProperties": false, + "required": [ + "file_paths" + ], + "properties": { + "file_paths": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "仓库相对文件路径；不允许目录、重复或越界路径。" + }, + "revision": { + "type": "string", + "default": "master", + "description": "目标分支，不接受 tag。" + }, + "commit_message": { + "type": "string", + "default": "Delete files" + } + } + }, + "DeleteFilesResponse": { + "type": "object", + "required": [ + "success", + "request_id", + "data" + ], + "properties": { + "success": { + "type": "boolean" + }, + "request_id": { + "type": "string" + }, + "data": { + "type": "object", + "required": [ + "deleted_files", + "failed_files", + "total_files", + "commit_sha" + ], + "properties": { + "deleted_files": { + "type": "array", + "items": { + "type": "string" + } + }, + "failed_files": { + "type": "array", + "maxItems": 0, + "items": { + "type": "string" + }, + "description": "成功时为空；任何失败整个请求失败，不产生部分删除。" + }, + "total_files": { + "type": "integer" + }, + "commit_sha": { + "type": "string" + } + } + } + } + }, + "DatasetScienceOptionItem": { + "type": "object", + "description": "单个可选项，包含 type（自描述分类）、id 和 label；label 可用于各处直接展示，提交参数使用 id", + "properties": { + "type": { + "type": "string", + "description": "选项所属分类（自描述）", + "examples": [ + "specialization" + ] + }, + "label": { + "type": "string", + "description": "可用于各处直接展示的名称，例如选项列表、筛选项和标签。提交参数请使用 id，不要使用 label。", + "examples": [ + "医学" + ] + }, + "id": { + "type": "string", + "description": "实际传入 API 的值", + "examples": [ + "medicine" + ] + } + }, + "required": [ + "type", + "id", + "label" + ] + }, + "DatasetScienceSubjectOption": { + "type": "object", + "description": "科学数据集一级学科及其下属二级专业方向", + "properties": { + "type": { + "type": "string", + "description": "选项所属分类（自描述），固定为 \"subject_area\"", + "examples": [ + "subject_area" + ] + }, + "label": { + "type": "string", + "description": "可用于各处直接展示的名称，例如选项列表、筛选项和标签。提交参数请使用 id，不要使用 label。", + "examples": [ + "生命科学" + ] + }, + "specializations": { + "type": "array", + "description": "该学科下的二级专业方向列表", + "items": { + "$ref": "#/components/schemas/DatasetScienceOptionItem" + } + }, + "id": { + "type": "string", + "description": "一级学科标识", + "examples": [ + "life-sciences" + ] + } + }, + "required": [ + "type", + "id", + "label", + "specializations" + ] + }, + "ScienceDatasetOptions": { + "type": "object", + "description": "科学数据集专区配置项枚举", + "properties": { + "subject_areas": { + "type": "array", + "description": "学科分类树（一级学科 + 二级专业方向）", + "items": { + "$ref": "#/components/schemas/DatasetScienceSubjectOption" + } + } + }, + "required": [ + "subject_areas" + ] + }, + "ScienceDatasetOptionsResponse": { + "type": "object", + "description": "科学数据集专区配置项枚举标准响应包裹格式", + "properties": { + "success": { + "type": "boolean", + "examples": [ + true + ] + }, + "data": { + "$ref": "#/components/schemas/ScienceDatasetOptions" + }, + "request_id": { + "type": "string", + "examples": [ + "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" + ] + } + }, + "required": [ + "success", + "data", + "request_id" + ] + }, + "ModelOptionsResponse": { + "type": "object", + "required": [ + "success", + "data", + "request_id" + ], + "properties": { + "success": { + "type": "boolean" + }, + "request_id": { + "type": "string" + }, + "data": { + "type": "object", + "additionalProperties": false, + "required": [ + "tasks", + "license" + ], + "properties": { + "tasks": { + "type": "array", + "description": "模型任务选项。id 是任务标识字符串，可用于创建模型、更新模型设置的 task，以及 GET /models 的 filter.task。", + "items": { + "type": "object", + "required": [ + "type", + "id", + "label" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "task" + ] + }, + "id": { + "type": "string", + "description": "接口接受的任务标识字符串。", + "examples": [ + "text-generation" + ] + }, + "label": { + "type": "string", + "description": "可用于各处直接展示的名称，例如选项列表、筛选项和标签。传参使用 id，不要使用 label。", + "examples": [ + "文本生成" + ] + } + } + } + }, + "license": { + "type": "array", + "description": "模型许可证选项。id 可直接作为创建或更新模型的 license 入参；other 仍需在创建时提供 license_name。", + "items": { + "type": "object", + "required": [ + "type", + "id", + "label" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "license" + ] + }, + "id": { + "type": "string", + "description": "接口接受的许可证标识字符串。", + "examples": [ + "apache-2.0", + "mit", + "other" + ] + }, + "label": { + "type": "string", + "description": "可用于各处直接展示的许可证名称。传参使用 id，不要使用 label。", + "examples": [ + "Apache License 2.0", + "MIT License" + ] + } + } + } + } + } + } + } + } + }, + "responses": { + "bad-request": { + "description": "请求参数错误", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "input-parameter-error": { + "$ref": "#/components/examples/input-parameter-error" + } + } + } + } + }, + "unauthorized": { + "description": "认证失败（token 无效、过期或未提供）", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "invalid-authentication": { + "$ref": "#/components/examples/invalid-authentication" + } + } + } + } + }, + "forbidden": { + "description": "权限不足（token 有效但无权执行该操作，或配额超限）", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "operation-not-allowed": { + "$ref": "#/components/examples/operation-not-allowed" + }, + "quota-limit-exceed": { + "$ref": "#/components/examples/quota-limit-exceed" + } + } + } + } + }, + "not-found": { + "description": "资源未找到", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "resource-not-found": { + "$ref": "#/components/examples/resource-not-found" + } + } + } + } + }, + "internal-server-error": { + "description": "内部服务错误", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "internal-server-error": { + "$ref": "#/components/examples/internal-server-error" + } + } + } + } + }, + "service-unavailable": { + "description": "服务不可用", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "service-unavailable": { + "$ref": "#/components/examples/service-unavailable" + } + } + } + } + }, + "get-current-user-success": { + "description": "成功获取当前用户信息", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UserResponse" + }, + "examples": { + "user-all-response": { + "$ref": "#/components/examples/user-all-response" + } + } + } + } + }, + "list-models-success": { + "description": "成功获取模型列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ModelListResponse" + }, + "examples": { + "list-models-response": { + "$ref": "#/components/examples/list-models-response" + } + } + } + } + }, + "get-model-success": { + "description": "成功获取模型详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ModelDetailResponse" + }, + "examples": { + "get-model-response": { + "$ref": "#/components/examples/get-model-response" + } + } + } + } + }, + "list-datasets-success": { + "description": "成功获取数据集列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DatasetListResponse" + }, + "examples": { + "list-datasets-response": { + "$ref": "#/components/examples/list-datasets-response" + } + } + } + } + }, + "get-dataset-success": { + "description": "成功获取数据集详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DatasetDetailResponse" + }, + "examples": { + "get-dataset-response": { + "$ref": "#/components/examples/get-dataset-response" + } + } + } + } + }, + "list-mcp-servers-success": { + "description": "成功获取MCP服务列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpServerListResponse" + }, + "examples": { + "list-mcp-servers-response": { + "$ref": "#/components/examples/list-mcp-servers-response" + } + } + } + } + }, + "list-operational-mcp-servers-success": { + "description": "成功获取用户托管MCP服务列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpServerOperationalListResponse" + }, + "examples": { + "list-operational-mcp-servers-response": { + "$ref": "#/components/examples/list-operational-mcp-servers-response" + } + } + } + } + }, + "get-mcp-server-success": { + "description": "成功获取MCP服务详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpServerDetailResponse" + }, + "examples": { + "get-mcp-server-response": { + "$ref": "#/components/examples/get-mcp-server-response" + } + } + } + } + }, + "deploy-mcp-server-success": { + "description": "成功部署MCP服务", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpDeployResponse" + }, + "examples": { + "deploy-mcp-server-response": { + "$ref": "#/components/examples/deploy-mcp-server-response" + } + } + } + } + }, + "undeploy-mcp-server-success": { + "description": "成功解除MCP服务部署", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/McpUndeployResponse" + }, + "examples": { + "undeploy-mcp-server-response": { + "$ref": "#/components/examples/undeploy-mcp-server-response" + } + } + } + } + }, + "create-studio-success": { + "description": "成功创建 Studio", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateStudioResponse" + }, + "examples": { + "create-studio-response": { + "$ref": "#/components/examples/create-studio-response" + } + } + } + } + }, + "get-studio-success": { + "description": "成功获取 Studio 详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetStudioResponse" + }, + "examples": { + "get-studio-response": { + "$ref": "#/components/examples/get-studio-response" + } + } + } + } + }, + "update-studio-settings-success": { + "description": "成功更新 Studio 设置", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetStudioResponse" + }, + "examples": { + "update-studio-settings-response": { + "$ref": "#/components/examples/get-studio-response" + } + } + } + } + }, + "studio-operation-success": { + "description": "Studio 操作成功", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessResponse" + } + } + } + }, + "studio-runtime-success": { + "description": "Studio 运行时操作成功", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StudioRuntimeResponse" + }, + "examples": { + "studio-runtime-response": { + "$ref": "#/components/examples/studio-runtime-response" + } + } + } + } + }, + "get-studio-logs-success": { + "description": "成功获取 Studio 日志", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StudioLogsResponse" + }, + "examples": { + "get-studio-logs-response": { + "$ref": "#/components/examples/get-studio-logs-response" + } + } + } + } + }, + "list-hardware-success": { + "description": "成功获取可用硬件配置列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListHardwareResponse" + } + } + } + }, + "list-sdk-versions-success": { + "description": "成功获取可用 SDK 版本列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListSDKVersionsResponse" + }, + "examples": { + "list-sdk-versions-response": { + "$ref": "#/components/examples/list-sdk-versions-response" + } + } + } + } + }, + "list-base-images-success": { + "description": "成功获取可用基础镜像列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListBaseImagesResponse" + } + } + } + }, + "list-studio-variables-success": { + "description": "成功获取明文变量列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListStudioVariablesResponse" + } + } + } + }, + "list-skills-success": { + "description": "成功获取技能列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillListResponse" + }, + "examples": { + "list-skills-response": { + "$ref": "#/components/examples/list-skills-response" + } + } + } + } + }, + "get-skill-success": { + "description": "成功获取技能详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillDetailResponse" + }, + "examples": { + "get-skill-response": { + "$ref": "#/components/examples/get-skill-response" + } + } + } + } + }, + "create-skill-success": { + "description": "成功创建技能", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateSkillResponse" + }, + "examples": { + "create-skill-response": { + "$ref": "#/components/examples/create-skill-response" + } + } + } + } + }, + "update-skill-settings-success": { + "description": "成功更新技能设置", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessResponse" + }, + "examples": { + "update-skill-settings-response": { + "$ref": "#/components/examples/update-skill-settings-response" + } + } + } + } + }, + "skill-bad-request": { + "description": "请求参数错误", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "skill-input-parameter-error": { + "$ref": "#/components/examples/skill-input-parameter-error" + } + } + } + } + }, + "skill-unauthorized": { + "description": "未通过认证", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "skill-invalid-authentication": { + "$ref": "#/components/examples/skill-invalid-authentication" + } + } + } + } + }, + "skill-not-found": { + "description": "技能未找到", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "skill-not-found": { + "$ref": "#/components/examples/skill-not-found" + } + } + } + } + }, + "skill-conflict": { + "description": "资源冲突（如 Skill 已存在）", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "skill-duplicate-entity": { + "$ref": "#/components/examples/skill-duplicate-entity" + } + } + } + } + }, + "skill-internal-server-error": { + "description": "内部服务错误", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "skill-internal-server-error": { + "$ref": "#/components/examples/skill-internal-server-error" + } + } + } + } + }, + "skill-service-unavailable": { + "description": "服务不可用", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "skill-service-unavailable": { + "$ref": "#/components/examples/skill-service-unavailable" + } + } + } + } + }, + "list-agents-success": { + "description": "成功获取 Agent 列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentListResponse" + }, + "examples": { + "list-agents-response": { + "$ref": "#/components/examples/list-agents-response" + } + } + } + } + }, + "create-agent-success": { + "description": "成功创建 Agent", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateAgentResponse" + }, + "examples": { + "create-agent-response": { + "$ref": "#/components/examples/create-agent-response" + } + } + } + } + }, + "get-agent-success": { + "description": "成功获取 Agent 详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetAgentResponse" + }, + "examples": { + "get-agent-response": { + "$ref": "#/components/examples/get-agent-response" + } + } + } + } + }, + "agent-bad-request": { + "description": "请求参数错误", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "agent-input-parameter-error": { + "$ref": "#/components/examples/agent-input-parameter-error" + } + } + } + } + }, + "agent-unauthorized": { + "description": "未通过认证", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "agent-invalid-authentication": { + "$ref": "#/components/examples/agent-invalid-authentication" + } + } + } + } + }, + "agent-forbidden": { + "description": "无权访问该 Agent", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "agent-operation-not-allowed": { + "$ref": "#/components/examples/agent-operation-not-allowed" + } + } + } + } + }, + "agent-quota-limit-exceeded": { + "description": "分页查询总量（page_number × page_size）超出上限", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "agent-quota-limit-exceed": { + "$ref": "#/components/examples/agent-quota-limit-exceed" + } + } + } + } + }, + "agent-not-found": { + "description": "Agent 未找到", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "agent-not-found": { + "$ref": "#/components/examples/agent-not-found" + } + } + } + } + }, + "agent-list-failed": { + "description": "Agent 列表查询失败", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "agent-list-failed": { + "$ref": "#/components/examples/agent-list-failed" + } + } + } + } + }, + "agent-internal-server-error": { + "description": "内部服务错误", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "agent-internal-server-error": { + "$ref": "#/components/examples/agent-internal-server-error" + } + } + } + } + }, + "upload-file-success": { + "description": "成功上传文件", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UploadFileResponse" + }, + "examples": { + "upload-file-response": { + "$ref": "#/components/examples/upload-file-response" + } + } + } + } + }, + "create-agent-identity-success": { + "description": "成功创建 Agent 身份", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentIdentityResponse" + }, + "examples": { + "create-agent-identity-response": { + "$ref": "#/components/examples/create-agent-identity-response" + } + } + } + } + }, + "get-agent-identity-success": { + "description": "成功获取 Agent 身份详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentIdentityResponse" + }, + "examples": { + "get-agent-identity-response": { + "$ref": "#/components/examples/get-agent-identity-response" + } + } + } + } + }, + "update-agent-identity-success": { + "description": "成功更新 Agent 身份设置", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentIdentityResponse" + }, + "examples": { + "update-agent-identity-response": { + "$ref": "#/components/examples/update-agent-identity-response" + } + } + } + } + }, + "list-agent-identities-success": { + "description": "成功获取 Agent 身份列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListAgentIdentitiesResponse" + }, + "examples": { + "list-agent-identities-response": { + "$ref": "#/components/examples/list-agent-identities-response" + } + } + } + } + }, + "reset-key-pair-success": { + "description": "成功重置 Agent 密钥对", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResetKeyPairResponse" + }, + "examples": { + "reset-key-pair-response": { + "$ref": "#/components/examples/reset-key-pair-response" + } + } + } + } + }, + "pause-agent-success": { + "description": "成功启用/停用 Agent", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PauseAgentResponse" + }, + "examples": { + "pause-agent-response": { + "$ref": "#/components/examples/pause-agent-response" + } + } + } + } + }, + "delete-agent-identity-success": { + "description": "成功删除 Agent 身份", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteAgentIdentityResponse" + }, + "examples": { + "delete-agent-identity-response": { + "$ref": "#/components/examples/delete-agent-identity-response" + } + } + } + } + }, + "issue-token-success": { + "description": "成功签发 Token", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TokenSignResponse" + }, + "examples": { + "issue-token-response": { + "$ref": "#/components/examples/issue-token-response" + } + } + } + } + }, + "list-token-records-success": { + "description": "成功获取 Token 签发记录列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListTokenRecordsResponse" + }, + "examples": { + "list-token-records-response": { + "$ref": "#/components/examples/list-token-records-response" + } + } + } + } + }, + "agentid-configuration-success": { + "description": "成功获取 Agent IDP OIDC 配置", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentIdConfiguration" + }, + "examples": { + "agentid-configuration-response": { + "$ref": "#/components/examples/agentid-configuration-response" + } + } + } + } + }, + "agentid-jwks-success": { + "description": "成功获取 JWKS", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JWKS" + }, + "examples": { + "agentid-jwks-response": { + "$ref": "#/components/examples/agentid-jwks-response" + } + } + } + } + }, + "get-collection-success": { + "description": "成功获取合集详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CollectionResponse" + }, + "examples": { + "get-collection-response": { + "$ref": "#/components/examples/get-collection-response" + } + } + } + } }, - "skill-bad-request": { - "description": "请求参数错误", + "create-collection-success": { + "description": "成功创建合集", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/CollectionMutationResponse" }, "examples": { - "skill-input-parameter-error": { - "$ref": "#/components/examples/skill-input-parameter-error" + "create-collection-response": { + "$ref": "#/components/examples/create-collection-response" } } } } }, - "skill-unauthorized": { - "description": "未通过认证", + "update-collection-success": { + "description": "成功更新合集", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/CollectionMutationResponse" }, "examples": { - "skill-invalid-authentication": { - "$ref": "#/components/examples/skill-invalid-authentication" + "update-collection-response": { + "$ref": "#/components/examples/update-collection-response" } } } } }, - "skill-not-found": { - "description": "技能未找到", + "delete-collection-success": { + "description": "成功删除合集", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/EmptyDataResponse" }, "examples": { - "skill-not-found": { - "$ref": "#/components/examples/skill-not-found" + "delete-collection-response": { + "$ref": "#/components/examples/delete-collection-response" } } } } }, - "skill-conflict": { - "description": "资源冲突（如 Skill 已存在）", + "list-collections-success": { + "description": "成功获取合集列表", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/CollectionListResponse" }, "examples": { - "skill-duplicate-entity": { - "$ref": "#/components/examples/skill-duplicate-entity" + "list-collections-response": { + "$ref": "#/components/examples/list-collections-response" } } } } }, - "skill-internal-server-error": { - "description": "内部服务错误", + "list-collection-items-success": { + "description": "成功获取合集条目列表", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/CollectionItemListResponse" }, "examples": { - "skill-internal-server-error": { - "$ref": "#/components/examples/skill-internal-server-error" + "list-collection-items-response": { + "$ref": "#/components/examples/list-collection-items-response" } } } } }, - "skill-service-unavailable": { - "description": "服务不可用", + "add-collection-items-success": { + "description": "成功添加合集条目（部分失败仍返回 success=true，见 failed_items）", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/AddCollectionItemsResponse" }, "examples": { - "skill-service-unavailable": { - "$ref": "#/components/examples/skill-service-unavailable" + "add-collection-items-response": { + "$ref": "#/components/examples/add-collection-items-response" } } } } }, - "upload-file-success": { - "description": "成功上传文件", + "update-collection-item-success": { + "description": "成功更新单个合集条目", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UploadFileResponse" + "$ref": "#/components/schemas/UpdateCollectionItemResponse" }, "examples": { - "upload-file-response": { - "$ref": "#/components/examples/upload-file-response" + "update-collection-item-response": { + "$ref": "#/components/examples/update-collection-item-response" } } } } }, - "create-agent-identity-success": { - "description": "成功创建 Agent 身份", + "remove-collection-item-success": { + "description": "成功移除合集条目", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AgentIdentityResponse" + "$ref": "#/components/schemas/EmptyDataResponse" }, "examples": { - "create-agent-identity-response": { - "$ref": "#/components/examples/create-agent-identity-response" + "remove-collection-item-response": { + "$ref": "#/components/examples/remove-collection-item-response" } } } } }, - "get-agent-identity-success": { - "description": "成功获取 Agent 身份详情", + "update-collection-items-success": { + "description": "成功批量更新合集条目（部分失败仍返回 success=true，见 failed_items）", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AgentIdentityResponse" + "$ref": "#/components/schemas/UpdateCollectionItemsResponse" }, "examples": { - "get-agent-identity-response": { - "$ref": "#/components/examples/get-agent-identity-response" + "update-collection-items-response": { + "$ref": "#/components/examples/update-collection-items-response" } } } } }, - "update-agent-identity-success": { - "description": "成功更新 Agent 身份设置", + "create-gallery-success": { + "description": "成功创建 Gallery", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AgentIdentityResponse" + "$ref": "#/components/schemas/CreateGalleryResponse" + }, + "examples": { + "create-gallery-response": { + "$ref": "#/components/examples/create-gallery-response" + } + } + } + } + }, + "create-agent-playground-instance-success": { + "description": "成功受理创建请求（异步创建，`status` 为 `starting`）", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentPlaygroundInstanceResponse" + }, + "examples": { + "create-agent-playground-instance-response": { + "$ref": "#/components/examples/create-agent-playground-instance-response" + } + } + } + } + }, + "create-agent-playground-instance-from-agent-hub-success": { + "description": "成功受理创建请求（`source` 为 `agent_hub`，`status` 为 `starting`）", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentPlaygroundInstanceResponse" + }, + "examples": { + "create-agent-playground-instance-from-agent-hub-response": { + "$ref": "#/components/examples/create-agent-playground-instance-from-agent-hub-response" + } + } + } + } + }, + "list-agent-playground-instances-success": { + "description": "成功获取实例列表", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentPlaygroundInstanceListResponse" + }, + "examples": { + "list-agent-playground-instances-response": { + "$ref": "#/components/examples/list-agent-playground-instances-response" + } + } + } + } + }, + "get-agent-playground-instance-success": { + "description": "成功获取实例详情", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentPlaygroundInstanceResponse" + }, + "examples": { + "get-agent-playground-instance-response": { + "$ref": "#/components/examples/get-agent-playground-instance-response" + } + } + } + } + }, + "update-agent-playground-instance-success": { + "description": "成功更新实例元信息", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentPlaygroundInstanceResponse" + }, + "examples": { + "update-agent-playground-instance-response": { + "$ref": "#/components/examples/update-agent-playground-instance-response" + } + } + } + } + }, + "delete-agent-playground-instance-success": { + "description": "成功受理删除（幂等）", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteAgentPlaygroundInstanceResponse" + }, + "examples": { + "delete-agent-playground-instance-response": { + "$ref": "#/components/examples/delete-agent-playground-instance-response" + } + } + } + } + }, + "pause-agent-playground-instance-success": { + "description": "成功受理暂停", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentPlaygroundInstanceResponse" + }, + "examples": { + "pause-agent-playground-instance-response": { + "$ref": "#/components/examples/pause-agent-playground-instance-response" + } + } + } + } + }, + "resume-agent-playground-instance-success": { + "description": "成功受理恢复", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentPlaygroundInstanceResponse" + }, + "examples": { + "resume-agent-playground-instance-response": { + "$ref": "#/components/examples/resume-agent-playground-instance-response" + } + } + } + } + }, + "restart-agent-playground-instance-success": { + "description": "成功受理重启（异步操作，`status` 为 `starting`）", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentPlaygroundInstanceResponse" + }, + "examples": { + "restart-agent-playground-instance-response": { + "$ref": "#/components/examples/restart-agent-playground-instance-response" + } + } + } + } + }, + "get-agent-playground-instance-agents-success": { + "description": "成功获取实例内 Agent 信息", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentPlaygroundAgentListResponse" + }, + "examples": { + "get-agent-playground-instance-agents-response": { + "$ref": "#/components/examples/get-agent-playground-instance-agents-response" + } + } + } + } + }, + "setup-agent-playground-instance-success": { + "description": "成功下发配置，返回最新的实例对象（下发不改变实例 `status`）", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentPlaygroundInstanceResponse" + }, + "examples": { + "setup-agent-playground-instance-response": { + "$ref": "#/components/examples/setup-agent-playground-instance-response" + } + } + } + } + }, + "import-to-agent-playground-instance-success": { + "description": "成功受理导入（`status` 为 `starting`，结果经一次性通知返回）", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentPlaygroundInstanceResponse" + }, + "examples": { + "import-to-agent-playground-instance-response": { + "$ref": "#/components/examples/import-to-agent-playground-instance-response" + } + } + } + } + }, + "get-agent-playground-instance-notifications-success": { + "description": "成功获取并消费未读通知", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentPlaygroundNotificationListResponse" + }, + "examples": { + "get-agent-playground-instance-notifications-response": { + "$ref": "#/components/examples/get-agent-playground-instance-notifications-response" + } + } + } + } + }, + "agent-playground-bad-request": { + "description": "参数非法、引用不存在、实例状态不满足前置条件、实例数超配额或内容安全不通过", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "examples": { + "agent-playground-input-parameter-error": { + "$ref": "#/components/examples/agent-playground-input-parameter-error" + } + } + } + } + }, + "agent-playground-unauthorized": { + "description": "未认证或 Token 无效", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" }, "examples": { - "update-agent-identity-response": { - "$ref": "#/components/examples/update-agent-identity-response" + "agent-playground-invalid-authentication": { + "$ref": "#/components/examples/agent-playground-invalid-authentication" } } } } }, - "list-agent-identities-success": { - "description": "成功获取 Agent 身份列表", + "agent-playground-forbidden": { + "description": "非资源 owner、跨账号操作或 Token 权限档不足", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ListAgentIdentitiesResponse" + "$ref": "#/components/schemas/ErrorResponse" }, "examples": { - "list-agent-identities-response": { - "$ref": "#/components/examples/list-agent-identities-response" + "agent-playground-operation-not-allowed": { + "$ref": "#/components/examples/agent-playground-operation-not-allowed" } } } } }, - "reset-key-pair-success": { - "description": "成功重置 Agent 密钥对", + "agent-playground-not-found": { + "description": "实例或 Agent Hub agent 不存在", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ResetKeyPairResponse" + "$ref": "#/components/schemas/ErrorResponse" }, "examples": { - "reset-key-pair-response": { - "$ref": "#/components/examples/reset-key-pair-response" + "agent-playground-resource-not-found": { + "$ref": "#/components/examples/agent-playground-resource-not-found" } } } } }, - "pause-agent-success": { - "description": "成功启用/停用 Agent", + "agent-playground-conflict": { + "description": "实例名重复；实例内同名 MCP 或 Skill 已存在；并发写冲突（可直接重试）", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/PauseAgentResponse" + "$ref": "#/components/schemas/ErrorResponse" }, "examples": { - "pause-agent-response": { - "$ref": "#/components/examples/pause-agent-response" + "agent-playground-duplicate-entity": { + "$ref": "#/components/examples/agent-playground-duplicate-entity" } } } } }, - "delete-agent-identity-success": { - "description": "成功删除 Agent 身份", + "agent-playground-internal-server-error": { + "description": "服务端内部错误，排查请携带 `request_id`", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/DeleteAgentIdentityResponse" + "$ref": "#/components/schemas/ErrorResponse" }, "examples": { - "delete-agent-identity-response": { - "$ref": "#/components/examples/delete-agent-identity-response" + "agent-playground-internal-server-error": { + "$ref": "#/components/examples/agent-playground-internal-server-error" } } } } }, - "issue-token-success": { - "description": "成功签发 Token", + "agent-playground-bad-gateway": { + "description": "实例内服务未就绪或实例内操作失败，可稍后重试", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/TokenSignResponse" + "$ref": "#/components/schemas/ErrorResponse" }, "examples": { - "issue-token-response": { - "$ref": "#/components/examples/issue-token-response" + "agent-playground-service-unavailable": { + "$ref": "#/components/examples/agent-playground-service-unavailable" } } } } }, - "list-token-records-success": { - "description": "成功获取 Token 签发记录列表", + "agent-playground-service-unavailable": { + "description": "实例内服务未就绪或实例内操作失败，可稍后重试", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ListTokenRecordsResponse" + "$ref": "#/components/schemas/ErrorResponse" }, "examples": { - "list-token-records-response": { - "$ref": "#/components/examples/list-token-records-response" + "agent-playground-service-unavailable": { + "$ref": "#/components/examples/agent-playground-service-unavailable" } } } } }, - "agentid-configuration-success": { - "description": "成功获取 Agent IDP OIDC 配置", + "agent-playground-request-timeout": { + "description": "实例内操作超时；操作可能仍在实例内继续执行", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AgentIdConfiguration" + "$ref": "#/components/schemas/ErrorResponse" }, "examples": { - "agentid-configuration-response": { - "$ref": "#/components/examples/agentid-configuration-response" + "agent-playground-request-timeout": { + "$ref": "#/components/examples/agent-playground-request-timeout" } } } } }, - "agentid-jwks-success": { - "description": "成功获取 JWKS", + "create-model-success": { + "description": "模型创建成功", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/JWKS" + "$ref": "#/components/schemas/CreateModelResponse" }, "examples": { - "agentid-jwks-response": { - "$ref": "#/components/examples/agentid-jwks-response" + "create-model-response": { + "$ref": "#/components/examples/model-repo-info-response" } } } } }, - "get-collection-success": { - "description": "成功获取合集详情", + "conflict": { + "description": "资源冲突", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CollectionResponse" + "$ref": "#/components/schemas/ErrorResponse" }, "examples": { - "get-collection-response": { - "$ref": "#/components/examples/get-collection-response" + "duplicate-entity": { + "$ref": "#/components/examples/duplicate-entity" } } } } }, - "create-collection-success": { - "description": "成功创建合集", + "set-model-category-success": { + "description": "模型类型设置/变更成功", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CollectionMutationResponse" + "$ref": "#/components/schemas/SetModelCategoryResponse" }, "examples": { - "create-collection-response": { - "$ref": "#/components/examples/create-collection-response" + "set-model-category-response": { + "$ref": "#/components/examples/model-repo-info-response" } } } } }, - "update-collection-success": { - "description": "成功更新合集", + "aigc-model-options-success": { + "description": "成功返回 AIGC 模型配置项枚举", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CollectionMutationResponse" + "$ref": "#/components/schemas/AigcModelOptionsResponse" }, "examples": { - "update-collection-response": { - "$ref": "#/components/examples/update-collection-response" + "aigc-model-options-response": { + "$ref": "#/components/examples/aigc-model-options-response" } } } } }, - "delete-collection-success": { - "description": "成功删除合集", + "too-many-requests": { + "description": "请求过于频繁", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/EmptyDataResponse" + "$ref": "#/components/schemas/ErrorResponse" }, "examples": { - "delete-collection-response": { - "$ref": "#/components/examples/delete-collection-response" + "rate-limit-exceed": { + "$ref": "#/components/examples/rate-limit-exceed" } } } } }, - "list-collections-success": { - "description": "成功获取合集列表", + "science-model-options-success": { + "description": "成功返回科学模型配置项枚举", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CollectionListResponse" + "$ref": "#/components/schemas/ScienceModelOptionsResponse" }, "examples": { - "list-collections-response": { - "$ref": "#/components/examples/list-collections-response" + "science-model-options-response": { + "$ref": "#/components/examples/science-model-options-response" } } } } }, - "list-collection-items-success": { - "description": "成功获取合集条目列表", + "list-studios-success": { + "description": "成功获取创空间列表", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CollectionItemListResponse" - }, - "examples": { - "list-collection-items-response": { - "$ref": "#/components/examples/list-collection-items-response" - } + "$ref": "#/components/schemas/ListStudiosResponse" } } } }, - "add-collection-items-success": { - "description": "成功添加合集条目（部分失败仍返回 success=true，见 failed_items）", + "get-balance-success": { + "description": "余额查询成功", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AddCollectionItemsResponse" + "$ref": "#/components/schemas/GetBalanceResponse" }, "examples": { - "add-collection-items-response": { - "$ref": "#/components/examples/add-collection-items-response" + "get-balance-response": { + "$ref": "#/components/examples/get-balance-response" } } } } + } + }, + "securitySchemes": { + "bearerAuth": { + "type": "http", + "scheme": "bearer", + "description": "ModelScope API Token" + } + }, + "parameters": { + "owner": { + "name": "owner", + "in": "path", + "description": "仓库 owner（组织或个人）", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "examples": [ + "OpenDataLab", + "Qwen", + "iic" + ] + } + }, + "repo_name": { + "name": "repo_name", + "in": "path", + "description": "仓库 repo_name 名称", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "examples": [ + "AICC", + "Qwen-Image-Edit-2511" + ] + } + }, + "mcp_server_id": { + "name": "id", + "in": "path", + "description": "服务ID。由平台后台收录的MCP，ID 默认为@author/server_name，当被社区用户认领后将会变更为user_name/server_name；由社区用户自发提交的MCP，ID 为user_name/server_name。", + "required": true, + "schema": { + "type": "string", + "examples": [ + "@modelcontextprotocol/fetch", + "username/server-name" + ] + } + }, + "studio_owner": { + "name": "owner", + "in": "path", + "description": "所有者（用户名或组织名）", + "required": true, + "schema": { + "type": "string", + "examples": [ + "username", + "organization" + ] + } + }, + "studio_repo_name": { + "name": "repo_name", + "in": "path", + "description": "仓库名称", + "required": true, + "schema": { + "type": "string", + "examples": [ + "my-demo-app" + ] + } + }, + "studio_log_type": { + "name": "log_type", + "in": "path", + "description": "日志类型：build（构建日志）或 run（运行日志）", + "required": true, + "schema": { + "type": "string", + "enum": [ + "build", + "run" + ], + "examples": [ + "run" + ] + } + }, + "skill_owner_path": { + "name": "owner", + "in": "path", + "description": "技能作者/所有者，支持 `@author` 格式（官方认证）或 `owner`（用户名或组织名），其中 `@` 无需 URL 编码", + "required": true, + "schema": { + "type": "string", + "minLength": 1 + }, + "examples": { + "author": { + "value": "@iSolver", + "summary": "官方认证作者" + }, + "user": { + "value": "iSolver", + "summary": "用户名或组织名" + } + } + }, + "skill_name_path": { + "name": "skill_name", + "in": "path", + "description": "技能英文名称，仅允许小写字母、数字和连字符", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9-]+$", + "examples": [ + "my-awesome-skill" + ] + } + }, + "agent_id": { + "name": "id", + "in": "path", + "description": "Agent ID，格式为 path/name（例如 my_org/weather-agent），其中 / 无需 URL 编码", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "examples": [ + "my_org/weather-agent" + ] + } + }, + "agent_playground_instance_uuid": { + "name": "uuid", + "in": "path", + "description": "实验场实例 ID", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "examples": [ + "51a7d9d2-a521-451a-baf5-367cb115a29a" + ] + } + }, + "collection_owner_path": { + "name": "owner", + "in": "path", + "description": "Collection 所有者（用户名或组织名）。具体长度与格式约束以当前产品页面实现为准。", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "examples": [ + "iSolver" + ] + } }, - "update-collection-item-success": { - "description": "成功更新单个合集条目", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/UpdateCollectionItemResponse" - }, - "examples": { - "update-collection-item-response": { - "$ref": "#/components/examples/update-collection-item-response" - } - } - } + "collection_slug_path": { + "name": "slug", + "in": "path", + "description": "Collection slug 尾段，由 owner 和 title 字段按一定规则自动生成；创建接口会返回该字段。", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "examples": [ + "best-nlp-models-6a3b2c" + ] } }, - "remove-collection-item-success": { - "description": "成功移除合集条目", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EmptyDataResponse" - }, - "examples": { - "remove-collection-item-response": { - "$ref": "#/components/examples/remove-collection-item-response" - } - } - } + "collection_item_type_path": { + "name": "item_type", + "in": "path", + "description": "资源类型：model / dataset / studio / paper / skill / mcp", + "required": true, + "schema": { + "type": "string", + "enum": [ + "model", + "dataset", + "studio", + "paper", + "skill", + "mcp" + ], + "examples": [ + "model" + ] } }, - "update-collection-items-success": { - "description": "成功批量更新合集条目（部分失败仍返回 success=true，见 failed_items）", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/UpdateCollectionItemsResponse" - }, - "examples": { - "update-collection-items-response": { - "$ref": "#/components/examples/update-collection-items-response" - } - } - } + "collection_item_object_id_path": { + "name": "item_object_id", + "in": "path", + "description": "资源标识，如 damo/nlp_bert_base。可包含 `/`（作为 items/{item_type}/ 之后的整段剩余路径），无需 URL 编码。", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "examples": [ + "damo/nlp_bert_base" + ] } - }, - "create-gallery-success": { - "description": "成功创建 Gallery", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateGalleryResponse" - }, - "examples": { - "create-gallery-response": { - "$ref": "#/components/examples/create-gallery-response" - } - } - } + } + }, + "examples": { + "user-all-response": { + "value": { + "success": true, + "data": { + "username": "tester", + "nickname": "测试用户", + "description": "这是一个测试用户", + "email": "test@modelscope.cn", + "avatar_url": "https://resouces.modelscope.cn/avatar/516aa51f-5fb7-49eb-930c-e90500ae13ca.png" + }, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" } }, - "list-studios-success": { - "description": "成功获取创空间列表", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ListStudiosResponse" - } + "list-models-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": { + "models": [ + { + "id": "iic/QwenLong-L1.5-30B-A3B", + "display_name": "QwenLong-L1.5-30B-A3B", + "description": "", + "downloads": 1466, + "likes": 25, + "license": "apache-2.0", + "tasks": [ + "text-generation" + ], + "created_at": "2025-12-14T12:26:15Z", + "last_modified": "2025-12-16T03:34:26Z", + "file_size": 61081671123, + "params": 30532122624, + "tags": [ + "license:apache-2.0", + "model_type:qwen3_moe", + "library:transformer", + "library:safetensors", + "library:pytorch", + "task:text-generation", + "deploy:swingdeploy" + ], + "private": false, + "gated": false + }, + { + "id": "ZhipuAI/AutoGLM-Phone-9B", + "display_name": "AutoGLM-Phone-9B", + "description": "", + "downloads": 28525, + "likes": 265, + "license": "MIT License", + "tasks": [ + "image-captioning" + ], + "created_at": "2025-12-08T08:00:28Z", + "last_modified": "2025-12-09T15:22:57Z", + "file_size": 20605693975, + "params": 5368163840, + "tags": [ + "license:MIT License", + "model_type:glm4v", + "library:safetensors", + "library:pytorch", + "task:image-captioning", + "deploy:swingdeploy" + ], + "private": false, + "gated": false + } + ], + "total_count": 43127, + "page_number": 1, + "page_size": 2 } } }, - "conflict": { - "description": "资源冲突", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "examples": { - "duplicate-entity": { - "$ref": "#/components/examples/duplicate-entity" - } - } + "get-model-response": { + "summary": "普通模型", + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd2", + "data": { + "created_at": "2026-09-21T06:36:57Z", + "description": "用于文本生成的模型", + "display_name": "示例模型", + "downloads": 0, + "file_size": 0, + "gated": false, + "id": "alice/my-model", + "last_modified": "2026-09-21T06:37:01Z", + "license": "apache-2.0", + "likes": 0, + "model_category": "base", + "owner": "alice", + "params": 0, + "private": true, + "readme": "# 示例模型\n这里是仓库介绍。", + "repo_name": "my-model", + "tags": [ + "license:apache-2.0" + ], + "tasks": [], + "visibility": "private" } } }, - "too-many-requests": { - "description": "请求过于频繁", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "examples": { - "rate-limit-exceed": { - "$ref": "#/components/examples/rate-limit-exceed" + "list-datasets-response": { + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": { + "datasets": [ + { + "id": "DAMO_NLP/jd", + "display_name": "商品评论情感预测", + "description": "该数据集包含从2011年1月1日到2014年3月31日（3年多）某电商网站的消费者购买行为，用户评分，评论和产品元数据，涵盖15个一级产品类别，987个二级产品类别，近2个百万用户，超过10万种产品和超过6,000万条评论。该数据集中的每个文本评论都包含三个子评论：正面评论，负面评论和整体评论。", + "file_size": 0, + "downloads": 53996, + "likes": 102, + "license": "Apache License 2.0", + "tasks": [ + "text-classification" + ], + "created_at": "2022-09-28T06:30:31Z", + "last_modified": "2022-10-30T03:36:39Z", + "tags": [ + "license:Apache License 2.0", + "task:text-classification" + ], + "private": false, + "gated": false, + "login_required": false + }, + { + "id": "iic/nlp_domain_classification_chinese_testset", + "display_name": "中文文本领域分类测试集", + "description": "中文文本领域分类测试集，涉及国民经济行业标准的18个行业，部分收集自公开数据", + "file_size": 0, + "downloads": 2401, + "likes": 23, + "license": "Apache License 2.0", + "tasks": [ + "text-classification" + ], + "created_at": "2023-02-06T09:21:21Z", + "last_modified": "2024-09-02T15:16:35Z", + "tags": [ + "license:Apache License 2.0", + "task:text-classification", + "custom_tag:文本领域", + "custom_tag:领域", + "custom_tag:文本分类" + ], + "private": false, + "gated": false, + "login_required": false } - } + ], + "total_count": 60, + "page_number": 1, + "page_size": 2 } } }, - "get-balance-success": { - "description": "余额查询成功", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/GetBalanceResponse" - }, - "examples": { - "get-balance-response": { - "$ref": "#/components/examples/get-balance-response" - } - } + "get-dataset-response": { + "summary": "普通数据集", + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd2", + "data": { + "created_at": "2025-11-26T11:01:59Z", + "dataset_category": "base", + "description": "示例数据集说明", + "display_name": "示例数据集", + "downloads": 7239, + "file_size": 121668031, + "gated": false, + "id": "alice/my-dataset", + "last_modified": "2026-05-06T12:10:16Z", + "license": "apache-2.0", + "likes": 9, + "login_required": false, + "owner": "alice", + "private": false, + "readme": "# 示例数据集\n这里是仓库介绍。", + "repo_name": "my-dataset", + "tags": [ + "license:apache-2.0", + "custom_tag:RoboCOIN", + "custom_tag:LeRobot" + ], + "tasks": [], + "visibility": "public" } } - } - }, - "securitySchemes": { - "bearerAuth": { - "type": "http", - "scheme": "bearer", - "description": "ModelScope API Token" - } - }, - "parameters": { - "owner": { - "name": "owner", - "in": "path", - "description": "仓库 owner（组织或个人）", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "examples": [ - "OpenDataLab", - "Qwen", - "iic" - ] - } }, - "repo_name": { - "name": "repo_name", - "in": "path", - "description": "仓库 repo_name 名称", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "examples": [ - "AICC", - "Qwen-Image-Edit-2511" - ] + "list-mcp-servers-response": { + "value": { + "success": true, + "data": { + "mcp_server_list": [ + { + "id": "@modelcontextprotocol/fetch", + "name": "Fetch网页内容抓取", + "chinese_name": "Fetch网页内容抓取", + "description": "该服务器使大型语言模型能够检索和处理网页内容，支持多种格式的内容提取和转换。", + "logo_url": "https://resources.modelscope.cn/mcp-cover-img/fetch.png", + "publisher": "@modelcontextprotocol/fetch", + "categories": [ + "browser-automation" + ], + "tags": [ + "browser-automation", + "web-scraping" + ], + "view_count": 30667, + "locales": { + "zh": { + "name": "Fetch网页内容抓取", + "description": "该服务器使大型语言模型能够检索和处理网页内容..." + }, + "en": { + "name": "Fetch", + "description": "A server that enables LLMs to retrieve and process web content..." + } + } + }, + { + "id": "@anthropic/claude-mcp", + "name": "Claude MCP", + "chinese_name": "Claude MCP服务", + "description": "Anthropic 官方 MCP 服务，提供 Claude 模型的标准化访问接口。", + "logo_url": "https://resources.modelscope.cn/mcp-cover-img/claude.png", + "publisher": "@anthropic/claude-mcp", + "categories": [ + "ai-assistant" + ], + "tags": [ + "ai-assistant", + "claude" + ], + "view_count": 15234, + "locales": { + "zh": { + "name": "Claude MCP服务", + "description": "Anthropic 官方 MCP 服务..." + }, + "en": { + "name": "Claude MCP", + "description": "Official Anthropic MCP service..." + } + } + } + ], + "total_count": 100 + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "mcp_server_id": { - "name": "id", - "in": "path", - "description": "服务ID。由平台后台收录的MCP，ID 默认为@author/server_name，当被社区用户认领后将会变更为user_name/server_name；由社区用户自发提交的MCP，ID 为user_name/server_name。", - "required": true, - "schema": { - "type": "string", - "examples": [ - "@modelcontextprotocol/fetch", - "username/server-name" - ] + "list-operational-mcp-servers-response": { + "value": { + "success": true, + "data": { + "mcp_server_list": [ + { + "id": "@modelcontextprotocol/fetch", + "name": "Fetch网页内容抓取", + "chinese_name": "Fetch网页内容抓取", + "description": "该服务器使大型语言模型能够检索和处理网页内容...", + "logo_url": "https://resources.modelscope.cn/mcp-cover-img/fetch.png", + "publisher": "@modelcontextprotocol/fetch", + "categories": [ + "browser-automation" + ], + "tags": [ + "browser-automation" + ], + "view_count": 30667, + "operational_urls": [ + { + "id": "platform-pool", + "url": "https://mcp.api-inference.modelscope.net/abc123/mcp", + "transport_type": "streamable_http", + "auth_required": false, + "expiration": "2025-10-01 21:00:00", + "accessible": true + } + ] + } + ], + "total_count": 1 + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "studio_owner": { - "name": "owner", - "in": "path", - "description": "所有者（用户名或组织名）", - "required": true, - "schema": { - "type": "string", - "examples": [ - "username", - "organization" - ] + "get-mcp-server-response": { + "value": { + "success": true, + "data": { + "id": "@modelcontextprotocol/fetch", + "name": "Fetch网页内容抓取", + "chinese_name": "Fetch网页内容抓取", + "description": "该服务器使大型语言模型能够检索和处理网页内容，支持多种格式的内容提取和转换。", + "logo_url": "https://resources.modelscope.cn/mcp-cover-img/fetch.png", + "publisher": "@modelcontextprotocol/fetch", + "author": "modelcontextprotocol", + "owner": "", + "readme": "# Fetch MCP Server\n\nA Model Context Protocol server that provides web content fetching capabilities.\n\n## Features\n\n- Fetch web pages and extract content\n- Support for multiple output formats\n- Automatic content cleaning and formatting", + "source_url": "https://github.com/modelcontextprotocol/servers/tree/main/src/fetch", + "github_stars": 1234, + "is_hosted": true, + "is_verified": true, + "categories": [ + "browser-automation" + ], + "tags": [ + "browser-automation", + "web-scraping" + ], + "view_count": 30667, + "env_schema": {}, + "server_config": [], + "operational_urls": [], + "locales": { + "zh": { + "name": "Fetch网页内容抓取", + "description": "该服务器使大型语言模型能够检索和处理网页内容...", + "readme": "# Fetch MCP Server\n\n提供网页内容抓取能力的 MCP 服务。" + }, + "en": { + "name": "Fetch", + "description": "A server that enables LLMs to retrieve and process web content...", + "readme": "# Fetch MCP Server\n\nA Model Context Protocol server that provides web content fetching capabilities." + } + } + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "studio_repo_name": { - "name": "repo_name", - "in": "path", - "description": "仓库名称", - "required": true, - "schema": { - "type": "string", - "examples": [ - "my-demo-app" - ] + "deploy-mcp-server-response": { + "value": { + "success": true, + "data": { + "id": "platform-pool", + "url": "https://mcp.api-inference.modelscope.net/abc123def456/mcp", + "transport_type": "streamable_http", + "auth_required": false, + "expiration": "2025-10-01 21:00:00" + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "studio_log_type": { - "name": "log_type", - "in": "path", - "description": "日志类型：build（构建日志）或 run（运行日志）", - "required": true, - "schema": { - "type": "string", - "enum": [ - "build", - "run" - ], - "examples": [ - "run" - ] + "undeploy-mcp-server-response": { + "value": { + "success": true, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "skill_owner_path": { - "name": "owner", - "in": "path", - "description": "技能作者/所有者，支持 `@author` 格式（官方认证）或 `owner`（用户名或组织名），其中 `@` 无需 URL 编码", - "required": true, - "schema": { - "type": "string", - "minLength": 1 - }, - "examples": { - "author": { - "value": "@iSolver", - "summary": "官方认证作者" + "create-studio-response": { + "value": { + "success": true, + "data": { + "id": "username/my-demo-app", + "repo_name": "my-demo-app", + "display_name": "我的演示应用", + "owner": "username", + "url": "https://modelscope.cn/studios/username/my-demo-app" }, - "user": { - "value": "iSolver", - "summary": "用户名或组织名" - } + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "skill_name_path": { - "name": "skill_name", - "in": "path", - "description": "技能英文名称，仅允许小写字母、数字和连字符", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 64, - "pattern": "^[a-z0-9-]+$", - "examples": [ - "my-awesome-skill" - ] + "get-studio-response": { + "value": { + "success": true, + "data": { + "id": "username/my-demo-app", + "repo_name": "my-demo-app", + "display_name": "我的演示应用", + "owner": "username", + "description": "一个演示创空间应用", + "likes": 100, + "view_count": 5000, + "tags": [ + "nlp", + "chatbot" + ], + "visibility": "public", + "private": false, + "last_modified": "2025-01-15T10:30:00Z", + "sdk_type": "gradio", + "sdk_version": "6.2.0", + "hardware": "platform/2v-cpu-16g-mem", + "base_image": "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0", + "license": "apache-2.0", + "host": "https://username-my-demo-app.ms.show", + "mcp_support": false, + "runtime": { + "status": "Running", + "active_config": { + "hardware": "platform/2v-cpu-16g-mem", + "base_image": "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0", + "sdk_type": "gradio", + "sdk_version": "6.2.0" + }, + "created_at": "2025-01-15T10:30:00Z" + } + }, + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "collection_owner_path": { - "name": "owner", - "in": "path", - "description": "Collection 所有者（用户名或组织名）。具体长度与格式约束以当前产品页面实现为准。", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "examples": [ - "iSolver" - ] + "studio-runtime-response": { + "value": { + "success": true, + "data": { + "status": "Running", + "active_config": { + "hardware": "platform/2v-cpu-16g-mem", + "base_image": "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0", + "sdk_type": "gradio", + "sdk_version": "6.2.0" + }, + "created_at": "2025-01-15T10:30:00Z" + }, + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "collection_slug_path": { - "name": "slug", - "in": "path", - "description": "Collection slug 尾段，由 owner 和 title 字段按一定规则自动生成；创建接口会返回该字段。", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "examples": [ - "best-nlp-models-6a3b2c" - ] + "get-studio-logs-response": { + "value": { + "success": true, + "data": { + "logs": [ + "2025-01-15 10:30:00 INFO Starting application...", + "2025-01-15 10:30:01 INFO Application started successfully" + ], + "page_num": 1, + "page_size": 100, + "total_count": 500, + "total_page_num": 5 + }, + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "collection_item_type_path": { - "name": "item_type", - "in": "path", - "description": "资源类型：model / dataset / studio / paper / skill / mcp", - "required": true, - "schema": { - "type": "string", - "enum": [ - "model", - "dataset", - "studio", - "paper", - "skill", - "mcp" - ], - "examples": [ - "model" - ] + "list-sdk-versions-response": { + "value": { + "success": true, + "data": { + "sdk_versions": [ + { + "sdk_type": "gradio", + "tag": "latest", + "version": "6.2.0" + }, + { + "sdk_type": "gradio", + "tag": "stable", + "version": "5.49.1" + } + ], + "total_count": 2 + }, + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "list-skills-response": { + "value": { + "success": true, + "request_id": "c5bca202-aa4f-478b-b7e2-c568160583cb", + "data": { + "skills": [ + { + "_id": "CIK3H+DYKlSAMlHnX60AUg==", + "id": "@AMap-Web/amap-lbs-skill", + "display_name": "高德地图综合服务Skill", + "description": "高德地图综合服务，支持POI搜索、路径规划、旅游规划、周边搜索和热力图数据可视化", + "owner": "", + "license": "MIT License", + "developer": "AMap-Web", + "source_url": "https://github.com/AMap-Web/amap-lbs-skill", + "category": "developer-tools", + "tags": [ + "category:developer-tools", + "license:MIT License", + "developer:AMap-Web", + "custom_tag:api-design", + "custom_tag:general-tools" + ], + "logo_url": "", + "view_count": 6144, + "downloads": 791, + "locales": { + "en": { + "description": "Amap comprehensive services, supporting POI search, route planning, travel planning, nearby search, and heatmap data visualization", + "category": "DevTools" + }, + "zh": { + "description": "高德地图综合服务，支持POI搜索、路径规划、旅游规划、周边搜索和热力图数据可视化", + "category": "开发工具" + } + }, + "private": false, + "custom_tag": [ + "api-design", + "general-tools" + ] + }, + { + "_id": "0ry9hAiAiJ+2pklbvQjepg==", + "id": "@steipete/summarize", + "display_name": "summarize", + "description": "使用summarize CLI（适用于网页、PDF、图片、音频、YouTube）来总结URL或文件。", + "owner": "steipete", + "license": "", + "developer": "steipete", + "source_url": "https://clawhub.ai/steipete/summarize", + "category": "ai-media", + "tags": [ + "category:ai-media", + "developer:steipete", + "custom_tag:image-video-gen", + "custom_tag:media-processing" + ], + "logo_url": "", + "view_count": 4696, + "downloads": 690, + "locales": { + "en": { + "description": "Summarize URLs or files with the summarize CLI (web, PDFs, images, audio, YouTube).", + "category": "MediaAI" + }, + "zh": { + "description": "使用summarize CLI（适用于网页、PDF、图片、音频、YouTube）来总结URL或文件。", + "category": "媒体处理" + } + }, + "private": false, + "custom_tag": [ + "image-video-gen", + "media-processing" + ] + } + ], + "total": 20, + "page_number": 1, + "page_size": 20 + } } }, - "collection_item_object_id_path": { - "name": "item_object_id", - "in": "path", - "description": "资源标识，如 damo/nlp_bert_base。可包含 `/`（作为 items/{item_type}/ 之后的整段剩余路径），无需 URL 编码。", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "examples": [ - "damo/nlp_bert_base" - ] + "get-skill-response": { + "value": { + "success": true, + "request_id": "105c898f-fd6d-437d-aa0b-9006036782da", + "data": { + "_id": "CIK3H+DYKlSAMlHnX60AUg==", + "id": "@AMap-Web/amap-lbs-skill", + "display_name": "高德地图综合服务Skill", + "description": "高德地图综合服务，支持POI搜索、路径规划、旅游规划、周边搜索和热力图数据可视化", + "owner": "", + "license": "MIT License", + "developer": "AMap-Web", + "source_url": "https://github.com/AMap-Web/amap-lbs-skill", + "category": "developer-tools", + "tags": [ + "category:developer-tools", + "license:MIT License", + "developer:AMap-Web", + "custom_tag:api-design", + "custom_tag:general-tools" + ], + "logo_url": "", + "view_count": 6144, + "downloads": 791, + "locales": { + "en": { + "description": "Amap comprehensive services, supporting POI search, route planning, travel planning, nearby search, and heatmap data visualization", + "category": "DevTools" + }, + "zh": { + "description": "高德地图综合服务，支持POI搜索、路径规划、旅游规划、周边搜索和热力图数据可视化", + "category": "开发工具" + } + }, + "private": false, + "custom_tag": [ + "api-design", + "general-tools" + ], + "install_command": [ + "npx skills add https://modelscope.cn/skills/@AMap-Web/amap-lbs-skill", + "curl -fsSL https://modelscope.cn/skills/install.sh | bash -s -- @AMap-Web/amap-lbs-skill" + ] + } } - } - }, - "examples": { - "user-all-response": { + }, + "create-skill-response": { "value": { "success": true, + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363", "data": { - "username": "tester", - "nickname": "测试用户", - "description": "这是一个测试用户", - "email": "test@modelscope.cn", - "avatar_url": "https://resouces.modelscope.cn/avatar/516aa51f-5fb7-49eb-930c-e90500ae13ca.png" - }, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd2" + "id": "iSolver/my-awesome-skill", + "name": "my-awesome-skill", + "display_name": "我的技能", + "owner": "iSolver", + "url": "https://modelscope.cn/skills/iSolver/my-awesome-skill" + } } }, - "list-models-response": { + "update-skill-settings-response": { "value": { "success": true, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "list-agents-response": { + "value": { + "success": true, + "request_id": "c5bca202-aa4f-478b-b7e2-c568160583cb", "data": { - "models": [ + "agents": [ { - "id": "iic/QwenLong-L1.5-30B-A3B", - "display_name": "QwenLong-L1.5-30B-A3B", - "description": "", - "downloads": 1466, - "likes": 25, - "license": "apache-2.0", - "tasks": [ - "text-generation" + "id": "my_org/weather-agent", + "name": "weather-agent", + "owner": "my_org", + "display_name": "天气助手", + "description": "一个可以查询天气的 Agent", + "framework": "ms-agent", + "logo_url": "https://example.com/logo.png", + "license": "Apache License 2.0", + "private": false, + "categories": [ + "tools" + ], + "custom_tags": [ + "weather", + "demo" ], - "created_at": "2025-12-14T12:26:15Z", - "last_modified": "2025-12-16T03:34:26Z", - "file_size": 61081671123, - "params": 30532122624, "tags": [ - "license:apache-2.0", - "model_type:qwen3_moe", - "library:transformer", - "library:safetensors", - "library:pytorch", - "task:text-generation", - "deploy:swingdeploy" + "category:tools", + "license:Apache License 2.0", + "framework:ms-agent", + "custom_tag:weather", + "custom_tag:demo" ], - "private": false, - "gated": false + "views": 319, + "downloads": 15, + "likes": 8, + "last_modified": "2026-07-16T10:30:00Z", + "created_at": "2026-07-01T08:00:00Z" }, { - "id": "ZhipuAI/AutoGLM-Phone-9B", - "display_name": "AutoGLM-Phone-9B", - "description": "", - "downloads": 28525, - "likes": 265, + "id": "zhangsan/deep-research", + "name": "deep-research", + "owner": "zhangsan", + "display_name": "深度研报助手", + "description": "多步检索与研报写作 Agent", + "framework": "qwenpaw", + "logo_url": "", "license": "MIT License", - "tasks": [ - "image-captioning" - ], - "created_at": "2025-12-08T08:00:28Z", - "last_modified": "2025-12-09T15:22:57Z", - "file_size": 20605693975, - "params": 5368163840, + "private": false, + "categories": null, + "custom_tags": null, "tags": [ "license:MIT License", - "model_type:glm4v", - "library:safetensors", - "library:pytorch", - "task:image-captioning", - "deploy:swingdeploy" + "framework:qwenpaw" ], - "private": false, - "gated": false + "views": 149, + "downloads": 3, + "likes": 1, + "last_modified": "2026-07-10T08:12:00Z", + "created_at": "2026-07-02T08:00:00Z" } ], - "total_count": 43127, + "total": 20, "page_number": 1, - "page_size": 2 + "page_size": 10 } } }, - "get-model-response": { + "create-agent-response": { "value": { "success": true, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "request_id": "2f1c8a3e-9b7d-4f21-a0c6-1e2d3f4a5b6c", "data": { - "id": "Qwen/Qwen-Image-Edit-2511", - "display_name": "Qwen-Image-Edit-2511", - "description": "", - "downloads": 20175, - "likes": 149, - "license": "apache-2.0", - "tasks": [ - "image-to-image" + "id": "my_org/weather-agent", + "name": "weather-agent", + "display_name": "天气助手", + "owner": "my_org", + "url": "https://modelscope.cn/agents/my_org/weather-agent" + } + } + }, + "get-agent-response": { + "value": { + "success": true, + "request_id": "7a2b1c3d-4e5f-6a7b-8c9d-0e1f2a3b4c5d", + "data": { + "id": "my_org/weather-agent", + "display_name": "天气助手", + "description": "一个可以查询天气的 Agent", + "framework": "ms-agent", + "logo_url": "https://example.com/logo.png", + "license": "Apache License 2.0", + "private": false, + "custom_tags": [ + "weather", + "demo" ], - "created_at": "2025-12-18T01:32:15Z", - "last_modified": "2025-12-23T14:09:52Z", - "file_size": 57720465119, - "params": 0, - "tags": [ - "license:apache-2.0", - "library:pytorch", - "library:safetensors", - "library:diffusers", - "task:image-to-image" + "categories": [ + "tools" ], - "private": false, - "gated": false, - "readme": "hello" + "last_modified": "2026-07-16T10:30:00Z", + "downloads": 15, + "likes": 8, + "mcps": [ + { + "name": "weather-mcp", + "description": "天气查询 MCP", + "tools": null + } + ], + "skills": [ + { + "name": "search-skill", + "description": "搜索技能", + "content": "# Search Skill\n..." + } + ], + "system_files": [ + { + "name": "agent.json", + "content": "{...}" + } + ], + "models": [ + { + "model_name": "qwen-max", + "provider_id": "dashscope", + "base_url": "https://dashscope.aliyuncs.com", + "protocol": "openai", + "provider": "DashScope" + } + ], + "readme": "# Weather Agent\n使用说明..." + } + } + }, + "upload-file-response": { + "value": { + "success": true, + "request_id": "99ee2083-63e0-46dc-b393-623c22645078", + "data": { + "id": "8c378570-8991-431b-a82c-96f3d0b4f0f4" } } }, - "list-datasets-response": { + "create-agent-identity-response": { + "value": { + "success": true, + "data": { + "agent_id": "agent_id:modelscope:agent_1234567890ab", + "agent_name": "my-agent", + "description": "用于演示的测试 Agent", + "token_expire_time": 3600, + "principal": { + "type": "user", + "id": "alice" + }, + "kid": "agent-key-001", + "public_key": { + "kty": "OKP", + "crv": "Ed25519", + "x": "f7QtzyEJhqFL8Jk5OYOq7010Ry9PipDGQkTzl7XtiEs", + "kid": "agent-key-001", + "alg": "EdDSA", + "use": "sig" + }, + "status": "active", + "create_time": "2026-06-08T10:00:00Z" + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "get-agent-identity-response": { + "value": { + "success": true, + "data": { + "agent_id": "agent_id:modelscope:agent_1234567890ab", + "agent_name": "my-agent", + "description": "用于演示的测试 Agent", + "token_expire_time": 3600, + "principal": { + "type": "user", + "id": "alice" + }, + "kid": "agent-key-001", + "public_key": { + "kty": "OKP", + "crv": "Ed25519", + "x": "f7QtzyEJhqFL8Jk5OYOq7010Ry9PipDGQkTzl7XtiEs", + "kid": "agent-key-001", + "alg": "EdDSA", + "use": "sig" + }, + "status": "active", + "create_time": "2026-06-08T10:00:00Z", + "update_time": "2026-06-08T10:30:00Z" + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "update-agent-identity-response": { + "value": { + "success": true, + "data": { + "agent_id": "agent_id:modelscope:agent_1234567890ab", + "agent_name": "my-agent-renamed", + "description": "更新后的描述", + "token_expire_time": 1800, + "principal": { + "type": "user", + "id": "alice" + }, + "kid": "agent-key-001", + "public_key": { + "kty": "OKP", + "crv": "Ed25519", + "x": "f7QtzyEJhqFL8Jk5OYOq7010Ry9PipDGQkTzl7XtiEs", + "kid": "agent-key-001", + "alg": "EdDSA", + "use": "sig" + }, + "status": "active", + "create_time": "2026-06-08T10:00:00Z", + "update_time": "2026-06-08T11:00:00Z" + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + } + }, + "list-agent-identities-response": { "value": { "success": true, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", "data": { - "datasets": [ + "agent_identities": [ { - "id": "DAMO_NLP/jd", - "display_name": "商品评论情感预测", - "description": "该数据集包含从2011年1月1日到2014年3月31日（3年多）某电商网站的消费者购买行为，用户评分，评论和产品元数据，涵盖15个一级产品类别，987个二级产品类别，近2个百万用户，超过10万种产品和超过6,000万条评论。该数据集中的每个文本评论都包含三个子评论：正面评论，负面评论和整体评论。", - "file_size": 0, - "downloads": 53996, - "likes": 102, - "license": "Apache License 2.0", - "tasks": [ - "text-classification" - ], - "created_at": "2022-09-28T06:30:31Z", - "last_modified": "2022-10-30T03:36:39Z", - "tags": [ - "license:Apache License 2.0", - "task:text-classification" - ], - "private": false, - "gated": false, - "login_required": false + "agent_id": "agent_id:modelscope:agent_1234567890ab", + "agent_name": "my-agent", + "kid": "agent-key-001", + "status": "active", + "token_expire_time": 3600, + "create_time": "2026-06-08T10:00:00Z" }, { - "id": "iic/nlp_domain_classification_chinese_testset", - "display_name": "中文文本领域分类测试集", - "description": "中文文本领域分类测试集，涉及国民经济行业标准的18个行业，部分收集自公开数据", - "file_size": 0, - "downloads": 2401, - "likes": 23, - "license": "Apache License 2.0", - "tasks": [ - "text-classification" - ], - "created_at": "2023-02-06T09:21:21Z", - "last_modified": "2024-09-02T15:16:35Z", - "tags": [ - "license:Apache License 2.0", - "task:text-classification", - "custom_tag:文本领域", - "custom_tag:领域", - "custom_tag:文本分类" - ], - "private": false, - "gated": false, - "login_required": false + "agent_id": "agent-cdef01234567", + "agent_name": "build-bot", + "kid": "agent-key-002", + "status": "paused", + "token_expire_time": 1800, + "create_time": "2026-06-07T15:20:00Z" } ], - "total_count": 60, + "total_count": 2, "page_number": 1, - "page_size": 2 - } + "page_size": 20 + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "get-dataset-response": { + "reset-key-pair-response": { "value": { "success": true, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", "data": { - "id": "OpenDataLab/AICC", - "display_name": "AICC", - "description": "", - "file_size": 0, - "downloads": 12119, - "likes": 7, - "license": "cc-by-4.0", - "tasks": [], - "created_at": "2025-11-25T14:30:22Z", - "last_modified": "2026-01-01T00:31:42Z", - "tags": [ - "license:cc-by-4.0", - "custom_tag:common-crawl", - "custom_tag:code", - "custom_tag:markdown", - "custom_tag:math", - "custom_tag:html-parsing" - ], - "private": false, - "gated": false, - "login_required": false, - "readme": "hello" - } + "agent_id": "agent_id:modelscope:agent_1234567890ab", + "kid": "agent-key-002", + "public_key": { + "kty": "OKP", + "crv": "Ed25519", + "x": "UnZsPxBV8ZyK5zhQlpnT1nf5GjxJQbpHQMwoS0jZeNc", + "kid": "agent-key-002", + "alg": "EdDSA", + "use": "sig" + } + }, + "message": "success", + "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "list-mcp-servers-response": { + "pause-agent-response": { "value": { "success": true, - "data": { - "mcp_server_list": [ - { - "id": "@modelcontextprotocol/fetch", - "name": "Fetch网页内容抓取", - "chinese_name": "Fetch网页内容抓取", - "description": "该服务器使大型语言模型能够检索和处理网页内容，支持多种格式的内容提取和转换。", - "logo_url": "https://resources.modelscope.cn/mcp-cover-img/fetch.png", - "publisher": "@modelcontextprotocol/fetch", - "categories": [ - "browser-automation" - ], - "tags": [ - "browser-automation", - "web-scraping" - ], - "view_count": 30667, - "locales": { - "zh": { - "name": "Fetch网页内容抓取", - "description": "该服务器使大型语言模型能够检索和处理网页内容..." - }, - "en": { - "name": "Fetch", - "description": "A server that enables LLMs to retrieve and process web content..." - } - } - }, - { - "id": "@anthropic/claude-mcp", - "name": "Claude MCP", - "chinese_name": "Claude MCP服务", - "description": "Anthropic 官方 MCP 服务，提供 Claude 模型的标准化访问接口。", - "logo_url": "https://resources.modelscope.cn/mcp-cover-img/claude.png", - "publisher": "@anthropic/claude-mcp", - "categories": [ - "ai-assistant" - ], - "tags": [ - "ai-assistant", - "claude" - ], - "view_count": 15234, - "locales": { - "zh": { - "name": "Claude MCP服务", - "description": "Anthropic 官方 MCP 服务..." - }, - "en": { - "name": "Claude MCP", - "description": "Official Anthropic MCP service..." - } - } - } - ], - "total_count": 100 - }, "message": "success", "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "list-operational-mcp-servers-response": { + "delete-agent-identity-response": { "value": { "success": true, - "data": { - "mcp_server_list": [ - { - "id": "@modelcontextprotocol/fetch", - "name": "Fetch网页内容抓取", - "chinese_name": "Fetch网页内容抓取", - "description": "该服务器使大型语言模型能够检索和处理网页内容...", - "logo_url": "https://resources.modelscope.cn/mcp-cover-img/fetch.png", - "publisher": "@modelcontextprotocol/fetch", - "categories": [ - "browser-automation" - ], - "tags": [ - "browser-automation" - ], - "view_count": 30667, - "operational_urls": [ - { - "id": "platform-pool", - "url": "https://mcp.api-inference.modelscope.net/abc123/mcp", - "transport_type": "streamable_http", - "auth_required": false, - "expiration": "2025-10-01 21:00:00", - "accessible": true - } - ] - } - ], - "total_count": 1 - }, "message": "success", "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "get-mcp-server-response": { + "issue-token-response": { "value": { "success": true, "data": { - "id": "@modelcontextprotocol/fetch", - "name": "Fetch网页内容抓取", - "chinese_name": "Fetch网页内容抓取", - "description": "该服务器使大型语言模型能够检索和处理网页内容，支持多种格式的内容提取和转换。", - "logo_url": "https://resources.modelscope.cn/mcp-cover-img/fetch.png", - "publisher": "@modelcontextprotocol/fetch", - "author": "modelcontextprotocol", - "owner": "", - "readme": "# Fetch MCP Server\n\nA Model Context Protocol server that provides web content fetching capabilities.\n\n## Features\n\n- Fetch web pages and extract content\n- Support for multiple output formats\n- Automatic content cleaning and formatting", - "source_url": "https://github.com/modelcontextprotocol/servers/tree/main/src/fetch", - "github_stars": 1234, - "is_hosted": true, - "is_verified": true, - "categories": [ - "browser-automation" - ], - "tags": [ - "browser-automation", - "web-scraping" - ], - "view_count": 30667, - "env_schema": {}, - "server_config": [], - "operational_urls": [], - "locales": { - "zh": { - "name": "Fetch网页内容抓取", - "description": "该服务器使大型语言模型能够检索和处理网页内容...", - "readme": "# Fetch MCP Server\n\n提供网页内容抓取能力的 MCP 服务。" - }, - "en": { - "name": "Fetch", - "description": "A server that enables LLMs to retrieve and process web content...", - "readme": "# Fetch MCP Server\n\nA Model Context Protocol server that provides web content fetching capabilities." - } - } + "access_token": "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCIsImtpZCI6ImlkcC1rZXktMDAxIn0.eyJpc3MiOiJodHRwczovL21vZGVsc2NvcGUuY24iLCJzdWIiOiJhZ2VudC0xMjM0NTY3ODkwYWIiLCJhdWQiOiJodWItYXBwLWFiY2RlZjEyMzQ1NiIsImV4cCI6MTcxNzQxMzYwMCwiaWF0IjoxNzE3NDEwMDAwLCJqdGkiOiJqdGktMTIzNDU2Nzg5MCJ9.dummy_signature_base64url", + "token_type": "Bearer", + "expire_at": 1717413600, + "jti": "jti-1234567890" }, "message": "success", "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "deploy-mcp-server-response": { + "list-token-records-response": { "value": { "success": true, "data": { - "id": "platform-pool", - "url": "https://mcp.api-inference.modelscope.net/abc123def456/mcp", - "transport_type": "streamable_http", - "auth_required": false, - "expiration": "2025-10-01 21:00:00" + "token_records": [ + { + "token_id": "jti-1234567890", + "audience": "hub-app-abcdef123456", + "issued_at": "2026-06-08T11:30:00Z", + "expire_at": "2026-06-08T12:30:00Z", + "status": "active", + "jwt": "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCIsImtpZCI6ImlkcC1rZXktMDAxIn0.eyJpc3MiOiJodHRwczovL21vZGVsc2NvcGUuY24iLCJzdWIiOiJhZ2VudC0xMjM0NTY3ODkwYWIiLCJhdWQiOiJodWItYXBwLWFiY2RlZjEyMzQ1NiIsImV4cCI6MTcxNzQxMzYwMCwiaWF0IjoxNzE3NDEwMDAwLCJqdGkiOiJqdGktMTIzNDU2Nzg5MCJ9.dummy_signature_base64url" + }, + { + "token_id": "jti-2345678901", + "audience": "hub-app-fedcba654321", + "issued_at": "2026-06-07T09:15:00Z", + "expire_at": "2026-06-07T10:15:00Z", + "status": "expired", + "jwt": "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCIsImtpZCI6ImlkcC1rZXktMDAxIn0.eyJpc3MiOiJodHRwczovL21vZGVsc2NvcGUuY24iLCJzdWIiOiJhZ2VudC0xMjM0NTY3ODkwYWIiLCJhdWQiOiJodWItYXBwLWZlZGNiYTY1NDMyMSIsImV4cCI6MTcxNzMyMjUwMCwiaWF0IjoxNzE3MzE4OTAwLCJqdGkiOiJqdGktMjM0NTY3ODkwMSJ9.dummy_signature_base64url" + } + ], + "total_count": 2, + "page_number": 1, + "page_size": 20 }, "message": "success", "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } }, - "undeploy-mcp-server-response": { + "agentid-configuration-response": { "value": { - "success": true, - "message": "success", - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + "issuer": "https://modelscope.cn", + "token_endpoint": "https://modelscope.cn/openapi/v1/agent_id/token", + "jwks_uri": "https://modelscope.cn/openapi/v1/agent_id/.well-known/agentid-jwks", + "registration_endpoint": "https://modelscope.cn/openapi/v1/agent_ids", + "activity_endpoint": "https://modelscope.cn/openapi/v1/agent_ids/{agent_id}/jwt_id_tokens", + "id_token_signing_alg_values_supported": "EdDSA" } }, - "create-studio-response": { + "agentid-jwks-response": { "value": { - "success": true, - "data": { - "id": "username/my-demo-app", - "repo_name": "my-demo-app", - "display_name": "我的演示应用", - "owner": "username", - "url": "https://modelscope.cn/studios/username/my-demo-app" - }, - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + "keys": [ + { + "kty": "OKP", + "crv": "Ed25519", + "x": "UnZsPxBV8ZyK5zhQlpnT1nf5GjxJQbpHQMwoS0jZeNc", + "kid": "idp-key-001", + "alg": "EdDSA", + "use": "sig" + }, + { + "kty": "OKP", + "crv": "Ed25519", + "x": "WoxPa7_oZOn1HtdPNn-3Btu7UKmtMVRUyQMlYMM0SnQ", + "kid": "idp-key-002", + "alg": "EdDSA", + "use": "sig" + } + ] } }, - "get-studio-response": { + "get-collection-response": { "value": { "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", "data": { - "id": "username/my-demo-app", - "repo_name": "my-demo-app", - "display_name": "我的演示应用", - "owner": "username", - "description": "一个演示创空间应用", - "likes": 100, - "view_count": 5000, - "tags": [ - "nlp", - "chatbot" - ], + "slug": "iSolver/best-nlp-models-6a3b2c", + "title": "最佳 NLP 模型合集", + "description": "精选的自然语言处理模型...", + "owner": "iSolver", "visibility": "public", - "private": false, - "last_modified": "2025-01-15T10:30:00Z", - "sdk_type": "gradio", - "sdk_version": "6.2.0", - "hardware": "platform/2v-cpu-16g-mem", - "base_image": "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0", - "license": "apache-2.0", - "host": "https://username-my-demo-app.ms.show", - "mcp_support": false, - "runtime": { - "status": "Running", - "active_config": { - "hardware": "platform/2v-cpu-16g-mem", - "base_image": "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0", - "sdk_type": "gradio", - "sdk_version": "6.2.0" + "theme": "Blue", + "item_count": 2, + "likes": 128, + "view_count": 1024, + "items": [ + { + "item_type": "model", + "item_object_id": "damo/nlp_bert_base", + "note": "这个模型在中文 NLP 任务上表现优异", + "position": 1, + "created_at": "2025-01-15T08:00:00Z", + "visibility": "public", + "gated": false, + "protected_mode": 2 }, - "created_at": "2025-01-15T10:30:00Z" - } - }, - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + { + "item_type": "dataset", + "item_object_id": "damo/nlp_corpus", + "note": null, + "position": 2, + "created_at": "2025-01-16T09:00:00Z", + "visibility": "public", + "gated": false, + "protected_mode": 2 + } + ], + "created_at": "2025-01-15T08:00:00Z", + "updated_at": "2025-04-20T10:30:00Z", + "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + } } }, - "studio-runtime-response": { + "create-collection-response": { "value": { "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", "data": { - "status": "Running", - "active_config": { - "hardware": "platform/2v-cpu-16g-mem", - "base_image": "ubuntu22.04-py311-torch2.9.1-modelscope1.35.0", - "sdk_type": "gradio", - "sdk_version": "6.2.0" - }, - "created_at": "2025-01-15T10:30:00Z" - }, - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + "slug": "iSolver/best-nlp-models-6a3b2c", + "title": "最佳 NLP 模型合集", + "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + } } }, - "get-studio-logs-response": { + "update-collection-response": { "value": { "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", "data": { - "logs": [ - "2025-01-15 10:30:00 INFO Starting application...", - "2025-01-15 10:30:01 INFO Application started successfully" - ], - "page_num": 1, - "page_size": 100, - "total_count": 500, - "total_page_num": 5 - }, - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + "slug": "iSolver/best-nlp-models-renamed-6a3b2c", + "title": "最佳 NLP 模型合集（已更名）", + "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-renamed-6a3b2c" + } } }, - "list-sdk-versions-response": { + "delete-collection-response": { "value": { "success": true, - "data": { - "sdk_versions": [ - { - "sdk_type": "gradio", - "tag": "latest", - "version": "6.2.0" - }, - { - "sdk_type": "gradio", - "tag": "stable", - "version": "5.49.1" - } - ], - "total_count": 2 - }, - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": {} } }, - "list-skills-response": { + "list-collections-response": { "value": { "success": true, - "request_id": "c5bca202-aa4f-478b-b7e2-c568160583cb", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", "data": { - "skills": [ - { - "_id": "CIK3H+DYKlSAMlHnX60AUg==", - "id": "@AMap-Web/amap-lbs-skill", - "display_name": "高德地图综合服务Skill", - "description": "高德地图综合服务，支持POI搜索、路径规划、旅游规划、周边搜索和热力图数据可视化", - "owner": "", - "license": "MIT License", - "developer": "AMap-Web", - "source_url": "https://github.com/AMap-Web/amap-lbs-skill", - "category": "developer-tools", - "tags": [ - "category:developer-tools", - "license:MIT License", - "developer:AMap-Web", - "custom_tag:api-design", - "custom_tag:general-tools" - ], - "logo_url": "", - "view_count": 6144, - "downloads": 791, - "locales": { - "en": { - "description": "Amap comprehensive services, supporting POI search, route planning, travel planning, nearby search, and heatmap data visualization", - "category": "DevTools" - }, - "zh": { - "description": "高德地图综合服务，支持POI搜索、路径规划、旅游规划、周边搜索和热力图数据可视化", - "category": "开发工具" - } - }, - "private": false, - "custom_tag": [ - "api-design", - "general-tools" - ] - }, + "collection_list": [ { - "_id": "0ry9hAiAiJ+2pklbvQjepg==", - "id": "@steipete/summarize", - "display_name": "summarize", - "description": "使用summarize CLI（适用于网页、PDF、图片、音频、YouTube）来总结URL或文件。", - "owner": "steipete", - "license": "", - "developer": "steipete", - "source_url": "https://clawhub.ai/steipete/summarize", - "category": "ai-media", - "tags": [ - "category:ai-media", - "developer:steipete", - "custom_tag:image-video-gen", - "custom_tag:media-processing" - ], - "logo_url": "", - "view_count": 4696, - "downloads": 690, - "locales": { - "en": { - "description": "Summarize URLs or files with the summarize CLI (web, PDFs, images, audio, YouTube).", - "category": "MediaAI" - }, - "zh": { - "description": "使用summarize CLI（适用于网页、PDF、图片、音频、YouTube）来总结URL或文件。", - "category": "媒体处理" - } - }, - "private": false, - "custom_tag": [ - "image-video-gen", - "media-processing" - ] + "slug": "iSolver/best-nlp-models-6a3b2c", + "title": "最佳 NLP 模型合集", + "description": "精选的自然语言处理模型...", + "owner": "iSolver", + "visibility": "public", + "theme": "Blue", + "item_count": 12, + "likes": 128, + "view_count": 1024, + "items": [], + "created_at": "2025-01-15T08:00:00Z", + "updated_at": "2025-04-20T10:30:00Z", + "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" } ], - "total": 20, + "total_count": 128, "page_number": 1, - "page_size": 20 + "page_size": 10 } } }, - "get-skill-response": { + "list-collection-items-response": { "value": { "success": true, - "request_id": "105c898f-fd6d-437d-aa0b-9006036782da", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", "data": { - "_id": "CIK3H+DYKlSAMlHnX60AUg==", - "id": "@AMap-Web/amap-lbs-skill", - "display_name": "高德地图综合服务Skill", - "description": "高德地图综合服务，支持POI搜索、路径规划、旅游规划、周边搜索和热力图数据可视化", - "owner": "", - "license": "MIT License", - "developer": "AMap-Web", - "source_url": "https://github.com/AMap-Web/amap-lbs-skill", - "category": "developer-tools", - "tags": [ - "category:developer-tools", - "license:MIT License", - "developer:AMap-Web", - "custom_tag:api-design", - "custom_tag:general-tools" - ], - "logo_url": "", - "view_count": 6144, - "downloads": 791, - "locales": { - "en": { - "description": "Amap comprehensive services, supporting POI search, route planning, travel planning, nearby search, and heatmap data visualization", - "category": "DevTools" - }, - "zh": { - "description": "高德地图综合服务，支持POI搜索、路径规划、旅游规划、周边搜索和热力图数据可视化", - "category": "开发工具" + "items": [ + { + "item_type": "model", + "item_object_id": "damo/nlp_bert_base", + "note": "这个模型在中文 NLP 任务上表现优异", + "position": 1, + "created_at": "2025-01-15T08:00:00Z", + "visibility": "public", + "gated": false, + "protected_mode": 2 } - }, - "private": false, - "custom_tag": [ - "api-design", - "general-tools" ], - "install_command": [ - "npx skills add https://modelscope.cn/skills/@AMap-Web/amap-lbs-skill", - "curl -fsSL https://modelscope.cn/skills/install.sh | bash -s -- @AMap-Web/amap-lbs-skill" - ] + "total_count": 50, + "page_number": 1, + "page_size": 10 } } }, - "create-skill-response": { + "add-collection-items-request": { + "value": { + "items": [ + { + "item_type": "model", + "item_object_id": "damo/nlp_bert_base_chinese", + "note": "推荐用于中文文本分类", + "position": 1 + }, + { + "item_type": "dataset", + "item_object_id": "damo/nlp_corpus" + } + ] + } + }, + "add-collection-items-response": { "value": { "success": true, - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", "data": { - "id": "iSolver/my-awesome-skill", - "name": "my-awesome-skill", - "display_name": "我的技能", - "owner": "iSolver", - "url": "https://modelscope.cn/skills/iSolver/my-awesome-skill" + "added_count": 2, + "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c", + "failed_items": [ + { + "item_type": "model", + "item_object_id": "not/exist-model", + "reason": "ResourceNotFound" + } + ] } } }, - "update-skill-settings-response": { + "update-collection-item-request": { "value": { - "success": true, - "message": "success", - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + "note": "更新后的备注", + "position": 2 } }, - "upload-file-response": { + "update-collection-item-response": { "value": { "success": true, - "request_id": "99ee2083-63e0-46dc-b393-623c22645078", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", "data": { - "id": "8c378570-8991-431b-a82c-96f3d0b4f0f4" + "item_type": "model", + "item_object_id": "damo/nlp_bert_base", + "note": "更新后的备注", + "position": 2, + "created_at": "2025-01-15T08:00:00Z", + "visibility": "public", + "gated": false, + "protected_mode": 2 } } }, - "create-agent-identity-response": { + "remove-collection-item-response": { "value": { "success": true, - "data": { - "agent_id": "agent_id:modelscope:agent_1234567890ab", - "agent_name": "my-agent", - "description": "用于演示的测试 Agent", - "token_expire_time": 3600, - "principal": { - "type": "user", - "id": "alice" - }, - "kid": "agent-key-001", - "public_key": { - "kty": "OKP", - "crv": "Ed25519", - "x": "f7QtzyEJhqFL8Jk5OYOq7010Ry9PipDGQkTzl7XtiEs", - "kid": "agent-key-001", - "alg": "EdDSA", - "use": "sig" - }, - "status": "active", - "create_time": "2026-06-08T10:00:00Z" - }, - "message": "success", - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "data": {} } }, - "get-agent-identity-response": { + "update-collection-items-request": { "value": { - "success": true, - "data": { - "agent_id": "agent_id:modelscope:agent_1234567890ab", - "agent_name": "my-agent", - "description": "用于演示的测试 Agent", - "token_expire_time": 3600, - "principal": { - "type": "user", - "id": "alice" - }, - "kid": "agent-key-001", - "public_key": { - "kty": "OKP", - "crv": "Ed25519", - "x": "f7QtzyEJhqFL8Jk5OYOq7010Ry9PipDGQkTzl7XtiEs", - "kid": "agent-key-001", - "alg": "EdDSA", - "use": "sig" + "items": [ + { + "item_type": "model", + "item_object_id": "damo/nlp_bert_base", + "position": 1 }, - "status": "active", - "create_time": "2026-06-08T10:00:00Z", - "update_time": "2026-06-08T10:30:00Z" - }, - "message": "success", - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + { + "item_type": "dataset", + "item_object_id": "damo/nlp_corpus", + "note": "更新备注", + "position": 2 + } + ] } }, - "update-agent-identity-response": { + "update-collection-items-response": { "value": { "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", "data": { - "agent_id": "agent_id:modelscope:agent_1234567890ab", - "agent_name": "my-agent-renamed", - "description": "更新后的描述", - "token_expire_time": 1800, - "principal": { - "type": "user", - "id": "alice" - }, - "kid": "agent-key-001", - "public_key": { - "kty": "OKP", - "crv": "Ed25519", - "x": "f7QtzyEJhqFL8Jk5OYOq7010Ry9PipDGQkTzl7XtiEs", - "kid": "agent-key-001", - "alg": "EdDSA", - "use": "sig" - }, - "status": "active", - "create_time": "2026-06-08T10:00:00Z", - "update_time": "2026-06-08T11:00:00Z" - }, - "message": "success", - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + "updated_count": 2, + "failed_items": [ + { + "item_type": "model", + "item_object_id": "not/exist-model", + "reason": "ItemNotInCollection" + } + ] + } } }, - "list-agent-identities-response": { + "create-gallery-request": { "value": { - "success": true, - "data": { - "agent_identities": [ - { - "agent_id": "agent_id:modelscope:agent_1234567890ab", - "agent_name": "my-agent", - "kid": "agent-key-001", - "status": "active", - "token_expire_time": 3600, - "create_time": "2026-06-08T10:00:00Z" - }, - { - "agent_id": "agent-cdef01234567", - "agent_name": "build-bot", - "kid": "agent-key-002", - "status": "paused", - "token_expire_time": 1800, - "create_time": "2026-06-07T15:20:00Z" - } - ], - "total_count": 2, - "page_number": 1, - "page_size": 20 - }, - "message": "success", - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + "name": "my-gallery", + "owner": "my-org", + "files": [ + "8c378570-8991-431b-a82c-96f3d0b4f0f4", + "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + ], + "entry_file": "notebook/demo.ipynb", + "private": false, + "category": "notebook", + "labels": [ + "demo", + "tutorial" + ], + "path": "my-custom-path" } }, - "reset-key-pair-response": { + "create-gallery-response": { "value": { "success": true, + "request_id": "req-xxxxxx", "data": { - "agent_id": "agent_id:modelscope:agent_1234567890ab", - "kid": "agent-key-002", - "public_key": { - "kty": "OKP", - "crv": "Ed25519", - "x": "UnZsPxBV8ZyK5zhQlpnT1nf5GjxJQbpHQMwoS0jZeNc", - "kid": "agent-key-002", - "alg": "EdDSA", - "use": "sig" + "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "url": "https://modelscope.cn/gallery/my-org/a1b2c3d4-e5f6-7890-abcd-ef1234567890" + } + } + }, + "create-agent-playground-instance-request": { + "value": { + "instance_name": "my-agent", + "display_name": "我的 Agent", + "description": "用于测试的 Agent Playground 实例", + "logo": "https://resources.modelscope.cn/avatar/my-agent.png", + "framework_type": "qwenpaw", + "models": [ + { + "provider_name": "ModelScope API-Inference", + "model_name": "Qwen/Qwen3.5-27B" } - }, - "message": "success", - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + ], + "mcps": [ + { + "name": "amap-maps", + "source": "custom", + "server_config": { + "mcpServers": { + "amap-maps": { + "type": "sse", + "url": "https://mcp.example.com/sse" + } + } + } + } + ], + "skills": [ + "@modelscope/web-search" + ], + "message": "帮我查一下明天的天气" } }, - "pause-agent-response": { + "create-agent-playground-instance-response": { "value": { "success": true, - "message": "success", - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + "request_id": "7d3f8f8a-8c1f-4d0b-8f1a-7b6a0d8a1a11", + "data": { + "uuid": "51a7d9d2-a521-451a-baf5-367cb115a29a", + "owner": "username", + "instance_name": "my-agent", + "display_name": "我的 Agent", + "description": "用于测试的 Agent Playground 实例", + "logo": "https://resources.modelscope.cn/avatar/my-agent.png", + "status": "starting", + "framework_type": "qwenpaw", + "source": "playground", + "framework_version": "2.2.0", + "agent_count": 0, + "model_count": 0, + "mcp_count": 0, + "skill_count": 0, + "gmt_created": "2026-09-17T15:47:19+08:00", + "gmt_modified": "2026-09-17T15:47:19+08:00", + "user_modified_at": "2026-09-17T15:47:19+08:00" + } } }, - "delete-agent-identity-response": { + "create-agent-playground-instance-from-agent-hub-request": { "value": { - "success": true, - "message": "success", - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + "instance_name": "playground-instance", + "display_name": "Playground Instance", + "description": "从 Agent Hub 导入的实例", + "framework_type": "qwenpaw", + "agent_owner": "jakexiao", + "agent_name": "FirstAgent", + "models": [ + { + "provider_name": "ModelScope API-Inference", + "model_name": "Qwen/Qwen3.5-27B" + } + ] } }, - "issue-token-response": { + "create-agent-playground-instance-from-agent-hub-response": { "value": { "success": true, + "request_id": "1b9d5c7e-2a34-4f6b-8d0e-3c5a7b9d1f24", "data": { - "access_token": "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCIsImtpZCI6ImlkcC1rZXktMDAxIn0.eyJpc3MiOiJodHRwczovL21vZGVsc2NvcGUuY24iLCJzdWIiOiJhZ2VudC0xMjM0NTY3ODkwYWIiLCJhdWQiOiJodWItYXBwLWFiY2RlZjEyMzQ1NiIsImV4cCI6MTcxNzQxMzYwMCwiaWF0IjoxNzE3NDEwMDAwLCJqdGkiOiJqdGktMTIzNDU2Nzg5MCJ9.dummy_signature_base64url", - "token_type": "Bearer", - "expire_at": 1717413600, - "jti": "jti-1234567890" - }, - "message": "success", - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + "uuid": "8f2c4a1e-7b63-4d95-a0c8-2e6f1b3d5a79", + "owner": "username", + "instance_name": "playground-instance", + "display_name": "Playground Instance", + "description": "从 Agent Hub 导入的实例", + "logo": "", + "status": "starting", + "framework_type": "qwenpaw", + "source": "agent_hub", + "framework_version": "2.2.0", + "agent_count": 0, + "model_count": 0, + "mcp_count": 0, + "skill_count": 0, + "gmt_created": "2026-09-18T10:12:05+08:00", + "gmt_modified": "2026-09-18T10:12:05+08:00", + "user_modified_at": "2026-09-18T10:12:05+08:00" + } } }, - "list-token-records-response": { + "list-agent-playground-instances-response": { "value": { "success": true, + "request_id": "bb971b6e-36f0-4c4c-a50e-ee3fb6d3ceba", "data": { - "token_records": [ + "instances": [ { - "token_id": "jti-1234567890", - "audience": "hub-app-abcdef123456", - "issued_at": "2026-06-08T11:30:00Z", - "expire_at": "2026-06-08T12:30:00Z", - "status": "active", - "jwt": "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCIsImtpZCI6ImlkcC1rZXktMDAxIn0.eyJpc3MiOiJodHRwczovL21vZGVsc2NvcGUuY24iLCJzdWIiOiJhZ2VudC0xMjM0NTY3ODkwYWIiLCJhdWQiOiJodWItYXBwLWFiY2RlZjEyMzQ1NiIsImV4cCI6MTcxNzQxMzYwMCwiaWF0IjoxNzE3NDEwMDAwLCJqdGkiOiJqdGktMTIzNDU2Nzg5MCJ9.dummy_signature_base64url" + "uuid": "51a7d9d2-a521-451a-baf5-367cb115a29a", + "owner": "username", + "instance_name": "my-agent", + "display_name": "我的 Agent", + "description": "用于测试的 Agent Playground 实例", + "logo": "https://resources.modelscope.cn/avatar/my-agent.png", + "status": "running", + "framework_type": "qwenpaw", + "source": "playground", + "framework_version": "2.2.0", + "agent_count": 2, + "model_count": 1, + "mcp_count": 3, + "skill_count": 17, + "gmt_created": "2026-09-17T15:47:19+08:00", + "gmt_modified": "2026-09-18T17:06:19+08:00", + "user_modified_at": "2026-09-17T15:47:19+08:00" }, { - "token_id": "jti-2345678901", - "audience": "hub-app-fedcba654321", - "issued_at": "2026-06-07T09:15:00Z", - "expire_at": "2026-06-07T10:15:00Z", - "status": "expired", - "jwt": "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCIsImtpZCI6ImlkcC1rZXktMDAxIn0.eyJpc3MiOiJodHRwczovL21vZGVsc2NvcGUuY24iLCJzdWIiOiJhZ2VudC0xMjM0NTY3ODkwYWIiLCJhdWQiOiJodWItYXBwLWZlZGNiYTY1NDMyMSIsImV4cCI6MTcxNzMyMjUwMCwiaWF0IjoxNzE3MzE4OTAwLCJqdGkiOiJqdGktMjM0NTY3ODkwMSJ9.dummy_signature_base64url" + "uuid": "8f2c4a1e-7b63-4d95-a0c8-2e6f1b3d5a79", + "owner": "username", + "instance_name": "playground-instance", + "display_name": "Playground Instance", + "description": "", + "logo": "", + "status": "sleeping", + "framework_type": "ms-agent", + "source": "agent_hub", + "framework_version": "1.7.0.dev35316159068", + "agent_count": 1, + "model_count": 1, + "mcp_count": 0, + "skill_count": 0, + "gmt_created": "2026-09-18T10:12:05+08:00", + "gmt_modified": "2026-09-18T18:30:41+08:00", + "user_modified_at": "2026-09-18T10:12:05+08:00" } ], "total_count": 2, "page_number": 1, - "page_size": 20 - }, - "message": "success", - "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" + "page_size": 10 + } } }, - "agentid-configuration-response": { + "get-agent-playground-instance-response": { "value": { - "issuer": "https://modelscope.cn", - "token_endpoint": "https://modelscope.cn/openapi/v1/agent_id/token", - "jwks_uri": "https://modelscope.cn/openapi/v1/agent_id/.well-known/agentid-jwks", - "registration_endpoint": "https://modelscope.cn/openapi/v1/agent_ids", - "activity_endpoint": "https://modelscope.cn/openapi/v1/agent_ids/{agent_id}/jwt_id_tokens", - "id_token_signing_alg_values_supported": "EdDSA" + "success": true, + "request_id": "a4c8e2f1-5b7d-4906-8e3a-1f2c4d6b8a05", + "data": { + "uuid": "51a7d9d2-a521-451a-baf5-367cb115a29a", + "owner": "username", + "instance_name": "my-agent", + "display_name": "我的 Agent", + "description": "用于测试的 Agent Playground 实例", + "logo": "https://resources.modelscope.cn/avatar/my-agent.png", + "status": "running", + "framework_type": "qwenpaw", + "source": "playground", + "framework_version": "2.2.0", + "agent_count": 2, + "model_count": 1, + "mcp_count": 3, + "skill_count": 17, + "gmt_created": "2026-09-17T15:47:19+08:00", + "gmt_modified": "2026-09-18T17:06:19+08:00", + "user_modified_at": "2026-09-17T15:47:19+08:00" + } } }, - "agentid-jwks-response": { + "update-agent-playground-instance-request": { "value": { - "keys": [ - { - "kty": "OKP", - "crv": "Ed25519", - "x": "UnZsPxBV8ZyK5zhQlpnT1nf5GjxJQbpHQMwoS0jZeNc", - "kid": "idp-key-001", - "alg": "EdDSA", - "use": "sig" - }, - { - "kty": "OKP", - "crv": "Ed25519", - "x": "WoxPa7_oZOn1HtdPNn-3Btu7UKmtMVRUyQMlYMM0SnQ", - "kid": "idp-key-002", - "alg": "EdDSA", - "use": "sig" - } - ] + "instance_name": "my-agent-renamed", + "display_name": "My Agent Renamed", + "description": "Updated description", + "logo": "https://resources.modelscope.cn/avatar/my-agent.png" } }, - "get-collection-response": { + "update-agent-playground-instance-response": { "value": { "success": true, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "request_id": "6e1a9c3b-8d42-4f07-b5e9-7a0c2d4f6b18", "data": { - "slug": "iSolver/best-nlp-models-6a3b2c", - "title": "最佳 NLP 模型合集", - "description": "精选的自然语言处理模型...", - "owner": "iSolver", - "visibility": "public", - "theme": "Blue", - "item_count": 2, - "likes": 128, - "view_count": 1024, - "items": [ - { - "item_type": "model", - "item_object_id": "damo/nlp_bert_base", - "note": "这个模型在中文 NLP 任务上表现优异", - "position": 1, - "created_at": "2025-01-15T08:00:00Z", - "visibility": "public", - "gated": false, - "protected_mode": 2 - }, - { - "item_type": "dataset", - "item_object_id": "damo/nlp_corpus", - "note": null, - "position": 2, - "created_at": "2025-01-16T09:00:00Z", - "visibility": "public", - "gated": false, - "protected_mode": 2 - } - ], - "created_at": "2025-01-15T08:00:00Z", - "updated_at": "2025-04-20T10:30:00Z", - "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + "uuid": "51a7d9d2-a521-451a-baf5-367cb115a29a", + "owner": "username", + "instance_name": "my-agent-renamed", + "display_name": "My Agent Renamed", + "description": "Updated description", + "logo": "https://resources.modelscope.cn/avatar/my-agent.png", + "status": "running", + "framework_type": "qwenpaw", + "source": "playground", + "framework_version": "2.2.0", + "agent_count": 2, + "model_count": 1, + "mcp_count": 3, + "skill_count": 17, + "gmt_created": "2026-09-17T15:47:19+08:00", + "gmt_modified": "2026-09-18T17:06:19+08:00", + "user_modified_at": "2026-09-18T17:06:19+08:00" } } }, - "create-collection-response": { + "delete-agent-playground-instance-response": { "value": { "success": true, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "request_id": "0d2b7a54-1f6c-4a2b-9e33-2c7f9c1d8b40", "data": { - "slug": "iSolver/best-nlp-models-6a3b2c", - "title": "最佳 NLP 模型合集", - "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" + "deleted": true } } }, - "update-collection-response": { + "pause-agent-playground-instance-response": { "value": { "success": true, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "request_id": "2f8b6d0a-4c19-4e73-9a5f-8b1d3e5c7a92", "data": { - "slug": "iSolver/best-nlp-models-renamed-6a3b2c", - "title": "最佳 NLP 模型合集（已更名）", - "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-renamed-6a3b2c" + "uuid": "51a7d9d2-a521-451a-baf5-367cb115a29a", + "owner": "username", + "instance_name": "my-agent", + "display_name": "我的 Agent", + "description": "用于测试的 Agent Playground 实例", + "logo": "https://resources.modelscope.cn/avatar/my-agent.png", + "status": "sleeping", + "framework_type": "qwenpaw", + "source": "playground", + "framework_version": "2.2.0", + "agent_count": 2, + "model_count": 1, + "mcp_count": 3, + "skill_count": 17, + "gmt_created": "2026-09-17T15:47:19+08:00", + "gmt_modified": "2026-09-18T18:30:41+08:00", + "user_modified_at": "2026-09-18T18:30:41+08:00" } } }, - "delete-collection-response": { + "resume-agent-playground-instance-response": { "value": { "success": true, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", - "data": {} + "request_id": "9c3e5a7b-1d24-4068-8f0a-6b2d4c8e1f37", + "data": { + "uuid": "51a7d9d2-a521-451a-baf5-367cb115a29a", + "owner": "username", + "instance_name": "my-agent", + "display_name": "我的 Agent", + "description": "用于测试的 Agent Playground 实例", + "logo": "https://resources.modelscope.cn/avatar/my-agent.png", + "status": "running", + "framework_type": "qwenpaw", + "source": "playground", + "framework_version": "2.2.0", + "agent_count": 2, + "model_count": 1, + "mcp_count": 3, + "skill_count": 17, + "gmt_created": "2026-09-17T15:47:19+08:00", + "gmt_modified": "2026-09-18T18:42:03+08:00", + "user_modified_at": "2026-09-18T18:42:03+08:00" + } } }, - "list-collections-response": { + "restart-agent-playground-instance-response": { "value": { "success": true, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "request_id": "5a7c9e1b-3f80-4d62-9c4e-0a8b2d6f4c13", "data": { - "collection_list": [ - { - "slug": "iSolver/best-nlp-models-6a3b2c", - "title": "最佳 NLP 模型合集", - "description": "精选的自然语言处理模型...", - "owner": "iSolver", - "visibility": "public", - "theme": "Blue", - "item_count": 12, - "likes": 128, - "view_count": 1024, - "items": [], - "created_at": "2025-01-15T08:00:00Z", - "updated_at": "2025-04-20T10:30:00Z", - "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c" - } - ], - "total_count": 128, - "page_number": 1, - "page_size": 10 + "uuid": "51a7d9d2-a521-451a-baf5-367cb115a29a", + "owner": "username", + "instance_name": "my-agent", + "display_name": "我的 Agent", + "description": "用于测试的 Agent Playground 实例", + "logo": "https://resources.modelscope.cn/avatar/my-agent.png", + "status": "starting", + "framework_type": "qwenpaw", + "source": "playground", + "framework_version": "2.2.0", + "agent_count": 2, + "model_count": 1, + "mcp_count": 3, + "skill_count": 17, + "gmt_created": "2026-09-17T15:47:19+08:00", + "gmt_modified": "2026-09-18T18:45:27+08:00", + "user_modified_at": "2026-09-18T18:45:27+08:00" } } }, - "list-collection-items-response": { + "get-agent-playground-instance-agents-response": { "value": { "success": true, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "request_id": "3c9a1b2d-4e5f-6a7b-8c9d-0e1f2a3b4c5d", "data": { - "items": [ + "agents": [ { - "item_type": "model", - "item_object_id": "damo/nlp_bert_base", - "note": "这个模型在中文 NLP 任务上表现优异", - "position": 1, - "created_at": "2025-01-15T08:00:00Z", - "visibility": "public", - "gated": false, - "protected_mode": 2 + "summary": { + "id": "default", + "name": "Default Agent", + "description": "", + "enabled": true, + "models": [ + { + "provider_id": "playground-openai-3f2a9c1d", + "provider": "Playground Openai (3f2a9c1d)", + "model": "Qwen/Qwen3.5-27B", + "type": "openai", + "base_url": "https://api-inference.modelscope.cn/v1", + "enabled": true + } + ] + }, + "detail": { + "id": "default", + "name": "Default Agent", + "description": "", + "models": [ + { + "provider_id": "playground-openai-3f2a9c1d", + "provider": "Playground Openai (3f2a9c1d)", + "model": "Qwen/Qwen3.5-27B", + "type": "openai", + "base_url": "https://api-inference.modelscope.cn/v1", + "enabled": true + } + ], + "persona": { + "AGENTS.md": { + "content": "# AGENTS.md\n\nYou are a helpful agent.", + "enabled": true + }, + "HEARTBEAT.md": { + "content": "", + "enabled": false + } + }, + "skills": [ + { + "name": "web-search", + "description": "", + "enabled": true, + "source": "builtin" + } + ], + "mcp_clients": [ + { + "key": "amap-maps", + "name": "amap-maps", + "enabled": true, + "transport": "sse", + "url": "https://mcp.example.com/sse", + "tools": [ + { + "name": "maps_geo", + "description": "地理编码" + } + ] + } + ] + } } ], - "total_count": 50, - "page_number": 1, - "page_size": 10 + "refreshed_at": 1789722407 } } }, - "add-collection-items-request": { + "setup-agent-playground-model-request": { "value": { - "items": [ + "agent_id": "default", + "provider_name": "ModelScope API-Inference", + "model_name": "Qwen/Qwen3.5-27B" + } + }, + "setup-agent-playground-mcp-request": { + "value": { + "agent_id": "default", + "name": "amap-maps", + "source": "custom", + "server_config": { + "mcpServers": { + "amap-maps": { + "type": "sse", + "url": "https://mcp.example.com/sse" + } + } + } + } + }, + "setup-agent-playground-skill-request": { + "value": { + "agent_id": "default", + "skill_ref": "@modelscope/web-search" + } + }, + "setup-agent-playground-instance-response": { + "value": { + "success": true, + "request_id": "8b4d2f6a-0c57-4e19-a3b8-5d7f9c1e2a64", + "data": { + "uuid": "51a7d9d2-a521-451a-baf5-367cb115a29a", + "owner": "username", + "instance_name": "my-agent", + "display_name": "我的 Agent", + "description": "用于测试的 Agent Playground 实例", + "logo": "https://resources.modelscope.cn/avatar/my-agent.png", + "status": "running", + "framework_type": "qwenpaw", + "source": "playground", + "framework_version": "2.2.0", + "agent_count": 2, + "model_count": 1, + "mcp_count": 3, + "skill_count": 17, + "gmt_created": "2026-09-17T15:47:19+08:00", + "gmt_modified": "2026-09-18T19:02:11+08:00", + "user_modified_at": "2026-09-17T15:47:19+08:00" + } + } + }, + "import-to-agent-playground-instance-request": { + "value": { + "models": [ { - "item_type": "model", - "item_object_id": "damo/nlp_bert_base_chinese", - "note": "推荐用于中文文本分类", - "position": 1 - }, + "provider_name": "ModelScope API-Inference", + "model_name": "Qwen/Qwen3.5-27B" + } + ], + "mcps": [ { - "item_type": "dataset", - "item_object_id": "damo/nlp_corpus" + "name": "amap-maps", + "source": "custom", + "server_config": { + "mcpServers": { + "amap-maps": { + "type": "sse", + "url": "https://mcp.example.com/sse" + } + } + } } - ] + ], + "skills": [ + "@modelscope/web-search" + ], + "message": "帮我查一下明天的天气" } }, - "add-collection-items-response": { + "import-to-agent-playground-instance-response": { + "value": { + "success": true, + "request_id": "c7e9a1b3-5d02-4f68-9a4c-1b3d5e7f9a02", + "data": { + "uuid": "51a7d9d2-a521-451a-baf5-367cb115a29a", + "owner": "username", + "instance_name": "my-agent", + "display_name": "我的 Agent", + "description": "用于测试的 Agent Playground 实例", + "logo": "https://resources.modelscope.cn/avatar/my-agent.png", + "status": "starting", + "framework_type": "qwenpaw", + "source": "playground", + "framework_version": "2.2.0", + "agent_count": 2, + "model_count": 1, + "mcp_count": 3, + "skill_count": 17, + "gmt_created": "2026-09-17T15:47:19+08:00", + "gmt_modified": "2026-09-18T19:15:38+08:00", + "user_modified_at": "2026-09-18T19:15:38+08:00" + } + } + }, + "get-agent-playground-instance-notifications-response": { "value": { "success": true, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", + "request_id": "f1e2d3c4-b5a6-4978-8a9b-0c1d2e3f4a5b", "data": { - "added_count": 2, - "url": "https://modelscope.cn/collections/iSolver/best-nlp-models-6a3b2c", - "failed_items": [ + "notifications": [ { - "item_type": "model", - "item_object_id": "not/exist-model", - "reason": "ResourceNotFound" + "id": 7, + "kind": "conversation", + "payload": { + "framework": "qwenpaw", + "agent_id": "default", + "agent_name": "Default", + "message": "帮我查一下明天的天气", + "conversation": { + "framework": "qwenpaw", + "agent_id": "default", + "agent_name": "Default", + "channel": "console", + "session_id": "pg-4150b1f6-chat-0", + "chat_id": "f9ef8968-a0eb-4e70-bca3-d40e93197e85", + "chat_name": "帮我查一下明天的天气" + } + }, + "gmt_created": "2026-07-12T15:32:47+08:00" + }, + { + "id": 8, + "kind": "init_error", + "payload": { + "item": "mcp:2", + "error": "invalid mcp server config", + "code": "InputParameterError" + }, + "gmt_created": "2026-07-12T15:33:02+08:00" } ] } } }, - "update-collection-item-request": { + "agent-playground-input-parameter-error": { "value": { - "note": "更新后的备注", - "position": 2 + "success": false, + "request_id": "bb971b6e-36f0-4c4c-a50e-ee3fb6d3ceba", + "code": "InputParameterError", + "message": "Input parameter is invalid: instance count limit reached (8/8), delete an existing instance first | 输入参数无效: 实例数量已达上限 (8/8)，请先删除一个实例" } }, - "update-collection-item-response": { + "agent-playground-invalid-authentication": { "value": { - "success": true, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", - "data": { - "item_type": "model", - "item_object_id": "damo/nlp_bert_base", - "note": "更新后的备注", - "position": 2, - "created_at": "2025-01-15T08:00:00Z", - "visibility": "public", - "gated": false, - "protected_mode": 2 - } + "success": false, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd2", + "code": "InvalidAuthentication", + "message": "Invalid authentication | 认证无效" } }, - "remove-collection-item-response": { + "agent-playground-operation-not-allowed": { "value": { - "success": true, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", - "data": {} + "success": false, + "request_id": "8a1c3e5f-7b92-4d06-8c4a-2e6b0d4f8a13", + "code": "OperationNotAllowed", + "message": "Operation not allowed | 无权执行该操作" } }, - "update-collection-items-request": { + "agent-playground-resource-not-found": { "value": { - "items": [ - { - "item_type": "model", - "item_object_id": "damo/nlp_bert_base", - "position": 1 - }, - { - "item_type": "dataset", - "item_object_id": "damo/nlp_corpus", - "note": "更新备注", - "position": 2 - } - ] + "success": false, + "request_id": "5f7a9c1e-3b0d-4682-9e4f-7c1a3d5b9e02", + "code": "ResourceNotFound", + "message": "Resource not found: instance | 资源不存在: 实例" } }, - "update-collection-items-response": { + "agent-playground-duplicate-entity": { "value": { - "success": true, - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd3", - "data": { - "updated_count": 2, - "failed_items": [ - { - "item_type": "model", - "item_object_id": "not/exist-model", - "reason": "ItemNotInCollection" - } - ] - } + "success": false, + "request_id": "1c3e5a7b-9d02-4f46-8a1b-3d5f7a9c1e04", + "code": "DuplicateEntity", + "message": "instance name already exists: my-agent | 实例名称已存在: my-agent" } }, - "create-gallery-request": { + "agent-playground-internal-server-error": { "value": { - "name": "my-gallery", - "owner": "my-org", - "files": [ - "8c378570-8991-431b-a82c-96f3d0b4f0f4", - "a1b2c3d4-e5f6-7890-abcd-ef1234567890" - ], - "entry_file": "notebook/demo.ipynb", - "private": false, - "category": "notebook", - "labels": [ - "demo", - "tutorial" - ], - "path": "my-custom-path" + "success": false, + "request_id": "9e1b3d5f-7a0c-4248-9b3e-5a7c9e1b3d5f", + "code": "InternalServerError", + "message": "Internal server error, please try again later. | 服务器内部错误，请稍后重试。" } }, - "create-gallery-response": { + "agent-playground-service-unavailable": { "value": { - "success": true, - "request_id": "req-xxxxxx", - "data": { - "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", - "url": "https://modelscope.cn/gallery/my-org/a1b2c3d4-e5f6-7890-abcd-ef1234567890" - } + "success": false, + "request_id": "2a4c6e8b-0d13-4957-8e2a-4c6e8b0d1357", + "code": "ServiceUnavailable", + "message": "Agent service in the instance is starting, please try again later. | 实例内 Agent 服务启动中，请稍后重试。" + } + }, + "agent-playground-request-timeout": { + "value": { + "success": false, + "request_id": "6b8d0f2a-4c15-4e79-9b1d-3f5a7c9e1b0d", + "code": "RequestTimeout", + "message": "Request timed out, please try again later. | 请求超时，请稍后重试。" } }, "invalid-authentication": { @@ -10071,12 +16685,17 @@ "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd9" } }, - "resource-not-found": { + "model-repo-info-response": { "value": { - "success": false, - "code": "ResourceNotFound", - "message": "Resource not found: model Qwen/Qwen3-72B", - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd5" + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd2", + "data": { + "id": "alice/my-anime-lora", + "repo_name": "my-anime-lora", + "display_name": "我的动漫 LoRA", + "owner": "alice", + "url": "https://modelscope.cn/models/alice/my-anime-lora" + } } }, "input-parameter-error": { @@ -10111,6 +16730,187 @@ "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd6" } }, + "resource-not-found": { + "value": { + "success": false, + "code": "ResourceNotFound", + "message": "Resource not found: model Qwen/Qwen3-72B", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd5" + } + }, + "aigc-model-options-response": { + "summary": "AIGC 配置选项（部分枚举）", + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd2", + "data": { + "aigc_types": [ + { + "label": "Checkpoint", + "type": "aigc_type", + "id": "Checkpoint" + }, + { + "label": "LoRA", + "type": "aigc_type", + "id": "LoRA" + }, + { + "label": "VAE", + "type": "aigc_type", + "id": "VAE" + } + ], + "base_architectures": [ + { + "label": "Z-Image", + "subtypes": [ + { + "label": "Z-Image-Turbo", + "type": "base_architecture_subtype", + "id": "Z_IMAGE_TURBO" + }, + { + "label": "Z-Image", + "type": "base_architecture_subtype", + "id": "Z_IMAGE" + } + ], + "type": "base_architecture", + "id": "Z_IMAGE_TURBO" + }, + { + "label": "Wan2.2", + "subtypes": [ + { + "label": "Wan2.2-TI2V-5B", + "type": "base_architecture_subtype", + "id": "WAN_VIDEO_2_2_TI2V_5_B" + } + ], + "type": "base_architecture", + "id": "WAN_VIDEO_2_2_TI2V_5_B" + }, + { + "label": "Ace-Step", + "subtypes": [ + { + "label": "Acestep-Xl-Sft", + "type": "base_architecture_subtype", + "id": "ACESTEP_XL_SFT" + }, + { + "label": "Acestep-Xl-Turbo", + "type": "base_architecture_subtype", + "id": "ACESTEP_XL_TURBO" + } + ], + "type": "base_architecture", + "id": "ACESTEP_XL" + } + ], + "content_elements": [ + { + "label": "logo图标", + "type": "content_element", + "id": "logo" + }, + { + "label": "商品", + "type": "content_element", + "id": "commodity" + } + ], + "content_functions": [ + { + "label": "人物加强", + "type": "content_function", + "id": "character-enhancement" + }, + { + "label": "画风加强", + "type": "content_function", + "id": "art-style-strong" + } + ], + "content_styles": [ + { + "label": "写实摄影", + "type": "content_style", + "id": "photography" + }, + { + "label": "插画设计", + "type": "content_style", + "id": "illustration-design" + } + ] + } + } + }, + "rate-limit-exceed": { + "value": { + "success": false, + "code": "RateLimitExceed", + "message": "Rate limit exceeded: 100 requests per minute", + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd7" + } + }, + "science-model-options-response": { + "summary": "科学模型学科选项（部分枚举）", + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd2", + "data": { + "subject_areas": [ + { + "label": "生命科学", + "specializations": [ + { + "label": "医学", + "type": "specialization", + "id": "medicine" + }, + { + "label": "RNA", + "type": "specialization", + "id": "rna" + }, + { + "label": "DNA", + "type": "specialization", + "id": "dna" + } + ], + "type": "subject_area", + "id": "life-sciences" + }, + { + "label": "物质科学", + "specializations": [ + { + "label": "分子结构", + "type": "specialization", + "id": "molecular-structure" + }, + { + "label": "材料结构", + "type": "specialization", + "id": "material-structure" + }, + { + "label": "化学反应", + "type": "specialization", + "id": "chemical-reaction" + } + ], + "type": "subject_area", + "id": "physical-science" + } + ] + } + } + }, "skill-input-parameter-error": { "value": { "success": false, @@ -10159,12 +16959,60 @@ "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375be2" } }, - "rate-limit-exceed": { + "agent-input-parameter-error": { "value": { "success": false, - "code": "RateLimitExceed", - "message": "Rate limit exceeded: 100 requests per minute", - "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd7" + "code": "InputParameterError", + "message": "Input parameter is invalid: invalid request body", + "request_id": "2f1c8a3e-9b7d-4f21-a0c6-1e2d3f4a5b6c" + } + }, + "agent-invalid-authentication": { + "value": { + "success": false, + "code": "InvalidAuthentication", + "message": "Invalid authentication: user not authenticated", + "request_id": "2f1c8a3e-9b7d-4f21-a0c6-1e2d3f4a5b6c" + } + }, + "agent-quota-limit-exceed": { + "value": { + "success": false, + "code": "QuotaLimitExceed", + "message": "Quota limit exceeded: quota limit exceed 2400 agents", + "request_id": "c5bca202-aa4f-478b-b7e2-c568160583cb" + } + }, + "agent-list-failed": { + "value": { + "success": false, + "code": "ResourceNotFound", + "message": "Resource not found: failed to list agents", + "request_id": "c5bca202-aa4f-478b-b7e2-c568160583cb" + } + }, + "agent-internal-server-error": { + "value": { + "success": false, + "code": "InternalServerError", + "message": "Internal server error: failed to create agent", + "request_id": "2f1c8a3e-9b7d-4f21-a0c6-1e2d3f4a5b6c" + } + }, + "agent-operation-not-allowed": { + "value": { + "success": false, + "code": "OperationNotAllowed", + "message": "Operation not allowed: no permission to access this agent", + "request_id": "7a2b1c3d-4e5f-6a7b-8c9d-0e1f2a3b4c5d" + } + }, + "agent-not-found": { + "value": { + "success": false, + "code": "ResourceNotFound", + "message": "Resource not found: agent my_org/weather-agent", + "request_id": "7a2b1c3d-4e5f-6a7b-8c9d-0e1f2a3b4c5d" } }, "get-balance-response": { @@ -10177,7 +17025,62 @@ }, "request_id": "c55709d2-4e09-4bd8-be84-d8518bc46363" } + }, + "science-dataset-options-response": { + "summary": "科学数据集学科选项（部分枚举）", + "value": { + "success": true, + "request_id": "4d3f4b7d-95be-4e6f-95eb-d75178375bd2", + "data": { + "subject_areas": [ + { + "label": "生命科学", + "specializations": [ + { + "label": "医学", + "type": "specialization", + "id": "medicine" + }, + { + "label": "RNA", + "type": "specialization", + "id": "rna" + }, + { + "label": "DNA", + "type": "specialization", + "id": "dna" + } + ], + "type": "subject_area", + "id": "life-sciences" + }, + { + "label": "物质科学", + "specializations": [ + { + "label": "分子结构", + "type": "specialization", + "id": "molecular-structure" + }, + { + "label": "材料结构", + "type": "specialization", + "id": "material-structure" + }, + { + "label": "化学反应", + "type": "specialization", + "id": "chemical-reaction" + } + ], + "type": "subject_area", + "id": "physical-science" + } + ] + } + } } } } -} +} \ No newline at end of file diff --git a/tests/test_compat_get_model_files.py b/tests/test_compat_get_model_files.py index 6f16696..222b625 100644 --- a/tests/test_compat_get_model_files.py +++ b/tests/test_compat_get_model_files.py @@ -149,31 +149,44 @@ def test_aigc_model_uploads_optional_readme_after_creation(self): api = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") aigc_model = _FakeAigcModel() - with mock.patch.object(api._api.legacy, "_request", return_value=_response()) as request: + with ( + mock.patch.object(api._api.legacy, "_request", return_value=_response()) as legacy_request, + mock.patch.object(api._api.openapi, "_request", return_value={}) as openapi_request, + ): api.create_model( "owner/aigc-model", aigc_model=aigc_model, readme_content="# Initial AIGC README\n", ) - assert request.call_count == 3 - create_call, ensure_repo_call, readme_commit_call = request.call_args_list + # The AIGC repository itself still goes through the dedicated legacy + # endpoint: no OpenAPI route accepts its payload. + assert legacy_request.call_count == 2 + create_call, readme_commit_call = legacy_request.call_args_list assert create_call.args == ("POST", "models/aigc") - assert ensure_repo_call.args == ("POST", "models") assert readme_commit_call.args == ( "POST", "repos/models/owner/aigc-model/commit/master", ) + # Only the uploader's "ensure the repo exists" step moved to OpenAPI. + assert [call.args for call in openapi_request.call_args_list] == [("POST", "/models")] commit_body = readme_commit_call.kwargs["json_body"] assert commit_body["commit_message"] == "Update README.md for AIGC version v1.0" operation = commit_body["actions"][0] assert operation["path"] == "README.md" assert operation["content"] == "IyBJbml0aWFsIEFJR0MgUkVBRE1FCg==" - def test_plain_model_keeps_generic_create_repo_path(self): + def test_plain_model_uses_the_openapi_create_endpoint(self): api = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") - with mock.patch.object(api._api.legacy, "_request", return_value=_response()) as request: + with ( + mock.patch.object( + api._api.openapi, + "_request", + return_value={"id": "owner/plain-model", "owner": "owner", "repo_name": "plain-model"}, + ) as openapi_request, + mock.patch.object(api._api.legacy, "_request", return_value=_response()) as legacy_request, + ): url = api.create_model( "owner/plain-model", visibility=1, @@ -182,16 +195,36 @@ def test_plain_model_keeps_generic_create_repo_path(self): aigc_model=None, ) - request.assert_called_once() - method, path = request.call_args.args - body = request.call_args.kwargs["json_body"] - assert (method, path) == ("POST", "models") - assert body["Path"] == "owner" - assert body["Name"] == "plain-model" + legacy_request.assert_not_called() + openapi_request.assert_called_once() + assert openapi_request.call_args.args == ("POST", "/models") + body = openapi_request.call_args.kwargs["json_body"] + assert body == { + "owner": "owner", + "repo_name": "plain-model", + "visibility": "private", + "display_name": "普通模型", + "license": "apache-2.0", + } assert "TagShowName" not in body assert "aigc_model" not in body assert url == "https://modelscope.cn/models/owner/plain-model" + def test_plain_model_falls_back_to_legacy_for_an_unlistable_license(self): + """``POST /models`` rejects licenses outside its enum; legacy accepts them.""" + api = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") + + with ( + mock.patch.object(api._api.openapi, "_request") as openapi_request, + mock.patch.object(api._api.legacy, "_request", return_value=_response()) as legacy_request, + ): + api.create_model("owner/plain-model", license="Unlicense", aigc_model=None) + + openapi_request.assert_not_called() + legacy_request.assert_called_once() + assert legacy_request.call_args.args == ("POST", "models") + assert legacy_request.call_args.kwargs["json_body"]["License"] == "unlicense" + def test_aigc_model_tag_uses_dedicated_endpoint(self): api = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") aigc_model = _FakeAigcModel() @@ -257,16 +290,19 @@ def test_aigc_model_tag_uploads_readme_before_snapshot(self): aigc_model = _FakeAigcModel() aigc_model.readme_content = "# Version v1.3\n\nCustom README content.\n" - with mock.patch.object(api._api.legacy, "_request", return_value=_response()) as request: + with ( + mock.patch.object(api._api.legacy, "_request", return_value=_response()) as legacy_request, + mock.patch.object(api._api.openapi, "_request", return_value={}), + ): api.create_model_tag( "owner/aigc-model", "v1.3", aigc_model=aigc_model, ) - assert request.call_count == 3 - create_repo_call, readme_commit_call, tag_call = request.call_args_list - assert create_repo_call.args == ("POST", "models") + # The README must be committed before the immutable tag snapshot. + assert legacy_request.call_count == 2 + readme_commit_call, tag_call = legacy_request.call_args_list assert readme_commit_call.args == ( "POST", "repos/models/owner/aigc-model/commit/master", @@ -286,7 +322,10 @@ def test_explicit_readme_content_overrides_aigc_model_value(self): aigc_model = _FakeAigcModel() aigc_model.readme_content = "model value" - with mock.patch.object(api._api.legacy, "_request", return_value=_response()) as request: + with ( + mock.patch.object(api._api.legacy, "_request", return_value=_response()) as legacy_request, + mock.patch.object(api._api.openapi, "_request", return_value={}), + ): api.create_model_tag( "owner/aigc-model", "v1.3", @@ -294,7 +333,7 @@ def test_explicit_readme_content_overrides_aigc_model_value(self): readme_content="explicit value", ) - commit_body = request.call_args_list[1].kwargs["json_body"] + commit_body = legacy_request.call_args_list[0].kwargs["json_body"] operation = commit_body["actions"][0] assert operation["content"] == "ZXhwbGljaXQgdmFsdWU=" diff --git a/tests/test_create_repo_routing.py b/tests/test_create_repo_routing.py new file mode 100644 index 0000000..b5ff4aa --- /dev/null +++ b/tests/test_create_repo_routing.py @@ -0,0 +1,444 @@ +"""Routing of ``create_repo`` between the OpenAPI and legacy create endpoints. + +The OpenAPI schemas are closed: ``visibility`` has no ``internal``, +``POST /models`` restricts ``license`` to a published set, and both endpoints +answer 400 for undeclared fields. Inputs the schema cannot express therefore +keep using the legacy endpoint, so a call that works today keeps working. These +tests pin which surface each input lands on, asserting the method and path +actually sent rather than a mock's return value. +""" + +from __future__ import annotations + +from unittest import mock + +import pytest + +from modelscope_hub import HubApi +from modelscope_hub import api as api_module +from modelscope_hub.constants import GatedMode, RepoType, RepoVisibility, Visibility +from modelscope_hub.errors import AlreadyExistsError, InvalidParameter, NotExistError, ServerError + + +@pytest.fixture +def api() -> HubApi: + return HubApi(endpoint="https://modelscope.cn", token="ms-test") + + +def _legacy_response(data=None): + response = mock.MagicMock() + response.json.return_value = {"Data": data or {}} + return response + + +class _Routes: + """Captures which surface a create call reached, and with what body.""" + + def __init__(self, openapi_request, legacy_request): + self.openapi = openapi_request + self.legacy = legacy_request + + @property + def went_openapi(self) -> bool: + return self.openapi.called and not self.legacy.called + + @property + def went_legacy(self) -> bool: + return self.legacy.called and not self.openapi.called + + @property + def openapi_body(self) -> dict: + return self.openapi.call_args.kwargs["json_body"] + + @property + def legacy_body(self) -> dict: + kwargs = self.legacy.call_args.kwargs + if "json_body" in kwargs: + return kwargs["json_body"] + # The dataset endpoint posts multipart form fields. + return {k: v[1] for k, v in kwargs["files"].items()} + + +@pytest.fixture +def routes(api: HubApi): + created = {"id": "alice/thing", "owner": "alice", "repo_name": "thing"} + with ( + mock.patch.object(api.openapi, "_request", return_value=created) as openapi_request, + mock.patch.object(api.legacy, "_request", return_value=_legacy_response(created)) as legacy_request, + ): + yield _Routes(openapi_request, legacy_request) + + +# --------------------------------------------------------------------------- +# Inputs the OpenAPI schema expresses +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("repo_type", ["model", "dataset"]) +@pytest.mark.parametrize( + ("visibility", "expected"), + [ + (None, None), + ("public", "public"), + ("private", "private"), + (Visibility.PUBLIC, "public"), + (Visibility.PRIVATE, "private"), + ("gated", "gated"), + ], +) +def test_ordinary_visibility_uses_openapi(api, routes, repo_type, visibility, expected): + api.create_repo("alice/thing", repo_type, visibility=visibility) + + assert routes.went_openapi + assert routes.openapi.call_args.args == ("POST", f"/{repo_type}s") + assert routes.openapi_body.get("visibility") == expected + + +def test_license_in_the_enum_uses_openapi(api, routes): + api.create_repo("alice/thing", "model", license="apache-2.0") + + assert routes.went_openapi + assert routes.openapi_body["license"] == "apache-2.0" + + +def test_display_name_license_other_and_task_reach_openapi(api, routes): + api.create_repo( + "alice/thing", + "model", + chinese_name="我的模型", + description="desc", + license="other", + license_name="Custom", + license_link="https://example.com/LICENSE", + task="text-generation", + ) + + assert routes.went_openapi + assert routes.openapi_body == { + "owner": "alice", + "repo_name": "thing", + "display_name": "我的模型", + "description": "desc", + "license": "other", + "license_name": "Custom", + "license_link": "https://example.com/LICENSE", + "task": "text-generation", + } + + +def test_omitted_license_is_not_sent(api, routes): + """The legacy fallback value ``Apache-2.0`` is outside the OpenAPI enum.""" + api.create_repo("alice/thing", "model") + + assert routes.went_openapi + assert "license" not in routes.openapi_body + + +def test_task_is_rejected_for_datasets(api, routes): + """Kept as a defensive guard; datasets are routed to legacy before this.""" + with pytest.raises(InvalidParameter, match="task is only supported"): + api._create_via_openapi( + RepoType.DATASET, + "alice", + "thing", + visibility=None, + repo_visibility=None, + license=None, + chinese_name=None, + description=None, + gated=None, + license_name=None, + license_link=None, + task="text-classification", + extra={}, + ) + + +# --------------------------------------------------------------------------- +# Gated access +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("mode", ["auto", "manual", GatedMode.MANUAL]) +def test_gated_mode_is_sent_with_gated_visibility(api, routes, mode): + api.create_repo("alice/thing", "model", visibility="gated", gated=mode) + + assert routes.went_openapi + body = routes.openapi_body + assert body["visibility"] == "gated" + assert body["gated_mode"] == str(mode) + + +def test_gated_without_visibility_implies_gated_visibility(api, routes): + api.create_repo("alice/thing", "model", gated="manual") + + assert routes.openapi_body["visibility"] == "gated" + assert routes.openapi_body["gated_mode"] == "manual" + + +def test_legacy_boolean_gated_mode_keeps_the_legacy_wire_format(api, routes): + """The historical boolean must produce the request it always produced. + + It carries no approval mode, and the two surfaces default differently: + legacy sends ``ProtectedMode=1`` and lets the server decide, while the + OpenAPI schema defaults ``gated_mode`` to ``auto``. Translating it could + silently turn a review-gated repository into an auto-approving one. + """ + api.create_repo("alice/thing", "model", gated_mode=True) + + assert routes.went_legacy + body = routes.legacy_body + assert body["Visibility"] == int(Visibility.PRIVATE) + assert body["ProtectedMode"] == 1 + + +def test_gated_mode_false_keeps_the_legacy_wire_format(api, routes): + """Preserves the legacy behaviour of ``--no-gated`` with no visibility.""" + api.create_repo("alice/thing", "model", gated_mode=False) + + assert routes.went_legacy + body = routes.legacy_body + assert body["Visibility"] == int(Visibility.PRIVATE) + assert body["ProtectedMode"] == 2 + + +def test_gated_mode_with_public_visibility_warns_and_is_ignored(api, routes, caplog, monkeypatch): + """Unchanged legacy behaviour: warn, ignore the gating, stay public.""" + monkeypatch.setattr(api_module.logger, "handlers", [*api_module.logger.handlers, caplog.handler]) + with caplog.at_level("WARNING", logger="modelscope_hub.api"): + api.create_repo("alice/thing", "model", visibility="public", gated_mode=True) + + assert routes.went_legacy + body = routes.legacy_body + assert body["Visibility"] == int(Visibility.PUBLIC) + assert "ProtectedMode" not in body + assert "only effective when visibility is PRIVATE" in caplog.text + + +def test_the_new_gated_parameter_goes_through_openapi(api, routes): + """``gated`` can state the approval mode, so it has no legacy equivalent.""" + api.create_repo("alice/thing", "model", gated="manual") + + assert routes.went_openapi + assert routes.openapi_body["visibility"] == "gated" + assert routes.openapi_body["gated_mode"] == "manual" + + +def test_gated_false_without_visibility_implies_private_on_openapi(api, routes): + api.create_repo("alice/thing", "model", gated=False) + + assert routes.went_openapi + assert routes.openapi_body["visibility"] == RepoVisibility.PRIVATE.value + assert "gated_mode" not in routes.openapi_body + + +def test_gated_with_public_visibility_warns_and_is_dropped(api, routes, caplog, monkeypatch): + """The endpoint 400s on the pair, so mirror the legacy warn-and-ignore.""" + # The SDK logger deliberately does not propagate, so caplog needs attaching. + monkeypatch.setattr(api_module.logger, "handlers", [*api_module.logger.handlers, caplog.handler]) + with caplog.at_level("WARNING", logger="modelscope_hub.api"): + api.create_repo("alice/thing", "model", visibility="public", gated="manual") + + assert routes.openapi_body["visibility"] == "public" + assert "gated_mode" not in routes.openapi_body + assert "only effective when visibility is gated" in caplog.text + + +def test_an_unknown_gated_value_is_rejected(api, routes): + with pytest.raises(InvalidParameter, match="gated must be"): + api.create_repo("alice/thing", "model", gated="sometimes") + + +@pytest.mark.parametrize( + ("extra", "expected_route"), + [({}, "openapi"), ({"create_default_config": True}, "legacy")], +) +def test_private_plus_an_approval_mode_means_gated_on_either_route(api, routes, extra, expected_route): + """``private`` + a mode is what the legacy encoding always called gated. + + The two routes must agree: adding an unrelated field should not change the + access policy the repository ends up with. + """ + api.create_repo("alice/thing", "model", visibility="private", gated="manual", **extra) + + if expected_route == "openapi": + assert routes.went_openapi + assert routes.openapi_body["visibility"] == "gated" + assert routes.openapi_body["gated_mode"] == "manual" + else: + assert routes.went_legacy + assert routes.legacy_body["Visibility"] == int(Visibility.PRIVATE) + assert routes.legacy_body["ProtectedMode"] == 1 + + +def test_gated_visibility_survives_a_forced_legacy_route(api, routes): + """``visibility="gated"`` states the intent through visibility alone, so the + legacy encoding has to restate it as ``ProtectedMode``.""" + api.create_repo("alice/thing", "model", visibility="gated", create_default_config=True) + + assert routes.went_legacy + assert routes.legacy_body["Visibility"] == int(Visibility.PRIVATE) + assert routes.legacy_body["ProtectedMode"] == 1 + + +# --------------------------------------------------------------------------- +# Inputs the OpenAPI schema cannot express -> legacy +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("visibility", ["internal", Visibility.INTERNAL, 3, "3"]) +def test_internal_visibility_falls_back_to_legacy(api, routes, visibility): + api.create_repo("alice/thing", "model", visibility=visibility) + + assert routes.went_legacy + assert routes.legacy.call_args.args == ("POST", "models") + assert routes.legacy_body["Visibility"] == int(Visibility.INTERNAL) + + +@pytest.mark.parametrize("license_value", ["bsd-2-clause", "unlicense", "my-company-eula"]) +def test_a_model_license_outside_the_enum_falls_back_to_legacy(api, routes, license_value): + api.create_repo("alice/thing", "model", license=license_value) + + assert routes.went_legacy + assert routes.legacy_body["License"] == license_value + + +def test_a_dataset_license_is_unconstrained_and_stays_on_openapi(api, routes): + """``POST /datasets`` declares no license enum, unlike ``POST /models``.""" + api.create_repo("alice/thing", "dataset", license="my-company-eula") + + assert routes.went_openapi + assert routes.openapi_body["license"] == "my-company-eula" + + +def test_an_undeclared_extra_field_falls_back_to_legacy(api, routes): + api.create_repo("alice/thing", "model", create_default_config=True) + + assert routes.went_legacy + assert routes.legacy_body["create_default_config"] is True + + +def test_protected_visibility_is_rejected_for_models(api, routes): + with pytest.raises(ValueError): + api.create_repo("alice/thing", "model", visibility="protected") + + +def test_task_is_routed_to_legacy_for_datasets(api, routes): + """Only the model schema declares ``task``; legacy took it through extra.""" + api.create_repo("alice/thing", "dataset", task="text-classification") + + assert routes.went_legacy + assert routes.legacy_body["task"] == "text-classification" + + +def test_the_newly_named_params_keep_their_legacy_key_names(api, routes): + """They were ``extra`` keys before the signature grew; renaming them on the + legacy wire would change a request that used to work.""" + api.create_repo( + "alice/thing", + "model", + gated_mode=True, # forces the legacy route + task="text-generation", + license_name="Custom", + license_link="https://example.com/LICENSE", + ) + + assert routes.went_legacy + body = routes.legacy_body + # Appended verbatim, after ProtectedMode, exactly as ``**extra`` always was. + assert [k for k in body if k not in ("Path", "Name", "Visibility", "License")] == [ + "ProtectedMode", + "task", + "license_name", + "license_link", + ] + assert "Task" not in body + assert "LicenseName" not in body + assert "LicenseLink" not in body + + +# --------------------------------------------------------------------------- +# Endpoint availability +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("status", [404, 501]) +def test_an_unpublished_endpoint_degrades_to_legacy(api, status): + error = NotExistError("absent", status_code=status) if status == 404 else ServerError("absent", status_code=status) + with ( + mock.patch.object(api.openapi, "_request", side_effect=error) as openapi_request, + mock.patch.object(api.legacy, "_request", return_value=_legacy_response()) as legacy_request, + ): + api.create_repo("alice/thing", "model") + + assert openapi_request.called + assert legacy_request.call_args.args == ("POST", "models") + + +def test_other_server_errors_are_not_swallowed(api): + with ( + mock.patch.object(api.openapi, "_request", side_effect=ServerError("boom", status_code=500)), + mock.patch.object(api.legacy, "_request", return_value=_legacy_response()) as legacy_request, + ): + with pytest.raises(ServerError): + api.create_repo("alice/thing", "model") + + legacy_request.assert_not_called() + + +def test_a_duplicate_dataset_reported_as_400_becomes_already_exists(api): + """``POST /datasets`` declares no 409, so callers would otherwise miss it.""" + rejection = InvalidParameter("dataset already exists", status_code=400) + with mock.patch.object(api.openapi, "_request", side_effect=rejection): + with pytest.raises(AlreadyExistsError): + api.create_repo("alice/thing", "dataset") + + +def test_a_genuine_bad_request_stays_invalid_parameter(api): + with mock.patch.object(api.openapi, "_request", side_effect=InvalidParameter("bad name", status_code=400)): + with pytest.raises(InvalidParameter) as excinfo: + api.create_repo("alice/thing", "dataset") + assert not isinstance(excinfo.value, AlreadyExistsError) + + +# --------------------------------------------------------------------------- +# Returned metadata +# --------------------------------------------------------------------------- +def test_request_values_backfill_the_sparse_create_response(api, routes): + """The create response carries identity only; legacy echoed the settings.""" + info = api.create_repo( + "alice/thing", + "model", + visibility="private", + license="mit", + description="desc", + ) + + assert info.repo_id == "alice/thing" + assert info.visibility == Visibility.PRIVATE + assert info.license == "mit" + assert info.description == "desc" + assert info.gated is False + + +@pytest.mark.parametrize( + ("kwargs", "expected"), + [ + ({"visibility": "private"}, Visibility.PRIVATE), + ({"visibility": "public"}, Visibility.PUBLIC), + ({"visibility": 5}, Visibility.PUBLIC), + # Gating alone implies a visibility the caller never named. + ({"gated": False}, Visibility.PRIVATE), + ({"gated": "manual"}, Visibility.PRIVATE), + ], +) +def test_the_backfilled_visibility_is_an_enum_not_a_bare_int(api, routes, kwargs, expected): + """Assigned after ``__init__``, so it bypasses the dataclass's coercion. + + ``1 == Visibility.PRIVATE`` holds for a plain int, so this asserts on + ``.label`` -- which only exists on the enum. + """ + info = api.create_repo("alice/thing", "model", **kwargs) + + assert isinstance(info.visibility, Visibility) + assert info.visibility.label == expected.label + + +def test_a_gated_repo_reports_private_visibility_and_its_mode(api, routes): + info = api.create_repo("alice/thing", "model", visibility="gated", gated="manual") + + assert info.visibility == Visibility.PRIVATE + assert info.gated is True + assert info.gated_mode == "manual" diff --git a/tests/test_dataset_category.py b/tests/test_dataset_category.py new file mode 100644 index 0000000..3144857 --- /dev/null +++ b/tests/test_dataset_category.py @@ -0,0 +1,191 @@ +"""The dataset zone endpoints, and the settings fields that are not writable. + +Mirrors the model zone rules with two differences worth pinning: the general +dataset category is ``base`` (not ``basic``, as earlier drafts had it), and +``login_required`` appears in a dataset's detail payload but no request schema +accepts it, so it cannot be set through the SDK at all. +""" + +from __future__ import annotations + +from unittest import mock + +import pytest + +from modelscope_hub import HubApi +from modelscope_hub import api as api_module +from modelscope_hub.errors import InvalidParameter, NotSupportedError + + +@pytest.fixture +def api() -> HubApi: + return HubApi(endpoint="https://modelscope.cn", token="ms-test") + + +@pytest.fixture +def request_mock(api: HubApi): + with mock.patch.object(api.openapi, "_request", return_value={"id": "alice/d"}) as m: + yield m + + +# --------------------------------------------------------------------------- +# PUT /category +# --------------------------------------------------------------------------- +def test_setting_a_zone_uses_put(api, request_mock): + api.set_dataset_category( + "alice/d", + dataset_category="science", + primary_subject_area="life-sciences", + secondary_specialization="dna", + ) + + assert request_mock.call_args.args == ("PUT", "/datasets/alice/d/category") + assert request_mock.call_args.kwargs["json_body"] == { + "dataset_category": "science", + "primary_subject_area": "life-sciences", + "secondary_specialization": "dna", + } + + +def test_the_general_category_is_base_not_basic(api, request_mock): + api.set_dataset_category("alice/d", dataset_category="base") + + assert request_mock.call_args.kwargs["json_body"] == {"dataset_category": "base"} + + +def test_basic_is_rejected_and_the_valid_values_are_named(api, request_mock): + with pytest.raises(InvalidParameter, match="'base', 'science'"): + api.set_dataset_category("alice/d", dataset_category="basic") + request_mock.assert_not_called() + + +# --------------------------------------------------------------------------- +# PATCH /category +# --------------------------------------------------------------------------- +def test_updating_a_zone_uses_patch(api, request_mock): + api.update_dataset_category("alice/d", secondary_specialization="medicine") + + assert request_mock.call_args.args == ("PATCH", "/datasets/alice/d/category") + assert request_mock.call_args.kwargs["json_body"] == {"secondary_specialization": "medicine"} + + +def test_the_category_discriminator_is_dropped_with_a_warning(api, request_mock, caplog, monkeypatch): + monkeypatch.setattr(api_module.logger, "handlers", [*api_module.logger.handlers, caplog.handler]) + with caplog.at_level("WARNING", logger="modelscope_hub.api"): + api.update_dataset_category("alice/d", dataset_category="science", cover_image="https://x/y.png") + + assert "dataset_category" not in request_mock.call_args.kwargs["json_body"] + assert "set_dataset_category" in caplog.text + + +def test_an_explicit_none_is_forwarded_to_clear_a_field(api, request_mock): + """Omitting a field leaves it unchanged; ``None`` clears it.""" + api.update_dataset_category("alice/d", cover_image=None) + + assert request_mock.call_args.kwargs["json_body"] == {"cover_image": None} + + +# --------------------------------------------------------------------------- +# PATCH /settings +# --------------------------------------------------------------------------- +def test_dataset_settings_reach_the_settings_endpoint(api, request_mock): + api.update_repo_settings("alice/d", "dataset", description="new", display_name="D") + + assert request_mock.call_args.args == ("PATCH", "/datasets/alice/d/settings") + assert request_mock.call_args.kwargs["json_body"] == {"description": "new", "display_name": "D"} + + +def test_model_settings_reach_the_settings_endpoint(api, request_mock): + api.update_repo_settings("alice/m", "model", task="text-generation") + + assert request_mock.call_args.args == ("PATCH", "/models/alice/m/settings") + assert request_mock.call_args.kwargs["json_body"] == {"task": "text-generation"} + + +@pytest.mark.parametrize("repo_type", ["model", "dataset"]) +def test_gated_settings_emit_both_wire_fields(api, request_mock, repo_type): + api.update_repo_settings("alice/x", repo_type, visibility="gated", gated="manual") + + assert request_mock.call_args.kwargs["json_body"] == {"visibility": "gated", "gated_mode": "manual"} + + +def test_naming_an_approval_mode_alone_turns_gating_on(api, request_mock): + """A mode without a visibility is a request to gate the repository.""" + api.update_repo_settings("alice/d", "dataset", gated="auto") + + assert request_mock.call_args.kwargs["json_body"] == {"visibility": "gated", "gated_mode": "auto"} + + +# The wire field is accepted alongside the convenience argument and forwarded +# as given, so the contract's own rules apply to it. +@pytest.mark.parametrize("mode", ["auto", "manual", ""]) +def test_a_native_gated_mode_is_forwarded_verbatim(api, request_mock, mode): + """An empty string is valid and means "default to auto" server-side.""" + api.update_repo_settings("alice/d", "dataset", visibility="gated", gated_mode=mode) + + assert request_mock.call_args.kwargs["json_body"] == {"visibility": "gated", "gated_mode": mode} + + +@pytest.mark.parametrize("bad", [None, True, False]) +def test_a_null_or_boolean_gated_mode_is_rejected(api, request_mock, bad): + """The schema rejects both; catching it here names the right replacement.""" + with pytest.raises(InvalidParameter, match="gated_mode must be"): + api.update_repo_settings("alice/d", "dataset", gated_mode=bad) + request_mock.assert_not_called() + + +def test_the_two_gated_spellings_cannot_be_combined(api, request_mock): + with pytest.raises(InvalidParameter, match="either gated or gated_mode"): + api.update_repo_settings("alice/d", "dataset", gated="auto", gated_mode="manual") + request_mock.assert_not_called() + + +@pytest.mark.parametrize("visibility", ["public", "private"]) +def test_gated_conflicting_with_visibility_is_refused(api, request_mock, visibility): + """The endpoint 400s on the pair; refuse against the caller's own arguments.""" + with pytest.raises(InvalidParameter, match="conflicts with visibility"): + api.update_repo_settings("alice/d", "dataset", visibility=visibility, gated="auto") + request_mock.assert_not_called() + + +def test_turning_gating_off_without_a_visibility_leaves_the_gated_state(api, request_mock): + api.update_repo_settings("alice/d", "dataset", gated=False) + + assert request_mock.call_args.kwargs["json_body"] == {"visibility": "private"} + + +def test_integer_visibility_is_translated_for_settings(api, request_mock): + api.update_repo_settings("alice/d", "dataset", visibility=1) + + assert request_mock.call_args.kwargs["json_body"] == {"visibility": "private"} + + +def test_internal_visibility_is_refused_for_settings(api, request_mock): + """Unlike create, there is no legacy settings endpoint to fall back to.""" + with pytest.raises(InvalidParameter, match="not supported"): + api.update_repo_settings("alice/d", "dataset", visibility="internal") + request_mock.assert_not_called() + + +def test_login_required_is_reported_as_unsupported(api, request_mock): + """No request schema accepts it, so silently dropping it would mislead.""" + with pytest.raises(NotSupportedError, match="login_required"): + api.update_repo_settings("alice/d", "dataset", login_required=True) + request_mock.assert_not_called() + + +def test_unsupported_repo_types_still_raise(api, request_mock): + with pytest.raises(NotSupportedError): + api.update_repo_settings("alice/x", "mcp", description="x") + + +# --------------------------------------------------------------------------- +# Repo type guards +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("repo_type", ["model", "studio", "skill"]) +def test_the_dataset_methods_reject_other_repo_types(api, request_mock, repo_type): + with pytest.raises(NotSupportedError): + api.set_dataset_category("alice/d", repo_type, dataset_category="base") + with pytest.raises(NotSupportedError): + api.update_dataset_category("alice/d", repo_type, cover_image=None) + request_mock.assert_not_called() diff --git a/tests/test_model_category.py b/tests/test_model_category.py new file mode 100644 index 0000000..4e8a66c --- /dev/null +++ b/tests/test_model_category.py @@ -0,0 +1,261 @@ +"""The model zone endpoints: ``PUT`` to change zone, ``PATCH`` to edit in place. + +The two verbs are not interchangeable, and their payload rules are close to +inverses of each other: + +* ``PUT`` sets the zone. Required fields follow from the *target* zone, and + fields belonging to another zone are rejected. +* ``PATCH`` edits the current zone. The server reads the zone itself and + rejects the discriminator, addresses AIGC versions by numeric id, and refuses + to touch an existing version's weights. + +Checks the SDK can make locally are made locally, so a request certain to fail +names the fix instead of costing a round trip for a generic 400. +""" + +from __future__ import annotations + +from unittest import mock + +import pytest + +from modelscope_hub import HubApi +from modelscope_hub import api as api_module +from modelscope_hub.constants import WEIGHT_FILE_SUFFIXES +from modelscope_hub.errors import InvalidParameter, NotSupportedError + +_AIGC = { + "aigc_type": "LoRA", + "base_model_architecture_type": "Z_IMAGE_TURBO", + "base_model_architecture_subtype": "Z_IMAGE_TURBO", + "version_name": "v1.0", + "weight_files": [{"path_in_repo": "model.safetensors"}], + "sample_images": [{"url": "https://example.com/a.png", "prompt": "a cat"}], +} + + +@pytest.fixture +def api() -> HubApi: + return HubApi(endpoint="https://modelscope.cn", token="ms-test") + + +@pytest.fixture +def request_mock(api: HubApi): + with mock.patch.object(api.openapi, "_request", return_value={"id": "alice/m"}) as m: + yield m + + +# --------------------------------------------------------------------------- +# PUT: verb, path, and the per-zone required fields +# --------------------------------------------------------------------------- +def test_setting_a_zone_uses_put(api, request_mock): + api.set_model_category("alice/m", model_category="base") + + assert request_mock.call_args.args == ("PUT", "/models/alice/m/category") + assert request_mock.call_args.kwargs["json_body"] == {"model_category": "base"} + + +def test_moving_to_aigc_sends_every_required_field(api, request_mock): + api.set_model_category("alice/m", model_category="aigc", **_AIGC) + + body = request_mock.call_args.kwargs["json_body"] + assert body["model_category"] == "aigc" + assert body["version_name"] == "v1.0" + assert body["weight_files"] == [{"path_in_repo": "model.safetensors"}] + assert body["sample_images"] == [{"url": "https://example.com/a.png", "prompt": "a cat"}] + + +def test_moving_to_science_needs_only_the_subject_areas(api, request_mock): + api.set_model_category( + "alice/m", + model_category="science", + primary_subject_area="life-sciences", + secondary_specialization="dna", + ) + + assert request_mock.call_args.kwargs["json_body"] == { + "model_category": "science", + "primary_subject_area": "life-sciences", + "secondary_specialization": "dna", + } + + +@pytest.mark.parametrize("missing", sorted(_AIGC)) +def test_a_missing_aigc_field_is_reported_before_the_request(api, request_mock, missing): + payload = {k: v for k, v in _AIGC.items() if k != missing} + + with pytest.raises(InvalidParameter, match=missing): + api.set_model_category("alice/m", model_category="aigc", **payload) + request_mock.assert_not_called() + + +@pytest.mark.parametrize("empty", [[], None]) +def test_empty_weight_files_and_sample_images_are_rejected(api, request_mock, empty): + for field in ("weight_files", "sample_images"): + with pytest.raises(InvalidParameter, match=field): + api.set_model_category("alice/m", model_category="aigc", **{**_AIGC, field: empty}) + request_mock.assert_not_called() + + +def test_an_unknown_category_names_the_valid_values(api, request_mock): + with pytest.raises(InvalidParameter, match="'base', 'aigc', 'science'"): + api.set_model_category("alice/m", model_category="basic") + request_mock.assert_not_called() + + +def test_an_unknown_aigc_type_names_the_valid_values(api, request_mock): + with pytest.raises(InvalidParameter, match="Checkpoint"): + api.set_model_category("alice/m", model_category="aigc", **{**_AIGC, "aigc_type": "lora"}) + request_mock.assert_not_called() + + +@pytest.mark.parametrize("suffix", [".safetensors", ".CKPT", ".Bin", ".onnx"]) +def test_recognised_weight_suffixes_are_accepted_case_insensitively(api, request_mock, suffix): + api.set_model_category( + "alice/m", + model_category="aigc", + **{**_AIGC, "weight_files": [{"path_in_repo": f"sub/dir/model{suffix}"}]}, + ) + + assert request_mock.called + + +def test_an_unrecognised_weight_suffix_lists_the_supported_ones(api, request_mock): + with pytest.raises(InvalidParameter) as excinfo: + api.set_model_category( + "alice/m", + model_category="aigc", + **{**_AIGC, "weight_files": [{"path_in_repo": "notes.txt"}]}, + ) + + message = str(excinfo.value) + assert "notes.txt" in message + for suffix in WEIGHT_FILE_SUFFIXES: + assert suffix in message + request_mock.assert_not_called() + + +def test_a_weight_file_without_a_path_is_rejected(api, request_mock): + with pytest.raises(InvalidParameter, match="path_in_repo"): + api.set_model_category("alice/m", model_category="aigc", **{**_AIGC, "weight_files": [{}]}) + + +def test_a_sample_image_without_a_url_is_rejected(api, request_mock): + with pytest.raises(InvalidParameter, match="url"): + api.set_model_category( + "alice/m", + model_category="aigc", + **{**_AIGC, "sample_images": [{"prompt": "no url"}]}, + ) + + +def test_the_retired_sample_image_text_key_is_rejected(api, request_mock): + """The schema takes ``prompt``/``negative_prompt``; ``text`` never existed.""" + with pytest.raises(InvalidParameter, match="prompt"): + api.set_model_category( + "alice/m", + model_category="aigc", + **{**_AIGC, "sample_images": [{"url": "https://example.com/a.png", "text": "cat"}]}, + ) + + +def test_an_overlong_version_name_is_rejected(api, request_mock): + with pytest.raises(InvalidParameter, match="version_name"): + api.set_model_category("alice/m", model_category="aigc", **{**_AIGC, "version_name": "v" * 21}) + + +# --------------------------------------------------------------------------- +# PATCH: verb, path, and the inverted payload rules +# --------------------------------------------------------------------------- +def test_updating_a_zone_uses_patch(api, request_mock): + api.update_model_category("alice/m", primary_subject_area="life-sciences") + + assert request_mock.call_args.args == ("PATCH", "/models/alice/m/category") + assert request_mock.call_args.kwargs["json_body"] == {"primary_subject_area": "life-sciences"} + + +def test_the_category_discriminator_is_dropped_with_a_warning(api, request_mock, caplog, monkeypatch): + """Earlier drafts required the field; the endpoint now 400s if it is present.""" + monkeypatch.setattr(api_module.logger, "handlers", [*api_module.logger.handlers, caplog.handler]) + with caplog.at_level("WARNING", logger="modelscope_hub.api"): + api.update_model_category("alice/m", model_category="aigc", trigger_words=["x"]) + + assert "model_category" not in request_mock.call_args.kwargs["json_body"] + assert "set_model_category" in caplog.text + + +@pytest.mark.parametrize("field", ["weight_files", "base_model_id"]) +def test_fields_only_put_can_change_are_refused(api, request_mock, field): + with pytest.raises(InvalidParameter, match="set_model_category"): + api.update_model_category("alice/m", **{field: "whatever"}) + request_mock.assert_not_called() + + +def test_version_name_is_a_resolution_hint_not_a_wire_field(api): + """The endpoint rejects ``version_name``; the SDK turns it into a lookup.""" + detail = {"versions": [{"version_name": "v1.0", "version_id": 42}]} + + with mock.patch.object(api.openapi, "_request", side_effect=[detail, {"id": "alice/m"}]) as request_mock: + api.update_model_category("alice/m", version_name="v1.0", trigger_words=["x"]) + + body = request_mock.call_args_list[1].kwargs["json_body"] + assert "version_name" not in body + assert body["version_id"] == 42 + + +def test_an_explicit_version_id_is_forwarded(api, request_mock): + api.update_model_category("alice/m", version_id=1070760, trigger_words=["anime"]) + + assert request_mock.call_args.kwargs["json_body"]["version_id"] == 1070760 + + +def test_a_science_update_sends_no_version_id(api, request_mock): + """The endpoint rejects ``version_id`` outside the AIGC zone.""" + api.update_model_category("alice/m", cover_image=None) + + assert "version_id" not in request_mock.call_args.kwargs["json_body"] + + +# --------------------------------------------------------------------------- +# version_name -> version_id resolution +# --------------------------------------------------------------------------- +def test_a_version_name_is_resolved_to_its_numeric_id(api): + detail = {"model_category": "aigc", "versions": [{"version_name": "v1.0", "version_id": 42}]} + + with mock.patch.object(api.openapi, "_request", side_effect=[detail, {"id": "alice/m"}]) as request_mock: + api.update_model_category("alice/m", version_name="v1.0", trigger_words=["anime"]) + + detail_call, update_call = request_mock.call_args_list + assert detail_call.args == ("GET", "/models/alice/m") + body = update_call.kwargs["json_body"] + assert body["version_id"] == 42 + assert "version_name" not in body + + +def test_an_unknown_version_name_lists_what_is_available(api): + detail = {"versions": [{"version_name": "v1.0", "version_id": 42}]} + + with mock.patch.object(api.openapi, "_request", return_value=detail): + with pytest.raises(InvalidParameter) as excinfo: + api.update_model_category("alice/m", version_name="v9.9", trigger_words=["x"]) + + assert "'v1.0'" in str(excinfo.value) + + +def test_an_explicit_version_id_skips_the_lookup(api, request_mock): + api.update_model_category("alice/m", version_id=7, version_name="v1.0", trigger_words=["x"]) + + request_mock.assert_called_once() + assert request_mock.call_args.kwargs["json_body"]["version_id"] == 7 + + +# --------------------------------------------------------------------------- +# Repo type guards +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("repo_type", ["dataset", "studio", "skill"]) +def test_the_model_methods_reject_other_repo_types(api, request_mock, repo_type): + with pytest.raises(NotSupportedError): + api.set_model_category("alice/m", repo_type, model_category="base") + with pytest.raises(NotSupportedError): + api.update_model_category("alice/m", repo_type, trigger_words=["x"]) + request_mock.assert_not_called() diff --git a/tests/test_model_detail_fields.py b/tests/test_model_detail_fields.py new file mode 100644 index 0000000..e4e0d1a --- /dev/null +++ b/tests/test_model_detail_fields.py @@ -0,0 +1,235 @@ +"""``get_model`` / ``get_dataset`` return the detail payload, not a ``RepoInfo``. + +The zone fields -- AIGC architecture and ``versions``, or the science subject +areas -- have no place on the ``RepoInfo`` dataclass, so ``get_repo`` drops +them. These methods hand back the server's payload as-is instead. + +Which keys are present depends on the category: the fields of the zones a +repository is *not* in are absent rather than null, so callers must use +``.get()`` and must not assume a fixed shape. +""" + +from __future__ import annotations + +from unittest import mock + +import pytest + +from modelscope_hub import HubApi +from modelscope_hub.compat import LegacyHubApi +from modelscope_hub.constants import RepoType + +_BASE_MODEL = { + "id": "alice/plain", + "owner": "alice", + "repo_name": "plain", + "model_category": "base", + "visibility": "public", + "downloads": 10, + "likes": 2, +} + +_AIGC_MODEL = { + "id": "alice/lora", + "owner": "alice", + "repo_name": "lora", + "model_category": "aigc", + "visibility": "public", + "aigc_type": "LoRA", + "base_model_architecture_type": "Z_IMAGE_TURBO", + "base_model_architecture_subtype": "Z_IMAGE_TURBO", + "content_categories": {"style": "ILLUSTRATION"}, + "versions": [ + { + "version_id": 1070760, + "version_name": "v1.0", + "weight_files": [{"path_in_repo": "model.safetensors", "size": 42, "sha": "abc"}], + "sample_images": [{"url": "https://example.com/a.png", "prompt": "a cat"}], + "trigger_words": ["anime"], + "created_at": "2026-06-22T10:00:00Z", + } + ], +} + +_SCIENCE_MODEL = { + "id": "alice/rna", + "owner": "alice", + "repo_name": "rna", + "model_category": "science", + "primary_subject_area": "life-sciences", + "secondary_specialization": "dna", + "cover_image": "https://example.com/cover.png", +} + +_SCIENCE_DATASET = { + "id": "alice/rna-seq", + "owner": "alice", + "repo_name": "rna-seq", + "dataset_category": "science", + "login_required": True, + "primary_subject_area": "life-sciences", + "secondary_specialization": "dna", +} + + +@pytest.fixture +def api() -> HubApi: + return HubApi(endpoint="https://modelscope.cn", token="ms-test") + + +# --------------------------------------------------------------------------- +# Routing +# --------------------------------------------------------------------------- +def test_get_model_reads_the_model_detail_endpoint(api): + with mock.patch.object(api.openapi, "_request", return_value=_AIGC_MODEL) as request_mock: + detail = api.get_model("alice/lora") + + assert request_mock.call_args.args == ("GET", "/models/alice/lora") + assert detail == _AIGC_MODEL + + +def test_get_dataset_reads_the_dataset_detail_endpoint(api): + with mock.patch.object(api.openapi, "_request", return_value=_SCIENCE_DATASET) as request_mock: + detail = api.get_dataset("alice/rna-seq") + + assert request_mock.call_args.args == ("GET", "/datasets/alice/rna-seq") + assert detail == _SCIENCE_DATASET + + +@pytest.mark.parametrize(("alias", "canonical"), [("model_info", "get_model"), ("dataset_info", "get_dataset")]) +def test_the_info_aliases_match_their_canonical_methods(api, alias, canonical): + with mock.patch.object(api.openapi, "_request", return_value=_BASE_MODEL) as request_mock: + from_alias = getattr(api, alias)("alice/plain") + from_canonical = getattr(api, canonical)("alice/plain") + + assert from_alias == from_canonical + assert request_mock.call_args_list[0].args == request_mock.call_args_list[1].args + + +# --------------------------------------------------------------------------- +# Zone fields survive, and are category-dependent +# --------------------------------------------------------------------------- +def test_aigc_versions_survive_with_their_numeric_ids(api): + """``update_model_category`` needs ``version_id``, so it must not be lost.""" + with mock.patch.object(api.openapi, "_request", return_value=_AIGC_MODEL): + detail = api.get_model("alice/lora") + + version = detail["versions"][0] + assert version["version_id"] == 1070760 + assert version["weight_files"][0]["sha"] == "abc" + assert version["sample_images"][0]["prompt"] == "a cat" + + +def test_a_base_model_carries_no_zone_fields(api): + with mock.patch.object(api.openapi, "_request", return_value=_BASE_MODEL): + detail = api.get_model("alice/plain") + + assert detail["model_category"] == "base" + for absent in ("aigc_type", "versions", "primary_subject_area", "cover_image"): + assert absent not in detail + + +def test_a_science_model_carries_subject_areas_and_no_versions(api): + with mock.patch.object(api.openapi, "_request", return_value=_SCIENCE_MODEL): + detail = api.get_model("alice/rna") + + assert detail["primary_subject_area"] == "life-sciences" + assert detail["cover_image"] == "https://example.com/cover.png" + assert "versions" not in detail + assert "aigc_type" not in detail + + +def test_an_aigc_model_carries_no_cover_image(api): + """Cover images belong to the science zone only.""" + with mock.patch.object(api.openapi, "_request", return_value=_AIGC_MODEL): + detail = api.get_model("alice/lora") + + assert "cover_image" not in detail + + +def test_login_required_is_readable_even_though_it_is_not_writable(api): + with mock.patch.object(api.openapi, "_request", return_value=_SCIENCE_DATASET): + detail = api.get_dataset("alice/rna-seq") + + assert detail["login_required"] is True + + +def test_a_non_mapping_payload_degrades_to_an_empty_dict(api): + with mock.patch.object(api.openapi, "_request", return_value=None): + assert api.get_model("alice/plain") == {} + assert api.get_dataset("alice/plain") == {} + + +# --------------------------------------------------------------------------- +# RepoInfo keeps the category, without gaining the zone payload +# --------------------------------------------------------------------------- +def test_get_repo_reports_the_category_but_not_the_zone_payload(api): + with mock.patch.object(api.openapi, "_request", return_value=_AIGC_MODEL): + info = api.get_repo("alice/lora", "model") + + assert info.model_category == "aigc" + assert info.repo_id == "alice/lora" + # ``versions`` has no dataclass field: use get_model() for it. + assert not hasattr(info, "versions") + + +def test_a_dataset_repo_info_reports_its_category(api): + with mock.patch.object(api.openapi, "_request", return_value=_SCIENCE_DATASET): + info = api.get_repo("alice/rna-seq", "dataset") + + assert info.dataset_category == "science" + assert info.login_required is True + + +def test_the_new_fields_are_omitted_from_to_dict_when_absent(api): + with mock.patch.object(api.openapi, "_request", return_value={"id": "alice/x", "private": False}): + info = api.get_repo("alice/x", "studio") + + as_dict = info.to_dict() + for optional in ("model_category", "dataset_category", "gated_mode"): + assert optional not in as_dict + + +# --------------------------------------------------------------------------- +# Compat surface keeps both shapes available +# --------------------------------------------------------------------------- +def test_compat_get_model_keeps_its_legacy_pascalcase_shape(api): + compat = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") + + with mock.patch.object(compat._api.openapi, "_request", return_value=_AIGC_MODEL): + out = compat.get_model("alice/lora") + + # Historical contract: a flattened RepoInfo with PascalCase keys. + assert out["Id"] == "alice/lora" + assert "model_category" in out # carried through as a RepoInfo field + + +def test_compat_get_model_detail_returns_the_raw_payload(api): + compat = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") + + with mock.patch.object(compat._api.openapi, "_request", return_value=_AIGC_MODEL): + out = compat.get_model_detail("alice/lora") + + assert out == _AIGC_MODEL + assert out["versions"][0]["version_id"] == 1070760 + + +def test_compat_get_dataset_detail_returns_the_raw_payload(api): + compat = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") + + with mock.patch.object(compat._api.openapi, "_request", return_value=_SCIENCE_DATASET): + out = compat.get_dataset_detail("alice/rna-seq") + + assert out == _SCIENCE_DATASET + + +def test_compat_get_dataset_still_returns_a_repo_info(): + """Deprecated but unchanged: it hands back the dataclass, not a dict.""" + compat = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") + + with mock.patch.object(compat._api.openapi, "_request", return_value=_SCIENCE_DATASET): + with pytest.deprecated_call(): + out = compat.get_dataset("alice/rna-seq") + + assert out.repo_id == "alice/rna-seq" + assert out.repo_type is RepoType.DATASET diff --git a/tests/test_model_options.py b/tests/test_model_options.py new file mode 100644 index 0000000..f7cf625 --- /dev/null +++ b/tests/test_model_options.py @@ -0,0 +1,207 @@ +"""The enum option endpoints: anonymous, cached, and ``{type, id, label}``. + +These four endpoints publish product configuration rather than account data, so +they take no credentials and must stay reachable without a token. Their values +change on a release cadence, so repeated reads are served from a short-lived +cache -- otherwise setting a category would re-fetch the whole enum tree on +every call. +""" + +from __future__ import annotations + +import importlib +from unittest import mock + +import pytest + +from modelscope_hub import HubApi + +_AIGC_OPTIONS = { + "aigc_types": [{"type": "aigc_type", "id": "LoRA", "label": "LoRA"}], + "base_architectures": [ + { + "type": "base_architecture", + "id": "Z_IMAGE_TURBO", + "label": "Z-Image Turbo", + "subtypes": [{"type": "base_architecture_subtype", "id": "Z_IMAGE_TURBO", "label": "Z-Image Turbo"}], + } + ], + "content_styles": [{"type": "content_style", "id": "ILLUSTRATION", "label": "插画"}], + "content_functions": [{"type": "content_function", "id": "TEXT_TO_IMAGE", "label": "文生图"}], + "content_elements": [{"type": "content_element", "id": "PERSON", "label": "人物"}], +} + +_SCIENCE_OPTIONS = { + "subject_areas": [ + { + "type": "subject_area", + "id": "life-sciences", + "label": "生命科学", + "specializations": [{"type": "specialization", "id": "dna", "label": "基因组学"}], + } + ] +} + +_MODEL_OPTIONS = { + "tasks": [{"type": "task", "id": "text-generation", "label": "Text Generation"}], + "license": [{"type": "license", "id": "apache-2.0", "label": "Apache License 2.0"}], +} + +# (HubApi method, request path, payload) +_ENDPOINTS = [ + ("list_aigc_model_options", "/model/aigc/options", _AIGC_OPTIONS), + ("list_science_model_options", "/model/science/options", _SCIENCE_OPTIONS), + ("list_science_dataset_options", "/dataset/science/options", _SCIENCE_OPTIONS), + ("list_model_options", "/models/options", _MODEL_OPTIONS), +] + + +@pytest.fixture +def api() -> HubApi: + return HubApi(endpoint="https://modelscope.cn", token="ms-test") + + +# --------------------------------------------------------------------------- +# Routing and anonymity +# --------------------------------------------------------------------------- +@pytest.mark.parametrize(("method", "path", "payload"), _ENDPOINTS) +def test_each_endpoint_is_reached_without_requiring_a_token(api, method, path, payload): + with mock.patch.object(api.openapi, "_request", return_value=payload) as request_mock: + result = getattr(api, method)() + + assert request_mock.call_args.args == ("GET", path) + assert request_mock.call_args.kwargs["require_token"] is False + assert result == payload + + +@pytest.mark.parametrize(("method", "path", "payload"), _ENDPOINTS) +def test_the_endpoints_work_with_no_credentials_at_all(method, path, payload): + anonymous = HubApi(endpoint="https://modelscope.cn", token=None) + + with mock.patch.object(anonymous.openapi, "_request", return_value=payload) as request_mock: + getattr(anonymous, method)() + + assert request_mock.call_args.kwargs["require_token"] is False + + +@pytest.mark.parametrize(("method", "_path", "_payload"), _ENDPOINTS) +def test_a_non_mapping_payload_degrades_to_an_empty_dict(api, method, _path, _payload): + with mock.patch.object(api.openapi, "_request", return_value=None): + assert getattr(api, method)() == {} + + +# --------------------------------------------------------------------------- +# Option item shape +# --------------------------------------------------------------------------- +def test_options_are_submitted_by_id_and_displayed_by_label(api): + with mock.patch.object(api.openapi, "_request", return_value=_AIGC_OPTIONS): + options = api.list_aigc_model_options() + + architecture = options["base_architectures"][0] + for key in ("type", "id", "label"): + assert key in architecture + assert "value" not in architecture # an earlier draft's spelling + assert architecture["subtypes"][0]["id"] == "Z_IMAGE_TURBO" + + +def test_the_science_tree_cascades_from_subject_area_to_specialization(api): + with mock.patch.object(api.openapi, "_request", return_value=_SCIENCE_OPTIONS): + tree = api.list_science_model_options() + + area = tree["subject_areas"][0] + assert area["id"] == "life-sciences" + assert [s["id"] for s in area["specializations"]] == ["dna"] + + +def test_model_options_publish_tasks_and_licenses(api): + with mock.patch.object(api.openapi, "_request", return_value=_MODEL_OPTIONS): + options = api.list_model_options() + + assert [t["id"] for t in options["tasks"]] == ["text-generation"] + assert [lic["id"] for lic in options["license"]] == ["apache-2.0"] + + +# --------------------------------------------------------------------------- +# Caching +# --------------------------------------------------------------------------- +def test_a_repeated_read_is_served_from_the_cache(api): + with mock.patch.object(api.openapi, "_request", return_value=_AIGC_OPTIONS) as request_mock: + first = api.list_aigc_model_options() + second = api.list_aigc_model_options() + + request_mock.assert_called_once() + assert first == second + + +def test_each_endpoint_is_cached_separately(api): + with mock.patch.object(api.openapi, "_request", side_effect=[_AIGC_OPTIONS, _SCIENCE_OPTIONS]) as request_mock: + api.list_aigc_model_options() + api.list_science_model_options() + + assert [call.args[1] for call in request_mock.call_args_list] == [ + "/model/aigc/options", + "/model/science/options", + ] + + +def test_clearing_the_cache_forces_a_refetch(api): + with mock.patch.object(api.openapi, "_request", return_value=_AIGC_OPTIONS) as request_mock: + api.list_aigc_model_options() + api.openapi.clear_options_cache() + api.list_aigc_model_options() + + assert request_mock.call_count == 2 + + +def test_two_clients_do_not_share_a_cache(api): + other = HubApi(endpoint="https://modelscope.cn", token="ms-test") + + with ( + mock.patch.object(api.openapi, "_request", return_value=_AIGC_OPTIONS) as first_mock, + mock.patch.object(other.openapi, "_request", return_value=_AIGC_OPTIONS) as second_mock, + ): + api.list_aigc_model_options() + other.list_aigc_model_options() + + first_mock.assert_called_once() + second_mock.assert_called_once() + + +def test_a_zero_ttl_disables_caching(monkeypatch): + """``MODELSCOPE_OPTIONS_CACHE_TTL=0`` is the documented opt-out.""" + monkeypatch.setenv("MODELSCOPE_OPTIONS_CACHE_TTL", "0") + import modelscope_hub.constants as constants_module + + importlib.reload(constants_module) + assert constants_module.OPTIONS_CACHE_TTL_SECONDS == 0 + + import modelscope_hub._openapi as openapi_module + + importlib.reload(openapi_module) + try: + client = openapi_module.OpenAPIClient() + with mock.patch.object(client, "_request", return_value=_AIGC_OPTIONS) as request_mock: + client.list_aigc_model_options() + client.list_aigc_model_options() + assert request_mock.call_count == 2 + finally: + # Restore the real modules so later tests see the default TTL. + monkeypatch.delenv("MODELSCOPE_OPTIONS_CACHE_TTL", raising=False) + importlib.reload(constants_module) + importlib.reload(openapi_module) + importlib.reload(importlib.import_module("modelscope_hub.api")) + importlib.reload(importlib.import_module("modelscope_hub")) + + +def test_an_expired_entry_is_refetched(api, monkeypatch): + import modelscope_hub._openapi as openapi_module + + monkeypatch.setattr(openapi_module, "OPTIONS_CACHE_TTL_SECONDS", 300) + clock = iter([0.0, 0.0, 1000.0, 1000.0]) + monkeypatch.setattr(openapi_module.time, "monotonic", lambda: next(clock)) + + with mock.patch.object(api.openapi, "_request", return_value=_AIGC_OPTIONS) as request_mock: + api.list_aigc_model_options() + api.list_aigc_model_options() + + assert request_mock.call_count == 2 diff --git a/tests/test_openapi_coverage.py b/tests/test_openapi_coverage.py index 344f87b..096d23d 100644 --- a/tests/test_openapi_coverage.py +++ b/tests/test_openapi_coverage.py @@ -24,20 +24,20 @@ _HTTP_METHODS = frozenset({"get", "post", "put", "delete", "patch", "head", "options"}) # Tags the SDK implements end to end. -_COVERED_TAGS = frozenset({"Agent-IDP", "MCP", "Studios"}) +_COVERED_TAGS = frozenset({"Agent-IDP", "Datasets", "MCP", "Models", "Studios"}) # Tags not implemented yet. Remove an entry as its tag lands; until then the # guard would otherwise fail on operations nobody has promised. _DEFERRED_TAGS = frozenset( { "User", - "Models", - "Datasets", "Skills", "Files", "Collections", "Galleries", "Magicube", + "AgentHub", + "Agent Playground", } ) diff --git a/tests/test_openapi_delete_files.py b/tests/test_openapi_delete_files.py new file mode 100644 index 0000000..a91b0c1 --- /dev/null +++ b/tests/test_openapi_delete_files.py @@ -0,0 +1,160 @@ +"""The OpenAPI delete-files endpoint, and why ``delete_files`` does not use it. + +The endpoint is strictly narrower than the commit-based deletion the SDK +already performs: it caps at 100 paths, takes no directories or glob patterns, +accepts only a branch, and is all-or-nothing. The commit path has none of those +limits, so :meth:`HubApi.delete_files` keeps using it; routing it here would +turn working calls into failures. + +The endpoint is still exposed on the OpenAPI client for callers that want its +``commit_sha`` and its strict guarantee. The regression guard below is the +point of this file: ``HubApi.delete_files`` must stay on the commit path. +""" + +from __future__ import annotations + +from unittest import mock + +import pytest + +from modelscope_hub import HubApi +from modelscope_hub.constants import OPENAPI_DELETE_FILES_MAX_PATHS +from modelscope_hub.errors import InvalidParameter + +_RESULT = { + "deleted_files": ["a.txt", "b.txt"], + "failed_files": [], + "total_files": 2, + "commit_sha": "a1b2c3", +} + + +@pytest.fixture +def api() -> HubApi: + return HubApi(endpoint="https://modelscope.cn", token="ms-test") + + +# --------------------------------------------------------------------------- +# The endpoint itself +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + ("method", "path"), + [ + ("delete_model_files", "/models/alice/m/files"), + ("delete_dataset_files", "/datasets/alice/m/files"), + ], +) +def test_the_request_uses_delete_with_a_json_body(api, method, path): + with mock.patch.object(api.openapi, "_request", return_value=_RESULT) as request_mock: + result = getattr(api.openapi, method)("alice", "m", {"file_paths": ["a.txt", "b.txt"]}) + + assert request_mock.call_args.args == ("DELETE", path) + assert request_mock.call_args.kwargs["json_body"] == {"file_paths": ["a.txt", "b.txt"]} + assert result["commit_sha"] == "a1b2c3" + + +def test_revision_and_commit_message_are_forwarded(api): + with mock.patch.object(api.openapi, "_request", return_value=_RESULT) as request_mock: + api.openapi.delete_model_files( + "alice", + "m", + {"file_paths": ["a.txt"], "revision": "dev", "commit_message": "tidy up"}, + ) + + body = request_mock.call_args.kwargs["json_body"] + assert body["revision"] == "dev" + assert body["commit_message"] == "tidy up" + + +def test_exactly_the_maximum_number_of_paths_is_accepted(api): + paths = [f"f{i}.txt" for i in range(OPENAPI_DELETE_FILES_MAX_PATHS)] + + with mock.patch.object(api.openapi, "_request", return_value=_RESULT) as request_mock: + api.openapi.delete_model_files("alice", "m", {"file_paths": paths}) + + assert len(request_mock.call_args.kwargs["json_body"]["file_paths"]) == OPENAPI_DELETE_FILES_MAX_PATHS + + +def test_too_many_paths_are_refused_before_the_request(api): + paths = [f"f{i}.txt" for i in range(OPENAPI_DELETE_FILES_MAX_PATHS + 1)] + + with mock.patch.object(api.openapi, "_request") as request_mock: + with pytest.raises(InvalidParameter, match="at most 100"): + api.openapi.delete_model_files("alice", "m", {"file_paths": paths}) + request_mock.assert_not_called() + + +@pytest.mark.parametrize("paths", [[], None, "a.txt"]) +def test_an_empty_or_non_list_path_set_is_refused(api, paths): + with mock.patch.object(api.openapi, "_request") as request_mock: + with pytest.raises(InvalidParameter, match="non-empty list"): + api.openapi.delete_dataset_files("alice", "m", {"file_paths": paths}) + request_mock.assert_not_called() + + +def test_duplicate_paths_are_refused(api): + with mock.patch.object(api.openapi, "_request") as request_mock: + with pytest.raises(InvalidParameter, match="duplicate"): + api.openapi.delete_model_files("alice", "m", {"file_paths": ["a.txt", "a.txt"]}) + request_mock.assert_not_called() + + +# --------------------------------------------------------------------------- +# Regression guard: HubApi.delete_files must not move +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("repo_type", ["model", "dataset"]) +def test_delete_files_still_commits_instead_of_calling_the_endpoint(api, repo_type): + with ( + mock.patch.object(api.uploader, "delete_files", return_value=_RESULT) as uploader_delete, + mock.patch.object(api.openapi, "_request") as openapi_request, + ): + api.delete_files("alice/m", repo_type, ["a.txt", "b.txt"]) + + openapi_request.assert_not_called() + uploader_delete.assert_called_once() + assert uploader_delete.call_args.kwargs["file_paths"] == ["a.txt", "b.txt"] + + +def test_delete_files_has_no_hundred_path_ceiling(api): + """The commit path splits on the server's own action limit, far above 100.""" + paths = [f"f{i}.txt" for i in range(250)] + + with ( + mock.patch.object(api.uploader, "delete_files", return_value=_RESULT) as uploader_delete, + mock.patch.object(api.openapi, "_request") as openapi_request, + ): + api.delete_files("alice/m", "model", paths) + + openapi_request.assert_not_called() + assert len(uploader_delete.call_args.kwargs["file_paths"]) == 250 + + +def test_delete_files_still_resolves_glob_patterns(api): + """The endpoint takes no patterns; the commit path resolves them remotely.""" + remote = [ + mock.Mock(path="keep.bin", type="blob"), + mock.Mock(path="drop/a.json", type="blob"), + mock.Mock(path="drop/b.json", type="blob"), + ] + + with ( + mock.patch.object(api, "list_repo_files", return_value=remote), + mock.patch.object(api.uploader, "delete_files", return_value=_RESULT) as uploader_delete, + mock.patch.object(api.openapi, "_request") as openapi_request, + ): + api.delete_files("alice/m", "model", delete_patterns="*.json") + + openapi_request.assert_not_called() + assert uploader_delete.call_args.kwargs["file_paths"] == ["drop/a.json", "drop/b.json"] + + +def test_delete_files_accepts_a_tag_revision(api): + """``revision`` is forwarded as given; the endpoint would reject a tag.""" + with ( + mock.patch.object(api.uploader, "delete_files", return_value=_RESULT) as uploader_delete, + mock.patch.object(api.openapi, "_request") as openapi_request, + ): + api.delete_files("alice/m", "model", ["a.txt"], revision="v1.0") + + openapi_request.assert_not_called() + assert uploader_delete.call_args.kwargs["revision"] == "v1.0" diff --git a/tests/test_repo_info_visibility.py b/tests/test_repo_info_visibility.py new file mode 100644 index 0000000..920d5a2 --- /dev/null +++ b/tests/test_repo_info_visibility.py @@ -0,0 +1,121 @@ +"""``RepoInfo.visibility`` must stay a :class:`Visibility` integer. + +The model and dataset detail payloads gained ``visibility: "gated"``, a third +string state the 1/3/5 integer encoding cannot express. Letting it fall through +as a raw string would break ``.label``, ``int()`` and equality against the enum +-- and the compat dict, which the legacy SDK reads as an integer. + +A gated repository maps to PRIVATE: that is what it is on the legacy wire +(``Visibility=PRIVATE`` plus ``ProtectedMode=1``), and the approval mode travels +separately in ``gated`` / ``gated_mode``. The server reports ``private: false`` +for gated repos, so without that mapping they would read back as PUBLIC. +""" + +from __future__ import annotations + +from unittest import mock + +import pytest + +from modelscope_hub import HubApi +from modelscope_hub.compat import LegacyHubApi +from modelscope_hub.constants import RepoType, Visibility + + +def _info(payload: dict, repo_type: RepoType = RepoType.MODEL): + return HubApi._repo_info_from_payload(payload, repo_type, owner_hint="alice", name_hint="thing") + + +# --------------------------------------------------------------------------- +# String visibility (current contract) +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + ("payload", "expected"), + [ + ({"visibility": "public", "private": False, "gated": False}, Visibility.PUBLIC), + ({"visibility": "private", "private": True, "gated": False}, Visibility.PRIVATE), + # A gated repo: the server says ``private: false`` yet it is not public. + ({"visibility": "gated", "private": False, "gated": True}, Visibility.PRIVATE), + ], +) +def test_string_visibility_maps_to_the_integer_enum(payload, expected): + info = _info(payload) + + assert info.visibility == expected + assert isinstance(info.visibility, Visibility) + + +def test_gated_visibility_implies_the_gated_flag(): + """``gated`` may be absent from a summary payload; visibility still implies it.""" + info = _info({"visibility": "gated"}) + + assert info.visibility == Visibility.PRIVATE + assert info.gated is True + + +def test_an_explicit_gated_flag_is_not_overwritten(): + info = _info({"visibility": "gated", "gated": True, "gated_mode": "manual"}) + + assert info.gated is True + assert info.gated_mode == "manual" + + +# --------------------------------------------------------------------------- +# Older payloads with no visibility field +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + ("payload", "expected"), + [ + ({"private": True}, Visibility.PRIVATE), + ({"private": False}, Visibility.PUBLIC), + # ``gated`` is checked first: a gated repo reports ``private: false``, + # so trusting that flag would report it as public. + ({"private": False, "gated": True}, Visibility.PRIVATE), + ({"gated": True}, Visibility.PRIVATE), + ], +) +def test_boolean_payloads_still_resolve(payload, expected): + info = _info(payload) + + assert info.visibility == expected + assert isinstance(info.visibility, Visibility) + + +def test_an_integer_visibility_is_preserved(): + assert _info({"Visibility": int(Visibility.INTERNAL)}).visibility == Visibility.INTERNAL + + +# --------------------------------------------------------------------------- +# Studio visibility must not regress +# --------------------------------------------------------------------------- +def test_studio_protected_is_preserved_verbatim(): + """``protected`` has no integer equivalent, so it is kept as the string.""" + info = _info({"visibility": "protected"}, RepoType.STUDIO) + + assert info.visibility == "protected" + + +# --------------------------------------------------------------------------- +# Compat surface +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("visibility", ["public", "private", "gated"]) +def test_the_compat_dict_reports_visibility_as_an_integer(visibility): + """The legacy SDK reads ``Visibility`` as an int, never an enum or string.""" + compat = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") + payload = {"id": "alice/thing", "visibility": visibility, "private": False} + + with mock.patch.object(compat._api.openapi, "_request", return_value=payload): + out = compat.get_model("alice/thing") + + assert isinstance(out["Visibility"], int) + assert out["Visibility"] in {int(v) for v in Visibility} + + +def test_a_gated_repo_is_not_reported_as_public_through_compat(): + compat = LegacyHubApi(endpoint="https://modelscope.cn", token="ms-test") + payload = {"id": "alice/thing", "visibility": "gated", "private": False, "gated": True} + + with mock.patch.object(compat._api.openapi, "_request", return_value=payload): + out = compat.get_model("alice/thing") + + assert out["Visibility"] == int(Visibility.PRIVATE) From 52e0df64a704aa01a895ddfe7b0bea9f19f5dc66 Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Thu, 29 Oct 2026 20:27:30 +0800 Subject: [PATCH 21/22] Describe repository deletion permission accurately and bump version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit delete_repo no longer claims to be deprecated for security reasons; it now documents the real constraint: a token needs repository deletion permission, and insufficient permission surfaces as 10010101003 (model) or 10020301011 (dataset). The legacy DeprecationWarning wording is kept as a warning so existing callers still fire, but it now states the permission requirement. - align README deletion notes and create_repo example - bump version to 0.6.0 Signed-off-by: 班扬 --- README.md | 12 +++++----- src/modelscope_hub/api.py | 33 +++++++++++++++------------- src/modelscope_hub/compat/hub_api.py | 13 ++++++++--- src/modelscope_hub/version.py | 2 +- 4 files changed, 35 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index bac0b6d..059fe5f 100644 --- a/README.md +++ b/README.md @@ -297,7 +297,7 @@ Models belong to the `base`, `aigc`, or `science` zone; datasets to `base` or the zone — the AIGC endpoint checks that each weight path already exists: ```python -api.create_repo("my-org/my-lora", "model") +api.create_repo("my-org/my-lora", "model", visibility="private") api.upload_folder("my-org/my-lora", "model", "./my-lora-files") options = api.list_aigc_model_options() # enum ids, no credentials needed @@ -540,9 +540,9 @@ ms-hub list --repo-type studio --owner my-org ms-hub delete my-org/my-model --repo-type model --yes ``` -> **Deprecation notice:** `delete_repo` / `ms-hub delete` emits a `DeprecationWarning` — programmatic repo deletion is restricted for security reasons and will be restored once token-scoped auth is available. Use the [web console](https://modelscope.cn) to delete repos. +> **Deletion permission:** `delete_repo` / `ms-hub delete` needs repository deletion permission; ordinary write permission alone is not sufficient. An admin-level token has been verified to work. Tokens without the required permission are refused (`10010101003` for models, `10020301011` for datasets) and directed to the [web console](https://modelscope.cn). Retrying a permission rejection with the same token will not help. > -> `delete_files` requires cookie-based session auth; API tokens may receive a 401 error. +> `delete_files` uses the legacy commit/session path. File-deletion permission is distinct from repository-deletion permission; a write token may delete files even when it cannot delete the repository.
ms-hub create options @@ -890,7 +890,7 @@ api = HubApi(token="...", endpoint="https://modelscope.ai") | | `get_model(repo_id)` / `model_info(repo_id)` | Model detail, including zone fields | | | `get_dataset(repo_id)` / `dataset_info(repo_id)` | Dataset detail, including science fields | | | `list_repos(repo_type, ...)` | Paginated listing (`model`, `dataset`, `studio`, `skill`, `mcp`) | -| | `delete_repo(repo_id, repo_type)` | Delete a repository *(deprecated — see note below)* | +| | `delete_repo(repo_id, repo_type)` | Delete a repository *(needs deletion permission — see note below)* | | | `repo_exists(repo_id, repo_type)` | Check existence | | **Zones** | `set_model_category(repo_id, model_category=...)` | Move a model into `base` / `aigc` / `science` | | | `update_model_category(repo_id, ...)` | Edit a model's zone fields in place | @@ -938,8 +938,8 @@ api = HubApi(token="...", endpoint="https://modelscope.ai") | | `clear_cache(cache_dir, ...)` | Free disk space | > **Deletion restrictions:** -> - `delete_repo` is deprecated for security reasons (emits `DeprecationWarning`). Will be restored with token-scoped auth. Use the [web console](https://modelscope.cn) instead. -> - `delete_files` requires cookie-based session auth; API tokens (`ms-...`) may receive a 401 error. +> - `delete_repo` needs repository deletion permission. Insufficient permission is rejected (`10010101003` / `10020301011`); use an authorised token or the [web console](https://modelscope.cn). An admin-level token has been verified to work. The SDK uses the legacy endpoint; the current OpenAPI contract has no repository-deletion route. +> - `delete_files` uses the legacy commit/session path and checks file-deletion permission separately. It does not inherit the OpenAPI file-deletion endpoint's 100-path limit. Editing a tag revision may be rejected by repository policy. > **Token permission levels:** every write method needs a token issued with > `write` permission or higher. A rejected write raises `PermissionDeniedError` diff --git a/src/modelscope_hub/api.py b/src/modelscope_hub/api.py index 097ffb3..9f5c277 100644 --- a/src/modelscope_hub/api.py +++ b/src/modelscope_hub/api.py @@ -2025,12 +2025,11 @@ def list_repos( def delete_repo(self, repo_id: str, repo_type: RepoTypeLike) -> None: """Delete a repository. - .. deprecated:: - Programmatic repository deletion is not currently supported by - the Hub API for security reasons. This method will be restored - in a future release once proper token-scoped authentication is - available. To delete a repository now, use the web console at - https://modelscope.cn. + Needs a token carrying repository deletion permission; an admin-level + token has been verified to work, while ordinary write permission does + not guarantee it. Do not infer the permission from the token's name or + kind. Deletion uses the legacy endpoint -- the current OpenAPI contract + exposes no repository-deletion route. Parameters ---------- @@ -2038,16 +2037,20 @@ def delete_repo(self, repo_id: str, repo_type: RepoTypeLike) -> None: Canonical ``owner/name`` identifier. repo_type : str or RepoType Repository type (``"model"``, ``"dataset"``, etc.). - """ - import warnings - warnings.warn( - "This function is deprecated due to security reasons, " - "and will be recovered in future versions with proper token authentication. " - "Please go to https://modelscope.cn to delete repositories via the web console.", - DeprecationWarning, - stacklevel=2, - ) + Raises + ------ + AuthenticationError or InvalidParameter + When the token lacks deletion permission. The Hub answers business + code ``10010101003`` for models and ``10020301011`` for datasets, + both saying the operation must be performed at + https://modelscope.cn. Retrying with the same token will not help; + use a token with deletion permission or the web console. + + Examples + -------- + >>> api.delete_repo("alice/my-model", "model") + """ rt = self._normalize_repo_type(repo_type) self._parse_repo_id(repo_id) self.legacy.delete_repo(repo_id=repo_id, repo_type=str(rt)) diff --git a/src/modelscope_hub/compat/hub_api.py b/src/modelscope_hub/compat/hub_api.py index 5d18331..43d1156 100644 --- a/src/modelscope_hub/compat/hub_api.py +++ b/src/modelscope_hub/compat/hub_api.py @@ -1090,10 +1090,17 @@ def delete_dataset( endpoint: str | None = None, token: str | None = None, ) -> None: - """Delete a dataset repository.""" + """Delete a dataset repository. + + Needs a token carrying deletion permission; see + :meth:`HubApi.delete_repo`. The warning is kept so callers that filter + on it keep behaving as before, but its wording now describes the actual + constraint rather than claiming the method is going away. + """ warnings.warn( - "This function is deprecated due to security reasons, " - "and will be recovered in future versions with proper token authentication.", + "delete_dataset requires a token with deletion permission. " + "Ordinary write permission alone may not authorise repository deletion; " + "use an admin-level token or delete via https://modelscope.cn.", DeprecationWarning, stacklevel=2, ) diff --git a/src/modelscope_hub/version.py b/src/modelscope_hub/version.py index 48b097a..b22f9da 100644 --- a/src/modelscope_hub/version.py +++ b/src/modelscope_hub/version.py @@ -1,3 +1,3 @@ """Version information for modelscope_hub.""" -__version__ = "0.5.1+main" +__version__ = "0.6.0+main" From f2283f0efc245b8f5e0005a922d1c35fd6c580be Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Thu, 29 Oct 2026 23:44:19 +0800 Subject: [PATCH 22/22] Keep gated_mode as the only public gating switch and drop request aliases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gated / approval_mode request parameters introduced on this branch are removed before release: neither can be preserved through the legacy endpoint, so keeping them would allow an explicit approval policy to be silently lost. - create_repo and update_repo_settings accept gated_mode as a boolean switch and reject gated / approval_mode with TypeError before any request is sent - the compat AIGC creation path applies the same rejection ahead of uploading - settings adapt the boolean to the OpenAPI access mode: True enables gating with the server's auto default, False returns to ordinary private, None is a no-op - create_repo keeps the legacy ProtectedMode=1/2 wire format for any explicit boolean, so v0.5.1 callers see no change - visibility accepts the historical enum or public/private labels only - README documents the switch, the removed aliases and the detail-read polling needed after resolving a new version name Signed-off-by: 班扬 --- README.md | 69 +++++-- src/modelscope_hub/api.py | 296 ++++++++------------------- src/modelscope_hub/compat/hub_api.py | 1 + tests/cli/test_repo.py | 19 ++ tests/test_compat_get_model_files.py | 25 +++ tests/test_create_repo_routing.py | 170 +++++++-------- tests/test_dataset_category.py | 75 ++++--- tests/test_user_agent.py | 13 ++ 8 files changed, 313 insertions(+), 355 deletions(-) diff --git a/README.md b/README.md index 059fe5f..c1f3854 100644 --- a/README.md +++ b/README.md @@ -271,21 +271,45 @@ api.create_repo("my-org/my-model", "model", visibility="private", license="apach ### Visibility and gated access -A repository is `public`, `private`, or `gated`. Gated repositories are -discoverable, but downloading requires an application, approved either -automatically (`"auto"`) or by a maintainer (`"manual"`): +The SDK keeps its historical `Visibility` integer enum: `PRIVATE=1`, +`INTERNAL=3`, `PUBLIC=5`. Application gating is a separate boolean switch, +`gated_mode=True` / `False`, and is not deprecated. It is effective for private +repositories; with explicit public/internal visibility it is warned about and +ignored. Either explicit boolean with no visibility still implies private. + +Use the same high-level spelling as v0.5.1. A gated repository requires an +approved application to download; the boolean switch does not select an +approval policy: ```python -api.create_repo("my-org/my-data", "dataset", visibility="gated", gated="manual") -api.update_repo_settings("my-org/my-data", "dataset", visibility="gated", gated="auto") +api.create_repo( + "my-org/my-data", "dataset", visibility="private", + gated_mode=True, +) -# Turn gating off by leaving the gated state. -api.update_repo_settings("my-org/my-data", "dataset", visibility="private") +# Turn the application switch off; the repository becomes ordinary private. +api.update_repo_settings("my-org/my-data", "dataset", gated_mode=False) ``` -The older boolean `gated_mode=True` / `gated_mode=False` still works and keeps -its original behaviour exactly, but it cannot express the approval mode. Prefer -`gated`. +Ordinary creation prefers OpenAPI. Explicit `gated_mode=True/False` creation +retains the legacy `Visibility` plus `ProtectedMode=1/2` request format to keep +v0.5.1 behaviour. Settings adapt the boolean to the OpenAPI access mode internally; +`True` enables gating with the server's `auto` approval default, `False` leaves an +ordinary private repository, and `None` leaves the control unchanged. + +High-level `visibility` accepts the historical enum/labels, not `"gated"`. +`approval_mode` and `gated` request parameters have been removed, including the +`--approval-mode` CLI flag; passing them through `**kwargs` raises `TypeError` +before any request. This does not remove the existing `RepoInfo.gated` boolean +response field or the native `gated_mode` string in new detail responses. +`get_repo()` preserves `Visibility` for models/datasets, while raw `get_model()` / +`get_dataset()` retain native OpenAPI fields. Native approval-policy strings +belong only to the low-level `api.openapi` payloads, not to the high-level +`gated_mode` boolean. + +```bash +ms-hub create my-org/my-data --repo-type dataset --visibility private --gated +``` `visibility="internal"` (visible within an organisation) has no equivalent in the OpenAPI surface, so those requests continue to use the legacy endpoint. @@ -312,6 +336,17 @@ api.set_model_category( sample_images=[{"url": "https://example.com/sample.png", "prompt": "a cat"}], ) +# Detail reads can lag behind writes; wait briefly before resolving a new version name. +import time + +for _ in range(10): + detail = api.get_model("my-org/my-lora") + if any(v.get("version_name") == "v1.0" for v in detail.get("versions", [])): + break + time.sleep(1) +else: + raise TimeoutError("The new version is not visible yet; retry the detail read later.") + # Edit a version's metadata in place, addressed by version name or numeric id. api.update_model_category("my-org/my-lora", version_name="v1.0", trigger_words=["anime"]) @@ -556,6 +591,7 @@ ms-hub delete my-org/my-model --repo-type model --yes | `--chinese-name` | no | Display name in Chinese | | `--description` | no | Repository description | | `--exist-ok` | no | No error if repository already exists | +| `--gated` / `--no-gated` | no | Historical application switch; explicit public/internal visibility takes precedence | | `--sdk-type` | no | Studio SDK: `gradio`, `streamlit`, `docker`, `static` | | `--sdk-version` | no | Studio SDK version | | `--base-image` | no | Studio base Docker image | @@ -1089,18 +1125,19 @@ calls continue to work; three observable differences remain: | Change | Detail | |--------|--------| -| Gated repositories in metadata | A gated repository is now reported as `Visibility.PRIVATE` with `gated=True`. Previously the server's `private: false` made it read back as `Visibility.PUBLIC`. The field stays a `Visibility` integer. | +| Gated repositories in metadata | The historical `Visibility` enum is preserved. New OpenAPI `"gated"` responses map to `Visibility.PRIVATE` with `gated=True`, rather than leaking a raw string as in 0.5.1 against that response. Older payloads without `visibility` could instead be misclassified as PUBLIC from `private: false`; both parsing cases are handled. | | Default license | When `license` is omitted the Hub applies its own default (`apache-2.0`) instead of the SDK sending `Apache-2.0`. Only affects calls that take the OpenAPI route. | | `LegacyHubApi.create_dataset(description=...)` | The argument was previously accepted and then dropped; it is now sent. | Unchanged on purpose: - **`gated_mode=True` / `gated_mode=False`** keep the exact legacy request - (`Visibility=PRIVATE` plus `ProtectedMode=1`/`2`), including the warn-and-ignore - when visibility is public. The boolean cannot name an approval mode and the two - surfaces default differently, so translating it could have turned a - review-gated repository into an auto-approving one. `ms-hub create --gated` - is therefore also byte-for-byte unchanged. + (`Visibility=PRIVATE` plus `ProtectedMode=1`/`2`), including warn-and-ignore for + explicit public/internal visibility and implicit private for either boolean + when visibility is omitted. `ms-hub create --gated/--no-gated` stays unchanged. +- High-level `approval_mode`, `gated`, `visibility="gated"` and string-valued + `gated_mode` are not supported. The removed request aliases must not leak into + legacy extra fields. Native OpenAPI payload/response fields remain unchanged. - `delete_files()` still commits deletions directly, so it keeps glob support and has no 100-path ceiling. - `visibility="internal"`, licenses outside the OpenAPI enum, a `task` on a diff --git a/src/modelscope_hub/api.py b/src/modelscope_hub/api.py index 9f5c277..15e38d4 100644 --- a/src/modelscope_hub/api.py +++ b/src/modelscope_hub/api.py @@ -51,7 +51,6 @@ WEIGHT_FILE_SUFFIXES, AigcModelType, DatasetCategory, - GatedMode, ModelCategory, RepoType, RepoVisibility, @@ -455,8 +454,8 @@ def _repo_info_from_payload( for key, value in data.items(): normalised[aliases.get(key, key)] = value - # The OpenAPI surface uses ``private`` bool for visibility. - # gated is orthogonal and does not affect visibility mapping. + # Normalize native access modes to the historical SDK enum. The server + # reports private=false for gated repos, so the private flag alone is insufficient. raw_visibility = normalised.get("visibility") if isinstance(raw_visibility, str): # Two string encodings arrive here. Studio payloads use @@ -860,7 +859,6 @@ def create_repo( chinese_name: str | None = None, description: str | None = None, gated_mode: bool | None = None, - gated: bool | str | GatedMode | None = None, license_name: str | None = None, license_link: str | None = None, task: str | None = None, @@ -883,7 +881,7 @@ def create_repo( One of ``"model"``, ``"dataset"``, ``"studio"``, ``"skill"``. visibility : int, str or Visibility, optional Visibility level. Accepts the integer wire encoding, a label - (``"public"`` / ``"private"`` / ``"gated"``) or a + (``"public"`` / ``"private"`` / ``"internal"``) or a :class:`Visibility` value. Defaults to public. license : str, optional SPDX-style license identifier (e.g. ``"apache-2.0"``). Values the @@ -894,14 +892,12 @@ def create_repo( description : str, optional Short description of the repository. gated_mode : bool, optional - Deprecated. ``True`` requests an application-gated repository and - ``False`` an ordinary private one. Prefer ``gated``, which can also - say *how* applications are approved. - gated : bool, "auto", "manual" or GatedMode, optional - Application-gated download policy. ``"auto"`` approves applications - automatically, ``"manual"`` requires review, ``True`` means - ``"auto"``, and ``False`` means not gated. Only meaningful when - visibility is gated or unset; ignored with a warning otherwise. + Historical application-access switch, not deprecated. Effective + only for private repositories; explicit public/internal visibility + takes precedence and the switch is warned about and ignored. + Either explicit boolean with omitted visibility implies private. + These calls retain the legacy ProtectedMode=1/2 request format. + This boolean does not select an approval policy. license_name : str, optional Custom license name. Required when ``license="other"``. license_link : str, optional @@ -940,14 +936,15 @@ def create_repo( >>> info.repo_id 'alice/llama-7b-finetuned' - Create a dataset that requires a reviewed application to download: + Create a private dataset with application-gated access: - >>> api.create_repo("alice/my-data", "dataset", visibility="gated", gated="manual") + >>> api.create_repo("alice/my-data", "dataset", visibility="private", gated_mode=True) Create a public Studio space: >>> api.create_repo("alice/chat-demo", repo_type="studio", visibility="public") """ + self._reject_removed_gating_parameters(extra) rt = self._normalize_repo_type(repo_type) if rt not in _CREATABLE_TYPES: supported = ", ".join(sorted(t.value for t in _CREATABLE_TYPES)) @@ -956,12 +953,7 @@ def create_repo( ) owner, name = self._parse_repo_id(repo_id) studio_visibility = StudioVisibility.parse(visibility) if rt is RepoType.STUDIO else None - repo_visibility = RepoVisibility.parse(visibility) if rt in _LEGACY_CREATE_FALLBACK_TYPES else None - vis = ( - None - if studio_visibility is not None or repo_visibility is RepoVisibility.GATED - else self._normalize_visibility(visibility) - ) + vis = None if studio_visibility is not None else self._normalize_visibility(visibility) if license is not None: license = _LICENSE_DISPLAY_TO_SPDX.get(license, license) @@ -971,12 +963,10 @@ def create_repo( owner, name, visibility=vis, - repo_visibility=repo_visibility, license=license, chinese_name=chinese_name, description=description, gated_mode=gated_mode, - gated=gated, license_name=license_name, license_link=license_link, task=task, @@ -1021,31 +1011,11 @@ def create_repo( return self._repo_info_from_payload(data, rt, owner_hint=owner, name_hint=name) @staticmethod - def _resolve_gated( - gated: bool | str | GatedMode | None, - gated_mode: bool | None, - ) -> GatedMode | bool | None: - """Collapse the two gated parameters into one value. - - ``gated`` is expressive (it can name the approval mode); ``gated_mode`` - is the older boolean and wins only when ``gated`` is not given. A - ``True`` from either resolves to :data:`GatedMode.AUTO`. - - For ``create_repo`` that resolved mode is only ever used as "gating was - requested": a call that used the boolean is routed to the legacy - endpoint, whose ``ProtectedMode`` field carries no mode at all, so the - historical behaviour is preserved exactly. - """ - value = gated if gated is not None else gated_mode - if value is None or value is False: - return value - if value is True: - return GatedMode.AUTO - mode = GatedMode.parse(value) - if mode is None: - allowed = ", ".join(repr(m.value) for m in GatedMode) - raise InvalidParameter(f"gated must be False, True, or one of {allowed} (got {value!r}).") - return mode + def _reject_removed_gating_parameters(parameters: Mapping[str, Any]) -> None: + """Reject removed request parameters before kwargs can reach a backend.""" + removed = sorted({"approval_mode", "gated"} & set(parameters)) + if removed: + raise TypeError(f"Unexpected keyword argument(s): {', '.join(removed)}; use gated_mode=True/False.") @staticmethod def _legacy_create_reason( @@ -1067,14 +1037,8 @@ def _legacy_create_reason( endpoint can serve the request. """ if gated_mode is not None: - # The historical boolean has no approval mode, and the two surfaces - # default differently: legacy sends ``ProtectedMode=1`` and lets the - # server decide, while the OpenAPI schema defaults ``gated_mode`` to - # ``auto``. Translating the boolean could therefore silently turn a - # repository that required review into one that auto-approves, so it - # keeps using the wire format it has always used. ``gated`` is the - # parameter that can state the mode, and it does go through OpenAPI. - return "gated_mode is the historical boolean; use gated= to reach the OpenAPI endpoint" + # Retain the historical boolean wire contract, not a new policy selector. + return "historical gated_mode boolean keeps the legacy ProtectedMode wire" if visibility is not None and visibility == int(Visibility.INTERNAL): return "visibility=internal is not part of the OpenAPI visibility enum" if rt is RepoType.MODEL and license is not None and license not in OPENAPI_MODEL_LICENSES: @@ -1097,23 +1061,18 @@ def _create_model_or_dataset( name: str, *, visibility: int | None, - repo_visibility: RepoVisibility | None, license: str | None, chinese_name: str | None, description: str | None, gated_mode: bool | None, - gated: bool | str | GatedMode | None, license_name: str | None, license_link: str | None, task: str | None, extra: Mapping[str, Any], ) -> RepoInfo: """Create a model or dataset, preferring OpenAPI over the legacy endpoint.""" - if repo_visibility is not None and repo_visibility is not RepoVisibility.GATED: - # ``public``/``private`` already round-tripped through the integer - # encoding; only ``gated`` needs the string form carried forward. - repo_visibility = None - resolved_gated = self._resolve_gated(gated, gated_mode) + if gated_mode is not None and not isinstance(gated_mode, bool): + raise InvalidParameter("gated_mode must be True, False or None, not an approval-policy string.") reason = self._legacy_create_reason( rt, visibility=visibility, @@ -1129,11 +1088,9 @@ def _create_model_or_dataset( owner, name, visibility=visibility, - repo_visibility=repo_visibility, license=license, chinese_name=chinese_name, description=description, - gated=resolved_gated, license_name=license_name, license_link=license_link, task=task, @@ -1152,11 +1109,9 @@ def _create_model_or_dataset( owner, name, visibility=visibility, - repo_visibility=repo_visibility, license=license, chinese_name=chinese_name, description=description, - gated=resolved_gated, gated_mode=gated_mode, # ``task`` / ``license_name`` / ``license_link`` are named parameters # only because the OpenAPI schema declares them. The legacy endpoint @@ -1178,11 +1133,9 @@ def _create_via_openapi( name: str, *, visibility: int | None, - repo_visibility: RepoVisibility | None, license: str | None, chinese_name: str | None, description: str | None, - gated: GatedMode | bool | None, license_name: str | None, license_link: str | None, task: str | None, @@ -1190,19 +1143,9 @@ def _create_via_openapi( ) -> RepoInfo: """Create a model or dataset via ``POST /models`` / ``POST /datasets``.""" payload: dict[str, Any] = {"owner": owner, "repo_name": name} - wire_visibility = self._openapi_visibility(rt, visibility, repo_visibility, gated) + wire_visibility = self._openapi_visibility(rt, visibility) if wire_visibility is not None: payload["visibility"] = wire_visibility.value - if isinstance(gated, GatedMode): - if wire_visibility is RepoVisibility.GATED: - payload["gated_mode"] = gated.value - else: - # The endpoint rejects ``gated_mode`` outside a gated repo, and - # the legacy path has always warned-and-ignored here. Keep that. - logger.warning( - "gated is only effective when visibility is gated, ignored (visibility=%s).", - wire_visibility.value if wire_visibility else "public", - ) if chinese_name is not None: payload["display_name"] = chinese_name # ``license`` is omitted rather than defaulted: the server's own default @@ -1240,9 +1183,7 @@ def _create_via_openapi( # The create response carries only identity fields, while the legacy one # echoed the settings back. Backfill from the request so a caller reading # ``info.visibility`` after create does not see None where it saw a value. - # Derived from the wire value rather than the raw argument: the argument - # may be absent (gating alone implies a visibility) and must not leak a - # plain ``int`` past the dataclass's enum coercion. + # Backfill enum instances, not bare ints, after dataclass construction. if info.visibility is None and wire_visibility is not None: info.visibility = Visibility.PUBLIC if wire_visibility is RepoVisibility.PUBLIC else Visibility.PRIVATE if info.license is None: @@ -1250,50 +1191,24 @@ def _create_via_openapi( if info.description is None: info.description = description if info.gated is None: - info.gated = wire_visibility is RepoVisibility.GATED - if info.gated_mode is None and isinstance(gated, GatedMode) and wire_visibility is RepoVisibility.GATED: - info.gated_mode = gated.value + info.gated = False return info @staticmethod - def _openapi_visibility( - rt: RepoType, - visibility: int | None, - repo_visibility: RepoVisibility | None, - gated: GatedMode | bool | None, - ) -> RepoVisibility | None: - """Pick the ``visibility`` string the create endpoints expect. - - Returns ``None`` to omit the field entirely, letting the server apply - its own default of ``public``. - - A private repository that also carries an approval mode *is* a gated - repository: the legacy encoding spelled exactly that as - ``Visibility=PRIVATE`` plus ``ProtectedMode=1``. Resolving it to plain - ``private`` would quietly drop the gating, and would disagree with the - legacy route for the same arguments. - """ - if repo_visibility is RepoVisibility.GATED: - return RepoVisibility.GATED - wants_gated = isinstance(gated, GatedMode) - if visibility is not None: - if visibility == int(Visibility.PRIVATE): - return RepoVisibility.GATED if wants_gated else RepoVisibility.PRIVATE - if visibility == int(Visibility.PUBLIC): - # Gating a public repository is contradictory; the caller is - # warned and the gating dropped, as the legacy path has always - # done. - return RepoVisibility.PUBLIC - raise InvalidParameter( - f"visibility={visibility!r} is not supported for repo_type={rt.value!r}. " - f"Expected one of: {', '.join(v.value for v in RepoVisibility)}." - ) - if gated is None: + def _openapi_visibility(rt: RepoType, visibility: int | None) -> RepoVisibility | None: + """Map historical ordinary visibility values to OpenAPI strings. + + None omits the field, retaining the public server default. Internal + visibility and explicit application switches are routed to legacy + before this helper is called. + """ + if visibility is None: return None - # Visibility was not stated but the gating intent was. The legacy path - # has always made that imply a non-public repository, so both a request - # for gating and an explicit opt-out land on a private repo. - return RepoVisibility.GATED if wants_gated else RepoVisibility.PRIVATE + if visibility == int(Visibility.PRIVATE): + return RepoVisibility.PRIVATE + if visibility == int(Visibility.PUBLIC): + return RepoVisibility.PUBLIC + raise InvalidParameter(f"visibility={visibility!r} is not supported by the OpenAPI {rt.value} create endpoint.") def _create_via_legacy( self, @@ -1302,11 +1217,9 @@ def _create_via_legacy( name: str, *, visibility: int | None, - repo_visibility: RepoVisibility | None, license: str | None, chinese_name: str | None, description: str | None, - gated: GatedMode | bool | None, gated_mode: bool | None, extra: Mapping[str, Any], ) -> RepoInfo: @@ -1318,10 +1231,6 @@ def _create_via_legacy( get must be the one they got before the OpenAPI route existed. """ vis = visibility - if repo_visibility is RepoVisibility.GATED: - # The legacy encoding has no ``gated`` state: it is PRIVATE plus - # ``ProtectedMode=1``. - vis = int(Visibility.PRIVATE) if rt is RepoType.DATASET: body: dict[str, Any] = { "Owner": owner, @@ -1341,17 +1250,15 @@ def _create_via_legacy( if description is not None: body["Description"] = description - # gated -> ProtectedMode wire field (1=gated, 2=off). - # gated only effective with PRIVATE; when vis=None + gated requested, - # implicitly set visibility to PRIVATE (user intent: gated repo). - wants_gated = isinstance(gated, GatedMode) or repo_visibility is RepoVisibility.GATED - if gated is not None or repo_visibility is RepoVisibility.GATED: + # Preserve v0.5.1: either explicit boolean with no visibility implies + # PRIVATE; public/internal visibility wins and ignores the switch. + if gated_mode is not None: if vis is None: vis = int(Visibility.PRIVATE) body["Visibility"] = vis - body["ProtectedMode"] = 1 if wants_gated else 2 + body["ProtectedMode"] = 1 if gated_mode else 2 elif vis == int(Visibility.PRIVATE): - body["ProtectedMode"] = 1 if wants_gated else 2 + body["ProtectedMode"] = 1 if gated_mode else 2 else: logger.warning("gated_mode is only effective when visibility is PRIVATE, ignored.") @@ -2925,7 +2832,8 @@ def update_repo_settings( Supported for ``model``, ``dataset``, ``studio`` and ``skill``. Only the keys you pass are changed; for models and datasets an explicit ``None`` - clears a field. Zone membership is not a setting -- use + clears metadata fields, while None for visibility/gated_mode leaves + those controls unchanged. Zone membership is not a setting -- use :meth:`set_model_category` / :meth:`set_dataset_category` for that. Parameters @@ -2936,8 +2844,13 @@ def update_repo_settings( ``"model"``, ``"dataset"``, ``"studio"`` or ``"skill"``. **settings : Any Setting key/value pairs forwarded to the backend. For models and - datasets: ``display_name``, ``license``, ``visibility``, ``gated`` / - ``gated_mode``, ``description``, and ``task`` (models only). + datasets: ``display_name``, ``license``, ``visibility``, ``description`` + and ``task`` (models only). ``gated_mode`` remains a boolean switch: + True enables application access, False disables it, None leaves it + unchanged. Omitted visibility with either explicit boolean implies + private; explicit public takes precedence and ignores the switch. + High-level visibility accepts the historical integer enum/labels, + not the native OpenAPI 'gated' access-mode string. Returns ------- @@ -2958,8 +2871,13 @@ def update_repo_settings( ... visibility="public", ... hardware="GPU-A10", ... ) - >>> api.update_repo_settings("alice/rna-seq", "dataset", visibility="gated", gated="manual") + >>> api.update_repo_settings( + ... "alice/rna-seq", "dataset", visibility="private", + ... gated_mode=True, + ... ) + >>> api.update_repo_settings("alice/rna-seq", "dataset", gated_mode=False) """ + self._reject_removed_gating_parameters(settings) rt = self._normalize_repo_type(repo_type) owner, name = self._parse_repo_id(repo_id) if rt is RepoType.STUDIO: @@ -2978,22 +2896,13 @@ def update_repo_settings( @classmethod def _repo_settings_payload(cls, rt: RepoType, settings: Mapping[str, Any]) -> dict[str, Any]: - """Translate model/dataset settings kwargs into the wire payload. + """Adapt the historical boolean switch and visibility to OpenAPI. - Two spellings are accepted and they are kept apart on purpose: - - * ``gated`` is the SDK's convenience argument (``True`` / ``False`` / - ``"auto"`` / ``"manual"``). It is translated into the endpoint's two - fields, including the visibility change that turning gating off - requires. - * ``gated_mode`` is the wire field itself. It is forwarded as given, so - the contract's own rules apply -- notably that an empty string is - valid and means "default to auto", while an explicit ``null`` or a - boolean is rejected. - - Settings the endpoint cannot change are rejected rather than dropped, - since silently dropping them would look like the update had succeeded. + None leaves the switch/visibility unchanged. True/False with omitted + visibility implies private. Native strings belong to the low-level + OpenAPI payload, not to this high-level boolean parameter. """ + cls._reject_removed_gating_parameters(settings) payload = dict(settings) unsupported = sorted(_UNSETTABLE_FIELDS & set(payload)) if unsupported: @@ -3002,62 +2911,33 @@ def _repo_settings_payload(cls, rt: RepoType, settings: Mapping[str, Any]) -> di f"the {rt.value} settings endpoint does not accept it. " f"It is reported read-only by get_{rt.value}()." ) - - has_native_mode = "gated_mode" in payload - native_mode = payload.pop("gated_mode", None) - gated = cls._resolve_gated(payload.pop("gated", None), None) - if has_native_mode and gated is not None: - raise InvalidParameter("Pass either gated or gated_mode, not both.") - - if "visibility" in payload: - raw = payload["visibility"] - if raw is None or raw == "": - # An explicit empty value clears the setting server-side. - payload["visibility"] = raw - else: - visibility = RepoVisibility.parse(raw) - if visibility is None: - normalised = cls._normalize_visibility(raw) - if normalised == int(Visibility.PUBLIC): - visibility = RepoVisibility.PUBLIC - elif normalised == int(Visibility.PRIVATE): - visibility = RepoVisibility.PRIVATE - else: - allowed = ", ".join(v.value for v in RepoVisibility) - raise InvalidParameter( - f"visibility={raw!r} is not supported for repo_type={rt.value!r}. " - f"Expected one of: {allowed}." - ) - payload["visibility"] = visibility.value - - if has_native_mode: - # Forwarded verbatim. The endpoint accepts ``auto``/``manual``/``""`` - # and rejects an explicit null or a boolean; the boolean is caught - # here because it is the historical ``gated_mode`` spelling and would - # otherwise reach the server as JSON ``true``/``false``. - if isinstance(native_mode, bool) or native_mode is None: - raise InvalidParameter( - f"gated_mode must be one of {', '.join(repr(m.value) for m in GatedMode)} or '' " - f"(got {native_mode!r}). Use gated=True/False to switch gating on or off." - ) - payload["gated_mode"] = native_mode - elif isinstance(gated, GatedMode): - target = payload.get("visibility") - if target is not None and target != RepoVisibility.GATED.value: - # The endpoint rejects the pair outright; refuse locally so the - # contradiction is reported against the caller's own arguments. + switch = payload.pop("gated_mode", None) + if switch is not None and not isinstance(switch, bool): + raise InvalidParameter("gated_mode must be True, False or None, not an approval-policy string.") + raw_visibility = payload.pop("visibility", None) + visibility = cls._normalize_visibility(raw_visibility) + if visibility is not None: + if visibility not in (int(Visibility.PRIVATE), int(Visibility.PUBLIC)): raise InvalidParameter( - f"gated={gated.value!r} conflicts with visibility={target!r}. " - f"Set visibility='gated' to keep an approval mode, or drop gated." + f"visibility={raw_visibility!r} is not supported by the {rt.value} settings endpoint; " + "there is no equivalent legacy settings endpoint." ) - if target is None: - # Naming a mode asks for gating, which is a visibility change. - payload["visibility"] = RepoVisibility.GATED.value - payload["gated_mode"] = gated.value - elif gated is False and "visibility" not in payload: - # Turning gating off is expressed by leaving the gated state, which - # the endpoint models as a visibility change. - payload["visibility"] = RepoVisibility.PRIVATE.value + wire_visibility = cls._openapi_visibility(rt, visibility) + assert wire_visibility is not None + payload["visibility"] = wire_visibility.value + if switch is not None: + if visibility is None: + visibility = int(Visibility.PRIVATE) + payload["visibility"] = RepoVisibility.PRIVATE.value + if visibility == int(Visibility.PRIVATE): + if switch: + # OpenAPI represents application access as a wire access mode. + payload["visibility"] = RepoVisibility.GATED.value + payload["gated_mode"] = "auto" + else: + payload["visibility"] = RepoVisibility.PRIVATE.value + else: + logger.warning("gated_mode is only effective when visibility is PRIVATE, ignored.") return payload # ================================================================== diff --git a/src/modelscope_hub/compat/hub_api.py b/src/modelscope_hub/compat/hub_api.py index 43d1156..78334a8 100644 --- a/src/modelscope_hub/compat/hub_api.py +++ b/src/modelscope_hub/compat/hub_api.py @@ -382,6 +382,7 @@ def create_model(self, model_id: str, **kwargs: Any) -> str: def _create_aigc_model(self, model_id: str, aigc_model: Any, kwargs: dict[str, Any]) -> str: """Create an AIGC model without changing the plain-model code path.""" + HubApi._reject_removed_gating_parameters(kwargs) token = kwargs.pop("token", None) endpoint = kwargs.pop("endpoint", None) visibility = kwargs.pop("visibility", None) diff --git a/tests/cli/test_repo.py b/tests/cli/test_repo.py index e4400a1..924a333 100644 --- a/tests/cli/test_repo.py +++ b/tests/cli/test_repo.py @@ -53,6 +53,16 @@ def test_invalid_visibility_rejected(self, parser): with pytest.raises(SystemExit): parser.parse_args(["create", "o/r", "--repo-type", "model", "--visibility", "secret"]) + @pytest.mark.parametrize("mode", ["auto", "manual"]) + def test_removed_approval_mode_flag(self, parser, mode): + with pytest.raises(SystemExit): + parser.parse_args(["create", "o/r", "--repo-type", "model", "--gated", "--approval-mode", mode]) + + def test_old_gated_flag_remains_boolean(self, parser): + args = parser.parse_args(["create", "o/r", "--repo-type", "model", "--gated"]) + assert args.gated is True + assert not hasattr(args, "approval_mode") + def test_license_flag(self, parser): args = parser.parse_args(["create", "o/r", "--repo-type", "model", "--license", "apache-2.0"]) assert args.license == "apache-2.0" @@ -339,6 +349,15 @@ def test_create_model(self, parser, mock_api, capsys): out = capsys.readouterr().out assert "Created" in out + @pytest.mark.parametrize("flag, expected", [("--gated", True), ("--no-gated", False)]) + def test_create_with_historical_boolean_flag(self, parser, mock_api, flag, expected): + args = parser.parse_args(["create", "owner/model", "--repo-type", "model", flag]) + with patch("modelscope_hub.cli.repo.make_api", return_value=mock_api): + CreateCommand(args).execute() + kwargs = mock_api.create_repo.call_args.kwargs + assert kwargs["gated_mode"] is expected + assert "gated" not in kwargs and "approval_mode" not in kwargs + def test_create_studio_with_extras(self, parser, mock_api, capsys): args = parser.parse_args( [ diff --git a/tests/test_compat_get_model_files.py b/tests/test_compat_get_model_files.py index 222b625..216a047 100644 --- a/tests/test_compat_get_model_files.py +++ b/tests/test_compat_get_model_files.py @@ -51,6 +51,31 @@ def _fake_files(): ] +@pytest.mark.parametrize("name", ["approval_mode", "gated"]) +@pytest.mark.parametrize("value", [None, "manual"]) +def test_legacy_aigc_rejects_removed_parameters_before_upload(name, value): + lha = LegacyHubApi(token="ms-test") + aigc = _FakeAigcModel() + with pytest.raises(TypeError, match=name): + lha.create_model("owner/repo", aigc_model=aigc, visibility=1, gated_mode=True, **{name: value}) + aigc.preupload_weights.assert_not_called() + aigc.upload_to_repo.assert_not_called() + + +@pytest.mark.parametrize("method", ["create_repo", "create_model", "update_model", "update_dataset"]) +@pytest.mark.parametrize("name", ["approval_mode", "gated"]) +def test_plain_compat_wrappers_reject_removed_request_parameters(method, name): + lha = LegacyHubApi(token="ms-test") + with ( + mock.patch.object(lha._api.legacy, "create_repo") as legacy, + mock.patch.object(lha._api.openapi, "_request") as native, + ): + with pytest.raises(TypeError, match=name): + getattr(lha, method)("owner/repo", **{name: None}) + legacy.assert_not_called() + native.assert_not_called() + + class TestGetModelFilesLegacyCompat: def test_revision_is_accepted_and_forwarded(self): lha = LegacyHubApi() diff --git a/tests/test_create_repo_routing.py b/tests/test_create_repo_routing.py index b5ff4aa..c8598f7 100644 --- a/tests/test_create_repo_routing.py +++ b/tests/test_create_repo_routing.py @@ -16,7 +16,7 @@ from modelscope_hub import HubApi from modelscope_hub import api as api_module -from modelscope_hub.constants import GatedMode, RepoType, RepoVisibility, Visibility +from modelscope_hub.constants import RepoType, Visibility from modelscope_hub.errors import AlreadyExistsError, InvalidParameter, NotExistError, ServerError @@ -81,7 +81,9 @@ def routes(api: HubApi): ("private", "private"), (Visibility.PUBLIC, "public"), (Visibility.PRIVATE, "private"), - ("gated", "gated"), + (1, "private"), + (5, "public"), + ("1", "private"), ], ) def test_ordinary_visibility_uses_openapi(api, routes, repo_type, visibility, expected): @@ -140,11 +142,9 @@ def test_task_is_rejected_for_datasets(api, routes): "alice", "thing", visibility=None, - repo_visibility=None, license=None, chinese_name=None, description=None, - gated=None, license_name=None, license_link=None, task="text-classification", @@ -155,30 +155,10 @@ def test_task_is_rejected_for_datasets(api, routes): # --------------------------------------------------------------------------- # Gated access # --------------------------------------------------------------------------- -@pytest.mark.parametrize("mode", ["auto", "manual", GatedMode.MANUAL]) -def test_gated_mode_is_sent_with_gated_visibility(api, routes, mode): - api.create_repo("alice/thing", "model", visibility="gated", gated=mode) - - assert routes.went_openapi - body = routes.openapi_body - assert body["visibility"] == "gated" - assert body["gated_mode"] == str(mode) - - -def test_gated_without_visibility_implies_gated_visibility(api, routes): - api.create_repo("alice/thing", "model", gated="manual") - - assert routes.openapi_body["visibility"] == "gated" - assert routes.openapi_body["gated_mode"] == "manual" - - def test_legacy_boolean_gated_mode_keeps_the_legacy_wire_format(api, routes): """The historical boolean must produce the request it always produced. - It carries no approval mode, and the two surfaces default differently: - legacy sends ``ProtectedMode=1`` and lets the server decide, while the - OpenAPI schema defaults ``gated_mode`` to ``auto``. Translating it could - silently turn a review-gated repository into an auto-approving one. + This pins the historical request format, not a new approval policy. """ api.create_repo("alice/thing", "model", gated_mode=True) @@ -211,70 +191,11 @@ def test_gated_mode_with_public_visibility_warns_and_is_ignored(api, routes, cap assert "only effective when visibility is PRIVATE" in caplog.text -def test_the_new_gated_parameter_goes_through_openapi(api, routes): - """``gated`` can state the approval mode, so it has no legacy equivalent.""" - api.create_repo("alice/thing", "model", gated="manual") - - assert routes.went_openapi - assert routes.openapi_body["visibility"] == "gated" - assert routes.openapi_body["gated_mode"] == "manual" - - -def test_gated_false_without_visibility_implies_private_on_openapi(api, routes): - api.create_repo("alice/thing", "model", gated=False) - - assert routes.went_openapi - assert routes.openapi_body["visibility"] == RepoVisibility.PRIVATE.value - assert "gated_mode" not in routes.openapi_body - - -def test_gated_with_public_visibility_warns_and_is_dropped(api, routes, caplog, monkeypatch): - """The endpoint 400s on the pair, so mirror the legacy warn-and-ignore.""" - # The SDK logger deliberately does not propagate, so caplog needs attaching. - monkeypatch.setattr(api_module.logger, "handlers", [*api_module.logger.handlers, caplog.handler]) - with caplog.at_level("WARNING", logger="modelscope_hub.api"): - api.create_repo("alice/thing", "model", visibility="public", gated="manual") - - assert routes.openapi_body["visibility"] == "public" - assert "gated_mode" not in routes.openapi_body - assert "only effective when visibility is gated" in caplog.text - - -def test_an_unknown_gated_value_is_rejected(api, routes): - with pytest.raises(InvalidParameter, match="gated must be"): - api.create_repo("alice/thing", "model", gated="sometimes") - - -@pytest.mark.parametrize( - ("extra", "expected_route"), - [({}, "openapi"), ({"create_default_config": True}, "legacy")], -) -def test_private_plus_an_approval_mode_means_gated_on_either_route(api, routes, extra, expected_route): - """``private`` + a mode is what the legacy encoding always called gated. - - The two routes must agree: adding an unrelated field should not change the - access policy the repository ends up with. - """ - api.create_repo("alice/thing", "model", visibility="private", gated="manual", **extra) - - if expected_route == "openapi": - assert routes.went_openapi - assert routes.openapi_body["visibility"] == "gated" - assert routes.openapi_body["gated_mode"] == "manual" - else: - assert routes.went_legacy - assert routes.legacy_body["Visibility"] == int(Visibility.PRIVATE) - assert routes.legacy_body["ProtectedMode"] == 1 - - -def test_gated_visibility_survives_a_forced_legacy_route(api, routes): - """``visibility="gated"`` states the intent through visibility alone, so the - legacy encoding has to restate it as ``ProtectedMode``.""" - api.create_repo("alice/thing", "model", visibility="gated", create_default_config=True) - - assert routes.went_legacy - assert routes.legacy_body["Visibility"] == int(Visibility.PRIVATE) - assert routes.legacy_body["ProtectedMode"] == 1 +@pytest.mark.parametrize("repo_type", ["model", "dataset"]) +def test_native_gated_visibility_is_not_a_high_level_input(api, routes, repo_type): + with pytest.raises(ValueError, match="Unknown visibility"): + api.create_repo("alice/thing", repo_type, visibility="gated") + assert not routes.openapi.called and not routes.legacy.called # --------------------------------------------------------------------------- @@ -419,9 +340,7 @@ def test_request_values_backfill_the_sparse_create_response(api, routes): ({"visibility": "private"}, Visibility.PRIVATE), ({"visibility": "public"}, Visibility.PUBLIC), ({"visibility": 5}, Visibility.PUBLIC), - # Gating alone implies a visibility the caller never named. - ({"gated": False}, Visibility.PRIVATE), - ({"gated": "manual"}, Visibility.PRIVATE), + ({"visibility": "1"}, Visibility.PRIVATE), ], ) def test_the_backfilled_visibility_is_an_enum_not_a_bare_int(api, routes, kwargs, expected): @@ -436,9 +355,66 @@ def test_the_backfilled_visibility_is_an_enum_not_a_bare_int(api, routes, kwargs assert info.visibility.label == expected.label -def test_a_gated_repo_reports_private_visibility_and_its_mode(api, routes): - info = api.create_repo("alice/thing", "model", visibility="gated", gated="manual") +def test_native_gated_response_preserves_historical_visibility_type(api): + with mock.patch.object( + api.openapi, + "get_model", + return_value={"visibility": "gated", "private": False, "gated": True, "gated_mode": "auto"}, + ): + info = api.get_repo("alice/thing", "model") + assert isinstance(info.visibility, Visibility) and info.visibility == Visibility.PRIVATE + assert info.gated is True and info.gated_mode == "auto" + assert not hasattr(info, "approval_mode") + - assert info.visibility == Visibility.PRIVATE - assert info.gated is True - assert info.gated_mode == "manual" +# Preserve the v0.5.1 switch/visibility contract rather than inventing aliases. +@pytest.mark.parametrize("repo_type", ["model", "dataset"]) +@pytest.mark.parametrize("switch", [True, False]) +@pytest.mark.parametrize("visibility", [None, "private", Visibility.PRIVATE, "public", "internal"]) +def test_historical_switch_matrix_is_unchanged(api, routes, repo_type, switch, visibility): + import warnings + + with warnings.catch_warnings(record=True) as recorded: + warnings.simplefilter("always") + api.create_repo("alice/thing", repo_type, visibility=visibility, gated_mode=switch) + assert routes.went_legacy + vis = ( + int(Visibility.PRIVATE) + if visibility is None + else int(Visibility.from_label(visibility)) + if isinstance(visibility, str) + else int(visibility) + ) + assert int(routes.legacy_body["Visibility"]) == vis + if vis == int(Visibility.PRIVATE): + assert int(routes.legacy_body["ProtectedMode"]) == (1 if switch else 2) + else: + assert "ProtectedMode" not in routes.legacy_body + assert not any(issubclass(w.category, DeprecationWarning) for w in recorded) + assert "approval_mode" not in routes.legacy_body + + +@pytest.mark.parametrize("repo_type", ["model", "dataset", "studio", "skill"]) +@pytest.mark.parametrize("name", ["approval_mode", "gated"]) +@pytest.mark.parametrize("value", [None, False, "manual"]) +def test_removed_parameters_cannot_leak_through_extra(api, routes, repo_type, name, value): + with pytest.raises(TypeError, match=name): + api.create_repo("alice/thing", repo_type, **{name: value}) + assert not routes.openapi.called and not routes.legacy.called + + +@pytest.mark.parametrize( + "method", + ["create_repo", "_create_model_or_dataset", "_create_via_openapi", "_create_via_legacy", "_openapi_visibility"], +) +def test_request_signatures_have_no_removed_aliases(method): + import inspect + + assert not {"approval_mode", "gated"} & set(inspect.signature(getattr(HubApi, method)).parameters) + + +@pytest.mark.parametrize("bad", ["auto", "manual", 1]) +def test_historical_switch_is_not_an_approval_string(api, routes, bad): + with pytest.raises(InvalidParameter, match="gated_mode must be"): + api.create_repo("alice/thing", "model", gated_mode=bad) + assert not routes.openapi.called and not routes.legacy.called diff --git a/tests/test_dataset_category.py b/tests/test_dataset_category.py index 3144857..436315c 100644 --- a/tests/test_dataset_category.py +++ b/tests/test_dataset_category.py @@ -103,55 +103,46 @@ def test_model_settings_reach_the_settings_endpoint(api, request_mock): @pytest.mark.parametrize("repo_type", ["model", "dataset"]) -def test_gated_settings_emit_both_wire_fields(api, request_mock, repo_type): - api.update_repo_settings("alice/x", repo_type, visibility="gated", gated="manual") +@pytest.mark.parametrize("visibility", [None, "private", 1, "1"]) +@pytest.mark.parametrize("switch", [True, False]) +def test_settings_use_historical_switch_and_visibility(api, request_mock, repo_type, visibility, switch): + api.update_repo_settings("alice/x", repo_type, visibility=visibility, gated_mode=switch) + expected = {"visibility": "gated", "gated_mode": "auto"} if switch else {"visibility": "private"} + assert request_mock.call_args.kwargs["json_body"] == expected - assert request_mock.call_args.kwargs["json_body"] == {"visibility": "gated", "gated_mode": "manual"} - -def test_naming_an_approval_mode_alone_turns_gating_on(api, request_mock): - """A mode without a visibility is a request to gate the repository.""" - api.update_repo_settings("alice/d", "dataset", gated="auto") - - assert request_mock.call_args.kwargs["json_body"] == {"visibility": "gated", "gated_mode": "auto"} - - -# The wire field is accepted alongside the convenience argument and forwarded -# as given, so the contract's own rules apply to it. @pytest.mark.parametrize("mode", ["auto", "manual", ""]) -def test_a_native_gated_mode_is_forwarded_verbatim(api, request_mock, mode): - """An empty string is valid and means "default to auto" server-side.""" - api.update_repo_settings("alice/d", "dataset", visibility="gated", gated_mode=mode) - - assert request_mock.call_args.kwargs["json_body"] == {"visibility": "gated", "gated_mode": mode} +def test_native_settings_payload_still_belongs_to_openapi(api, request_mock, mode): + payload = {"visibility": "gated", "gated_mode": mode} + api.openapi.update_dataset_settings("alice", "d", payload) + assert request_mock.call_args.kwargs["json_body"] == payload -@pytest.mark.parametrize("bad", [None, True, False]) -def test_a_null_or_boolean_gated_mode_is_rejected(api, request_mock, bad): - """The schema rejects both; catching it here names the right replacement.""" +@pytest.mark.parametrize("bad", ["auto", "manual", "", 0, []]) +def test_settings_reject_non_boolean_switches(api, request_mock, bad): with pytest.raises(InvalidParameter, match="gated_mode must be"): api.update_repo_settings("alice/d", "dataset", gated_mode=bad) request_mock.assert_not_called() -def test_the_two_gated_spellings_cannot_be_combined(api, request_mock): - with pytest.raises(InvalidParameter, match="either gated or gated_mode"): - api.update_repo_settings("alice/d", "dataset", gated="auto", gated_mode="manual") +@pytest.mark.parametrize("name", ["approval_mode", "gated"]) +@pytest.mark.parametrize("value", [None, False, "manual"]) +@pytest.mark.parametrize("repo_type", ["model", "dataset", "studio", "skill"]) +def test_removed_settings_parameters_are_rejected_before_sending(api, request_mock, repo_type, name, value): + with pytest.raises(TypeError, match=name): + api.update_repo_settings("alice/d", repo_type, **{name: value}) request_mock.assert_not_called() -@pytest.mark.parametrize("visibility", ["public", "private"]) -def test_gated_conflicting_with_visibility_is_refused(api, request_mock, visibility): - """The endpoint 400s on the pair; refuse against the caller's own arguments.""" - with pytest.raises(InvalidParameter, match="conflicts with visibility"): - api.update_repo_settings("alice/d", "dataset", visibility=visibility, gated="auto") - request_mock.assert_not_called() - +def test_settings_none_is_not_a_new_gating_policy(api, request_mock): + api.update_repo_settings("alice/d", "dataset", gated_mode=None, visibility=None, description="same") + assert request_mock.call_args.kwargs["json_body"] == {"description": "same"} -def test_turning_gating_off_without_a_visibility_leaves_the_gated_state(api, request_mock): - api.update_repo_settings("alice/d", "dataset", gated=False) - assert request_mock.call_args.kwargs["json_body"] == {"visibility": "private"} +def test_settings_do_not_accept_native_gated_visibility(api, request_mock): + with pytest.raises(ValueError, match="Unknown visibility"): + api.update_repo_settings("alice/d", "dataset", visibility="gated") + request_mock.assert_not_called() def test_integer_visibility_is_translated_for_settings(api, request_mock): @@ -179,6 +170,22 @@ def test_unsupported_repo_types_still_raise(api, request_mock): api.update_repo_settings("alice/x", "mcp", description="x") +# --------------------------------------------------------------------------- +# High-level historical switch, not an approval-policy selector +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("repo_type", ["model", "dataset"]) +def test_false_switch_changes_to_private_without_native_boolean(api, request_mock, repo_type): + api.update_repo_settings("alice/x", repo_type, gated_mode=False) + assert request_mock.call_args.kwargs["json_body"] == {"visibility": "private"} + + +def test_public_visibility_wins_over_only_boolean_switch(api, request_mock, caplog, monkeypatch): + monkeypatch.setattr(api_module.logger, "handlers", [*api_module.logger.handlers, caplog.handler]) + api.update_repo_settings("alice/x", "model", visibility="public", gated_mode=True) + assert request_mock.call_args.kwargs["json_body"] == {"visibility": "public"} + assert "ignored" in caplog.text + + # --------------------------------------------------------------------------- # Repo type guards # --------------------------------------------------------------------------- diff --git a/tests/test_user_agent.py b/tests/test_user_agent.py index 0e01658..367bae1 100644 --- a/tests/test_user_agent.py +++ b/tests/test_user_agent.py @@ -329,3 +329,16 @@ def fake_download(self, *args, **kwargs): else: assert download("alice/m", **kwargs) == "local-path" assert len(seen) == 1 + + +@pytest.mark.parametrize("switch", [True, False]) +@responses.activate +def test_compat_historical_switch_uses_prepared_request_marker(switch): + """Per-call credentials preserve the SDK/compat UA on historical creation.""" + responses.add(responses.POST, f"{CN}/api/v1/models", json={"Data": {"Id": "alice/m"}}, status=201) + api = LegacyHubApi(endpoint=CN, token=TOKEN) + api.create_repo("alice/m", visibility=1, gated_mode=switch, token=TOKEN, endpoint=CN) + prepared = responses.calls[0].request + assert isinstance(prepared, requests.PreparedRequest) + assert "invoked_by/compat" in _assert_sdk(prepared.headers["User-Agent"]) + assert prepared.headers["Authorization"] == f"Bearer {TOKEN}"