diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 9ea22e025..06e2e545c 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -62,7 +62,12 @@ DandiInstance, EmbargoStatus, ) -from .exceptions import HTTP404Error, NotFoundError, SchemaVersionError +from .exceptions import ( + HTTP404Error, + NotFoundError, + SchemaVersionError, + ZarrIngestionError, +) from .keyring_utils import keyring_lookup, keyring_save from .misctypes import ( Digest, @@ -126,6 +131,29 @@ def set_asset_schema_key(metadata: dict[str, Any]) -> None: metadata["schemaKey"] = "Asset" +def zarr_ingestion_deferred_until(zarr_data: dict[str, Any]) -> str | None: + """ + Given a Zarr's data from the server (as returned by ``GET + /zarr/{zarr_id}/``), return the time until which the server deferred its + ingestion, or `None` if it did not. + + dandi-archive#3018 defers the ingestion of a Zarr while any of its + unexpired upload URLs could still change it, leaving it ``"Uploaded"`` with + the time of the next attempt in ``"ingestion_deferred_until"``. The field + stays set while that attempt runs (``"Ingesting"``), and is cleared once + the Zarr is ingested or modified. + """ + # Servers without dandi-archive#3018 lack the field and never defer + # ingestion; .get() can be replaced by indexing once all known instances + # deploy that feature. + if zarr_data["status"] == "Uploaded" and ( + deferred_until := zarr_data.get("ingestion_deferred_until") + ): + assert isinstance(deferred_until, str) + return deferred_until + return None + + class AssetType(Enum): """ .. versionadded:: 0.36.0 @@ -2073,7 +2101,7 @@ def rmfiles(self, files: Iterable[RemoteZarrEntry], reingest: bool = True) -> No If ``reingest`` is true, after performing the deletion, the client triggers a recalculation of the Zarr's checksum and waits for it to - complete. + complete, unless the server defers it (see `wait_for_ingestion()`). """ # Don't bother checking that the entries are actually files or even # belong to this Zarr, as if they're not, the server will return an @@ -2085,11 +2113,54 @@ def rmfiles(self, files: Iterable[RemoteZarrEntry], reingest: bool = True) -> No ) if reingest: self.client.post(f"/zarr/{self.zarr}/finalize/") - while True: - sleep(2) - r = self.client.get(f"/zarr/{self.zarr}/") - if r["status"] == "Complete": - break + r = self.wait_for_ingestion() + if deferred_until := zarr_ingestion_deferred_until(r): + lgr.info( + "Server deferred ingestion of Zarr %s until %s", + self.zarr, + deferred_until, + ) + + def wait_for_ingestion(self) -> dict[str, Any]: + """ + Wait for the server to finish ingesting the Zarr after it has been + finalized, or to defer its ingestion. + + Returns the Zarr's data from the server (as returned by + ``GET /zarr/{zarr_id}/``) once its ``"status"`` is either + ``"Complete"``, in which case its ``"checksum"`` is set, or + ``"Uploaded"`` with ``"ingestion_deferred_until"`` set. In the latter + case, the server ingests the Zarr on its own at the given time, so + there is no point in waiting for it. + + Raises `ZarrIngestionError` if the Zarr becomes ``"Pending"`` again, + i.e., something else modified it in the meantime. + """ + while True: + sleep(2) + r: dict[str, Any] = self.client.get(f"/zarr/{self.zarr}/") + status = r["status"] + if status == "Complete": + if "ingestion_deferred_until" not in r: + # See zarr_ingestion_deferred_until() for when this can be + # removed + lgr.debug( + "Server does not report deferred Zarr ingestion" + " (dandi-archive#3018); assuming it never defers it" + ) + return r + if deferred_until := zarr_ingestion_deferred_until(r): + lgr.debug( + "Server deferred ingestion of Zarr %s until %s", + self.zarr, + deferred_until, + ) + return r + if status == "Pending": + raise ZarrIngestionError( + f"Zarr {self.zarr} was modified while waiting for the" + " server to ingest it" + ) class RemoteAsset(BaseRemoteAsset): diff --git a/dandi/exceptions.py b/dandi/exceptions.py index f323d2a42..67c1f85ce 100644 --- a/dandi/exceptions.py +++ b/dandi/exceptions.py @@ -97,3 +97,9 @@ class UploadValidationError(UploadError): """An upload could not proceed because an asset failed validation.""" pass + + +class ZarrIngestionError(RuntimeError): + """A finalized Zarr did not reach a state in which its ingestion is done or deferred.""" + + pass diff --git a/dandi/files/zarr.py b/dandi/files/zarr.py index 696522b55..468eb06f7 100644 --- a/dandi/files/zarr.py +++ b/dandi/files/zarr.py @@ -42,6 +42,7 @@ RemoteZarrEntry, RESTFullAPIClient, set_asset_schema_key, + zarr_ingestion_deferred_until, ) from dandi.exceptions import UploadError from dandi.metadata.core import get_default_metadata @@ -588,7 +589,11 @@ def iter_upload( `RemoteAsset`. If the local Zarr is bit-identical to the remote (nothing to upload or delete), the terminal status is instead ``"skipped"`` with a ``"message"`` of ``"identical"`` and the same - ``"asset"`` key. + ``"asset"`` key. If the server deferred ingesting the Zarr (see + `RemoteZarrAsset.wait_for_ingestion()`), the upload does not wait + for it, the local checksum is verified against one computed from + the remote file listing instead of the server's, and the terminal + ``"message"`` says until when ingestion is deferred. """ asset_path = metadata.setdefault("path", self.path) set_asset_schema_key(metadata) @@ -661,6 +666,61 @@ def mkzarr() -> str: mismatched = True first_run = True unchanged = False + # Until when the server deferred ingesting the Zarr, if it did + deferred_until: str | None = None + + def ingest(finalize: bool) -> Generator[dict, None, bool]: + """ + Wait for the server to ingest the Zarr, after finalizing it if + ``finalize`` is true, and verify its checksum against the local + one. Returns whether they mismatch. + + If the server defers ingesting the Zarr, it is not waited for, and + the local checksum is verified against one computed from the + remote file listing instead, which is what the server computes its + checksum from. + """ + nonlocal deferred_until + lgr.debug("%s: Waiting for server to calculate Zarr checksum", asset_path) + yield {"status": "server calculating checksum"} + if finalize: + client.post(f"/zarr/{zarr_id}/finalize/") + r = a.wait_for_ingestion() + if deferred_until := zarr_ingestion_deferred_until(r): + lgr.info( + "%s: Server deferred ingestion until %s because some of the" + " Zarr's upload URLs could still change it", + asset_path, + deferred_until, + ) + if zarr_mode == "patch": + lgr.info( + "%s: Skipping local checksum verification (patch mode)", + asset_path, + ) + return False + our_checksum = str(zcc.process()) + if deferred_until is not None: + lgr.debug( + "%s: Computing Zarr checksum from remote file listing", asset_path + ) + remote_checksum = _remote_zarr_checksum(a) + source = "remote file listing" + else: + remote_checksum = r["checksum"] + source = "server" + if our_checksum == remote_checksum: + return False + lgr.info( + "%s: Asset checksum mismatch (local: %s; %s: %s); redoing upload", + asset_path, + our_checksum, + source, + remote_checksum, + ) + yield {"status": "Checksum mismatch"} + return True + while mismatched: zcc = ZarrChecksumTree() old_zarr_entries: dict[str, RemoteZarrEntry] = { @@ -750,9 +810,26 @@ def mkzarr() -> str: len(old_zarr_entries) if zarr_mode == "full" else 0 ) if n_add == 0 and n_modify == 0 and n_delete == 0: - mismatched = False - unchanged = True - break + zarr_info = client.get(f"/zarr/{zarr_id}/") + status = zarr_info["status"] + deferred_until = zarr_ingestion_deferred_until(zarr_info) + if status == "Complete" or deferred_until is not None: + mismatched = False + unchanged = True + break + # The remote entries match, but an earlier upload ended + # before finalizing the Zarr ("Pending") or before the + # server ingested it ("Uploaded" or "Ingesting") + lgr.info( + "%s: Remote Zarr matches local but has not been" + " ingested (status: %s); %s", + asset_path, + status, + "finalizing" if status == "Pending" else "waiting for it", + ) + mismatched = yield from ingest(finalize=status == "Pending") + first_run = False + continue bytes_add = sum(e.size for e in to_upload.fresh_entries) bytes_modify = sum(it.size for it in to_upload.digested_entries) clauses = [] @@ -919,38 +996,7 @@ def mkzarr() -> str: "file" if n_remote_only == 1 else "files", ) if changed: - lgr.debug( - "%s: Waiting for server to calculate Zarr checksum", asset_path - ) - yield {"status": "server calculating checksum"} - client.post(f"/zarr/{zarr_id}/finalize/") - while True: - sleep(2) - r = client.get(f"/zarr/{zarr_id}/") - if r["status"] == "Complete": - if zarr_mode == "patch": - lgr.info( - "%s: Skipping local checksum verification" - " (patch mode)", - asset_path, - ) - mismatched = False - else: - our_checksum = str(zcc.process()) - server_checksum = r["checksum"] - if our_checksum == server_checksum: - mismatched = False - else: - mismatched = True - lgr.info( - "%s: Asset checksum mismatch (local: %s;" - " server: %s); redoing upload", - asset_path, - our_checksum, - server_checksum, - ) - yield {"status": "Checksum mismatch"} - break + mismatched = yield from ingest(finalize=True) elif mismatched and not first_run: lgr.error( "%s: Previous upload loop resulted in checksum mismatch," @@ -969,12 +1015,33 @@ def mkzarr() -> str: unchanged, first_run, ) + deferral = ( + f"server ingestion deferred until {deferred_until}" + if deferred_until is not None + else None + ) if unchanged: lgr.info("%s: Zarr already matches remote; nothing to upload", asset_path) - yield {"status": "skipped", "message": "identical", "asset": a} + message = "identical" if deferral is None else f"identical; {deferral}" + yield {"status": "skipped", "message": message, "asset": a} else: lgr.info("%s: Asset successfully uploaded", asset_path) - yield {"status": "done", "asset": a} + done: dict[str, Any] = {"status": "done", "asset": a} + if deferral is not None: + done["message"] = deferral + yield done + + +def _remote_zarr_checksum(asset: RemoteZarrAsset) -> str: + """ + Compute the DANDI Zarr checksum of a remote Zarr from the listing of its + files, which, unlike the checksum the server reports, does not have to + wait for the server to ingest the Zarr + """ + tree = ZarrChecksumTree() + for entry in asset.iterfiles(): + tree.add_leaf(Path(str(entry)), entry.size, entry.digest.value) + return str(tree.process()) def _handle_failed_items_and_raise( diff --git a/dandi/tests/data/dandiarchive-docker/docker-compose.yml b/dandi/tests/data/dandiarchive-docker/docker-compose.yml index 95167115f..743135b9e 100644 --- a/dandi/tests/data/dandiarchive-docker/docker-compose.yml +++ b/dandi/tests/data/dandiarchive-docker/docker-compose.yml @@ -39,6 +39,15 @@ services: DJANGO_DANDI_INSTANCE_NAME: ${DANDI_TESTS_INSTANCE_NAME:-DANDI} DJANGO_DANDI_INSTANCE_IDENTIFIER: ${DANDI_TESTS_INSTANCE_IDENTIFIER:-RRID:ABC_123456} DJANGO_DANDI_DOI_API_PREFIX: ${DANDI_TESTS_DOI_PREFIX:-10.80507} + # dandi-archive defers ingestion of a Zarr until every upload URL that + # could still change it has expired, and retries deferred ingestion + # periodically (https://github.com/dandi/dandi-archive/pull/3018). + # Shorten the expiry from the default hour, and the retry interval from + # the default 5 minutes, so that tests which overwrite or delete recently + # uploaded Zarr entries do not wait that long. Ignored by servers + # without that feature. + DJANGO_DANDI_ZARR_UPLOAD_URL_EXPIRATION: "30" + DJANGO_DANDI_DEFERRED_INGESTION_RETRY_INTERVAL: "5" ports: - "127.0.0.1:8000:8000" diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index ac6534ba3..afa659837 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -37,18 +37,21 @@ dandiset_metadata_file, ) from ..dandiapi import ( + BaseRemoteZarrAsset, DandiAPIClient, RemoteAsset, RemoteBlobAsset, RemoteDandiset, RemoteZarrAsset, + RemoteZarrEntry, Version, VersionStatus, ) from ..download import download -from ..exceptions import NotFoundError, SchemaVersionError +from ..exceptions import NotFoundError, SchemaVersionError, ZarrIngestionError from ..files import GenericAsset, dandi_file from ..files.bases import _SCHEMA_BAREASSET_HAS_DATASTANDARD +from ..misctypes import Digest from ..utils import list_paths # These tests exercise the 0.8.0 -> 0.7.0 downgrade wiring, which requires @@ -1177,3 +1180,156 @@ def test_all_public_api() -> None: assert exported == names assert {"DandiAPIClient", "RemoteDandiset", "RemoteAsset"} <= names assert not {"APIBase", "RemoteDandisetData", "ZarrEntryServerData"} & names + + +def _zarr_asset_for_mocked_server() -> BaseRemoteZarrAsset: + _mock_server_info(dandischema.consts.DANDI_SCHEMA_VERSION) + now = datetime.now(timezone.utc) + return BaseRemoteZarrAsset.model_validate( + { + "client": DandiAPIClient("https://test.nil/api"), + "asset_id": "abc", + "path": "sample.zarr", + "size": 0, + "created": now, + "modified": now, + "zarr": "zarr-id", + } + ) + + +_MOCK_ZARR_URL = "https://test.nil/api/zarr/zarr-id/" + + +def _zarr_polls() -> int: + return sum(c.request.url == _MOCK_ZARR_URL for c in responses.calls) + + +def _mock_zarr_states(*states: dict[str, Any]) -> None: + for state in states: + responses.add( + responses.GET, + _MOCK_ZARR_URL, + json={"zarr_id": "zarr-id", "checksum": None, **state}, + ) + + +@pytest.mark.ai_generated +@responses.activate +def test_wait_for_ingestion_without_deferral_support( + caplog: pytest.LogCaptureFixture, mocker: MockerFixture +) -> None: + """A server without dandi-archive#3018 never defers; wait for "Complete".""" + mocker.patch.object(dandiapi, "sleep") + _mock_zarr_states( + {"status": "Uploaded"}, + {"status": "Ingesting"}, + {"status": "Complete", "checksum": "abc-1--1"}, + ) + r = _zarr_asset_for_mocked_server().wait_for_ingestion() + assert r["status"] == "Complete" + assert r["checksum"] == "abc-1--1" + assert _zarr_polls() == 3 + assert "Server does not report deferred Zarr ingestion" in caplog.text + + +@pytest.mark.ai_generated +@responses.activate +def test_wait_for_ingestion_complete( + caplog: pytest.LogCaptureFixture, mocker: MockerFixture +) -> None: + mocker.patch.object(dandiapi, "sleep") + _mock_zarr_states( + {"status": "Uploaded", "ingestion_deferred_until": None}, + {"status": "Ingesting", "ingestion_deferred_until": None}, + { + "status": "Complete", + "checksum": "abc-1--1", + "ingestion_deferred_until": None, + }, + ) + r = _zarr_asset_for_mocked_server().wait_for_ingestion() + assert r["status"] == "Complete" + assert _zarr_polls() == 3 + assert "Server does not report deferred Zarr ingestion" not in caplog.text + + +@pytest.mark.ai_generated +@responses.activate +def test_wait_for_ingestion_deferred(mocker: MockerFixture) -> None: + """A deferred ingestion is not waited for.""" + mocker.patch.object(dandiapi, "sleep") + deferred_until = "2026-11-17T13:00:00Z" + _mock_zarr_states( + {"status": "Uploaded", "ingestion_deferred_until": None}, + {"status": "Ingesting", "ingestion_deferred_until": None}, + {"status": "Uploaded", "ingestion_deferred_until": deferred_until}, + ) + r = _zarr_asset_for_mocked_server().wait_for_ingestion() + assert r["status"] == "Uploaded" + assert r["ingestion_deferred_until"] == deferred_until + assert _zarr_polls() == 3 + + +@pytest.mark.ai_generated +@responses.activate +def test_wait_for_ingestion_retried_after_deferral(mocker: MockerFixture) -> None: + """ + The server keeps "ingestion_deferred_until" while retrying a deferred + ingestion; a Zarr being ingested again is waited for. + """ + mocker.patch.object(dandiapi, "sleep") + deferred_until = "2026-11-17T13:00:00Z" + _mock_zarr_states( + {"status": "Ingesting", "ingestion_deferred_until": deferred_until}, + { + "status": "Complete", + "checksum": "abc-1--1", + "ingestion_deferred_until": None, + }, + ) + r = _zarr_asset_for_mocked_server().wait_for_ingestion() + assert r["status"] == "Complete" + assert _zarr_polls() == 2 + + +@pytest.mark.ai_generated +@responses.activate +def test_wait_for_ingestion_modified_meanwhile(mocker: MockerFixture) -> None: + mocker.patch.object(dandiapi, "sleep") + _mock_zarr_states( + {"status": "Uploaded", "ingestion_deferred_until": None}, + {"status": "Pending", "ingestion_deferred_until": None}, + ) + with pytest.raises(ZarrIngestionError, match="zarr-id was modified"): + _zarr_asset_for_mocked_server().wait_for_ingestion() + + +@pytest.mark.ai_generated +@responses.activate +def test_rmfiles_reingest_deferred( + caplog: pytest.LogCaptureFixture, mocker: MockerFixture +) -> None: + """``rmfiles(reingest=True)`` does not wait for a deferred ingestion.""" + mocker.patch.object(dandiapi, "sleep") + asset = _zarr_asset_for_mocked_server() + responses.add(responses.DELETE, f"{_MOCK_ZARR_URL}files/", status=204) + responses.add(responses.POST, f"{_MOCK_ZARR_URL}finalize/", status=204) + deferred_until = "2026-11-18T13:00:00Z" + _mock_zarr_states( + {"status": "Uploaded", "ingestion_deferred_until": deferred_until}, + ) + entry = RemoteZarrEntry( + client=asset.client, + zarr_id=asset.zarr, + parts=("arr_0", "c", "0"), + modified=datetime.now(timezone.utc), + digest=Digest(algorithm=DigestType.md5, value="0" * 32), + size=1, + ) + asset.rmfiles([entry]) + assert _zarr_polls() == 1 + assert ( + f"Server deferred ingestion of Zarr zarr-id until {deferred_until}" + in caplog.text + ) diff --git a/dandi/tests/test_upload.py b/dandi/tests/test_upload.py index 03dfdb710..437f247b7 100644 --- a/dandi/tests/test_upload.py +++ b/dandi/tests/test_upload.py @@ -1,14 +1,18 @@ from __future__ import annotations +from base64 import b64encode from collections import defaultdict +from collections.abc import Callable from datetime import datetime, timezone import os from pathlib import Path from shutil import copyfile, rmtree +from time import monotonic, sleep from typing import Any from unittest.mock import Mock from urllib.parse import urlparse +import anys import numpy as np import pynwb import pytest @@ -32,6 +36,7 @@ from ..download import download from ..exceptions import NotFoundError, UploadError from ..files import LocalFileAsset +from ..files import zarr as zarr_files from ..files.zarr import ZarrAsset from ..pynwb_utils import make_nwb_file from ..upload import UploadExisting, UploadValidation @@ -466,6 +471,30 @@ def test_upload_bids_zarr( bids_zarr_dandiset.upload() +def _wait_for_zarr_ingestion( + sample_dandiset: SampleDandiset, path: str = "sample.zarr" +) -> None: + """ + Wait for the server to ingest the Zarr asset at ``path``. + + Rewriting or deleting entries of a Zarr while the URLs they were uploaded + with are still valid makes dandi-archive#3018 defer ingesting it until + those URLs expire, and ``dandi upload`` does not wait for that. In the + test archive instance, the URLs expire after 30 seconds, and deferred + ingestion is retried every 5 seconds, so this takes well under a minute. + The deadline leaves the rest of pytest's timeout for the test itself, so + that a Zarr which is never ingested fails with its status. + """ + asset = sample_dandiset.dandiset.get_asset_by_path(path) + assert isinstance(asset, RemoteZarrAsset) + deadline = monotonic() + 180 + while ( + status := sample_dandiset.client.get(f"/zarr/{asset.zarr}/")["status"] + ) != "Complete": + assert monotonic() < deadline, f"Zarr {asset.zarr} still {status!r}" + sleep(2) + + def _replace_zarr_and_check( tmp_path: Path, zarr_dandiset: SampleDandiset, zarr_id: str ) -> None: @@ -475,6 +504,7 @@ def _replace_zarr_and_check( asset = zarr_dandiset.dandiset.get_asset_by_path("sample.zarr") assert isinstance(asset, RemoteZarrAsset) assert asset.zarr == zarr_id + _wait_for_zarr_ingestion(zarr_dandiset) download(zarr_dandiset.dandiset.version_api_url, tmp_path) assert_dirtrees_eq( zarr_dandiset.dspath / "sample.zarr", @@ -514,6 +544,7 @@ def test_upload_different_zarr_entry_conflicts( (zf / "changed-size.txt").write_text("This is a test of the upload code.\n") (zf / "changed-type").write_text("This is now a file.\n") new_dandiset.upload(validation=UploadValidation.SKIP) + _wait_for_zarr_ingestion(new_dandiset) download(new_dandiset.dandiset.version_api_url, tmp_path) assert_dirtrees_eq(zf, tmp_path / new_dandiset.dandiset_id / "sample.zarr") @@ -530,6 +561,7 @@ def test_upload_different_zarr_file_to_parent_dir( (zf / "foo").mkdir() (zf / "foo" / "bar").write_text("This is under what used to be a file.\n") new_dandiset.upload(validation=UploadValidation.SKIP) + _wait_for_zarr_ingestion(new_dandiset) download(new_dandiset.dandiset.version_api_url, tmp_path) assert_dirtrees_eq(zf, tmp_path / new_dandiset.dandiset_id / "sample.zarr") @@ -649,83 +681,178 @@ def mock_put(self, url, **kwargs): assert "arr_1" not in url, f"URL {url} should have been retried but wasn't" +def _fake_zarr_entry_uploads( + monkeypatch: pytest.MonkeyPatch, + fake: Callable[[str], tuple[int, bytes] | None], +) -> None: + """ + Make uploads of Zarr entries to storage get the status code and content + ``fake`` returns for their URL, unless it returns `None`. The responses + are faked by `requests.Session.request`, below the retry logic of + `RESTFullAPIClient.request()`. + """ + original_request = requests.Session.request + + def mock_request( + self: requests.Session, method: str, url: str, *args: Any, **kwargs: Any + ) -> requests.Response: + if ( + method == "PUT" + and "dandi-dandisets" in url + and (faked := fake(url)) is not None + ): + resp = requests.Response() + resp.status_code, resp._content = faked + resp.url = url + return resp + return original_request(self, method, url, *args, **kwargs) + + monkeypatch.setattr(requests.Session, "request", mock_request) + + @pytest.mark.ai_generated def test_zarr_upload_400_timeout_retry( + mocker: MockerFixture, new_dandiset: SampleDandiset, monkeypatch: pytest.MonkeyPatch, ) -> None: - """Test that 400 RequestTimeout errors trigger automatic retry via tenacity""" - # Create test Zarr - zarr_path = new_dandiset.dspath / "test.zarr" - zarr.save(zarr_path, np.arange(100), np.arange(100, 0, -1)) - - # Track request attempts - request_attempts: defaultdict[str, int] = defaultdict(int) - original_request = RESTFullAPIClient.request + """ + A 400 RequestTimeout response to uploading a Zarr entry is retried by + `RESTFullAPIClient.request()` (https://github.com/dandi/dandi-cli/issues/1662) + """ + zarr.save(new_dandiset.dspath / "test.zarr", np.arange(100), np.arange(100, 0, -1)) - def mock_request(self, method, path, **kwargs): - # Track attempts for each request - urlpath = urlparse(path).path if path.startswith("http") else path - request_attempts[urlpath] += 1 + # Attempts to upload each Zarr entry, by the path of its URL + put_attempts: defaultdict[str, int] = defaultdict(int) - # Simulate 400 timeout on first attempt for files containing "arr_0" - if method == "PUT" and "arr_0" in path and request_attempts[urlpath] == 1: - # Return a mock response that will trigger the retry_if condition - resp = Mock(spec=requests.Response) - resp.status_code = 400 - resp.text = ( - '\n' - "RequestTimeout" - "Your socket connection to the server " - "was not read from or written to within the timeout period. " - "Idle connections will be closed." - "1111111111111111" - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" - "" + def fake(url: str) -> tuple[int, bytes] | None: + # Respond to the first attempt to upload each entry of arr_0 with a + # timeout + urlpath = urlparse(url).path + put_attempts[urlpath] += 1 + if "arr_0" in urlpath and put_attempts[urlpath] == 1: + return ( + 400, + b'\n' + b"RequestTimeout" + b"Your socket connection to the server" + b" was not read from or written to within the timeout period." + b" Idle connections will be closed." + b"1111111111111111" + b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + b"", ) - resp.headers = {} - resp.content = resp.text.encode() - resp.json = Mock(side_effect=ValueError("No JSON")) - return resp + return None - # Otherwise, call the original method - return original_request(self, method, path, **kwargs) - - # Apply the mock - monkeypatch.setattr(RESTFullAPIClient, "request", mock_request) + _fake_zarr_entry_uploads(monkeypatch, fake) + events = _spy_zarr_iter_upload(mocker) - # Upload the Zarr new_dandiset.upload() - # Verify the upload succeeded (asset,) = new_dandiset.dandiset.get_assets() assert isinstance(asset, RemoteZarrAsset) assert asset.asset_type is AssetType.ZARR assert asset.path == "test.zarr" - # Verify that arr_0 files were retried automatically by tenacity - # Filter for PUT requests to S3/minio (not API endpoints) - arr_0_urls = [ - url for url in request_attempts if "arr_0" in url and "dandi-dandisets" in url - ] - assert len(arr_0_urls) > 0, "Expected to find arr_0 files" - - for url in arr_0_urls: - # Each arr_0 file should have been attempted twice (initial + 1 retry) + arr_0_paths = [p for p in put_attempts if "arr_0" in p] + assert arr_0_paths, "Expected to find arr_0 files" + for p in put_attempts: + expected = 2 if p in arr_0_paths else 1 assert ( - request_attempts[url] == 2 - ), f"Expected {url} to be retried once, got {request_attempts[url]} attempts" - - # Verify non-arr_0 S3/minio URLs were not retried - non_arr_0_urls = [ - url - for url in request_attempts - if "arr_0" not in url and "dandi-dandisets" in url and request_attempts[url] > 0 - ] - for url in non_arr_0_urls: - assert ( - request_attempts[url] == 1 - ), f"URL {url} should not have been retried but had {request_attempts[url]} attempts" + put_attempts[p] == expected + ), f"Expected {expected} attempt(s) to upload {p}, got {put_attempts[p]}" + # The second attempts were the retries, not uploads in a second pass after + # the Zarr's checksum did not match, as would happen if the timed-out + # entries had not been retried + assert not any(e["status"] == "Checksum mismatch" for e in events) + # Nor did the server defer ingesting the Zarr for upload URLs left unused + # (dandi-archive#3018), as it would if they had not been retried + assert events[-1] == {"status": "done", "asset": anys.AnyInstance(RemoteZarrAsset)} + + +@pytest.mark.ai_generated +def test_upload_zarr_lost_entry( + caplog: pytest.LogCaptureFixture, + mocker: MockerFixture, + new_dandiset: SampleDandiset, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """ + An entry which the client believes it uploaded, but which is missing from + the remote Zarr, is detected by the Zarr's checksum and uploaded again. + + Against a server which defers ingesting a Zarr with unused upload URLs + (dandi-archive#3018), as it does here for the lost entry's, the checksum + is computed from the remote file listing rather than by the server. + """ + zarr_path = new_dandiset.dspath / "test.zarr" + zarr.save(zarr_path, np.arange(100), np.arange(100, 0, -1)) + lost: list[str] = [] + + def fake(url: str) -> tuple[int, bytes] | None: + # Report success for the first upload of an entry of arr_0 without + # uploading it + if "arr_0" in (urlpath := urlparse(url).path) and not lost: + lost.append(urlpath) + return (200, b"") + return None + + _fake_zarr_entry_uploads(monkeypatch, fake) + events = _spy_zarr_iter_upload(mocker) + + new_dandiset.upload() + + assert lost + assert [e["status"] for e in events].count("Checksum mismatch") == 1 + asset = new_dandiset.dandiset.get_asset_by_path("test.zarr") + assert isinstance(asset, RemoteZarrAsset) + zarr_info = new_dandiset.client.get(f"/zarr/{asset.zarr}/") + if "ingestion_deferred_until" in zarr_info: + assert "remote file listing" in caplog.text + # Once the lost entry is uploaded again, none of the Zarr's upload URLs can + # change it, so the server does not defer ingesting it + assert events[-1] == {"status": "done", "asset": anys.AnyInstance(RemoteZarrAsset)} + assert zarr_info["status"] == "Complete" + download(new_dandiset.dandiset.version_api_url, tmp_path) + assert_dirtrees_eq(zarr_path, tmp_path / new_dandiset.dandiset_id / "test.zarr") + + +@pytest.mark.ai_generated +def test_upload_zarr_deferred_ingestion_faked( + mocker: MockerFixture, new_dandiset: SampleDandiset +) -> None: + """ + When the server defers ingesting a Zarr, the upload does not wait for it, + verifies the Zarr against its remote file listing instead, and reports the + deferral. The deferral is faked, so that this also runs against servers + without dandi-archive#3018 (cf. `test_upload_zarr_deferred_ingestion`). + """ + zarr.save(new_dandiset.dspath / "test.zarr", np.arange(100), np.arange(100, 0, -1)) + deferred_until = "2026-11-18T13:00:00Z" + original_wait = RemoteZarrAsset.wait_for_ingestion + + def wait_for_ingestion(self: RemoteZarrAsset) -> dict[str, Any]: + return { + **original_wait(self), + "status": "Uploaded", + "checksum": None, + "ingestion_deferred_until": deferred_until, + } + + mocker.patch.object(RemoteZarrAsset, "wait_for_ingestion", wait_for_ingestion) + remote_checksum = mocker.spy(zarr_files, "_remote_zarr_checksum") + events = _spy_zarr_iter_upload(mocker) + + new_dandiset.upload() + + assert remote_checksum.call_count == 1 + assert not any(e["status"] == "Checksum mismatch" for e in events) + assert events[-1] == { + "status": "done", + "message": f"server ingestion deferred until {deferred_until}", + "asset": anys.AnyInstance(RemoteZarrAsset), + } @pytest.mark.ai_generated @@ -1179,3 +1306,86 @@ def test_upload_identical_files_in_parallel(new_dandiset: SampleDandiset) -> Non ) == 1 ) + + +@pytest.mark.ai_generated +def test_upload_unchanged_unfinalized_zarr( + mocker: MockerFixture, zarr_dandiset: SampleDandiset +) -> None: + """ + A rerun of an upload which ended before finalizing the Zarr finalizes it, + rather than reporting the matching Zarr as identical and leaving it + "Pending". + """ + client = zarr_dandiset.client + (asset,) = zarr_dandiset.dandiset.get_assets() + assert isinstance(asset, RemoteZarrAsset) + checksum = client.get(f"/zarr/{asset.zarr}/")["checksum"] + # Requesting an upload URL, as the interrupted upload would have done, + # makes the Zarr "Pending". The URL is for the content already there, so + # it cannot change the Zarr (and does not defer its ingestion). + entry = next(asset.iterfiles()) + client.post( + f"/zarr/{asset.zarr}/files/", + json=[ + { + "path": str(entry), + "base64md5": b64encode(bytes.fromhex(entry.digest.value)).decode(), + } + ], + ) + assert client.get(f"/zarr/{asset.zarr}/")["status"] == "Pending" + + events = _spy_zarr_iter_upload(mocker) + zarr_dandiset.upload(validation="skip") + + assert events[-1]["status"] == "done" + zarr_info = client.get(f"/zarr/{asset.zarr}/") + assert zarr_info["status"] == "Complete" + assert zarr_info["checksum"] == checksum + + +@pytest.mark.ai_generated +def test_upload_zarr_deferred_ingestion( + mocker: MockerFixture, zarr_dandiset: SampleDandiset +) -> None: + """ + Modifying an entry while the URL it was uploaded with is still valid makes + dandi-archive#3018 defer ingesting the Zarr. The upload does not wait for + it, but says until when it is deferred, as does a rerun. + """ + client = zarr_dandiset.client + (asset,) = zarr_dandiset.dandiset.get_assets() + assert isinstance(asset, RemoteZarrAsset) + if "ingestion_deferred_until" not in client.get(f"/zarr/{asset.zarr}/"): + pytest.skip("Server does not defer Zarr ingestion") + # Modify a chunk, whose path depends on the Zarr format version + chunk = next( + p + for p in (zarr_dandiset.dspath / "sample.zarr" / "arr_0").rglob("*") + if p.is_file() and not p.name.startswith(".") and p.name != "zarr.json" + ) + chunk.write_bytes(b"x" * 64) + + events = _spy_zarr_iter_upload(mocker) + zarr_dandiset.upload(validation="skip") + + zarr_info = client.get(f"/zarr/{asset.zarr}/") + if (deferred_until := zarr_info["ingestion_deferred_until"]) is None: + # The URL expired before the modification was uploaded + pytest.skip("Server did not defer Zarr ingestion") + assert zarr_info["status"] == "Uploaded" + assert events[-1] == { + "status": "done", + "message": f"server ingestion deferred until {deferred_until}", + "asset": anys.AnyInstance(RemoteZarrAsset), + } + assert not any(e["status"] == "Checksum mismatch" for e in events) + + events.clear() + zarr_dandiset.upload(validation="skip") + assert events[-1] == { + "status": "skipped", + "message": f"identical; server ingestion deferred until {deferred_until}", + "asset": anys.AnyInstance(RemoteZarrAsset), + }