From df29d21f741636a2401ff4105a15020dbe976a16 Mon Sep 17 00:00:00 2001 From: naglepuff Date: Fri, 13 Nov 2026 14:50:45 -0500 Subject: [PATCH 1/7] Add deferred ingestion env vars to test env docker --- dandi/tests/data/dandiarchive-docker/docker-compose.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dandi/tests/data/dandiarchive-docker/docker-compose.yml b/dandi/tests/data/dandiarchive-docker/docker-compose.yml index 95167115f..0ec13484f 100644 --- a/dandi/tests/data/dandiarchive-docker/docker-compose.yml +++ b/dandi/tests/data/dandiarchive-docker/docker-compose.yml @@ -39,6 +39,13 @@ services: DJANGO_DANDI_INSTANCE_NAME: ${DANDI_TESTS_INSTANCE_NAME:-DANDI} DJANGO_DANDI_INSTANCE_IDENTIFIER: ${DANDI_TESTS_INSTANCE_IDENTIFIER:-RRID:ABC_123456} DJANGO_DANDI_DOI_API_PREFIX: ${DANDI_TESTS_DOI_PREFIX:-10.80507} + # dandi-archive defers ingestion of a Zarr until every upload URL that + # could still change it has expired + # (https://github.com/dandi/dandi-archive/pull/3018). Shorten the expiry + # from the default hour so that tests which overwrite or delete recently + # uploaded Zarr entries do not wait that long. Ignored by servers + # without that feature. + DJANGO_DANDI_ZARR_UPLOAD_URL_EXPIRATION: "30" ports: - "127.0.0.1:8000:8000" From d18008a40077da9f09d09122b986740e7fe5bfe4 Mon Sep 17 00:00:00 2001 From: naglepuff Date: Tue, 17 Nov 2026 17:58:03 -0500 Subject: [PATCH 2/7] Report deferred Zarr ingestion instead of waiting for it dandi-archive#3018 defers ingesting a Zarr while any unexpired upload URL could still change it, leaving it "Uploaded" with "ingestion_deferred_until" set; the server ingests it on its own once those URLs expire. Polling for "Complete" without a timeout made `dandi upload` hang for up to an hour in that case. - Add `BaseRemoteZarrAsset.wait_for_ingestion()`, which returns once the Zarr is "Complete" or its ingestion is deferred, and raises `ZarrIngestionError` if the Zarr becomes "Pending" again. Use it in `iter_upload()` and `rmfiles(reingest=True)`. - On a deferral, `iter_upload()` skips the local checksum verification and reports until when ingestion is deferred. - When there is nothing to upload, check the Zarr's status rather than always reporting it as identical: finalize it if an earlier upload ended before doing so ("Pending"), wait for it if it is being ingested, and report a deferral. - Servers without dandi-archive#3018 do not report "ingestion_deferred_until" and keep the prior behavior. - In tests that download a Zarr after rewriting or deleting recently uploaded entries, wait for its ingestion, with a longer timeout until the test archive instance can shorten the interval for retrying deferred ingestion. Co-Authored-By: Claude Opus 5.5 --- dandi/dandiapi.py | 78 ++++++++++++++++++++-- dandi/exceptions.py | 6 ++ dandi/files/zarr.py | 121 +++++++++++++++++++++++----------- dandi/tests/test_dandiapi.py | 120 +++++++++++++++++++++++++++++++++- dandi/tests/test_upload.py | 122 +++++++++++++++++++++++++++++++++++ 5 files changed, 401 insertions(+), 46 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 9ea22e025..e33f126e2 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -62,7 +62,12 @@ DandiInstance, EmbargoStatus, ) -from .exceptions import HTTP404Error, NotFoundError, SchemaVersionError +from .exceptions import ( + HTTP404Error, + NotFoundError, + SchemaVersionError, + ZarrIngestionError, +) from .keyring_utils import keyring_lookup, keyring_save from .misctypes import ( Digest, @@ -126,6 +131,29 @@ def set_asset_schema_key(metadata: dict[str, Any]) -> None: metadata["schemaKey"] = "Asset" +def zarr_ingestion_deferred_until(zarr_data: dict[str, Any]) -> str | None: + """ + Given a Zarr's data from the server (as returned by ``GET + /zarr/{zarr_id}/``), return the time until which the server deferred its + ingestion, or `None` if it did not. + + dandi-archive#3018 defers the ingestion of a Zarr while any of its + unexpired upload URLs could still change it, leaving it ``"Uploaded"`` with + the time of the next attempt in ``"ingestion_deferred_until"``. The field + stays set while that attempt runs (``"Ingesting"``), and is cleared once + the Zarr is ingested or modified. + """ + # Servers without dandi-archive#3018 lack the field and never defer + # ingestion; .get() can be replaced by indexing once all known instances + # deploy that feature. + if zarr_data["status"] == "Uploaded" and ( + deferred_until := zarr_data.get("ingestion_deferred_until") + ): + assert isinstance(deferred_until, str) + return deferred_until + return None + + class AssetType(Enum): """ .. versionadded:: 0.36.0 @@ -2073,7 +2101,7 @@ def rmfiles(self, files: Iterable[RemoteZarrEntry], reingest: bool = True) -> No If ``reingest`` is true, after performing the deletion, the client triggers a recalculation of the Zarr's checksum and waits for it to - complete. + complete, unless the server defers it (see `wait_for_ingestion()`). """ # Don't bother checking that the entries are actually files or even # belong to this Zarr, as if they're not, the server will return an @@ -2085,11 +2113,47 @@ def rmfiles(self, files: Iterable[RemoteZarrEntry], reingest: bool = True) -> No ) if reingest: self.client.post(f"/zarr/{self.zarr}/finalize/") - while True: - sleep(2) - r = self.client.get(f"/zarr/{self.zarr}/") - if r["status"] == "Complete": - break + r = self.wait_for_ingestion() + if deferred_until := zarr_ingestion_deferred_until(r): + lgr.info( + "Server deferred ingestion of Zarr %s until %s", + self.zarr, + deferred_until, + ) + + def wait_for_ingestion(self) -> dict[str, Any]: + """ + Wait for the server to finish ingesting the Zarr after it has been + finalized, or to defer its ingestion. + + Returns the Zarr's data from the server (as returned by + ``GET /zarr/{zarr_id}/``) once its ``"status"`` is either + ``"Complete"``, in which case its ``"checksum"`` is set, or + ``"Uploaded"`` with ``"ingestion_deferred_until"`` set. In the latter + case, the server ingests the Zarr on its own at the given time, so + there is no point in waiting for it. + + Raises `ZarrIngestionError` if the Zarr becomes ``"Pending"`` again, + i.e., something else modified it in the meantime. + """ + while True: + sleep(2) + r: dict[str, Any] = self.client.get(f"/zarr/{self.zarr}/") + status = r["status"] + if status == "Complete": + return r + if deferred_until := zarr_ingestion_deferred_until(r): + lgr.debug( + "Server deferred ingestion of Zarr %s until %s", + self.zarr, + deferred_until, + ) + return r + if status == "Pending": + raise ZarrIngestionError( + f"Zarr {self.zarr} was modified while waiting for the" + " server to ingest it" + ) class RemoteAsset(BaseRemoteAsset): diff --git a/dandi/exceptions.py b/dandi/exceptions.py index f323d2a42..67c1f85ce 100644 --- a/dandi/exceptions.py +++ b/dandi/exceptions.py @@ -97,3 +97,9 @@ class UploadValidationError(UploadError): """An upload could not proceed because an asset failed validation.""" pass + + +class ZarrIngestionError(RuntimeError): + """A finalized Zarr did not reach a state in which its ingestion is done or deferred.""" + + pass diff --git a/dandi/files/zarr.py b/dandi/files/zarr.py index 696522b55..43d421e8c 100644 --- a/dandi/files/zarr.py +++ b/dandi/files/zarr.py @@ -42,6 +42,7 @@ RemoteZarrEntry, RESTFullAPIClient, set_asset_schema_key, + zarr_ingestion_deferred_until, ) from dandi.exceptions import UploadError from dandi.metadata.core import get_default_metadata @@ -588,7 +589,10 @@ def iter_upload( `RemoteAsset`. If the local Zarr is bit-identical to the remote (nothing to upload or delete), the terminal status is instead ``"skipped"`` with a ``"message"`` of ``"identical"`` and the same - ``"asset"`` key. + ``"asset"`` key. If the server deferred ingesting the Zarr (see + `RemoteZarrAsset.wait_for_ingestion()`), the upload does not wait + for it, the local checksum is not verified, and the terminal + ``"message"`` says until when ingestion is deferred. """ asset_path = metadata.setdefault("path", self.path) set_asset_schema_key(metadata) @@ -661,6 +665,51 @@ def mkzarr() -> str: mismatched = True first_run = True unchanged = False + # Until when the server deferred ingesting the Zarr, if it did + deferred_until: str | None = None + + def ingest(finalize: bool) -> Generator[dict, None, bool]: + """ + Wait for the server to ingest the Zarr, after finalizing it if + ``finalize`` is true, and verify its checksum against the local + one. Returns whether they mismatch. + """ + nonlocal deferred_until + lgr.debug("%s: Waiting for server to calculate Zarr checksum", asset_path) + yield {"status": "server calculating checksum"} + if finalize: + client.post(f"/zarr/{zarr_id}/finalize/") + r = a.wait_for_ingestion() + if deferred_until := zarr_ingestion_deferred_until(r): + lgr.info( + "%s: Server deferred ingestion until %s because upload" + " URLs for entries that have since been changed or deleted" + " have not expired yet; skipping local checksum" + " verification", + asset_path, + deferred_until, + ) + return False + if zarr_mode == "patch": + lgr.info( + "%s: Skipping local checksum verification (patch mode)", + asset_path, + ) + return False + our_checksum = str(zcc.process()) + server_checksum = r["checksum"] + if our_checksum == server_checksum: + return False + lgr.info( + "%s: Asset checksum mismatch (local: %s; server: %s);" + " redoing upload", + asset_path, + our_checksum, + server_checksum, + ) + yield {"status": "Checksum mismatch"} + return True + while mismatched: zcc = ZarrChecksumTree() old_zarr_entries: dict[str, RemoteZarrEntry] = { @@ -750,9 +799,27 @@ def mkzarr() -> str: len(old_zarr_entries) if zarr_mode == "full" else 0 ) if n_add == 0 and n_modify == 0 and n_delete == 0: - mismatched = False - unchanged = True - break + zarr_info = client.get(f"/zarr/{zarr_id}/") + status = zarr_info["status"] + if status == "Complete" or ( + deferred_until := zarr_ingestion_deferred_until(zarr_info) + ): + mismatched = False + unchanged = True + break + # The remote entries match, but an earlier upload ended + # before finalizing the Zarr ("Pending") or before the + # server ingested it ("Uploaded" or "Ingesting") + lgr.info( + "%s: Remote Zarr matches local but has not been" + " ingested (status: %s); %s", + asset_path, + status, + "finalizing" if status == "Pending" else "waiting for it", + ) + mismatched = yield from ingest(finalize=status == "Pending") + first_run = False + continue bytes_add = sum(e.size for e in to_upload.fresh_entries) bytes_modify = sum(it.size for it in to_upload.digested_entries) clauses = [] @@ -919,38 +986,7 @@ def mkzarr() -> str: "file" if n_remote_only == 1 else "files", ) if changed: - lgr.debug( - "%s: Waiting for server to calculate Zarr checksum", asset_path - ) - yield {"status": "server calculating checksum"} - client.post(f"/zarr/{zarr_id}/finalize/") - while True: - sleep(2) - r = client.get(f"/zarr/{zarr_id}/") - if r["status"] == "Complete": - if zarr_mode == "patch": - lgr.info( - "%s: Skipping local checksum verification" - " (patch mode)", - asset_path, - ) - mismatched = False - else: - our_checksum = str(zcc.process()) - server_checksum = r["checksum"] - if our_checksum == server_checksum: - mismatched = False - else: - mismatched = True - lgr.info( - "%s: Asset checksum mismatch (local: %s;" - " server: %s); redoing upload", - asset_path, - our_checksum, - server_checksum, - ) - yield {"status": "Checksum mismatch"} - break + mismatched = yield from ingest(finalize=True) elif mismatched and not first_run: lgr.error( "%s: Previous upload loop resulted in checksum mismatch," @@ -969,12 +1005,21 @@ def mkzarr() -> str: unchanged, first_run, ) + deferral = ( + f"server ingestion deferred until {deferred_until}" + if deferred_until is not None + else None + ) if unchanged: lgr.info("%s: Zarr already matches remote; nothing to upload", asset_path) - yield {"status": "skipped", "message": "identical", "asset": a} + message = "identical" if deferral is None else f"identical; {deferral}" + yield {"status": "skipped", "message": message, "asset": a} else: lgr.info("%s: Asset successfully uploaded", asset_path) - yield {"status": "done", "asset": a} + done: dict[str, Any] = {"status": "done", "asset": a} + if deferral is not None: + done["message"] = deferral + yield done def _handle_failed_items_and_raise( diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index ac6534ba3..6ce44e3ab 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -37,6 +37,7 @@ dandiset_metadata_file, ) from ..dandiapi import ( + BaseRemoteZarrAsset, DandiAPIClient, RemoteAsset, RemoteBlobAsset, @@ -46,7 +47,7 @@ VersionStatus, ) from ..download import download -from ..exceptions import NotFoundError, SchemaVersionError +from ..exceptions import NotFoundError, SchemaVersionError, ZarrIngestionError from ..files import GenericAsset, dandi_file from ..files.bases import _SCHEMA_BAREASSET_HAS_DATASTANDARD from ..utils import list_paths @@ -1177,3 +1178,120 @@ def test_all_public_api() -> None: assert exported == names assert {"DandiAPIClient", "RemoteDandiset", "RemoteAsset"} <= names assert not {"APIBase", "RemoteDandisetData", "ZarrEntryServerData"} & names + + +def _zarr_asset_for_mocked_server() -> BaseRemoteZarrAsset: + _mock_server_info(dandischema.consts.DANDI_SCHEMA_VERSION) + now = datetime.now(timezone.utc) + return BaseRemoteZarrAsset.model_validate( + { + "client": DandiAPIClient("https://test.nil/api"), + "asset_id": "abc", + "path": "sample.zarr", + "size": 0, + "created": now, + "modified": now, + "zarr": "zarr-id", + } + ) + + +_MOCK_ZARR_URL = "https://test.nil/api/zarr/zarr-id/" + + +def _zarr_polls() -> int: + return sum(c.request.url == _MOCK_ZARR_URL for c in responses.calls) + + +def _mock_zarr_states(*states: dict[str, Any]) -> None: + for state in states: + responses.add( + responses.GET, + _MOCK_ZARR_URL, + json={"zarr_id": "zarr-id", "checksum": None, **state}, + ) + + +@pytest.mark.ai_generated +@responses.activate +def test_wait_for_ingestion_without_deferral_support(mocker: MockerFixture) -> None: + """A server without dandi-archive#3018 never defers; wait for "Complete".""" + mocker.patch.object(dandiapi, "sleep") + _mock_zarr_states( + {"status": "Uploaded"}, + {"status": "Ingesting"}, + {"status": "Complete", "checksum": "abc-1--1"}, + ) + r = _zarr_asset_for_mocked_server().wait_for_ingestion() + assert r["status"] == "Complete" + assert r["checksum"] == "abc-1--1" + assert _zarr_polls() == 3 + + +@pytest.mark.ai_generated +@responses.activate +def test_wait_for_ingestion_complete(mocker: MockerFixture) -> None: + mocker.patch.object(dandiapi, "sleep") + _mock_zarr_states( + {"status": "Uploaded", "ingestion_deferred_until": None}, + {"status": "Ingesting", "ingestion_deferred_until": None}, + { + "status": "Complete", + "checksum": "abc-1--1", + "ingestion_deferred_until": None, + }, + ) + r = _zarr_asset_for_mocked_server().wait_for_ingestion() + assert r["status"] == "Complete" + assert _zarr_polls() == 3 + + +@pytest.mark.ai_generated +@responses.activate +def test_wait_for_ingestion_deferred(mocker: MockerFixture) -> None: + """A deferred ingestion is not waited for.""" + mocker.patch.object(dandiapi, "sleep") + deferred_until = "2026-11-17T13:00:00Z" + _mock_zarr_states( + {"status": "Uploaded", "ingestion_deferred_until": None}, + {"status": "Ingesting", "ingestion_deferred_until": None}, + {"status": "Uploaded", "ingestion_deferred_until": deferred_until}, + ) + r = _zarr_asset_for_mocked_server().wait_for_ingestion() + assert r["status"] == "Uploaded" + assert r["ingestion_deferred_until"] == deferred_until + assert _zarr_polls() == 3 + + +@pytest.mark.ai_generated +@responses.activate +def test_wait_for_ingestion_retried_after_deferral(mocker: MockerFixture) -> None: + """ + The server keeps "ingestion_deferred_until" while retrying a deferred + ingestion; a Zarr being ingested again is waited for. + """ + mocker.patch.object(dandiapi, "sleep") + deferred_until = "2026-11-17T13:00:00Z" + _mock_zarr_states( + {"status": "Ingesting", "ingestion_deferred_until": deferred_until}, + { + "status": "Complete", + "checksum": "abc-1--1", + "ingestion_deferred_until": None, + }, + ) + r = _zarr_asset_for_mocked_server().wait_for_ingestion() + assert r["status"] == "Complete" + assert _zarr_polls() == 2 + + +@pytest.mark.ai_generated +@responses.activate +def test_wait_for_ingestion_modified_meanwhile(mocker: MockerFixture) -> None: + mocker.patch.object(dandiapi, "sleep") + _mock_zarr_states( + {"status": "Uploaded", "ingestion_deferred_until": None}, + {"status": "Pending", "ingestion_deferred_until": None}, + ) + with pytest.raises(ZarrIngestionError, match="zarr-id was modified"): + _zarr_asset_for_mocked_server().wait_for_ingestion() diff --git a/dandi/tests/test_upload.py b/dandi/tests/test_upload.py index 03dfdb710..eb5fbd18c 100644 --- a/dandi/tests/test_upload.py +++ b/dandi/tests/test_upload.py @@ -1,14 +1,17 @@ from __future__ import annotations +from base64 import b64encode from collections import defaultdict from datetime import datetime, timezone import os from pathlib import Path from shutil import copyfile, rmtree +from time import monotonic, sleep from typing import Any from unittest.mock import Mock from urllib.parse import urlparse +import anys import numpy as np import pynwb import pytest @@ -466,6 +469,35 @@ def test_upload_bids_zarr( bids_zarr_dandiset.upload() +#: Timeout for tests which rewrite or delete recently uploaded Zarr entries and +#: then need the Zarr ingested (see `_wait_for_zarr_ingestion()`). Shorten it +#: once the test archive instance also shortens the interval for retrying +#: deferred ingestion. +ZARR_DEFERRED_INGESTION_TIMEOUT = 600 + + +def _wait_for_zarr_ingestion( + sample_dandiset: SampleDandiset, path: str = "sample.zarr" +) -> None: + """ + Wait for the server to ingest the Zarr asset at ``path``. + + Rewriting or deleting entries of a Zarr while the URLs they were uploaded + with are still valid makes dandi-archive#3018 defer ingesting it until + those URLs expire, and ``dandi upload`` does not wait for that. In the + test archive instance, the URLs expire after 30 seconds, and deferred + ingestion is retried every 5 minutes. + """ + asset = sample_dandiset.dandiset.get_asset_by_path(path) + assert isinstance(asset, RemoteZarrAsset) + deadline = monotonic() + ZARR_DEFERRED_INGESTION_TIMEOUT - 30 + while ( + status := sample_dandiset.client.get(f"/zarr/{asset.zarr}/")["status"] + ) != "Complete": + assert monotonic() < deadline, f"Zarr {asset.zarr} still {status!r}" + sleep(5) + + def _replace_zarr_and_check( tmp_path: Path, zarr_dandiset: SampleDandiset, zarr_id: str ) -> None: @@ -475,6 +507,7 @@ def _replace_zarr_and_check( asset = zarr_dandiset.dandiset.get_asset_by_path("sample.zarr") assert isinstance(asset, RemoteZarrAsset) assert asset.zarr == zarr_id + _wait_for_zarr_ingestion(zarr_dandiset) download(zarr_dandiset.dandiset.version_api_url, tmp_path) assert_dirtrees_eq( zarr_dandiset.dspath / "sample.zarr", @@ -482,12 +515,14 @@ def _replace_zarr_and_check( ) +@pytest.mark.timeout(ZARR_DEFERRED_INGESTION_TIMEOUT) def test_upload_different_zarr(tmp_path: Path, zarr_dandiset: SampleDandiset) -> None: asset = zarr_dandiset.dandiset.get_asset_by_path("sample.zarr") assert isinstance(asset, RemoteZarrAsset) _replace_zarr_and_check(tmp_path, zarr_dandiset, asset.zarr) +@pytest.mark.timeout(ZARR_DEFERRED_INGESTION_TIMEOUT) def test_upload_loose_zarr(tmp_path: Path, zarr_dandiset: SampleDandiset) -> None: asset = zarr_dandiset.dandiset.get_asset_by_path("sample.zarr") assert isinstance(asset, RemoteZarrAsset) @@ -496,6 +531,7 @@ def test_upload_loose_zarr(tmp_path: Path, zarr_dandiset: SampleDandiset) -> Non _replace_zarr_and_check(tmp_path, zarr_dandiset, zarr_id) +@pytest.mark.timeout(ZARR_DEFERRED_INGESTION_TIMEOUT) def test_upload_different_zarr_entry_conflicts( tmp_path: Path, new_dandiset: SampleDandiset ) -> None: @@ -514,10 +550,12 @@ def test_upload_different_zarr_entry_conflicts( (zf / "changed-size.txt").write_text("This is a test of the upload code.\n") (zf / "changed-type").write_text("This is now a file.\n") new_dandiset.upload(validation=UploadValidation.SKIP) + _wait_for_zarr_ingestion(new_dandiset) download(new_dandiset.dandiset.version_api_url, tmp_path) assert_dirtrees_eq(zf, tmp_path / new_dandiset.dandiset_id / "sample.zarr") +@pytest.mark.timeout(ZARR_DEFERRED_INGESTION_TIMEOUT) def test_upload_different_zarr_file_to_parent_dir( tmp_path: Path, new_dandiset: SampleDandiset ) -> None: @@ -530,6 +568,7 @@ def test_upload_different_zarr_file_to_parent_dir( (zf / "foo").mkdir() (zf / "foo" / "bar").write_text("This is under what used to be a file.\n") new_dandiset.upload(validation=UploadValidation.SKIP) + _wait_for_zarr_ingestion(new_dandiset) download(new_dandiset.dandiset.version_api_url, tmp_path) assert_dirtrees_eq(zf, tmp_path / new_dandiset.dandiset_id / "sample.zarr") @@ -1179,3 +1218,86 @@ def test_upload_identical_files_in_parallel(new_dandiset: SampleDandiset) -> Non ) == 1 ) + + +@pytest.mark.ai_generated +def test_upload_unchanged_unfinalized_zarr( + mocker: MockerFixture, zarr_dandiset: SampleDandiset +) -> None: + """ + A rerun of an upload which ended before finalizing the Zarr finalizes it, + rather than reporting the matching Zarr as identical and leaving it + "Pending". + """ + client = zarr_dandiset.client + (asset,) = zarr_dandiset.dandiset.get_assets() + assert isinstance(asset, RemoteZarrAsset) + checksum = client.get(f"/zarr/{asset.zarr}/")["checksum"] + # Requesting an upload URL, as the interrupted upload would have done, + # makes the Zarr "Pending". The URL is for the content already there, so + # it cannot change the Zarr (and does not defer its ingestion). + entry = next(asset.iterfiles()) + client.post( + f"/zarr/{asset.zarr}/files/", + json=[ + { + "path": str(entry), + "base64md5": b64encode(bytes.fromhex(entry.digest.value)).decode(), + } + ], + ) + assert client.get(f"/zarr/{asset.zarr}/")["status"] == "Pending" + + events = _spy_zarr_iter_upload(mocker) + zarr_dandiset.upload(validation="skip") + + assert events[-1]["status"] == "done" + zarr_info = client.get(f"/zarr/{asset.zarr}/") + assert zarr_info["status"] == "Complete" + assert zarr_info["checksum"] == checksum + + +@pytest.mark.ai_generated +def test_upload_zarr_deferred_ingestion( + mocker: MockerFixture, zarr_dandiset: SampleDandiset +) -> None: + """ + Modifying an entry while the URL it was uploaded with is still valid makes + dandi-archive#3018 defer ingesting the Zarr. The upload does not wait for + it, but says until when it is deferred, as does a rerun. + """ + client = zarr_dandiset.client + (asset,) = zarr_dandiset.dandiset.get_assets() + assert isinstance(asset, RemoteZarrAsset) + if "ingestion_deferred_until" not in client.get(f"/zarr/{asset.zarr}/"): + pytest.skip("Server does not defer Zarr ingestion") + # Modify a chunk, whose path depends on the Zarr format version + chunk = next( + p + for p in (zarr_dandiset.dspath / "sample.zarr" / "arr_0").rglob("*") + if p.is_file() and not p.name.startswith(".") and p.name != "zarr.json" + ) + chunk.write_bytes(b"x" * 64) + + events = _spy_zarr_iter_upload(mocker) + zarr_dandiset.upload(validation="skip") + + zarr_info = client.get(f"/zarr/{asset.zarr}/") + if (deferred_until := zarr_info["ingestion_deferred_until"]) is None: + # The URL expired before the modification was uploaded + pytest.skip("Server did not defer Zarr ingestion") + assert zarr_info["status"] == "Uploaded" + assert events[-1] == { + "status": "done", + "message": f"server ingestion deferred until {deferred_until}", + "asset": anys.AnyInstance(RemoteZarrAsset), + } + assert not any(e["status"] == "Checksum mismatch" for e in events) + + events.clear() + zarr_dandiset.upload(validation="skip") + assert events[-1] == { + "status": "skipped", + "message": f"identical; server ingestion deferred until {deferred_until}", + "asset": anys.AnyInstance(RemoteZarrAsset), + } From 30240076f3ee1414bef1d1e3f9d3a7ff4ffa3a4f Mon Sep 17 00:00:00 2001 From: naglepuff Date: Wed, 18 Nov 2026 11:34:23 -0500 Subject: [PATCH 3/7] Shorten the deferred Zarr ingestion retry interval in tests dandi-archive#3018 now makes the interval for retrying deferred Zarr ingestion configurable. Set it to 5 seconds in the test archive instance, so that tests which download a Zarr after rewriting or deleting recently uploaded entries wait well under a minute for its ingestion, and drop their longer timeout. Co-Authored-By: Claude Opus 5.5 --- .../dandiarchive-docker/docker-compose.yml | 8 +++++--- dandi/tests/test_upload.py | 19 +++++-------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/dandi/tests/data/dandiarchive-docker/docker-compose.yml b/dandi/tests/data/dandiarchive-docker/docker-compose.yml index 0ec13484f..743135b9e 100644 --- a/dandi/tests/data/dandiarchive-docker/docker-compose.yml +++ b/dandi/tests/data/dandiarchive-docker/docker-compose.yml @@ -40,12 +40,14 @@ services: DJANGO_DANDI_INSTANCE_IDENTIFIER: ${DANDI_TESTS_INSTANCE_IDENTIFIER:-RRID:ABC_123456} DJANGO_DANDI_DOI_API_PREFIX: ${DANDI_TESTS_DOI_PREFIX:-10.80507} # dandi-archive defers ingestion of a Zarr until every upload URL that - # could still change it has expired - # (https://github.com/dandi/dandi-archive/pull/3018). Shorten the expiry - # from the default hour so that tests which overwrite or delete recently + # could still change it has expired, and retries deferred ingestion + # periodically (https://github.com/dandi/dandi-archive/pull/3018). + # Shorten the expiry from the default hour, and the retry interval from + # the default 5 minutes, so that tests which overwrite or delete recently # uploaded Zarr entries do not wait that long. Ignored by servers # without that feature. DJANGO_DANDI_ZARR_UPLOAD_URL_EXPIRATION: "30" + DJANGO_DANDI_DEFERRED_INGESTION_RETRY_INTERVAL: "5" ports: - "127.0.0.1:8000:8000" diff --git a/dandi/tests/test_upload.py b/dandi/tests/test_upload.py index eb5fbd18c..bbf6e4af9 100644 --- a/dandi/tests/test_upload.py +++ b/dandi/tests/test_upload.py @@ -469,13 +469,6 @@ def test_upload_bids_zarr( bids_zarr_dandiset.upload() -#: Timeout for tests which rewrite or delete recently uploaded Zarr entries and -#: then need the Zarr ingested (see `_wait_for_zarr_ingestion()`). Shorten it -#: once the test archive instance also shortens the interval for retrying -#: deferred ingestion. -ZARR_DEFERRED_INGESTION_TIMEOUT = 600 - - def _wait_for_zarr_ingestion( sample_dandiset: SampleDandiset, path: str = "sample.zarr" ) -> None: @@ -486,16 +479,18 @@ def _wait_for_zarr_ingestion( with are still valid makes dandi-archive#3018 defer ingesting it until those URLs expire, and ``dandi upload`` does not wait for that. In the test archive instance, the URLs expire after 30 seconds, and deferred - ingestion is retried every 5 minutes. + ingestion is retried every 5 seconds, so this takes well under a minute. + The deadline leaves the rest of pytest's timeout for the test itself, so + that a Zarr which is never ingested fails with its status. """ asset = sample_dandiset.dandiset.get_asset_by_path(path) assert isinstance(asset, RemoteZarrAsset) - deadline = monotonic() + ZARR_DEFERRED_INGESTION_TIMEOUT - 30 + deadline = monotonic() + 180 while ( status := sample_dandiset.client.get(f"/zarr/{asset.zarr}/")["status"] ) != "Complete": assert monotonic() < deadline, f"Zarr {asset.zarr} still {status!r}" - sleep(5) + sleep(2) def _replace_zarr_and_check( @@ -515,14 +510,12 @@ def _replace_zarr_and_check( ) -@pytest.mark.timeout(ZARR_DEFERRED_INGESTION_TIMEOUT) def test_upload_different_zarr(tmp_path: Path, zarr_dandiset: SampleDandiset) -> None: asset = zarr_dandiset.dandiset.get_asset_by_path("sample.zarr") assert isinstance(asset, RemoteZarrAsset) _replace_zarr_and_check(tmp_path, zarr_dandiset, asset.zarr) -@pytest.mark.timeout(ZARR_DEFERRED_INGESTION_TIMEOUT) def test_upload_loose_zarr(tmp_path: Path, zarr_dandiset: SampleDandiset) -> None: asset = zarr_dandiset.dandiset.get_asset_by_path("sample.zarr") assert isinstance(asset, RemoteZarrAsset) @@ -531,7 +524,6 @@ def test_upload_loose_zarr(tmp_path: Path, zarr_dandiset: SampleDandiset) -> Non _replace_zarr_and_check(tmp_path, zarr_dandiset, zarr_id) -@pytest.mark.timeout(ZARR_DEFERRED_INGESTION_TIMEOUT) def test_upload_different_zarr_entry_conflicts( tmp_path: Path, new_dandiset: SampleDandiset ) -> None: @@ -555,7 +547,6 @@ def test_upload_different_zarr_entry_conflicts( assert_dirtrees_eq(zf, tmp_path / new_dandiset.dandiset_id / "sample.zarr") -@pytest.mark.timeout(ZARR_DEFERRED_INGESTION_TIMEOUT) def test_upload_different_zarr_file_to_parent_dir( tmp_path: Path, new_dandiset: SampleDandiset ) -> None: From 86b3edb013ca74a07f94243f52c6c6a2a50fc7e4 Mon Sep 17 00:00:00 2001 From: naglepuff Date: Wed, 18 Nov 2026 12:41:07 -0500 Subject: [PATCH 4/7] Test the retry of Zarr uploads on a 400 RequestTimeout below the retry logic test_zarr_upload_400_timeout_retry replaced RESTFullAPIClient.request() itself, so its fake 400 bypassed the retry logic there, and the entries it was returned for were never uploaded. It passed only because a second pass of the upload, after the Zarr's checksum did not match, uploaded them again; against dandi-archive#3018, which defers ingesting a Zarr with unused upload URLs, there is no such second pass, and it failed. Return the fake 400 from requests.Session.request() instead, so that it is retried by RESTFullAPIClient.request(), and check that the upload needed neither a second pass nor had its ingestion deferred. Co-Authored-By: Claude Opus 5.5 --- dandi/tests/test_upload.py | 111 +++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 60 deletions(-) diff --git a/dandi/tests/test_upload.py b/dandi/tests/test_upload.py index bbf6e4af9..e9649ce50 100644 --- a/dandi/tests/test_upload.py +++ b/dandi/tests/test_upload.py @@ -681,81 +681,72 @@ def mock_put(self, url, **kwargs): @pytest.mark.ai_generated def test_zarr_upload_400_timeout_retry( + mocker: MockerFixture, new_dandiset: SampleDandiset, monkeypatch: pytest.MonkeyPatch, ) -> None: - """Test that 400 RequestTimeout errors trigger automatic retry via tenacity""" - # Create test Zarr + """ + A 400 RequestTimeout response to uploading a Zarr entry is retried by + `RESTFullAPIClient.request()` (https://github.com/dandi/dandi-cli/issues/1662) + """ zarr_path = new_dandiset.dspath / "test.zarr" zarr.save(zarr_path, np.arange(100), np.arange(100, 0, -1)) - # Track request attempts - request_attempts: defaultdict[str, int] = defaultdict(int) - original_request = RESTFullAPIClient.request - - def mock_request(self, method, path, **kwargs): - # Track attempts for each request - urlpath = urlparse(path).path if path.startswith("http") else path - request_attempts[urlpath] += 1 - - # Simulate 400 timeout on first attempt for files containing "arr_0" - if method == "PUT" and "arr_0" in path and request_attempts[urlpath] == 1: - # Return a mock response that will trigger the retry_if condition - resp = Mock(spec=requests.Response) - resp.status_code = 400 - resp.text = ( - '\n' - "RequestTimeout" - "Your socket connection to the server " - "was not read from or written to within the timeout period. " - "Idle connections will be closed." - "1111111111111111" - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" - "" - ) - resp.headers = {} - resp.content = resp.text.encode() - resp.json = Mock(side_effect=ValueError("No JSON")) - return resp - - # Otherwise, call the original method - return original_request(self, method, path, **kwargs) - - # Apply the mock - monkeypatch.setattr(RESTFullAPIClient, "request", mock_request) + # Attempts to upload each Zarr entry, by the path of its URL + put_attempts: defaultdict[str, int] = defaultdict(int) + original_request = requests.Session.request + + def mock_request( + self: requests.Session, method: str, url: str, *args: Any, **kwargs: Any + ) -> requests.Response: + # Respond to the first attempt to upload each entry of arr_0 with a + # timeout, below the retry logic of RESTFullAPIClient.request(), so + # that it is retried there + if method == "PUT" and "dandi-dandisets" in url: + urlpath = urlparse(url).path + put_attempts[urlpath] += 1 + if "arr_0" in urlpath and put_attempts[urlpath] == 1: + resp = requests.Response() + resp.status_code = 400 + resp.reason = "Bad Request" + resp.url = url + resp._content = ( + b'\n' + b"RequestTimeout" + b"Your socket connection to the server" + b" was not read from or written to within the timeout period." + b" Idle connections will be closed." + b"1111111111111111" + b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + b"" + ) + return resp + return original_request(self, method, url, *args, **kwargs) + + monkeypatch.setattr(requests.Session, "request", mock_request) + events = _spy_zarr_iter_upload(mocker) - # Upload the Zarr new_dandiset.upload() - # Verify the upload succeeded (asset,) = new_dandiset.dandiset.get_assets() assert isinstance(asset, RemoteZarrAsset) assert asset.asset_type is AssetType.ZARR assert asset.path == "test.zarr" - # Verify that arr_0 files were retried automatically by tenacity - # Filter for PUT requests to S3/minio (not API endpoints) - arr_0_urls = [ - url for url in request_attempts if "arr_0" in url and "dandi-dandisets" in url - ] - assert len(arr_0_urls) > 0, "Expected to find arr_0 files" - - for url in arr_0_urls: - # Each arr_0 file should have been attempted twice (initial + 1 retry) + arr_0_paths = [p for p in put_attempts if "arr_0" in p] + assert arr_0_paths, "Expected to find arr_0 files" + for p in put_attempts: + expected = 2 if p in arr_0_paths else 1 assert ( - request_attempts[url] == 2 - ), f"Expected {url} to be retried once, got {request_attempts[url]} attempts" - - # Verify non-arr_0 S3/minio URLs were not retried - non_arr_0_urls = [ - url - for url in request_attempts - if "arr_0" not in url and "dandi-dandisets" in url and request_attempts[url] > 0 - ] - for url in non_arr_0_urls: - assert ( - request_attempts[url] == 1 - ), f"URL {url} should not have been retried but had {request_attempts[url]} attempts" + put_attempts[p] == expected + ), f"Expected {expected} attempt(s) to upload {p}, got {put_attempts[p]}" + # The second attempts were the retries, not uploads in a second pass after + # the Zarr's checksum did not match, as would happen if the timed-out + # entries had not been retried + assert not any(e["status"] == "Checksum mismatch" for e in events) + # Nor did the server defer ingesting the Zarr for upload URLs left unused + # (dandi-archive#3018), as it would if they had not been retried + assert events[-1] == {"status": "done", "asset": anys.AnyInstance(RemoteZarrAsset)} @pytest.mark.ai_generated From ed676469ef7b02a45cff05621d1f79000887e971 Mon Sep 17 00:00:00 2001 From: naglepuff Date: Wed, 18 Nov 2026 13:23:42 -0500 Subject: [PATCH 5/7] Verify a Zarr whose ingestion is deferred against its remote file listing When the server defers ingesting a Zarr (dandi-archive#3018), it has no checksum yet, so the upload skipped verifying the remote Zarr against the local one. An entry that the client believed it uploaded but which is missing remotely then went unnoticed: once its unused upload URL expires, the server ingests the incomplete Zarr as "Complete". Instead, compute the remote checksum from the listing of the Zarr's files, which comes from storage directly and is what the server computes its checksum from, and redo the upload on a mismatch as before. Also reset the deferral when there is nothing to upload and the Zarr is "Complete", so that a deferral left over from an earlier pass is not reported. Test that a lost entry is uploaded again, sharing the faking of entry uploads below the retry logic with test_zarr_upload_400_timeout_retry. Co-Authored-By: Claude Opus 5.5 --- dandi/files/zarr.py | 50 ++++++++++----- dandi/tests/test_upload.py | 126 ++++++++++++++++++++++++++++--------- 2 files changed, 133 insertions(+), 43 deletions(-) diff --git a/dandi/files/zarr.py b/dandi/files/zarr.py index 43d421e8c..468eb06f7 100644 --- a/dandi/files/zarr.py +++ b/dandi/files/zarr.py @@ -591,7 +591,8 @@ def iter_upload( ``"skipped"`` with a ``"message"`` of ``"identical"`` and the same ``"asset"`` key. If the server deferred ingesting the Zarr (see `RemoteZarrAsset.wait_for_ingestion()`), the upload does not wait - for it, the local checksum is not verified, and the terminal + for it, the local checksum is verified against one computed from + the remote file listing instead of the server's, and the terminal ``"message"`` says until when ingestion is deferred. """ asset_path = metadata.setdefault("path", self.path) @@ -673,6 +674,11 @@ def ingest(finalize: bool) -> Generator[dict, None, bool]: Wait for the server to ingest the Zarr, after finalizing it if ``finalize`` is true, and verify its checksum against the local one. Returns whether they mismatch. + + If the server defers ingesting the Zarr, it is not waited for, and + the local checksum is verified against one computed from the + remote file listing instead, which is what the server computes its + checksum from. """ nonlocal deferred_until lgr.debug("%s: Waiting for server to calculate Zarr checksum", asset_path) @@ -682,14 +688,11 @@ def ingest(finalize: bool) -> Generator[dict, None, bool]: r = a.wait_for_ingestion() if deferred_until := zarr_ingestion_deferred_until(r): lgr.info( - "%s: Server deferred ingestion until %s because upload" - " URLs for entries that have since been changed or deleted" - " have not expired yet; skipping local checksum" - " verification", + "%s: Server deferred ingestion until %s because some of the" + " Zarr's upload URLs could still change it", asset_path, deferred_until, ) - return False if zarr_mode == "patch": lgr.info( "%s: Skipping local checksum verification (patch mode)", @@ -697,15 +700,23 @@ def ingest(finalize: bool) -> Generator[dict, None, bool]: ) return False our_checksum = str(zcc.process()) - server_checksum = r["checksum"] - if our_checksum == server_checksum: + if deferred_until is not None: + lgr.debug( + "%s: Computing Zarr checksum from remote file listing", asset_path + ) + remote_checksum = _remote_zarr_checksum(a) + source = "remote file listing" + else: + remote_checksum = r["checksum"] + source = "server" + if our_checksum == remote_checksum: return False lgr.info( - "%s: Asset checksum mismatch (local: %s; server: %s);" - " redoing upload", + "%s: Asset checksum mismatch (local: %s; %s: %s); redoing upload", asset_path, our_checksum, - server_checksum, + source, + remote_checksum, ) yield {"status": "Checksum mismatch"} return True @@ -801,9 +812,8 @@ def ingest(finalize: bool) -> Generator[dict, None, bool]: if n_add == 0 and n_modify == 0 and n_delete == 0: zarr_info = client.get(f"/zarr/{zarr_id}/") status = zarr_info["status"] - if status == "Complete" or ( - deferred_until := zarr_ingestion_deferred_until(zarr_info) - ): + deferred_until = zarr_ingestion_deferred_until(zarr_info) + if status == "Complete" or deferred_until is not None: mismatched = False unchanged = True break @@ -1022,6 +1032,18 @@ def ingest(finalize: bool) -> Generator[dict, None, bool]: yield done +def _remote_zarr_checksum(asset: RemoteZarrAsset) -> str: + """ + Compute the DANDI Zarr checksum of a remote Zarr from the listing of its + files, which, unlike the checksum the server reports, does not have to + wait for the server to ingest the Zarr + """ + tree = ZarrChecksumTree() + for entry in asset.iterfiles(): + tree.add_leaf(Path(str(entry)), entry.size, entry.digest.value) + return str(tree.process()) + + def _handle_failed_items_and_raise( executor: ThreadPoolExecutor, failed_items: list, futures: list ) -> None: diff --git a/dandi/tests/test_upload.py b/dandi/tests/test_upload.py index e9649ce50..acd2b6afc 100644 --- a/dandi/tests/test_upload.py +++ b/dandi/tests/test_upload.py @@ -2,6 +2,7 @@ from base64 import b64encode from collections import defaultdict +from collections.abc import Callable from datetime import datetime, timezone import os from pathlib import Path @@ -679,6 +680,35 @@ def mock_put(self, url, **kwargs): assert "arr_1" not in url, f"URL {url} should have been retried but wasn't" +def _fake_zarr_entry_uploads( + monkeypatch: pytest.MonkeyPatch, + fake: Callable[[str], tuple[int, bytes] | None], +) -> None: + """ + Make uploads of Zarr entries to storage get the status code and content + ``fake`` returns for their URL, unless it returns `None`. The responses + are faked by `requests.Session.request`, below the retry logic of + `RESTFullAPIClient.request()`. + """ + original_request = requests.Session.request + + def mock_request( + self: requests.Session, method: str, url: str, *args: Any, **kwargs: Any + ) -> requests.Response: + if ( + method == "PUT" + and "dandi-dandisets" in url + and (faked := fake(url)) is not None + ): + resp = requests.Response() + resp.status_code, resp._content = faked + resp.url = url + return resp + return original_request(self, method, url, *args, **kwargs) + + monkeypatch.setattr(requests.Session, "request", mock_request) + + @pytest.mark.ai_generated def test_zarr_upload_400_timeout_retry( mocker: MockerFixture, @@ -689,41 +719,31 @@ def test_zarr_upload_400_timeout_retry( A 400 RequestTimeout response to uploading a Zarr entry is retried by `RESTFullAPIClient.request()` (https://github.com/dandi/dandi-cli/issues/1662) """ - zarr_path = new_dandiset.dspath / "test.zarr" - zarr.save(zarr_path, np.arange(100), np.arange(100, 0, -1)) + zarr.save(new_dandiset.dspath / "test.zarr", np.arange(100), np.arange(100, 0, -1)) # Attempts to upload each Zarr entry, by the path of its URL put_attempts: defaultdict[str, int] = defaultdict(int) - original_request = requests.Session.request - def mock_request( - self: requests.Session, method: str, url: str, *args: Any, **kwargs: Any - ) -> requests.Response: + def fake(url: str) -> tuple[int, bytes] | None: # Respond to the first attempt to upload each entry of arr_0 with a - # timeout, below the retry logic of RESTFullAPIClient.request(), so - # that it is retried there - if method == "PUT" and "dandi-dandisets" in url: - urlpath = urlparse(url).path - put_attempts[urlpath] += 1 - if "arr_0" in urlpath and put_attempts[urlpath] == 1: - resp = requests.Response() - resp.status_code = 400 - resp.reason = "Bad Request" - resp.url = url - resp._content = ( - b'\n' - b"RequestTimeout" - b"Your socket connection to the server" - b" was not read from or written to within the timeout period." - b" Idle connections will be closed." - b"1111111111111111" - b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" - b"" - ) - return resp - return original_request(self, method, url, *args, **kwargs) + # timeout + urlpath = urlparse(url).path + put_attempts[urlpath] += 1 + if "arr_0" in urlpath and put_attempts[urlpath] == 1: + return ( + 400, + b'\n' + b"RequestTimeout" + b"Your socket connection to the server" + b" was not read from or written to within the timeout period." + b" Idle connections will be closed." + b"1111111111111111" + b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + b"", + ) + return None - monkeypatch.setattr(requests.Session, "request", mock_request) + _fake_zarr_entry_uploads(monkeypatch, fake) events = _spy_zarr_iter_upload(mocker) new_dandiset.upload() @@ -749,6 +769,54 @@ def mock_request( assert events[-1] == {"status": "done", "asset": anys.AnyInstance(RemoteZarrAsset)} +@pytest.mark.ai_generated +def test_upload_zarr_lost_entry( + caplog: pytest.LogCaptureFixture, + mocker: MockerFixture, + new_dandiset: SampleDandiset, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """ + An entry which the client believes it uploaded, but which is missing from + the remote Zarr, is detected by the Zarr's checksum and uploaded again. + + Against a server which defers ingesting a Zarr with unused upload URLs + (dandi-archive#3018), as it does here for the lost entry's, the checksum + is computed from the remote file listing rather than by the server. + """ + zarr_path = new_dandiset.dspath / "test.zarr" + zarr.save(zarr_path, np.arange(100), np.arange(100, 0, -1)) + lost: list[str] = [] + + def fake(url: str) -> tuple[int, bytes] | None: + # Report success for the first upload of an entry of arr_0 without + # uploading it + if "arr_0" in (urlpath := urlparse(url).path) and not lost: + lost.append(urlpath) + return (200, b"") + return None + + _fake_zarr_entry_uploads(monkeypatch, fake) + events = _spy_zarr_iter_upload(mocker) + + new_dandiset.upload() + + assert lost + assert [e["status"] for e in events].count("Checksum mismatch") == 1 + asset = new_dandiset.dandiset.get_asset_by_path("test.zarr") + assert isinstance(asset, RemoteZarrAsset) + zarr_info = new_dandiset.client.get(f"/zarr/{asset.zarr}/") + if "ingestion_deferred_until" in zarr_info: + assert "remote file listing" in caplog.text + # Once the lost entry is uploaded again, none of the Zarr's upload URLs can + # change it, so the server does not defer ingesting it + assert events[-1] == {"status": "done", "asset": anys.AnyInstance(RemoteZarrAsset)} + assert zarr_info["status"] == "Complete" + download(new_dandiset.dandiset.version_api_url, tmp_path) + assert_dirtrees_eq(zarr_path, tmp_path / new_dandiset.dandiset_id / "test.zarr") + + @pytest.mark.ai_generated def test_zarr_upload_403_batch_retry_reduces_parallelism( new_dandiset: SampleDandiset, From 5f80d962371c755513f1f8957f19a95afdd2ed34 Mon Sep 17 00:00:00 2001 From: naglepuff Date: Wed, 18 Nov 2026 13:36:07 -0500 Subject: [PATCH 6/7] Log at DEBUG when the server does not report deferred Zarr ingestion As DEVELOPMENT.md asks for the use of a fallback for servers without a dandi-archive feature, here dandi-archive#3018. It is logged once per wait for ingestion, when the Zarr is ingested, rather than on every poll. Co-Authored-By: Claude Opus 5.5 --- dandi/dandiapi.py | 7 +++++++ dandi/tests/test_dandiapi.py | 10 ++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index e33f126e2..06e2e545c 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -2141,6 +2141,13 @@ def wait_for_ingestion(self) -> dict[str, Any]: r: dict[str, Any] = self.client.get(f"/zarr/{self.zarr}/") status = r["status"] if status == "Complete": + if "ingestion_deferred_until" not in r: + # See zarr_ingestion_deferred_until() for when this can be + # removed + lgr.debug( + "Server does not report deferred Zarr ingestion" + " (dandi-archive#3018); assuming it never defers it" + ) return r if deferred_until := zarr_ingestion_deferred_until(r): lgr.debug( diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 6ce44e3ab..cedc1c341 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -1214,7 +1214,9 @@ def _mock_zarr_states(*states: dict[str, Any]) -> None: @pytest.mark.ai_generated @responses.activate -def test_wait_for_ingestion_without_deferral_support(mocker: MockerFixture) -> None: +def test_wait_for_ingestion_without_deferral_support( + caplog: pytest.LogCaptureFixture, mocker: MockerFixture +) -> None: """A server without dandi-archive#3018 never defers; wait for "Complete".""" mocker.patch.object(dandiapi, "sleep") _mock_zarr_states( @@ -1226,11 +1228,14 @@ def test_wait_for_ingestion_without_deferral_support(mocker: MockerFixture) -> N assert r["status"] == "Complete" assert r["checksum"] == "abc-1--1" assert _zarr_polls() == 3 + assert "Server does not report deferred Zarr ingestion" in caplog.text @pytest.mark.ai_generated @responses.activate -def test_wait_for_ingestion_complete(mocker: MockerFixture) -> None: +def test_wait_for_ingestion_complete( + caplog: pytest.LogCaptureFixture, mocker: MockerFixture +) -> None: mocker.patch.object(dandiapi, "sleep") _mock_zarr_states( {"status": "Uploaded", "ingestion_deferred_until": None}, @@ -1244,6 +1249,7 @@ def test_wait_for_ingestion_complete(mocker: MockerFixture) -> None: r = _zarr_asset_for_mocked_server().wait_for_ingestion() assert r["status"] == "Complete" assert _zarr_polls() == 3 + assert "Server does not report deferred Zarr ingestion" not in caplog.text @pytest.mark.ai_generated From 41adc217ff22f4346145484de3bd0dd492c6afcc Mon Sep 17 00:00:00 2001 From: naglepuff Date: Wed, 18 Nov 2026 14:30:14 -0500 Subject: [PATCH 7/7] Cover deferred Zarr ingestion in CI by faking the deferral CI runs the integration tests against a dandi-archive image built from master, which does not have dandi-archive#3018 yet, so the code handling a deferred ingestion never ran there, and codecov/patch failed. These tests mock the behavior of dandi-archive#3018 to bump coverage until CI's image includes it: - test_upload_zarr_deferred_ingestion_faked makes RemoteZarrAsset.wait_for_ingestion() report a deferral, whatever the server did, so that the upload verifies the Zarr against its remote file listing (from the real server) and reports the deferral. - test_rmfiles_reingest_deferred mocks the server's responses to check that rmfiles(reingest=True) logs a deferral rather than waiting. test_upload_zarr_deferred_ingestion and test_upload_zarr_lost_entry still exercise a real deferral against servers with dandi-archive#3018. Co-Authored-By: Claude Opus 5.5 --- dandi/tests/test_dandiapi.py | 32 ++++++++++++++++++++++++++++++ dandi/tests/test_upload.py | 38 ++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index cedc1c341..afa659837 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -43,6 +43,7 @@ RemoteBlobAsset, RemoteDandiset, RemoteZarrAsset, + RemoteZarrEntry, Version, VersionStatus, ) @@ -50,6 +51,7 @@ from ..exceptions import NotFoundError, SchemaVersionError, ZarrIngestionError from ..files import GenericAsset, dandi_file from ..files.bases import _SCHEMA_BAREASSET_HAS_DATASTANDARD +from ..misctypes import Digest from ..utils import list_paths # These tests exercise the 0.8.0 -> 0.7.0 downgrade wiring, which requires @@ -1301,3 +1303,33 @@ def test_wait_for_ingestion_modified_meanwhile(mocker: MockerFixture) -> None: ) with pytest.raises(ZarrIngestionError, match="zarr-id was modified"): _zarr_asset_for_mocked_server().wait_for_ingestion() + + +@pytest.mark.ai_generated +@responses.activate +def test_rmfiles_reingest_deferred( + caplog: pytest.LogCaptureFixture, mocker: MockerFixture +) -> None: + """``rmfiles(reingest=True)`` does not wait for a deferred ingestion.""" + mocker.patch.object(dandiapi, "sleep") + asset = _zarr_asset_for_mocked_server() + responses.add(responses.DELETE, f"{_MOCK_ZARR_URL}files/", status=204) + responses.add(responses.POST, f"{_MOCK_ZARR_URL}finalize/", status=204) + deferred_until = "2026-11-18T13:00:00Z" + _mock_zarr_states( + {"status": "Uploaded", "ingestion_deferred_until": deferred_until}, + ) + entry = RemoteZarrEntry( + client=asset.client, + zarr_id=asset.zarr, + parts=("arr_0", "c", "0"), + modified=datetime.now(timezone.utc), + digest=Digest(algorithm=DigestType.md5, value="0" * 32), + size=1, + ) + asset.rmfiles([entry]) + assert _zarr_polls() == 1 + assert ( + f"Server deferred ingestion of Zarr zarr-id until {deferred_until}" + in caplog.text + ) diff --git a/dandi/tests/test_upload.py b/dandi/tests/test_upload.py index acd2b6afc..437f247b7 100644 --- a/dandi/tests/test_upload.py +++ b/dandi/tests/test_upload.py @@ -36,6 +36,7 @@ from ..download import download from ..exceptions import NotFoundError, UploadError from ..files import LocalFileAsset +from ..files import zarr as zarr_files from ..files.zarr import ZarrAsset from ..pynwb_utils import make_nwb_file from ..upload import UploadExisting, UploadValidation @@ -817,6 +818,43 @@ def fake(url: str) -> tuple[int, bytes] | None: assert_dirtrees_eq(zarr_path, tmp_path / new_dandiset.dandiset_id / "test.zarr") +@pytest.mark.ai_generated +def test_upload_zarr_deferred_ingestion_faked( + mocker: MockerFixture, new_dandiset: SampleDandiset +) -> None: + """ + When the server defers ingesting a Zarr, the upload does not wait for it, + verifies the Zarr against its remote file listing instead, and reports the + deferral. The deferral is faked, so that this also runs against servers + without dandi-archive#3018 (cf. `test_upload_zarr_deferred_ingestion`). + """ + zarr.save(new_dandiset.dspath / "test.zarr", np.arange(100), np.arange(100, 0, -1)) + deferred_until = "2026-11-18T13:00:00Z" + original_wait = RemoteZarrAsset.wait_for_ingestion + + def wait_for_ingestion(self: RemoteZarrAsset) -> dict[str, Any]: + return { + **original_wait(self), + "status": "Uploaded", + "checksum": None, + "ingestion_deferred_until": deferred_until, + } + + mocker.patch.object(RemoteZarrAsset, "wait_for_ingestion", wait_for_ingestion) + remote_checksum = mocker.spy(zarr_files, "_remote_zarr_checksum") + events = _spy_zarr_iter_upload(mocker) + + new_dandiset.upload() + + assert remote_checksum.call_count == 1 + assert not any(e["status"] == "Checksum mismatch" for e in events) + assert events[-1] == { + "status": "done", + "message": f"server ingestion deferred until {deferred_until}", + "asset": anys.AnyInstance(RemoteZarrAsset), + } + + @pytest.mark.ai_generated def test_zarr_upload_403_batch_retry_reduces_parallelism( new_dandiset: SampleDandiset,