diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py
index 9ea22e025..06e2e545c 100644
--- a/dandi/dandiapi.py
+++ b/dandi/dandiapi.py
@@ -62,7 +62,12 @@
DandiInstance,
EmbargoStatus,
)
-from .exceptions import HTTP404Error, NotFoundError, SchemaVersionError
+from .exceptions import (
+ HTTP404Error,
+ NotFoundError,
+ SchemaVersionError,
+ ZarrIngestionError,
+)
from .keyring_utils import keyring_lookup, keyring_save
from .misctypes import (
Digest,
@@ -126,6 +131,29 @@ def set_asset_schema_key(metadata: dict[str, Any]) -> None:
metadata["schemaKey"] = "Asset"
+def zarr_ingestion_deferred_until(zarr_data: dict[str, Any]) -> str | None:
+ """
+ Given a Zarr's data from the server (as returned by ``GET
+ /zarr/{zarr_id}/``), return the time until which the server deferred its
+ ingestion, or `None` if it did not.
+
+ dandi-archive#3018 defers the ingestion of a Zarr while any of its
+ unexpired upload URLs could still change it, leaving it ``"Uploaded"`` with
+ the time of the next attempt in ``"ingestion_deferred_until"``. The field
+ stays set while that attempt runs (``"Ingesting"``), and is cleared once
+ the Zarr is ingested or modified.
+ """
+ # Servers without dandi-archive#3018 lack the field and never defer
+ # ingestion; .get() can be replaced by indexing once all known instances
+ # deploy that feature.
+ if zarr_data["status"] == "Uploaded" and (
+ deferred_until := zarr_data.get("ingestion_deferred_until")
+ ):
+ assert isinstance(deferred_until, str)
+ return deferred_until
+ return None
+
+
class AssetType(Enum):
"""
.. versionadded:: 0.36.0
@@ -2073,7 +2101,7 @@ def rmfiles(self, files: Iterable[RemoteZarrEntry], reingest: bool = True) -> No
If ``reingest`` is true, after performing the deletion, the client
triggers a recalculation of the Zarr's checksum and waits for it to
- complete.
+ complete, unless the server defers it (see `wait_for_ingestion()`).
"""
# Don't bother checking that the entries are actually files or even
# belong to this Zarr, as if they're not, the server will return an
@@ -2085,11 +2113,54 @@ def rmfiles(self, files: Iterable[RemoteZarrEntry], reingest: bool = True) -> No
)
if reingest:
self.client.post(f"/zarr/{self.zarr}/finalize/")
- while True:
- sleep(2)
- r = self.client.get(f"/zarr/{self.zarr}/")
- if r["status"] == "Complete":
- break
+ r = self.wait_for_ingestion()
+ if deferred_until := zarr_ingestion_deferred_until(r):
+ lgr.info(
+ "Server deferred ingestion of Zarr %s until %s",
+ self.zarr,
+ deferred_until,
+ )
+
+ def wait_for_ingestion(self) -> dict[str, Any]:
+ """
+ Wait for the server to finish ingesting the Zarr after it has been
+ finalized, or to defer its ingestion.
+
+ Returns the Zarr's data from the server (as returned by
+ ``GET /zarr/{zarr_id}/``) once its ``"status"`` is either
+ ``"Complete"``, in which case its ``"checksum"`` is set, or
+ ``"Uploaded"`` with ``"ingestion_deferred_until"`` set. In the latter
+ case, the server ingests the Zarr on its own at the given time, so
+ there is no point in waiting for it.
+
+ Raises `ZarrIngestionError` if the Zarr becomes ``"Pending"`` again,
+ i.e., something else modified it in the meantime.
+ """
+ while True:
+ sleep(2)
+ r: dict[str, Any] = self.client.get(f"/zarr/{self.zarr}/")
+ status = r["status"]
+ if status == "Complete":
+ if "ingestion_deferred_until" not in r:
+ # See zarr_ingestion_deferred_until() for when this can be
+ # removed
+ lgr.debug(
+ "Server does not report deferred Zarr ingestion"
+ " (dandi-archive#3018); assuming it never defers it"
+ )
+ return r
+ if deferred_until := zarr_ingestion_deferred_until(r):
+ lgr.debug(
+ "Server deferred ingestion of Zarr %s until %s",
+ self.zarr,
+ deferred_until,
+ )
+ return r
+ if status == "Pending":
+ raise ZarrIngestionError(
+ f"Zarr {self.zarr} was modified while waiting for the"
+ " server to ingest it"
+ )
class RemoteAsset(BaseRemoteAsset):
diff --git a/dandi/exceptions.py b/dandi/exceptions.py
index f323d2a42..67c1f85ce 100644
--- a/dandi/exceptions.py
+++ b/dandi/exceptions.py
@@ -97,3 +97,9 @@ class UploadValidationError(UploadError):
"""An upload could not proceed because an asset failed validation."""
pass
+
+
+class ZarrIngestionError(RuntimeError):
+ """A finalized Zarr did not reach a state in which its ingestion is done or deferred."""
+
+ pass
diff --git a/dandi/files/zarr.py b/dandi/files/zarr.py
index 696522b55..468eb06f7 100644
--- a/dandi/files/zarr.py
+++ b/dandi/files/zarr.py
@@ -42,6 +42,7 @@
RemoteZarrEntry,
RESTFullAPIClient,
set_asset_schema_key,
+ zarr_ingestion_deferred_until,
)
from dandi.exceptions import UploadError
from dandi.metadata.core import get_default_metadata
@@ -588,7 +589,11 @@ def iter_upload(
`RemoteAsset`. If the local Zarr is bit-identical to the remote
(nothing to upload or delete), the terminal status is instead
``"skipped"`` with a ``"message"`` of ``"identical"`` and the same
- ``"asset"`` key.
+ ``"asset"`` key. If the server deferred ingesting the Zarr (see
+ `RemoteZarrAsset.wait_for_ingestion()`), the upload does not wait
+ for it, the local checksum is verified against one computed from
+ the remote file listing instead of the server's, and the terminal
+ ``"message"`` says until when ingestion is deferred.
"""
asset_path = metadata.setdefault("path", self.path)
set_asset_schema_key(metadata)
@@ -661,6 +666,61 @@ def mkzarr() -> str:
mismatched = True
first_run = True
unchanged = False
+ # Until when the server deferred ingesting the Zarr, if it did
+ deferred_until: str | None = None
+
+ def ingest(finalize: bool) -> Generator[dict, None, bool]:
+ """
+ Wait for the server to ingest the Zarr, after finalizing it if
+ ``finalize`` is true, and verify its checksum against the local
+ one. Returns whether they mismatch.
+
+ If the server defers ingesting the Zarr, it is not waited for, and
+ the local checksum is verified against one computed from the
+ remote file listing instead, which is what the server computes its
+ checksum from.
+ """
+ nonlocal deferred_until
+ lgr.debug("%s: Waiting for server to calculate Zarr checksum", asset_path)
+ yield {"status": "server calculating checksum"}
+ if finalize:
+ client.post(f"/zarr/{zarr_id}/finalize/")
+ r = a.wait_for_ingestion()
+ if deferred_until := zarr_ingestion_deferred_until(r):
+ lgr.info(
+ "%s: Server deferred ingestion until %s because some of the"
+ " Zarr's upload URLs could still change it",
+ asset_path,
+ deferred_until,
+ )
+ if zarr_mode == "patch":
+ lgr.info(
+ "%s: Skipping local checksum verification (patch mode)",
+ asset_path,
+ )
+ return False
+ our_checksum = str(zcc.process())
+ if deferred_until is not None:
+ lgr.debug(
+ "%s: Computing Zarr checksum from remote file listing", asset_path
+ )
+ remote_checksum = _remote_zarr_checksum(a)
+ source = "remote file listing"
+ else:
+ remote_checksum = r["checksum"]
+ source = "server"
+ if our_checksum == remote_checksum:
+ return False
+ lgr.info(
+ "%s: Asset checksum mismatch (local: %s; %s: %s); redoing upload",
+ asset_path,
+ our_checksum,
+ source,
+ remote_checksum,
+ )
+ yield {"status": "Checksum mismatch"}
+ return True
+
while mismatched:
zcc = ZarrChecksumTree()
old_zarr_entries: dict[str, RemoteZarrEntry] = {
@@ -750,9 +810,26 @@ def mkzarr() -> str:
len(old_zarr_entries) if zarr_mode == "full" else 0
)
if n_add == 0 and n_modify == 0 and n_delete == 0:
- mismatched = False
- unchanged = True
- break
+ zarr_info = client.get(f"/zarr/{zarr_id}/")
+ status = zarr_info["status"]
+ deferred_until = zarr_ingestion_deferred_until(zarr_info)
+ if status == "Complete" or deferred_until is not None:
+ mismatched = False
+ unchanged = True
+ break
+ # The remote entries match, but an earlier upload ended
+ # before finalizing the Zarr ("Pending") or before the
+ # server ingested it ("Uploaded" or "Ingesting")
+ lgr.info(
+ "%s: Remote Zarr matches local but has not been"
+ " ingested (status: %s); %s",
+ asset_path,
+ status,
+ "finalizing" if status == "Pending" else "waiting for it",
+ )
+ mismatched = yield from ingest(finalize=status == "Pending")
+ first_run = False
+ continue
bytes_add = sum(e.size for e in to_upload.fresh_entries)
bytes_modify = sum(it.size for it in to_upload.digested_entries)
clauses = []
@@ -919,38 +996,7 @@ def mkzarr() -> str:
"file" if n_remote_only == 1 else "files",
)
if changed:
- lgr.debug(
- "%s: Waiting for server to calculate Zarr checksum", asset_path
- )
- yield {"status": "server calculating checksum"}
- client.post(f"/zarr/{zarr_id}/finalize/")
- while True:
- sleep(2)
- r = client.get(f"/zarr/{zarr_id}/")
- if r["status"] == "Complete":
- if zarr_mode == "patch":
- lgr.info(
- "%s: Skipping local checksum verification"
- " (patch mode)",
- asset_path,
- )
- mismatched = False
- else:
- our_checksum = str(zcc.process())
- server_checksum = r["checksum"]
- if our_checksum == server_checksum:
- mismatched = False
- else:
- mismatched = True
- lgr.info(
- "%s: Asset checksum mismatch (local: %s;"
- " server: %s); redoing upload",
- asset_path,
- our_checksum,
- server_checksum,
- )
- yield {"status": "Checksum mismatch"}
- break
+ mismatched = yield from ingest(finalize=True)
elif mismatched and not first_run:
lgr.error(
"%s: Previous upload loop resulted in checksum mismatch,"
@@ -969,12 +1015,33 @@ def mkzarr() -> str:
unchanged,
first_run,
)
+ deferral = (
+ f"server ingestion deferred until {deferred_until}"
+ if deferred_until is not None
+ else None
+ )
if unchanged:
lgr.info("%s: Zarr already matches remote; nothing to upload", asset_path)
- yield {"status": "skipped", "message": "identical", "asset": a}
+ message = "identical" if deferral is None else f"identical; {deferral}"
+ yield {"status": "skipped", "message": message, "asset": a}
else:
lgr.info("%s: Asset successfully uploaded", asset_path)
- yield {"status": "done", "asset": a}
+ done: dict[str, Any] = {"status": "done", "asset": a}
+ if deferral is not None:
+ done["message"] = deferral
+ yield done
+
+
+def _remote_zarr_checksum(asset: RemoteZarrAsset) -> str:
+ """
+ Compute the DANDI Zarr checksum of a remote Zarr from the listing of its
+ files, which, unlike the checksum the server reports, does not have to
+ wait for the server to ingest the Zarr
+ """
+ tree = ZarrChecksumTree()
+ for entry in asset.iterfiles():
+ tree.add_leaf(Path(str(entry)), entry.size, entry.digest.value)
+ return str(tree.process())
def _handle_failed_items_and_raise(
diff --git a/dandi/tests/data/dandiarchive-docker/docker-compose.yml b/dandi/tests/data/dandiarchive-docker/docker-compose.yml
index 95167115f..743135b9e 100644
--- a/dandi/tests/data/dandiarchive-docker/docker-compose.yml
+++ b/dandi/tests/data/dandiarchive-docker/docker-compose.yml
@@ -39,6 +39,15 @@ services:
DJANGO_DANDI_INSTANCE_NAME: ${DANDI_TESTS_INSTANCE_NAME:-DANDI}
DJANGO_DANDI_INSTANCE_IDENTIFIER: ${DANDI_TESTS_INSTANCE_IDENTIFIER:-RRID:ABC_123456}
DJANGO_DANDI_DOI_API_PREFIX: ${DANDI_TESTS_DOI_PREFIX:-10.80507}
+ # dandi-archive defers ingestion of a Zarr until every upload URL that
+ # could still change it has expired, and retries deferred ingestion
+ # periodically (https://github.com/dandi/dandi-archive/pull/3018).
+ # Shorten the expiry from the default hour, and the retry interval from
+ # the default 5 minutes, so that tests which overwrite or delete recently
+ # uploaded Zarr entries do not wait that long. Ignored by servers
+ # without that feature.
+ DJANGO_DANDI_ZARR_UPLOAD_URL_EXPIRATION: "30"
+ DJANGO_DANDI_DEFERRED_INGESTION_RETRY_INTERVAL: "5"
ports:
- "127.0.0.1:8000:8000"
diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py
index ac6534ba3..afa659837 100644
--- a/dandi/tests/test_dandiapi.py
+++ b/dandi/tests/test_dandiapi.py
@@ -37,18 +37,21 @@
dandiset_metadata_file,
)
from ..dandiapi import (
+ BaseRemoteZarrAsset,
DandiAPIClient,
RemoteAsset,
RemoteBlobAsset,
RemoteDandiset,
RemoteZarrAsset,
+ RemoteZarrEntry,
Version,
VersionStatus,
)
from ..download import download
-from ..exceptions import NotFoundError, SchemaVersionError
+from ..exceptions import NotFoundError, SchemaVersionError, ZarrIngestionError
from ..files import GenericAsset, dandi_file
from ..files.bases import _SCHEMA_BAREASSET_HAS_DATASTANDARD
+from ..misctypes import Digest
from ..utils import list_paths
# These tests exercise the 0.8.0 -> 0.7.0 downgrade wiring, which requires
@@ -1177,3 +1180,156 @@ def test_all_public_api() -> None:
assert exported == names
assert {"DandiAPIClient", "RemoteDandiset", "RemoteAsset"} <= names
assert not {"APIBase", "RemoteDandisetData", "ZarrEntryServerData"} & names
+
+
+def _zarr_asset_for_mocked_server() -> BaseRemoteZarrAsset:
+ _mock_server_info(dandischema.consts.DANDI_SCHEMA_VERSION)
+ now = datetime.now(timezone.utc)
+ return BaseRemoteZarrAsset.model_validate(
+ {
+ "client": DandiAPIClient("https://test.nil/api"),
+ "asset_id": "abc",
+ "path": "sample.zarr",
+ "size": 0,
+ "created": now,
+ "modified": now,
+ "zarr": "zarr-id",
+ }
+ )
+
+
+_MOCK_ZARR_URL = "https://test.nil/api/zarr/zarr-id/"
+
+
+def _zarr_polls() -> int:
+ return sum(c.request.url == _MOCK_ZARR_URL for c in responses.calls)
+
+
+def _mock_zarr_states(*states: dict[str, Any]) -> None:
+ for state in states:
+ responses.add(
+ responses.GET,
+ _MOCK_ZARR_URL,
+ json={"zarr_id": "zarr-id", "checksum": None, **state},
+ )
+
+
+@pytest.mark.ai_generated
+@responses.activate
+def test_wait_for_ingestion_without_deferral_support(
+ caplog: pytest.LogCaptureFixture, mocker: MockerFixture
+) -> None:
+ """A server without dandi-archive#3018 never defers; wait for "Complete"."""
+ mocker.patch.object(dandiapi, "sleep")
+ _mock_zarr_states(
+ {"status": "Uploaded"},
+ {"status": "Ingesting"},
+ {"status": "Complete", "checksum": "abc-1--1"},
+ )
+ r = _zarr_asset_for_mocked_server().wait_for_ingestion()
+ assert r["status"] == "Complete"
+ assert r["checksum"] == "abc-1--1"
+ assert _zarr_polls() == 3
+ assert "Server does not report deferred Zarr ingestion" in caplog.text
+
+
+@pytest.mark.ai_generated
+@responses.activate
+def test_wait_for_ingestion_complete(
+ caplog: pytest.LogCaptureFixture, mocker: MockerFixture
+) -> None:
+ mocker.patch.object(dandiapi, "sleep")
+ _mock_zarr_states(
+ {"status": "Uploaded", "ingestion_deferred_until": None},
+ {"status": "Ingesting", "ingestion_deferred_until": None},
+ {
+ "status": "Complete",
+ "checksum": "abc-1--1",
+ "ingestion_deferred_until": None,
+ },
+ )
+ r = _zarr_asset_for_mocked_server().wait_for_ingestion()
+ assert r["status"] == "Complete"
+ assert _zarr_polls() == 3
+ assert "Server does not report deferred Zarr ingestion" not in caplog.text
+
+
+@pytest.mark.ai_generated
+@responses.activate
+def test_wait_for_ingestion_deferred(mocker: MockerFixture) -> None:
+ """A deferred ingestion is not waited for."""
+ mocker.patch.object(dandiapi, "sleep")
+ deferred_until = "2026-11-17T13:00:00Z"
+ _mock_zarr_states(
+ {"status": "Uploaded", "ingestion_deferred_until": None},
+ {"status": "Ingesting", "ingestion_deferred_until": None},
+ {"status": "Uploaded", "ingestion_deferred_until": deferred_until},
+ )
+ r = _zarr_asset_for_mocked_server().wait_for_ingestion()
+ assert r["status"] == "Uploaded"
+ assert r["ingestion_deferred_until"] == deferred_until
+ assert _zarr_polls() == 3
+
+
+@pytest.mark.ai_generated
+@responses.activate
+def test_wait_for_ingestion_retried_after_deferral(mocker: MockerFixture) -> None:
+ """
+ The server keeps "ingestion_deferred_until" while retrying a deferred
+ ingestion; a Zarr being ingested again is waited for.
+ """
+ mocker.patch.object(dandiapi, "sleep")
+ deferred_until = "2026-11-17T13:00:00Z"
+ _mock_zarr_states(
+ {"status": "Ingesting", "ingestion_deferred_until": deferred_until},
+ {
+ "status": "Complete",
+ "checksum": "abc-1--1",
+ "ingestion_deferred_until": None,
+ },
+ )
+ r = _zarr_asset_for_mocked_server().wait_for_ingestion()
+ assert r["status"] == "Complete"
+ assert _zarr_polls() == 2
+
+
+@pytest.mark.ai_generated
+@responses.activate
+def test_wait_for_ingestion_modified_meanwhile(mocker: MockerFixture) -> None:
+ mocker.patch.object(dandiapi, "sleep")
+ _mock_zarr_states(
+ {"status": "Uploaded", "ingestion_deferred_until": None},
+ {"status": "Pending", "ingestion_deferred_until": None},
+ )
+ with pytest.raises(ZarrIngestionError, match="zarr-id was modified"):
+ _zarr_asset_for_mocked_server().wait_for_ingestion()
+
+
+@pytest.mark.ai_generated
+@responses.activate
+def test_rmfiles_reingest_deferred(
+ caplog: pytest.LogCaptureFixture, mocker: MockerFixture
+) -> None:
+ """``rmfiles(reingest=True)`` does not wait for a deferred ingestion."""
+ mocker.patch.object(dandiapi, "sleep")
+ asset = _zarr_asset_for_mocked_server()
+ responses.add(responses.DELETE, f"{_MOCK_ZARR_URL}files/", status=204)
+ responses.add(responses.POST, f"{_MOCK_ZARR_URL}finalize/", status=204)
+ deferred_until = "2026-11-18T13:00:00Z"
+ _mock_zarr_states(
+ {"status": "Uploaded", "ingestion_deferred_until": deferred_until},
+ )
+ entry = RemoteZarrEntry(
+ client=asset.client,
+ zarr_id=asset.zarr,
+ parts=("arr_0", "c", "0"),
+ modified=datetime.now(timezone.utc),
+ digest=Digest(algorithm=DigestType.md5, value="0" * 32),
+ size=1,
+ )
+ asset.rmfiles([entry])
+ assert _zarr_polls() == 1
+ assert (
+ f"Server deferred ingestion of Zarr zarr-id until {deferred_until}"
+ in caplog.text
+ )
diff --git a/dandi/tests/test_upload.py b/dandi/tests/test_upload.py
index 03dfdb710..437f247b7 100644
--- a/dandi/tests/test_upload.py
+++ b/dandi/tests/test_upload.py
@@ -1,14 +1,18 @@
from __future__ import annotations
+from base64 import b64encode
from collections import defaultdict
+from collections.abc import Callable
from datetime import datetime, timezone
import os
from pathlib import Path
from shutil import copyfile, rmtree
+from time import monotonic, sleep
from typing import Any
from unittest.mock import Mock
from urllib.parse import urlparse
+import anys
import numpy as np
import pynwb
import pytest
@@ -32,6 +36,7 @@
from ..download import download
from ..exceptions import NotFoundError, UploadError
from ..files import LocalFileAsset
+from ..files import zarr as zarr_files
from ..files.zarr import ZarrAsset
from ..pynwb_utils import make_nwb_file
from ..upload import UploadExisting, UploadValidation
@@ -466,6 +471,30 @@ def test_upload_bids_zarr(
bids_zarr_dandiset.upload()
+def _wait_for_zarr_ingestion(
+ sample_dandiset: SampleDandiset, path: str = "sample.zarr"
+) -> None:
+ """
+ Wait for the server to ingest the Zarr asset at ``path``.
+
+ Rewriting or deleting entries of a Zarr while the URLs they were uploaded
+ with are still valid makes dandi-archive#3018 defer ingesting it until
+ those URLs expire, and ``dandi upload`` does not wait for that. In the
+ test archive instance, the URLs expire after 30 seconds, and deferred
+ ingestion is retried every 5 seconds, so this takes well under a minute.
+ The deadline leaves the rest of pytest's timeout for the test itself, so
+ that a Zarr which is never ingested fails with its status.
+ """
+ asset = sample_dandiset.dandiset.get_asset_by_path(path)
+ assert isinstance(asset, RemoteZarrAsset)
+ deadline = monotonic() + 180
+ while (
+ status := sample_dandiset.client.get(f"/zarr/{asset.zarr}/")["status"]
+ ) != "Complete":
+ assert monotonic() < deadline, f"Zarr {asset.zarr} still {status!r}"
+ sleep(2)
+
+
def _replace_zarr_and_check(
tmp_path: Path, zarr_dandiset: SampleDandiset, zarr_id: str
) -> None:
@@ -475,6 +504,7 @@ def _replace_zarr_and_check(
asset = zarr_dandiset.dandiset.get_asset_by_path("sample.zarr")
assert isinstance(asset, RemoteZarrAsset)
assert asset.zarr == zarr_id
+ _wait_for_zarr_ingestion(zarr_dandiset)
download(zarr_dandiset.dandiset.version_api_url, tmp_path)
assert_dirtrees_eq(
zarr_dandiset.dspath / "sample.zarr",
@@ -514,6 +544,7 @@ def test_upload_different_zarr_entry_conflicts(
(zf / "changed-size.txt").write_text("This is a test of the upload code.\n")
(zf / "changed-type").write_text("This is now a file.\n")
new_dandiset.upload(validation=UploadValidation.SKIP)
+ _wait_for_zarr_ingestion(new_dandiset)
download(new_dandiset.dandiset.version_api_url, tmp_path)
assert_dirtrees_eq(zf, tmp_path / new_dandiset.dandiset_id / "sample.zarr")
@@ -530,6 +561,7 @@ def test_upload_different_zarr_file_to_parent_dir(
(zf / "foo").mkdir()
(zf / "foo" / "bar").write_text("This is under what used to be a file.\n")
new_dandiset.upload(validation=UploadValidation.SKIP)
+ _wait_for_zarr_ingestion(new_dandiset)
download(new_dandiset.dandiset.version_api_url, tmp_path)
assert_dirtrees_eq(zf, tmp_path / new_dandiset.dandiset_id / "sample.zarr")
@@ -649,83 +681,178 @@ def mock_put(self, url, **kwargs):
assert "arr_1" not in url, f"URL {url} should have been retried but wasn't"
+def _fake_zarr_entry_uploads(
+ monkeypatch: pytest.MonkeyPatch,
+ fake: Callable[[str], tuple[int, bytes] | None],
+) -> None:
+ """
+ Make uploads of Zarr entries to storage get the status code and content
+ ``fake`` returns for their URL, unless it returns `None`. The responses
+ are faked by `requests.Session.request`, below the retry logic of
+ `RESTFullAPIClient.request()`.
+ """
+ original_request = requests.Session.request
+
+ def mock_request(
+ self: requests.Session, method: str, url: str, *args: Any, **kwargs: Any
+ ) -> requests.Response:
+ if (
+ method == "PUT"
+ and "dandi-dandisets" in url
+ and (faked := fake(url)) is not None
+ ):
+ resp = requests.Response()
+ resp.status_code, resp._content = faked
+ resp.url = url
+ return resp
+ return original_request(self, method, url, *args, **kwargs)
+
+ monkeypatch.setattr(requests.Session, "request", mock_request)
+
+
@pytest.mark.ai_generated
def test_zarr_upload_400_timeout_retry(
+ mocker: MockerFixture,
new_dandiset: SampleDandiset,
monkeypatch: pytest.MonkeyPatch,
) -> None:
- """Test that 400 RequestTimeout errors trigger automatic retry via tenacity"""
- # Create test Zarr
- zarr_path = new_dandiset.dspath / "test.zarr"
- zarr.save(zarr_path, np.arange(100), np.arange(100, 0, -1))
-
- # Track request attempts
- request_attempts: defaultdict[str, int] = defaultdict(int)
- original_request = RESTFullAPIClient.request
+ """
+ A 400 RequestTimeout response to uploading a Zarr entry is retried by
+ `RESTFullAPIClient.request()` (https://github.com/dandi/dandi-cli/issues/1662)
+ """
+ zarr.save(new_dandiset.dspath / "test.zarr", np.arange(100), np.arange(100, 0, -1))
- def mock_request(self, method, path, **kwargs):
- # Track attempts for each request
- urlpath = urlparse(path).path if path.startswith("http") else path
- request_attempts[urlpath] += 1
+ # Attempts to upload each Zarr entry, by the path of its URL
+ put_attempts: defaultdict[str, int] = defaultdict(int)
- # Simulate 400 timeout on first attempt for files containing "arr_0"
- if method == "PUT" and "arr_0" in path and request_attempts[urlpath] == 1:
- # Return a mock response that will trigger the retry_if condition
- resp = Mock(spec=requests.Response)
- resp.status_code = 400
- resp.text = (
- '\n'
- "RequestTimeout"
- "Your socket connection to the server "
- "was not read from or written to within the timeout period. "
- "Idle connections will be closed."
- "1111111111111111"
- "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
- ""
+ def fake(url: str) -> tuple[int, bytes] | None:
+ # Respond to the first attempt to upload each entry of arr_0 with a
+ # timeout
+ urlpath = urlparse(url).path
+ put_attempts[urlpath] += 1
+ if "arr_0" in urlpath and put_attempts[urlpath] == 1:
+ return (
+ 400,
+ b'\n'
+ b"RequestTimeout"
+ b"Your socket connection to the server"
+ b" was not read from or written to within the timeout period."
+ b" Idle connections will be closed."
+ b"1111111111111111"
+ b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+ b"",
)
- resp.headers = {}
- resp.content = resp.text.encode()
- resp.json = Mock(side_effect=ValueError("No JSON"))
- return resp
+ return None
- # Otherwise, call the original method
- return original_request(self, method, path, **kwargs)
-
- # Apply the mock
- monkeypatch.setattr(RESTFullAPIClient, "request", mock_request)
+ _fake_zarr_entry_uploads(monkeypatch, fake)
+ events = _spy_zarr_iter_upload(mocker)
- # Upload the Zarr
new_dandiset.upload()
- # Verify the upload succeeded
(asset,) = new_dandiset.dandiset.get_assets()
assert isinstance(asset, RemoteZarrAsset)
assert asset.asset_type is AssetType.ZARR
assert asset.path == "test.zarr"
- # Verify that arr_0 files were retried automatically by tenacity
- # Filter for PUT requests to S3/minio (not API endpoints)
- arr_0_urls = [
- url for url in request_attempts if "arr_0" in url and "dandi-dandisets" in url
- ]
- assert len(arr_0_urls) > 0, "Expected to find arr_0 files"
-
- for url in arr_0_urls:
- # Each arr_0 file should have been attempted twice (initial + 1 retry)
+ arr_0_paths = [p for p in put_attempts if "arr_0" in p]
+ assert arr_0_paths, "Expected to find arr_0 files"
+ for p in put_attempts:
+ expected = 2 if p in arr_0_paths else 1
assert (
- request_attempts[url] == 2
- ), f"Expected {url} to be retried once, got {request_attempts[url]} attempts"
-
- # Verify non-arr_0 S3/minio URLs were not retried
- non_arr_0_urls = [
- url
- for url in request_attempts
- if "arr_0" not in url and "dandi-dandisets" in url and request_attempts[url] > 0
- ]
- for url in non_arr_0_urls:
- assert (
- request_attempts[url] == 1
- ), f"URL {url} should not have been retried but had {request_attempts[url]} attempts"
+ put_attempts[p] == expected
+ ), f"Expected {expected} attempt(s) to upload {p}, got {put_attempts[p]}"
+ # The second attempts were the retries, not uploads in a second pass after
+ # the Zarr's checksum did not match, as would happen if the timed-out
+ # entries had not been retried
+ assert not any(e["status"] == "Checksum mismatch" for e in events)
+ # Nor did the server defer ingesting the Zarr for upload URLs left unused
+ # (dandi-archive#3018), as it would if they had not been retried
+ assert events[-1] == {"status": "done", "asset": anys.AnyInstance(RemoteZarrAsset)}
+
+
+@pytest.mark.ai_generated
+def test_upload_zarr_lost_entry(
+ caplog: pytest.LogCaptureFixture,
+ mocker: MockerFixture,
+ new_dandiset: SampleDandiset,
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ """
+ An entry which the client believes it uploaded, but which is missing from
+ the remote Zarr, is detected by the Zarr's checksum and uploaded again.
+
+ Against a server which defers ingesting a Zarr with unused upload URLs
+ (dandi-archive#3018), as it does here for the lost entry's, the checksum
+ is computed from the remote file listing rather than by the server.
+ """
+ zarr_path = new_dandiset.dspath / "test.zarr"
+ zarr.save(zarr_path, np.arange(100), np.arange(100, 0, -1))
+ lost: list[str] = []
+
+ def fake(url: str) -> tuple[int, bytes] | None:
+ # Report success for the first upload of an entry of arr_0 without
+ # uploading it
+ if "arr_0" in (urlpath := urlparse(url).path) and not lost:
+ lost.append(urlpath)
+ return (200, b"")
+ return None
+
+ _fake_zarr_entry_uploads(monkeypatch, fake)
+ events = _spy_zarr_iter_upload(mocker)
+
+ new_dandiset.upload()
+
+ assert lost
+ assert [e["status"] for e in events].count("Checksum mismatch") == 1
+ asset = new_dandiset.dandiset.get_asset_by_path("test.zarr")
+ assert isinstance(asset, RemoteZarrAsset)
+ zarr_info = new_dandiset.client.get(f"/zarr/{asset.zarr}/")
+ if "ingestion_deferred_until" in zarr_info:
+ assert "remote file listing" in caplog.text
+ # Once the lost entry is uploaded again, none of the Zarr's upload URLs can
+ # change it, so the server does not defer ingesting it
+ assert events[-1] == {"status": "done", "asset": anys.AnyInstance(RemoteZarrAsset)}
+ assert zarr_info["status"] == "Complete"
+ download(new_dandiset.dandiset.version_api_url, tmp_path)
+ assert_dirtrees_eq(zarr_path, tmp_path / new_dandiset.dandiset_id / "test.zarr")
+
+
+@pytest.mark.ai_generated
+def test_upload_zarr_deferred_ingestion_faked(
+ mocker: MockerFixture, new_dandiset: SampleDandiset
+) -> None:
+ """
+ When the server defers ingesting a Zarr, the upload does not wait for it,
+ verifies the Zarr against its remote file listing instead, and reports the
+ deferral. The deferral is faked, so that this also runs against servers
+ without dandi-archive#3018 (cf. `test_upload_zarr_deferred_ingestion`).
+ """
+ zarr.save(new_dandiset.dspath / "test.zarr", np.arange(100), np.arange(100, 0, -1))
+ deferred_until = "2026-11-18T13:00:00Z"
+ original_wait = RemoteZarrAsset.wait_for_ingestion
+
+ def wait_for_ingestion(self: RemoteZarrAsset) -> dict[str, Any]:
+ return {
+ **original_wait(self),
+ "status": "Uploaded",
+ "checksum": None,
+ "ingestion_deferred_until": deferred_until,
+ }
+
+ mocker.patch.object(RemoteZarrAsset, "wait_for_ingestion", wait_for_ingestion)
+ remote_checksum = mocker.spy(zarr_files, "_remote_zarr_checksum")
+ events = _spy_zarr_iter_upload(mocker)
+
+ new_dandiset.upload()
+
+ assert remote_checksum.call_count == 1
+ assert not any(e["status"] == "Checksum mismatch" for e in events)
+ assert events[-1] == {
+ "status": "done",
+ "message": f"server ingestion deferred until {deferred_until}",
+ "asset": anys.AnyInstance(RemoteZarrAsset),
+ }
@pytest.mark.ai_generated
@@ -1179,3 +1306,86 @@ def test_upload_identical_files_in_parallel(new_dandiset: SampleDandiset) -> Non
)
== 1
)
+
+
+@pytest.mark.ai_generated
+def test_upload_unchanged_unfinalized_zarr(
+ mocker: MockerFixture, zarr_dandiset: SampleDandiset
+) -> None:
+ """
+ A rerun of an upload which ended before finalizing the Zarr finalizes it,
+ rather than reporting the matching Zarr as identical and leaving it
+ "Pending".
+ """
+ client = zarr_dandiset.client
+ (asset,) = zarr_dandiset.dandiset.get_assets()
+ assert isinstance(asset, RemoteZarrAsset)
+ checksum = client.get(f"/zarr/{asset.zarr}/")["checksum"]
+ # Requesting an upload URL, as the interrupted upload would have done,
+ # makes the Zarr "Pending". The URL is for the content already there, so
+ # it cannot change the Zarr (and does not defer its ingestion).
+ entry = next(asset.iterfiles())
+ client.post(
+ f"/zarr/{asset.zarr}/files/",
+ json=[
+ {
+ "path": str(entry),
+ "base64md5": b64encode(bytes.fromhex(entry.digest.value)).decode(),
+ }
+ ],
+ )
+ assert client.get(f"/zarr/{asset.zarr}/")["status"] == "Pending"
+
+ events = _spy_zarr_iter_upload(mocker)
+ zarr_dandiset.upload(validation="skip")
+
+ assert events[-1]["status"] == "done"
+ zarr_info = client.get(f"/zarr/{asset.zarr}/")
+ assert zarr_info["status"] == "Complete"
+ assert zarr_info["checksum"] == checksum
+
+
+@pytest.mark.ai_generated
+def test_upload_zarr_deferred_ingestion(
+ mocker: MockerFixture, zarr_dandiset: SampleDandiset
+) -> None:
+ """
+ Modifying an entry while the URL it was uploaded with is still valid makes
+ dandi-archive#3018 defer ingesting the Zarr. The upload does not wait for
+ it, but says until when it is deferred, as does a rerun.
+ """
+ client = zarr_dandiset.client
+ (asset,) = zarr_dandiset.dandiset.get_assets()
+ assert isinstance(asset, RemoteZarrAsset)
+ if "ingestion_deferred_until" not in client.get(f"/zarr/{asset.zarr}/"):
+ pytest.skip("Server does not defer Zarr ingestion")
+ # Modify a chunk, whose path depends on the Zarr format version
+ chunk = next(
+ p
+ for p in (zarr_dandiset.dspath / "sample.zarr" / "arr_0").rglob("*")
+ if p.is_file() and not p.name.startswith(".") and p.name != "zarr.json"
+ )
+ chunk.write_bytes(b"x" * 64)
+
+ events = _spy_zarr_iter_upload(mocker)
+ zarr_dandiset.upload(validation="skip")
+
+ zarr_info = client.get(f"/zarr/{asset.zarr}/")
+ if (deferred_until := zarr_info["ingestion_deferred_until"]) is None:
+ # The URL expired before the modification was uploaded
+ pytest.skip("Server did not defer Zarr ingestion")
+ assert zarr_info["status"] == "Uploaded"
+ assert events[-1] == {
+ "status": "done",
+ "message": f"server ingestion deferred until {deferred_until}",
+ "asset": anys.AnyInstance(RemoteZarrAsset),
+ }
+ assert not any(e["status"] == "Checksum mismatch" for e in events)
+
+ events.clear()
+ zarr_dandiset.upload(validation="skip")
+ assert events[-1] == {
+ "status": "skipped",
+ "message": f"identical; server ingestion deferred until {deferred_until}",
+ "asset": anys.AnyInstance(RemoteZarrAsset),
+ }