Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 78 additions & 7 deletions dandi/dandiapi.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,12 @@
DandiInstance,
EmbargoStatus,
)
from .exceptions import HTTP404Error, NotFoundError, SchemaVersionError
from .exceptions import (
HTTP404Error,
NotFoundError,
SchemaVersionError,
ZarrIngestionError,
)
from .keyring_utils import keyring_lookup, keyring_save
from .misctypes import (
Digest,
Expand Down Expand Up @@ -126,6 +131,29 @@ def set_asset_schema_key(metadata: dict[str, Any]) -> None:
metadata["schemaKey"] = "Asset"


def zarr_ingestion_deferred_until(zarr_data: dict[str, Any]) -> str | None:
"""
Given a Zarr's data from the server (as returned by ``GET
/zarr/{zarr_id}/``), return the time until which the server deferred its
ingestion, or `None` if it did not.

dandi-archive#3018 defers the ingestion of a Zarr while any of its
unexpired upload URLs could still change it, leaving it ``"Uploaded"`` with
the time of the next attempt in ``"ingestion_deferred_until"``. The field
stays set while that attempt runs (``"Ingesting"``), and is cleared once
the Zarr is ingested or modified.
"""
# Servers without dandi-archive#3018 lack the field and never defer
# ingestion; .get() can be replaced by indexing once all known instances
# deploy that feature.
if zarr_data["status"] == "Uploaded" and (
deferred_until := zarr_data.get("ingestion_deferred_until")
):
assert isinstance(deferred_until, str)
return deferred_until
return None


class AssetType(Enum):
"""
.. versionadded:: 0.36.0
Expand Down Expand Up @@ -2073,7 +2101,7 @@ def rmfiles(self, files: Iterable[RemoteZarrEntry], reingest: bool = True) -> No

If ``reingest`` is true, after performing the deletion, the client
triggers a recalculation of the Zarr's checksum and waits for it to
complete.
complete, unless the server defers it (see `wait_for_ingestion()`).
"""
# Don't bother checking that the entries are actually files or even
# belong to this Zarr, as if they're not, the server will return an
Expand All @@ -2085,11 +2113,54 @@ def rmfiles(self, files: Iterable[RemoteZarrEntry], reingest: bool = True) -> No
)
if reingest:
self.client.post(f"/zarr/{self.zarr}/finalize/")
while True:
sleep(2)
r = self.client.get(f"/zarr/{self.zarr}/")
if r["status"] == "Complete":
break
r = self.wait_for_ingestion()
if deferred_until := zarr_ingestion_deferred_until(r):
lgr.info(
"Server deferred ingestion of Zarr %s until %s",
self.zarr,
deferred_until,
)

def wait_for_ingestion(self) -> dict[str, Any]:
"""
Wait for the server to finish ingesting the Zarr after it has been
finalized, or to defer its ingestion.

Returns the Zarr's data from the server (as returned by
``GET /zarr/{zarr_id}/``) once its ``"status"`` is either
``"Complete"``, in which case its ``"checksum"`` is set, or
``"Uploaded"`` with ``"ingestion_deferred_until"`` set. In the latter
case, the server ingests the Zarr on its own at the given time, so
there is no point in waiting for it.

Raises `ZarrIngestionError` if the Zarr becomes ``"Pending"`` again,
i.e., something else modified it in the meantime.
"""
while True:
sleep(2)
r: dict[str, Any] = self.client.get(f"/zarr/{self.zarr}/")
status = r["status"]
if status == "Complete":
if "ingestion_deferred_until" not in r:
# See zarr_ingestion_deferred_until() for when this can be
# removed
lgr.debug(
"Server does not report deferred Zarr ingestion"
" (dandi-archive#3018); assuming it never defers it"
)
return r
if deferred_until := zarr_ingestion_deferred_until(r):
lgr.debug(
"Server deferred ingestion of Zarr %s until %s",
self.zarr,
deferred_until,
)
return r
if status == "Pending":
raise ZarrIngestionError(
f"Zarr {self.zarr} was modified while waiting for the"
" server to ingest it"
)


class RemoteAsset(BaseRemoteAsset):
Expand Down
6 changes: 6 additions & 0 deletions dandi/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,3 +97,9 @@ class UploadValidationError(UploadError):
"""An upload could not proceed because an asset failed validation."""

pass


class ZarrIngestionError(RuntimeError):
"""A finalized Zarr did not reach a state in which its ingestion is done or deferred."""

pass
143 changes: 105 additions & 38 deletions dandi/files/zarr.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
RemoteZarrEntry,
RESTFullAPIClient,
set_asset_schema_key,
zarr_ingestion_deferred_until,
)
from dandi.exceptions import UploadError
from dandi.metadata.core import get_default_metadata
Expand Down Expand Up @@ -588,7 +589,11 @@ def iter_upload(
`RemoteAsset`. If the local Zarr is bit-identical to the remote
(nothing to upload or delete), the terminal status is instead
``"skipped"`` with a ``"message"`` of ``"identical"`` and the same
``"asset"`` key.
``"asset"`` key. If the server deferred ingesting the Zarr (see
`RemoteZarrAsset.wait_for_ingestion()`), the upload does not wait
for it, the local checksum is verified against one computed from
the remote file listing instead of the server's, and the terminal
``"message"`` says until when ingestion is deferred.
"""
asset_path = metadata.setdefault("path", self.path)
set_asset_schema_key(metadata)
Expand Down Expand Up @@ -661,6 +666,61 @@ def mkzarr() -> str:
mismatched = True
first_run = True
unchanged = False
# Until when the server deferred ingesting the Zarr, if it did
deferred_until: str | None = None

def ingest(finalize: bool) -> Generator[dict, None, bool]:
"""
Wait for the server to ingest the Zarr, after finalizing it if
``finalize`` is true, and verify its checksum against the local
one. Returns whether they mismatch.

If the server defers ingesting the Zarr, it is not waited for, and
the local checksum is verified against one computed from the
remote file listing instead, which is what the server computes its
checksum from.
"""
nonlocal deferred_until
lgr.debug("%s: Waiting for server to calculate Zarr checksum", asset_path)
yield {"status": "server calculating checksum"}
if finalize:
client.post(f"/zarr/{zarr_id}/finalize/")
r = a.wait_for_ingestion()
if deferred_until := zarr_ingestion_deferred_until(r):
lgr.info(
"%s: Server deferred ingestion until %s because some of the"
" Zarr's upload URLs could still change it",
asset_path,
deferred_until,
)
if zarr_mode == "patch":
lgr.info(
"%s: Skipping local checksum verification (patch mode)",
asset_path,
)
return False
our_checksum = str(zcc.process())
if deferred_until is not None:
lgr.debug(
"%s: Computing Zarr checksum from remote file listing", asset_path
)
remote_checksum = _remote_zarr_checksum(a)
source = "remote file listing"
else:
remote_checksum = r["checksum"]
source = "server"
if our_checksum == remote_checksum:
return False
lgr.info(
"%s: Asset checksum mismatch (local: %s; %s: %s); redoing upload",
asset_path,
our_checksum,
source,
remote_checksum,
)
yield {"status": "Checksum mismatch"}
return True

while mismatched:
zcc = ZarrChecksumTree()
old_zarr_entries: dict[str, RemoteZarrEntry] = {
Expand Down Expand Up @@ -750,9 +810,26 @@ def mkzarr() -> str:
len(old_zarr_entries) if zarr_mode == "full" else 0
)
if n_add == 0 and n_modify == 0 and n_delete == 0:
mismatched = False
unchanged = True
break
zarr_info = client.get(f"/zarr/{zarr_id}/")
status = zarr_info["status"]
deferred_until = zarr_ingestion_deferred_until(zarr_info)
if status == "Complete" or deferred_until is not None:
mismatched = False
unchanged = True
break
# The remote entries match, but an earlier upload ended
# before finalizing the Zarr ("Pending") or before the
# server ingested it ("Uploaded" or "Ingesting")
lgr.info(
"%s: Remote Zarr matches local but has not been"
" ingested (status: %s); %s",
asset_path,
status,
"finalizing" if status == "Pending" else "waiting for it",
)
mismatched = yield from ingest(finalize=status == "Pending")
first_run = False
continue
bytes_add = sum(e.size for e in to_upload.fresh_entries)
bytes_modify = sum(it.size for it in to_upload.digested_entries)
clauses = []
Expand Down Expand Up @@ -919,38 +996,7 @@ def mkzarr() -> str:
"file" if n_remote_only == 1 else "files",
)
if changed:
lgr.debug(
"%s: Waiting for server to calculate Zarr checksum", asset_path
)
yield {"status": "server calculating checksum"}
client.post(f"/zarr/{zarr_id}/finalize/")
while True:
sleep(2)
r = client.get(f"/zarr/{zarr_id}/")
if r["status"] == "Complete":
if zarr_mode == "patch":
lgr.info(
"%s: Skipping local checksum verification"
" (patch mode)",
asset_path,
)
mismatched = False
else:
our_checksum = str(zcc.process())
server_checksum = r["checksum"]
if our_checksum == server_checksum:
mismatched = False
else:
mismatched = True
lgr.info(
"%s: Asset checksum mismatch (local: %s;"
" server: %s); redoing upload",
asset_path,
our_checksum,
server_checksum,
)
yield {"status": "Checksum mismatch"}
break
mismatched = yield from ingest(finalize=True)
elif mismatched and not first_run:
lgr.error(
"%s: Previous upload loop resulted in checksum mismatch,"
Expand All @@ -969,12 +1015,33 @@ def mkzarr() -> str:
unchanged,
first_run,
)
deferral = (
f"server ingestion deferred until {deferred_until}"
if deferred_until is not None
else None
)
if unchanged:
lgr.info("%s: Zarr already matches remote; nothing to upload", asset_path)
yield {"status": "skipped", "message": "identical", "asset": a}
message = "identical" if deferral is None else f"identical; {deferral}"
yield {"status": "skipped", "message": message, "asset": a}
else:
lgr.info("%s: Asset successfully uploaded", asset_path)
yield {"status": "done", "asset": a}
done: dict[str, Any] = {"status": "done", "asset": a}
if deferral is not None:
done["message"] = deferral
yield done


def _remote_zarr_checksum(asset: RemoteZarrAsset) -> str:
"""
Compute the DANDI Zarr checksum of a remote Zarr from the listing of its
files, which, unlike the checksum the server reports, does not have to
wait for the server to ingest the Zarr
"""
tree = ZarrChecksumTree()
for entry in asset.iterfiles():
tree.add_leaf(Path(str(entry)), entry.size, entry.digest.value)
return str(tree.process())


def _handle_failed_items_and_raise(
Expand Down
9 changes: 9 additions & 0 deletions dandi/tests/data/dandiarchive-docker/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,15 @@ services:
DJANGO_DANDI_INSTANCE_NAME: ${DANDI_TESTS_INSTANCE_NAME:-DANDI}
DJANGO_DANDI_INSTANCE_IDENTIFIER: ${DANDI_TESTS_INSTANCE_IDENTIFIER:-RRID:ABC_123456}
DJANGO_DANDI_DOI_API_PREFIX: ${DANDI_TESTS_DOI_PREFIX:-10.80507}
# dandi-archive defers ingestion of a Zarr until every upload URL that
# could still change it has expired, and retries deferred ingestion
# periodically (https://github.com/dandi/dandi-archive/pull/3018).
# Shorten the expiry from the default hour, and the retry interval from
# the default 5 minutes, so that tests which overwrite or delete recently
# uploaded Zarr entries do not wait that long. Ignored by servers
# without that feature.
DJANGO_DANDI_ZARR_UPLOAD_URL_EXPIRATION: "30"
DJANGO_DANDI_DEFERRED_INGESTION_RETRY_INTERVAL: "5"
ports:
- "127.0.0.1:8000:8000"

Expand Down
Loading
Loading