diff --git a/dispatch/README.md b/dispatch/README.md index e57f9da..bdaaaf8 100644 --- a/dispatch/README.md +++ b/dispatch/README.md @@ -5,7 +5,7 @@ It is repo-level infra, not a lab — it doesn't do any conversion itself, it ju ## What one run does, per registered project -1. `dandi download` the project's incoming dandiset (from the `ember-dandi` instance, the only archive this pipeline uses) into `//`. Runs inside `--dandi-image` (through Apptainer), not directly on the runner host. +1. `dandi download` the project's incoming dandiset (from the `ember-dandi` instance, the only archive this pipeline uses) into `//`. Runs inside `--dandi-image` (through Apptainer), not directly on the runner host. With `--sync do` it also deletes local files that the archive no longer has, so a recording the lab deleted or renamed doesn't linger as a stale session. Dispatch logs each deleted file and then removes the folders that leaves empty. 2. Discover its sessions (per `sessions.json`'s spec for the project) and diff them against the project's manifest (`//.ingest_state.json`) to find sessions with no conversion recorded yet. 3. If there are new sessions, **or** the conversion's code or config has changed since the manifest was last written, run the lab's conversion command. That covers the conversion script and every repository file `convert_command` names as `{repo_root}/...`, such as a batch driver or a config (sha256, so any edit forces a full reprocess via `overwrite_flag`). A project whose command names no other file is hashed by its script alone. If the project names a `container_image`, this step runs inside it (through Apptainer) instead of directly on the runner host — the image holds only the lab's runtime environment (e.g. FFmpeg for Kemere), not the code or data, which are bind-mounted in at run time from the same host paths. Otherwise it runs directly on the runner host, which must then already have whatever the conversion script needs installed. diff --git a/dispatch/dispatch.py b/dispatch/dispatch.py index 271edde..36b697e 100644 --- a/dispatch/dispatch.py +++ b/dispatch/dispatch.py @@ -2,7 +2,8 @@ """Cron entrypoint that drives one pass of the ingest pipeline for every registered project (see projects.json): - 1. ``dandi download`` the incoming dandiset into ``/``. + 1. ``dandi download`` the incoming dandiset into ``/``, + deleting local files the archive no longer has (``--sync do``). 2. Diff its sessions (dispatch/sessions.json's spec for the project) against the project's manifest to find sessions that haven't been converted yet with the current script. @@ -143,9 +144,41 @@ def git_head(repo_root: Path, /) -> str | None: DANDI_CACHE_ENV = {"DANDI_CACHE": "ignore"} +def local_files(root: Path, /) -> set[Path]: + """Every file under *root* outside hidden directories, which is what ``dandi download --sync`` can delete.""" + files = set() + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [name for name in dirnames if not name.startswith(".")] + files.update(Path(dirpath) / name for name in filenames if not name.startswith(".")) + return files + + +def remove_empty_dirs(root: Path, /) -> list[Path]: + """Remove the directories under *root* that hold nothing, deepest first, and return them. + + ``dandi download --sync`` deletes the files of assets gone from the archive but leaves their + directories, and an empty one can still match a project's session globs. + """ + walked = [] + for dirpath, dirnames, _ in os.walk(root): + dirnames[:] = [name for name in dirnames if not name.startswith(".")] + walked.append(Path(dirpath)) + removed = [] + for path in reversed(walked): + if path != root and not any(path.iterdir()): + path.rmdir() + removed.append(path) + return removed + + def dandi_download( project: Project, incoming_dir: Path, *, tracking: TrackingDataset, dandi_image: str, dry_run: bool ) -> None: + """Refresh the local copy of the incoming dandiset, deleting what the archive no longer has. + + Otherwise a file the lab deleted or renamed on the archive would linger here, still looking + like a session to convert or wait on. Each deleted file is logged. + """ url = f"dandi://{DANDI_INSTANCE}/{project.incoming_dandiset_id}" if not dry_run: incoming_dir.parent.mkdir(parents=True, exist_ok=True) @@ -156,8 +189,17 @@ def dandi_download( literal_env=DANDI_CACHE_ENV, ) cmd = ["apptainer", "exec", "--cleanenv", str(image.path)] - cmd += ["dandi", "download", "-o", str(incoming_dir.parent), "-e", "refresh", url] + cmd += ["dandi", "download", "-o", str(incoming_dir.parent), "-e", "refresh", "--sync", "do", url] + before = set() if dry_run else local_files(incoming_dir) run(cmd, env=env, dry_run=dry_run) + if dry_run: + return + deleted = sorted(before - local_files(incoming_dir)) + for path in deleted: + log.info("[%s] deleted %s, which the archive no longer has", project.key, path.relative_to(incoming_dir)) + emptied = remove_empty_dirs(incoming_dir) + if deleted or emptied: + log.info("[%s] deleted %d file(s) and %d emptied folder(s)", project.key, len(deleted), len(emptied)) def record_validation( diff --git a/dispatch/sessions.json b/dispatch/sessions.json index 6a0dc0d..c58e0e8 100644 --- a/dispatch/sessions.json +++ b/dispatch/sessions.json @@ -38,7 +38,10 @@ "exclude": [] }, "zhang/ferret-hyperscanning": { - "include": ["sourcedata/raw/Ferret_Hyperscanning/*/*/*.rec"], + "include": [ + "sourcedata/raw/Ferret_Hyperscanning/*/*/*.rec", + "sourcedata/raw/Ferret_Hyperscanning/*/*/[Ss]ess*/*.rec" + ], "exclude": [] } } diff --git a/dispatch/tests/test_dispatch.py b/dispatch/tests/test_dispatch.py index 68bc888..4abdb6f 100644 --- a/dispatch/tests/test_dispatch.py +++ b/dispatch/tests/test_dispatch.py @@ -1,4 +1,5 @@ import json +import logging import sys from pathlib import Path @@ -237,6 +238,8 @@ def test_dandi_download_runs_in_container(tmp_path, monkeypatch, tracking): str(incoming_root), "-e", "refresh", + "--sync", + "do", "dandi://ember-dandi/000001", ] env = kwargs["env"] @@ -246,6 +249,41 @@ def test_dandi_download_runs_in_container(tmp_path, monkeypatch, tracking): assert incoming_root.is_dir() # created ahead of the bind +def test_dandi_download_logs_deleted_files_and_removes_emptied_folders(tmp_path, monkeypatch, tracking, caplog): + incoming_dir = tmp_path / "incoming" / "000001" + stale = incoming_dir / "raw" / "ses-old" / "inner" / "data.bin" + kept = incoming_dir / "raw" / "ses-1" / "data.bin" + hidden = incoming_dir / ".cache" / "empty" + for path in (stale, kept): + path.parent.mkdir(parents=True) + path.write_text("x") + hidden.mkdir(parents=True) + # What `dandi download --sync do` does to an asset gone from the archive. + monkeypatch.setattr(dispatch.subprocess, "run", lambda cmd, **kwargs: stale.unlink()) + + project = make_project() + with caplog.at_level(logging.INFO, logger="dispatch"): + dispatch.dandi_download(project, incoming_dir, tracking=tracking, dandi_image=DANDI_IMAGE, dry_run=False) + + assert not (incoming_dir / "raw" / "ses-old").exists() + assert kept.is_file() + assert hidden.is_dir() + assert "deleted raw/ses-old/inner/data.bin" in caplog.text + assert "deleted 1 file(s) and 2 emptied folder(s)" in caplog.text + + +def test_dandi_download_dry_run_deletes_nothing(tmp_path, monkeypatch, tracking): + empty = tmp_path / "incoming" / "000001" / "raw" / "ses-old" + empty.mkdir(parents=True) + calls = record_calls(monkeypatch) + + project = make_project() + dispatch.dandi_download(project, empty.parents[1], tracking=tracking, dandi_image=DANDI_IMAGE, dry_run=True) + + assert calls == [] + assert empty.is_dir() + + def test_dandi_upload_runs_in_container_and_is_recorded(tmp_path, monkeypatch, tracking): standardized_dir = tmp_path / "standardized" / "000002" standardized_dir.mkdir(parents=True) diff --git a/labs/zhang/ferret-hyperscanning/OPEN_QUESTIONS.md b/labs/zhang/ferret-hyperscanning/OPEN_QUESTIONS.md index 09ce9f0..2e51d02 100644 --- a/labs/zhang/ferret-hyperscanning/OPEN_QUESTIONS.md +++ b/labs/zhang/ferret-hyperscanning/OPEN_QUESTIONS.md @@ -20,7 +20,6 @@ with a pointer to the change. | [Q10](#q10-electrodes) | Electrode type, geometry, coordinates, filtering, references | Deferred | Electrode table detail | | [Q11](#q11-accelerometer) | Accelerometer units and real sample rate | Deferred | Accelerometer stream | | [Q12](#q12-session-log-as-an-input) | Session log as a committed CSV, and its parsing rules | Implemented as proposed, unconfirmed | Channel map selection, notes | -| [Q13](#q13-two-recordings-on-one-date) | Layout of days with two recordings of one pair | Implemented as proposed, unverified | Video matching | | [Q14](#q14-sessions-to-exclude) | Should sessions without usable data be excluded? | Deferred, all converted | Which sessions publish | | [Q15](#q15-calibration-videos) | Calibration videos in the NWB or as plain assets? | Implemented as proposed, unconfirmed | Calibration video placement | | [Q16](#q16-investigate-video-transcoding) | Investigate video transcoding | Deferred to a follow-up | Video size and format | @@ -128,16 +127,6 @@ notes. hour below 8 were read as afternoon. - **Ask the lab:** confirm both parsing rules. -### Q13. Two recordings on one date - -Twelve date-pair combinations have two session-log rows. - -- **Current handling:** each `.rec` is matched to the videos whose filename - timestamp is nearest (`match_videos`), and to the session-log row whose - start time is nearest (`find_session_row`). -- **Needs:** a real example of such a day, to confirm that both recordings - share one `Videos/` directory. - ### Q15. Calibration videos - **Current handling:** the calibration videos are in the NWB as @@ -153,3 +142,4 @@ Twelve date-pair combinations have two session-log rows. | Q2 | Flat or nested lab layout | Nested, `labs/zhang/ferret-hyperscanning/` | Request 2 in `prompts/initial.md` | | Q3 | Data standard | NWB, one file per subject per session, DANDI layout | Request 2 | | Q4 | Video handling | Keep the AVIs unchanged as external files, transcode later | Request 2 | +| Q13 | Layout of days with two recordings of one pair | Each recording sits with its own `Videos/` in a `Session_1/`, `Session_2/` (or `Session1/`, `Session2/`) folder of the date folder, 22 recordings over 11 days. `batch_convert.discover_recordings` reads them | Request 13 | diff --git a/labs/zhang/ferret-hyperscanning/code/README.md b/labs/zhang/ferret-hyperscanning/code/README.md index e4afd4b..c7b57f0 100644 --- a/labs/zhang/ferret-hyperscanning/code/README.md +++ b/labs/zhang/ferret-hyperscanning/code/README.md @@ -40,8 +40,38 @@ before writing, which is what DANDI resolves after upload. Videos/ cam--0000.avi behavior video, one per camera cam-cal--0000.avi ChArUco calibration video, one per camera + Session_1/ only on a day with two recordings of the pair + .rec/ laid out as above + Videos/ that recording's own videos + Session_2/ ``` +The lab's spelling varies in places, and these variants are read as the +standard names: + +| Variant | Example | +| --- | --- | +| A session token between the headstages and the date | `0235HS3_0237HS4_Session1_20260308_142653`, also `sess1` and `Sess1` | +| `Merge` glued to the date | `0235HS5_0236HS8_Merge20260423_142917` | +| No underscore before `merged` | `…_0237HS4merged.rec` | +| `Calibration` for `cal` in a video name | `Cam14-Calibration-04012026121856-0000.avi` | +| `Session1/` for `Session_1/` | `0235-0236/03.18.2026/Session1/` | +| Merged files loose in the `.rec` folder, outside the animals' folders | `0235HS3_0237HS4_20260330_150816.rec/…_0235HS3_merged.rec` | + +A recording whose names contradict its own files is read through its entry +under `corrections` in `config.yaml`. An entry can replace the recording +name that the animals, headstages and session are read from, or point at an +animal's merged file. Its `reason` lands in the NWB `notes`. Each entry was +checked against the merged files' headers, which carry the recording's start +time and the serial of the headstage that recorded them. A corrected file is +used only while it exists, so a rename on EMBER takes over once the lab +fixes the name. Dispatch hashes `config.yaml`, so adding an entry +reconverts every recording on the next run. + +A recording that fits none of this is reported in the run log and left +unconverted. The lab issue lists the cases: +https://github.com/brain-bbqs/data-ingest-task-force/issues/62. + ## Mapping | Source | Output | @@ -54,6 +84,7 @@ before writing, which is what DANDI resolves after upload. | `Videos/cam--0000.avi` | `acquisition/BehaviorVideoCam` via NeuroConv's `ExternalVideoInterface` (external `ImageSeries`, rate and frame count from the file header, `starting_time` 0.0 PROVISIONAL), file placed as `sub-/sub-_ses-