From f8e17452be267fe6750e46ac918b3cc697dd0b34 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 00:04:29 +0000 Subject: [PATCH] Reported BIG_ARENA sessions with bad source data as pending Every BIG_ARENA session failed the 2026-11-11 scheduled ingest because its sync.csv disagrees with its files: listed media missing, a malformed name (channel_0_0), or audio timing outside sync.tolerance. Those checks now raise SourceDataError, which batch_convert reports as pending, so the session is retried each night without failing the run until the lab fixes it (#60). Co-Authored-By: Claude Code 2.1.319 / Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018Gd63VAaXEtYzbNRniezVW --- dispatch/README.md | 2 +- labs/sanes/code/_sanes_to_nwb.py | 32 ++++++++++++++----------- labs/sanes/code/batch_convert.py | 17 +++++++++++-- labs/sanes/prompts/big-arena-2024-12.md | 8 +++++++ labs/sanes/tests/test_batch_convert.py | 18 ++++++++++++++ 5 files changed, 60 insertions(+), 17 deletions(-) diff --git a/dispatch/README.md b/dispatch/README.md index e22da1f..e57f9da 100644 --- a/dispatch/README.md +++ b/dispatch/README.md @@ -148,7 +148,7 @@ Moving R34 onto the shared converter changed its output from `autogenerated/ses- `sanes/ssa-gt` is set to `upload_validation: ignore` too, as its requester decided. Its multisubject NWB files cannot validate yet, since DANDI's `check_subject_exists` stays CRITICAL on a multisubject file even with a full animal roster, and its subject age is still a placeholder. Because validation does not gate its upload, every upload is preceded by a recorded `dandi validate` whose full report lands in `records/sanes/ssa-gt/-validate/` of the tracking dataset. Problems there raise a warning, not a failure. -The two BIG_ARENA entries keep the default `upload_validation`, so a refused NWB file's external videos still upload (see `upload_validation` above) and the upload is retried every run (step 4). On the runner, each of them would publish its MP4s without the NWB that references them and then fail on every run. Keep `sanes/big-arena-2024-12` and `sanes/big-arena-2025-03` off real cron runs with `--only` the ready projects. +The two BIG_ARENA entries keep the default `upload_validation`, so a refused NWB file's external videos still upload (see `upload_validation` above) and the upload is retried every run (step 4). On the runner, each of them would publish its MP4s without the NWB that references them and then fail on every run. Keep `sanes/big-arena-2024-12` and `sanes/big-arena-2025-03` off real cron runs with `--only` the ready projects. A session whose `sync.csv` disagrees with its files (a listed file missing, a malformed name, media timing outside `sync.tolerance`) is reported as pending, not failed, until the lab fixes the source data. Every dataset config is still marked PROVISIONAL. See `labs/sanes/README.md` and `labs/sanes/OPEN_QUESTIONS.md` before treating any Sanes output as final. diff --git a/labs/sanes/code/_sanes_to_nwb.py b/labs/sanes/code/_sanes_to_nwb.py index b42f7dd..8466558 100644 --- a/labs/sanes/code/_sanes_to_nwb.py +++ b/labs/sanes/code/_sanes_to_nwb.py @@ -61,6 +61,10 @@ ISO_AGE = re.compile(rf"{_ISO_DURATION}(?:/(?:{_ISO_DURATION})?)?") +class SourceDataError(ValueError): + """The lab's source files are incomplete or inconsistent, so the session waits on the lab rather than on code.""" + + def load_cfg(config_path, /): cfg = yaml.safe_load(config_path.read_text()) return cfg @@ -640,14 +644,14 @@ def _read_sync_manifest(*, folder, timezone): with (folder / "sync.csv").open(newline="") as stream: reader = csv.DictReader(stream) if reader.fieldnames != ["video", "audio", "timestamp"]: - raise ValueError("sync.csv must have video,audio,timestamp columns") + raise SourceDataError("sync.csv must have video,audio,timestamp columns") for record in reader: values = {field: _manifest_list(value=record[field], field=field) for field in reader.fieldnames} if len(values["timestamp"]) != 2: - raise ValueError("Each sync.csv timestamp must contain start and end") + raise SourceDataError("Each sync.csv timestamp must contain start and end") start, stop = [datetime.datetime.fromisoformat(value) for value in values["timestamp"]] if any(value.tzinfo is not None for value in (start, stop)): - raise ValueError("Expected local timestamps without offsets in sync.csv") + raise SourceDataError("Expected local timestamps without offsets in sync.csv") audio = {} videos = {} segment_ids = set() @@ -658,24 +662,24 @@ def _read_sync_manifest(*, folder, timezone): for name in values[kind]: match = re.fullmatch(pattern, name) if match is None or name in seen or match[1] in destination: - raise ValueError(f"Invalid or duplicate sync.csv filename: {name}") + raise SourceDataError(f"Invalid or duplicate sync.csv filename: {name}") seen.add(name) segment_ids.add(match[2]) path = folder / (name + suffix) if not path.is_file(): - raise ValueError(f"Missing manifest media: {path}") + raise SourceDataError(f"Missing manifest media: {path}") destination[match[1]] = path if set(audio) != {f"{index:02d}" for index in range(8)} or set(videos) != set(SYNC_CAMERAS): - raise ValueError("Each sync.csv row needs eight microphones and five camera views") + raise SourceDataError("Each sync.csv row needs eight microphones and five camera views") if len(segment_ids) != 1 or stop <= start: - raise ValueError("Inconsistent segment suffixes or invalid time interval in sync.csv") + raise SourceDataError("Inconsistent segment suffixes or invalid time interval in sync.csv") rows.append({"start": start, "stop": stop, "segment": segment_ids.pop(), "audio": audio, "videos": videos}) if not rows: - raise ValueError("sync.csv contains no segments") + raise SourceDataError("sync.csv contains no segments") rows.sort(key=lambda row: row["start"]) for previous, current in zip(rows, rows[1:]): if previous["stop"] != current["start"]: - raise ValueError("sync.csv segments overlap or have a gap") + raise SourceDataError("sync.csv segments overlap or have a gap") readme = (folder / "README.txt").read_text() descriptions = [] for line in readme.splitlines(): @@ -683,7 +687,7 @@ def _read_sync_manifest(*, folder, timezone): continue groups = line.split(" - ") if len(groups) != 3: - raise ValueError("Malformed synchronization description in README.txt") + raise SourceDataError("Malformed synchronization description in README.txt") videos, audio, timestamps = [group.strip("[]").split(", ") for group in groups] descriptions.append((set(videos), set(audio), timestamps)) expected = [ @@ -695,21 +699,21 @@ def _read_sync_manifest(*, folder, timezone): for row in rows ] if len(descriptions) != len(expected) or any(item not in descriptions for item in expected): - raise ValueError("README.txt synchronization descriptions disagree with sync.csv") + raise SourceDataError("README.txt synchronization descriptions disagree with sync.csv") zone = zoneinfo.ZoneInfo(timezone) for row in rows: for key in ("start", "stop"): local = row[key] aware = local.replace(tzinfo=zone) if aware.utcoffset() != local.replace(tzinfo=zone, fold=1).utcoffset(): - raise ValueError("Ambiguous or nonexistent local time in sync.csv. Confirm the timestamp offsets.") + raise SourceDataError("Ambiguous or nonexistent local time in sync.csv. Confirm the timestamp offsets.") row[key] = aware origin = rows[0]["start"].timestamp() for row in rows: row["offset"] = row["start"].timestamp() - origin row["duration"] = row["stop"].timestamp() - row["start"].timestamp() if row["duration"] <= 0: - raise ValueError("Non-positive elapsed segment duration") + raise SourceDataError("Non-positive elapsed segment duration") included = {path.name for row in rows for kind in ("audio", "videos") for path in row[kind].values()} for path in sorted(folder.iterdir()): if path.name not in {"README.txt", "sync.csv"} and path.name not in included: @@ -785,7 +789,7 @@ def _probe_sync_video(path, /): def _check_sync_timing(*, actual, expected, tolerance, description): residual = actual - expected if not numpy.isfinite(residual) or abs(residual) > tolerance + 1e-6: - raise ValueError(f"Timing conflict for {description}: measured {actual:g}s, expected {expected:g}s") + raise SourceDataError(f"Timing conflict for {description}: measured {actual:g}s, expected {expected:g}s") return residual diff --git a/labs/sanes/code/batch_convert.py b/labs/sanes/code/batch_convert.py index ed8c72e..917d7ef 100644 --- a/labs/sanes/code/batch_convert.py +++ b/labs/sanes/code/batch_convert.py @@ -20,7 +20,9 @@ the output. A failed session does not stop the batch, and the exit code reports whether any failed. An ``experiment_*`` folder with no supported layout is reported as pending, since its media may not be -uploaded yet, and does not fail the batch. With ``--results`` the batch writes the sessions it left +uploaded yet, and does not fail the batch. So is a session whose ``sync.csv`` disagrees with its files (a +listed file missing, a malformed name, media timing outside ``sync.tolerance``), which waits on the lab to fix +the source data. With ``--results`` the batch writes the sessions it left unconverted, as ``{"pending": {name: reason}, "failed": {name: reason}}`` keyed by folder name. Dispatch fills it through its ``{results}`` placeholder, then keeps and uploads what did convert. @@ -43,7 +45,14 @@ # that directory on sys.path and import the sibling by name. sys.path.insert(0, str(Path(__file__).resolve().parent)) -from _sanes_to_nwb import SYNC_FOLDER, build_nwb, load_cfg, multisubject_path, session_label # noqa: E402 +from _sanes_to_nwb import ( # noqa: E402 + SYNC_FOLDER, + SourceDataError, + build_nwb, + load_cfg, + multisubject_path, + session_label, +) UNSUPPORTED_REASON = f"No idx_* chunks or {SYNC_FOLDER}/ folder yet" @@ -128,6 +137,10 @@ def convert_batch(*, incoming_dir, standardized_dir, config_path, overwrite=Fals for session_dir in tqdm.tqdm(pending, desc="Converting sessions", unit="session"): try: output_path = build_nwb(session_dir=session_dir, output_dir=output_dir, config_path=config_path) + except SourceDataError as error: + print(f"PENDING: {session_dir.name}: {error}", flush=True) + unconverted[session_dir.name] = str(error) + continue except Exception as error: traceback.print_exception(error) print(f"FAILED: {session_dir}: {error}", file=sys.stderr, flush=True) diff --git a/labs/sanes/prompts/big-arena-2024-12.md b/labs/sanes/prompts/big-arena-2024-12.md index 4124b77..ebe3b90 100644 --- a/labs/sanes/prompts/big-arena-2024-12.md +++ b/labs/sanes/prompts/big-arena-2024-12.md @@ -150,3 +150,11 @@ From the reviewer (CodyCBakerPhD), answering Q16, the three parts of Q17, and th > "Now: a collection with a different layout can also join, if its output follows the same conventions and the converter gains a second reader for the new layout." yeah that sounds fine The audio is now one eight-column series with `ndx-sound>=0.3.0`. The plan is kept as `big-arena-2024-12-plan.md` beside this record, amended to match the implementation. Q16 and Q17 are resolved. + +## Request 8 — Skip BIG_ARENA sessions whose source data disagrees with sync.csv + +Every BIG_ARENA session failed the 2026-11-11 scheduled ingest: files listed in `sync.csv` were missing, a name read `channel_0_0`, or audio timing fell outside `sync.tolerance`. Asked whether to report those as pending or take BIG_ARENA off the nightly run: + +> hmmm I see well skip those for now and raise an issue about them + +Recorded in https://github.com/brain-bbqs/data-ingest-task-force/issues/60. diff --git a/labs/sanes/tests/test_batch_convert.py b/labs/sanes/tests/test_batch_convert.py index a634399..bd04cad 100644 --- a/labs/sanes/tests/test_batch_convert.py +++ b/labs/sanes/tests/test_batch_convert.py @@ -15,8 +15,10 @@ CODE = Path(__file__).resolve().parents[1] / "code" sys.path.insert(0, str(CODE)) +sys.path.insert(0, str(Path(__file__).resolve().parent)) import batch_convert # noqa: E402 +import synthetic # noqa: E402 pytestmark = pytest.mark.ai_generated @@ -153,3 +155,19 @@ def test_colliding_session_labels_are_rejected(tmp_path, recorded_calls): ) assert actual == 1 assert recorded_calls == [] + + +@pytest.mark.ai_generated +def test_source_data_problems_are_pending_not_failed(tmp_path): + session = synthetic.write_sync_session(root=tmp_path / "in", segments=2) + missing = session / "concatenated_data_cam_mic_sync" / "video_underground_001.mp4" + missing.unlink() + actual = batch_convert.convert_batch( + incoming_dir=tmp_path / "in", + standardized_dir=tmp_path / "out", + config_path=CODE / "config_big_arena_2024_12.yaml", + results_path=tmp_path / "results.json", + ) + assert actual == 0 + results = json.loads((tmp_path / "results.json").read_text()) + assert results == {"failed": {}, "pending": {"experiment_109": f"Missing manifest media: {missing}"}}