Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dispatch/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ Moving R34 onto the shared converter changed its output from `autogenerated/ses-

`sanes/ssa-gt` is set to `upload_validation: ignore` too, as its requester decided. Its multisubject NWB files cannot validate yet, since DANDI's `check_subject_exists` stays CRITICAL on a multisubject file even with a full animal roster, and its subject age is still a placeholder. Because validation does not gate its upload, every upload is preceded by a recorded `dandi validate` whose full report lands in `records/sanes/ssa-gt/<stamp>-validate/` of the tracking dataset. Problems there raise a warning, not a failure.

The two BIG_ARENA entries keep the default `upload_validation`, so a refused NWB file's external videos still upload (see `upload_validation` above) and the upload is retried every run (step 4). On the runner, each of them would publish its MP4s without the NWB that references them and then fail on every run. Keep `sanes/big-arena-2024-12` and `sanes/big-arena-2025-03` off real cron runs with `--only` the ready projects.
The two BIG_ARENA entries keep the default `upload_validation`, so a refused NWB file's external videos still upload (see `upload_validation` above) and the upload is retried every run (step 4). On the runner, each of them would publish its MP4s without the NWB that references them and then fail on every run. Keep `sanes/big-arena-2024-12` and `sanes/big-arena-2025-03` off real cron runs with `--only` the ready projects. A session whose `sync.csv` disagrees with its files (a listed file missing, a malformed name, media timing outside `sync.tolerance`) is reported as pending, not failed, until the lab fixes the source data.

Every dataset config is still marked PROVISIONAL. See `labs/sanes/README.md` and `labs/sanes/OPEN_QUESTIONS.md` before treating any Sanes output as final.

Expand Down
32 changes: 18 additions & 14 deletions labs/sanes/code/_sanes_to_nwb.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,10 @@
ISO_AGE = re.compile(rf"{_ISO_DURATION}(?:/(?:{_ISO_DURATION})?)?")


class SourceDataError(ValueError):
"""The lab's source files are incomplete or inconsistent, so the session waits on the lab rather than on code."""


def load_cfg(config_path, /):
cfg = yaml.safe_load(config_path.read_text())
return cfg
Expand Down Expand Up @@ -640,14 +644,14 @@ def _read_sync_manifest(*, folder, timezone):
with (folder / "sync.csv").open(newline="") as stream:
reader = csv.DictReader(stream)
if reader.fieldnames != ["video", "audio", "timestamp"]:
raise ValueError("sync.csv must have video,audio,timestamp columns")
raise SourceDataError("sync.csv must have video,audio,timestamp columns")
for record in reader:
values = {field: _manifest_list(value=record[field], field=field) for field in reader.fieldnames}
if len(values["timestamp"]) != 2:
raise ValueError("Each sync.csv timestamp must contain start and end")
raise SourceDataError("Each sync.csv timestamp must contain start and end")
start, stop = [datetime.datetime.fromisoformat(value) for value in values["timestamp"]]
if any(value.tzinfo is not None for value in (start, stop)):
raise ValueError("Expected local timestamps without offsets in sync.csv")
raise SourceDataError("Expected local timestamps without offsets in sync.csv")
audio = {}
videos = {}
segment_ids = set()
Expand All @@ -658,32 +662,32 @@ def _read_sync_manifest(*, folder, timezone):
for name in values[kind]:
match = re.fullmatch(pattern, name)
if match is None or name in seen or match[1] in destination:
raise ValueError(f"Invalid or duplicate sync.csv filename: {name}")
raise SourceDataError(f"Invalid or duplicate sync.csv filename: {name}")
seen.add(name)
segment_ids.add(match[2])
path = folder / (name + suffix)
if not path.is_file():
raise ValueError(f"Missing manifest media: {path}")
raise SourceDataError(f"Missing manifest media: {path}")
destination[match[1]] = path
if set(audio) != {f"{index:02d}" for index in range(8)} or set(videos) != set(SYNC_CAMERAS):
raise ValueError("Each sync.csv row needs eight microphones and five camera views")
raise SourceDataError("Each sync.csv row needs eight microphones and five camera views")
if len(segment_ids) != 1 or stop <= start:
raise ValueError("Inconsistent segment suffixes or invalid time interval in sync.csv")
raise SourceDataError("Inconsistent segment suffixes or invalid time interval in sync.csv")
rows.append({"start": start, "stop": stop, "segment": segment_ids.pop(), "audio": audio, "videos": videos})
if not rows:
raise ValueError("sync.csv contains no segments")
raise SourceDataError("sync.csv contains no segments")
rows.sort(key=lambda row: row["start"])
for previous, current in zip(rows, rows[1:]):
if previous["stop"] != current["start"]:
raise ValueError("sync.csv segments overlap or have a gap")
raise SourceDataError("sync.csv segments overlap or have a gap")
readme = (folder / "README.txt").read_text()
descriptions = []
for line in readme.splitlines():
if not line.startswith("[video_"):
continue
groups = line.split(" - ")
if len(groups) != 3:
raise ValueError("Malformed synchronization description in README.txt")
raise SourceDataError("Malformed synchronization description in README.txt")
videos, audio, timestamps = [group.strip("[]").split(", ") for group in groups]
descriptions.append((set(videos), set(audio), timestamps))
expected = [
Expand All @@ -695,21 +699,21 @@ def _read_sync_manifest(*, folder, timezone):
for row in rows
]
if len(descriptions) != len(expected) or any(item not in descriptions for item in expected):
raise ValueError("README.txt synchronization descriptions disagree with sync.csv")
raise SourceDataError("README.txt synchronization descriptions disagree with sync.csv")
zone = zoneinfo.ZoneInfo(timezone)
for row in rows:
for key in ("start", "stop"):
local = row[key]
aware = local.replace(tzinfo=zone)
if aware.utcoffset() != local.replace(tzinfo=zone, fold=1).utcoffset():
raise ValueError("Ambiguous or nonexistent local time in sync.csv. Confirm the timestamp offsets.")
raise SourceDataError("Ambiguous or nonexistent local time in sync.csv. Confirm the timestamp offsets.")
row[key] = aware
origin = rows[0]["start"].timestamp()
for row in rows:
row["offset"] = row["start"].timestamp() - origin
row["duration"] = row["stop"].timestamp() - row["start"].timestamp()
if row["duration"] <= 0:
raise ValueError("Non-positive elapsed segment duration")
raise SourceDataError("Non-positive elapsed segment duration")
included = {path.name for row in rows for kind in ("audio", "videos") for path in row[kind].values()}
for path in sorted(folder.iterdir()):
if path.name not in {"README.txt", "sync.csv"} and path.name not in included:
Expand Down Expand Up @@ -785,7 +789,7 @@ def _probe_sync_video(path, /):
def _check_sync_timing(*, actual, expected, tolerance, description):
residual = actual - expected
if not numpy.isfinite(residual) or abs(residual) > tolerance + 1e-6:
raise ValueError(f"Timing conflict for {description}: measured {actual:g}s, expected {expected:g}s")
raise SourceDataError(f"Timing conflict for {description}: measured {actual:g}s, expected {expected:g}s")
return residual


Expand Down
17 changes: 15 additions & 2 deletions labs/sanes/code/batch_convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@
the output. A failed session does not stop the batch, and the exit code reports whether any failed.

An ``experiment_*`` folder with no supported layout is reported as pending, since its media may not be
uploaded yet, and does not fail the batch. With ``--results`` the batch writes the sessions it left
uploaded yet, and does not fail the batch. So is a session whose ``sync.csv`` disagrees with its files (a
listed file missing, a malformed name, media timing outside ``sync.tolerance``), which waits on the lab to fix
the source data. With ``--results`` the batch writes the sessions it left
unconverted, as ``{"pending": {name: reason}, "failed": {name: reason}}`` keyed by folder name. Dispatch
fills it through its ``{results}`` placeholder, then keeps and uploads what did convert.

Expand All @@ -43,7 +45,14 @@
# that directory on sys.path and import the sibling by name.
sys.path.insert(0, str(Path(__file__).resolve().parent))

from _sanes_to_nwb import SYNC_FOLDER, build_nwb, load_cfg, multisubject_path, session_label # noqa: E402
from _sanes_to_nwb import ( # noqa: E402
SYNC_FOLDER,
SourceDataError,
build_nwb,
load_cfg,
multisubject_path,
session_label,
)

UNSUPPORTED_REASON = f"No idx_* chunks or {SYNC_FOLDER}/ folder yet"

Expand Down Expand Up @@ -128,6 +137,10 @@ def convert_batch(*, incoming_dir, standardized_dir, config_path, overwrite=Fals
for session_dir in tqdm.tqdm(pending, desc="Converting sessions", unit="session"):
try:
output_path = build_nwb(session_dir=session_dir, output_dir=output_dir, config_path=config_path)
except SourceDataError as error:
print(f"PENDING: {session_dir.name}: {error}", flush=True)
unconverted[session_dir.name] = str(error)
continue
except Exception as error:
traceback.print_exception(error)
print(f"FAILED: {session_dir}: {error}", file=sys.stderr, flush=True)
Expand Down
8 changes: 8 additions & 0 deletions labs/sanes/prompts/big-arena-2024-12.md
Original file line number Diff line number Diff line change
Expand Up @@ -150,3 +150,11 @@ From the reviewer (CodyCBakerPhD), answering Q16, the three parts of Q17, and th
> "Now: a collection with a different layout can also join, if its output follows the same conventions and the converter gains a second reader for the new layout." yeah that sounds fine

The audio is now one eight-column series with `ndx-sound>=0.3.0`. The plan is kept as `big-arena-2024-12-plan.md` beside this record, amended to match the implementation. Q16 and Q17 are resolved.

## Request 8 — Skip BIG_ARENA sessions whose source data disagrees with sync.csv

Every BIG_ARENA session failed the 2026-11-11 scheduled ingest: files listed in `sync.csv` were missing, a name read `channel_0_0`, or audio timing fell outside `sync.tolerance`. Asked whether to report those as pending or take BIG_ARENA off the nightly run:

> hmmm I see well skip those for now and raise an issue about them

Recorded in https://github.com/brain-bbqs/data-ingest-task-force/issues/60.
18 changes: 18 additions & 0 deletions labs/sanes/tests/test_batch_convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@

CODE = Path(__file__).resolve().parents[1] / "code"
sys.path.insert(0, str(CODE))
sys.path.insert(0, str(Path(__file__).resolve().parent))

import batch_convert # noqa: E402
import synthetic # noqa: E402

pytestmark = pytest.mark.ai_generated

Expand Down Expand Up @@ -153,3 +155,19 @@ def test_colliding_session_labels_are_rejected(tmp_path, recorded_calls):
)
assert actual == 1
assert recorded_calls == []


@pytest.mark.ai_generated
def test_source_data_problems_are_pending_not_failed(tmp_path):
session = synthetic.write_sync_session(root=tmp_path / "in", segments=2)
missing = session / "concatenated_data_cam_mic_sync" / "video_underground_001.mp4"
missing.unlink()
actual = batch_convert.convert_batch(
incoming_dir=tmp_path / "in",
standardized_dir=tmp_path / "out",
config_path=CODE / "config_big_arena_2024_12.yaml",
results_path=tmp_path / "results.json",
)
assert actual == 0
results = json.loads((tmp_path / "results.json").read_text())
assert results == {"failed": {}, "pending": {"experiment_109": f"Missing manifest media: {missing}"}}
Loading