From c693ec4d17f945d3793c8286806495fbc2fd5e12 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 15:48:18 +0000 Subject: [PATCH 1/2] Read the Zhang lab's naming variants and same-day Session folders The Zhang converter skipped recordings over spelling differences: - Calibration videos named CamNN-Calibration-... (60 files) are now read as calibration videos alongside CamNN-cal-.... - Recording names with a session token (Session1, sess1, Sess1) or a Merge prefix on the date are now recognized. The session label stays the date and time. - A merged file without the underscore before "merged" is now found. - A day's two recordings in Session_1/ and Session_2/ (or Session1/) subfolders, each with its own Videos/, are now discovered. This answers Q13. On the current archive this takes the convertible recordings from 74 to 101 of 111. The animal and headstage mismatches in #62 stay pending. Co-Authored-By: Claude Code 2.1.328 / Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018Gd63VAaXEtYzbNRniezVW --- .../ferret-hyperscanning/OPEN_QUESTIONS.md | 12 +--- .../zhang/ferret-hyperscanning/code/README.md | 23 ++++++- .../_zhang_ferret_hyperscanning_to_nwb.py | 28 +++++++-- .../code/batch_convert.py | 29 +++++---- .../ferret-hyperscanning/prompts/initial.md | 14 +++++ .../tests/test_batch_convert.py | 60 ++++++++++++++++--- 6 files changed, 130 insertions(+), 36 deletions(-) diff --git a/labs/zhang/ferret-hyperscanning/OPEN_QUESTIONS.md b/labs/zhang/ferret-hyperscanning/OPEN_QUESTIONS.md index 09ce9f0..2e51d02 100644 --- a/labs/zhang/ferret-hyperscanning/OPEN_QUESTIONS.md +++ b/labs/zhang/ferret-hyperscanning/OPEN_QUESTIONS.md @@ -20,7 +20,6 @@ with a pointer to the change. | [Q10](#q10-electrodes) | Electrode type, geometry, coordinates, filtering, references | Deferred | Electrode table detail | | [Q11](#q11-accelerometer) | Accelerometer units and real sample rate | Deferred | Accelerometer stream | | [Q12](#q12-session-log-as-an-input) | Session log as a committed CSV, and its parsing rules | Implemented as proposed, unconfirmed | Channel map selection, notes | -| [Q13](#q13-two-recordings-on-one-date) | Layout of days with two recordings of one pair | Implemented as proposed, unverified | Video matching | | [Q14](#q14-sessions-to-exclude) | Should sessions without usable data be excluded? | Deferred, all converted | Which sessions publish | | [Q15](#q15-calibration-videos) | Calibration videos in the NWB or as plain assets? | Implemented as proposed, unconfirmed | Calibration video placement | | [Q16](#q16-investigate-video-transcoding) | Investigate video transcoding | Deferred to a follow-up | Video size and format | @@ -128,16 +127,6 @@ notes. hour below 8 were read as afternoon. - **Ask the lab:** confirm both parsing rules. -### Q13. Two recordings on one date - -Twelve date-pair combinations have two session-log rows. - -- **Current handling:** each `.rec` is matched to the videos whose filename - timestamp is nearest (`match_videos`), and to the session-log row whose - start time is nearest (`find_session_row`). -- **Needs:** a real example of such a day, to confirm that both recordings - share one `Videos/` directory. - ### Q15. Calibration videos - **Current handling:** the calibration videos are in the NWB as @@ -153,3 +142,4 @@ Twelve date-pair combinations have two session-log rows. | Q2 | Flat or nested lab layout | Nested, `labs/zhang/ferret-hyperscanning/` | Request 2 in `prompts/initial.md` | | Q3 | Data standard | NWB, one file per subject per session, DANDI layout | Request 2 | | Q4 | Video handling | Keep the AVIs unchanged as external files, transcode later | Request 2 | +| Q13 | Layout of days with two recordings of one pair | Each recording sits with its own `Videos/` in a `Session_1/`, `Session_2/` (or `Session1/`, `Session2/`) folder of the date folder, 22 recordings over 11 days. `batch_convert.discover_recordings` reads them | Request 13 | diff --git a/labs/zhang/ferret-hyperscanning/code/README.md b/labs/zhang/ferret-hyperscanning/code/README.md index e4afd4b..acbf154 100644 --- a/labs/zhang/ferret-hyperscanning/code/README.md +++ b/labs/zhang/ferret-hyperscanning/code/README.md @@ -40,8 +40,27 @@ before writing, which is what DANDI resolves after upload. Videos/ cam--0000.avi behavior video, one per camera cam-cal--0000.avi ChArUco calibration video, one per camera + Session_1/ only on a day with two recordings of the pair + .rec/ laid out as above + Videos/ that recording's own videos + Session_2/ ``` +The lab's spelling varies in places, and these variants are read as the +standard names: + +| Variant | Example | +| --- | --- | +| A session token between the headstages and the date | `0235HS3_0237HS4_Session1_20260308_142653`, also `sess1` and `Sess1` | +| `Merge` glued to the date | `0235HS5_0236HS8_Merge20260423_142917` | +| No underscore before `merged` | `…_0237HS4merged.rec` | +| `Calibration` for `cal` in a video name | `Cam14-Calibration-04012026121856-0000.avi` | +| `Session1/` for `Session_1/` | `0235-0236/03.18.2026/Session1/` | + +A recording whose name or merged files fit none of these is reported in the +run log and left unconverted. The lab issue lists the remaining cases: +https://github.com/brain-bbqs/data-ingest-task-force/issues/62. + ## Mapping | Source | Output | @@ -99,7 +118,9 @@ filesystem with the incoming tree, copied otherwise. Each lives in one function of `_zhang_ferret_hyperscanning_to_nwb.py`: - **Session label** (`derive_session_label`): the basename's date and time - joined as `20260401T130419`. + joined as `20260401T130419`. A session token or `Merge` prefix in the + basename does not change it, since the time already tells same-day + recordings apart. - **Subject labels** (`derive_subject_labels`): the two four-digit ids from `HS_HS`, in basename order, checked against the pair directory by `parse_rec_identity`. diff --git a/labs/zhang/ferret-hyperscanning/code/_zhang_ferret_hyperscanning_to_nwb.py b/labs/zhang/ferret-hyperscanning/code/_zhang_ferret_hyperscanning_to_nwb.py index a269d73..c13be43 100644 --- a/labs/zhang/ferret-hyperscanning/code/_zhang_ferret_hyperscanning_to_nwb.py +++ b/labs/zhang/ferret-hyperscanning/code/_zhang_ferret_hyperscanning_to_nwb.py @@ -16,6 +16,13 @@ cam--0000.avi cam-cal--0000.avi +A day with two recordings of one pair holds each, with its own ``Videos/``, +in a ``Session_1/`` or ``Session1/`` subfolder of the date folder. The lab's +spelling varies in places, and these variants are accepted: a session token +in the stem (``_Session1_``, ``_sess1_``, ``_Sess1_``), ``Merge`` glued to +the date (``_Merge20260423_``), ``merged.rec`` without its underscore +(``…_0237HS4merged.rec``), and ``Calibration`` for ``cal`` in a video name. + The recording becomes two NWB files, one per animal, in the DANDI layout, with the shared videos placed next to the lower-numbered animal's file and referenced from both:: @@ -64,12 +71,15 @@ REC_NAME_PATTERN = re.compile( r"^(?P\d{4})HS(?P\d+)_(?P\d{4})HS(?P\d+)" - r"_(?P\d{8})_(?PHS_HS`, in basename order, checked against the pair - directory by `parse_rec_identity`. + directory by `parse_rec_identity`. A correction's `name` stands in for + the basename. - **Channel split** (`headstage_channel_ids`): the first headstage in the basename takes hardware channels 0 to 31, the second 32 to 63. PROVISIONAL until a real file's `SpikeConfiguration` confirms it. diff --git a/labs/zhang/ferret-hyperscanning/code/_zhang_ferret_hyperscanning_to_nwb.py b/labs/zhang/ferret-hyperscanning/code/_zhang_ferret_hyperscanning_to_nwb.py index c13be43..f3ca0d1 100644 --- a/labs/zhang/ferret-hyperscanning/code/_zhang_ferret_hyperscanning_to_nwb.py +++ b/labs/zhang/ferret-hyperscanning/code/_zhang_ferret_hyperscanning_to_nwb.py @@ -22,6 +22,10 @@ in the stem (``_Session1_``, ``_sess1_``, ``_Sess1_``), ``Merge`` glued to the date (``_Merge20260423_``), ``merged.rec`` without its underscore (``…_0237HS4merged.rec``), and ``Calibration`` for ``cal`` in a video name. +A merged file sitting loose in the ``.rec`` directory, outside its animal's +folder, is found too. A recording whose names contradict its files is read +through its entry under ``corrections`` in the config, which can rename it +or point at an animal's merged file. The recording becomes two NWB files, one per animal, in the DANDI layout, with the shared videos placed next to the lower-numbered animal's file and referenced from both:: @@ -80,6 +84,7 @@ r"^cam(?P\d+)(?P-cal(?:ibration)?)?-(?P\d{14})-\d{4}\.avi$", re.IGNORECASE ) MERGED_SUFFIXES = ("_merged.rec", "merged.rec") +CORRECTION_KEYS = frozenset({"name", "merged", "reason"}) VIDEO_STAMP_FORMAT = "%m%d%Y%H%M%S" VIDEOS_DIRNAME = "Videos" VIDEO_KINDS = ("video", "calibration") @@ -101,13 +106,29 @@ class Headstage: headstage: str position: int + @property + def tag(self): + """``0236HS3``, as folder and file names label this animal's headstage.""" + tag = f"{self.subject}HS{self.headstage}" + return tag + @dataclasses.dataclass(frozen=True) class RecordingIdentity: + """A recording's animals and session, read from ``name``. + + ``name`` is the uploaded recording name unless a correction replaces it. A + correction can also give an animal's merged file in ``merged_files``, + keyed by headstage tag, and says why in ``correction_reason``. + """ + rec_path: Path headstages: tuple[Headstage, Headstage] session_label: str name_start: datetime.datetime + name: str + merged_files: dict[str, Path] = dataclasses.field(default_factory=dict) + correction_reason: str | None = None @property def pair(self): @@ -137,13 +158,21 @@ def resolve_rec_file(rec, /): def merged_rec_path(*, identity, headstage): """The Trodes file holding *headstage*'s ephys, merged from its datalogger. - The expected spelling is returned when no spelling exists, so a missing file is reported by its expected name. + A correction's file is taken while it exists. Otherwise the file is looked for in the headstage's folder, then + loose in the recording folder, named after the recording as uploaded or as corrected. The expected spelling is + returned when none exists, so a missing file is reported by its expected name. """ - tag = f"{headstage.subject}HS{headstage.headstage}" - candidates = [ - identity.rec_path.parent / tag / f"{identity.rec_path.stem}_{tag}{suffix}" for suffix in MERGED_SUFFIXES + tag = headstage.tag + rec_dir = identity.rec_path.parent + stems = dict.fromkeys([identity.rec_path.stem, identity.name]) + named = [ + folder / f"{stem}_{tag}{suffix}" + for folder in (rec_dir / tag, rec_dir) + for stem in stems + for suffix in MERGED_SUFFIXES ] - path = next((candidate for candidate in candidates if candidate.is_file()), candidates[0]) + corrected = [identity.merged_files[tag]] if tag in identity.merged_files else [] + path = next((candidate for candidate in [*corrected, *named] if candidate.is_file()), named[0]) return path @@ -168,12 +197,34 @@ def derive_subject_labels(rec_stem, /): return headstages -def parse_rec_identity(rec, /): +def recording_corrections(cfg, /): + """The config's per-recording corrections, keyed by the recording name as uploaded, each checked.""" + corrections = cfg.get("corrections") or {} + for uploaded, correction in corrections.items(): + unknown = sorted(set(correction) - CORRECTION_KEYS) + if unknown: + raise ValueError(f"correction for {uploaded} has unknown keys {unknown}") + if not correction.get("reason"): + raise ValueError(f"correction for {uploaded} gives no reason") + name = correction.get("name", uploaded) + if REC_NAME_PATTERN.match(name) is None: + raise ValueError(f"correction for {uploaded} names unrecognized recording {name!r}") + tags = {headstage.tag for headstage in derive_subject_labels(name)} + strays = sorted(set(correction.get("merged") or {}) - tags) + if strays: + raise ValueError(f"correction for {uploaded} gives merged files for {strays}, not headstages of {name}") + return corrections + + +def parse_rec_identity(rec, /, *, corrections=None): + """The animals, headstages and session of a recording, from its name or its entry in *corrections*.""" rec_path = resolve_rec_file(rec) - match = REC_NAME_PATTERN.match(rec_path.stem) + correction = (corrections or {}).get(rec_path.stem) or {} + name = correction.get("name", rec_path.stem) + match = REC_NAME_PATTERN.match(name) if match is None: raise ValueError(f"not a recognized recording name: {rec_path.name!r}") - headstages = derive_subject_labels(rec_path.stem) + headstages = derive_subject_labels(name) container = rec_path.parent.parent date_dir = container.parent if SESSION_DIRNAME_PATTERN.match(container.name) else container pair_dirname = date_dir.parent.name @@ -181,11 +232,15 @@ def parse_rec_identity(rec, /): if PAIR_DIRNAME_PATTERN.match(pair_dirname) and pair_dirname != expected_pair: raise ValueError(f"recording {rec_path.name} names pair {expected_pair} but sits under {pair_dirname}/") name_start = datetime.datetime.strptime(match["date"] + match["time"], "%Y%m%d%H%M%S") + merged_files = {tag: rec_path.parent / relative for tag, relative in (correction.get("merged") or {}).items()} identity = RecordingIdentity( rec_path=rec_path, headstages=headstages, - session_label=derive_session_label(rec_path.stem), + session_label=derive_session_label(name), name_start=name_start, + name=name, + merged_files=merged_files, + correction_reason=correction.get("reason"), ) return identity @@ -217,17 +272,15 @@ def session_start_time(*, identity, cfg): return start_time -def match_videos(rec, /): - """Per camera, the behavior and calibration videos nearest this recording's timestamp. +def match_videos(identity, /): + """Per camera, the behavior and calibration videos nearest the recording's timestamp. Returns ``(matched, unmatched)``: ``matched`` maps a camera number to a ``{kind: path}`` dict, ``unmatched`` lists files in ``Videos/`` whose names the pattern does not recognize. A missing ``Videos/`` directory yields nothing matched. """ - rec_path = resolve_rec_file(rec) - identity = parse_rec_identity(rec_path) - videos_dir = rec_path.parent.parent / VIDEOS_DIRNAME + videos_dir = identity.rec_path.parent.parent / VIDEOS_DIRNAME matched: dict[int, dict[str, Path]] = {} unmatched: list[Path] = [] if not videos_dir.is_dir(): @@ -276,6 +329,18 @@ def find_session_row(rows, /, *, pair, start): return nearest +def session_notes(*, identity, session_row, cfg): + """The NWB ``notes``: the pair, the session log's usable-data text and notes, and any naming correction.""" + notes = cfg["session"]["notes_template"].format( + pair=identity.pair, + usable_data=(session_row or {}).get("usable_data") or "not logged", + notes=(session_row or {}).get("notes") or "none", + ) + if identity.correction_reason: + notes += f" Source names corrected for {identity.rec_path.parent.name}: {identity.correction_reason}" + return notes + + def select_channel_map(*, belly_up, cfg): key = "belly_up" if belly_up else "belly_down" channel_map = cfg["ecephys"]["channel_maps"][key] @@ -437,11 +502,7 @@ def build_nwbfile(*, identity, headstage, cfg, placed_videos, nwb_path, belly_up device_name = cfg["devices"]["headstage"]["name_template"].format(headstage=headstage.headstage) session_cfg = cfg["session"] channel_map_name = "Belly-up" if belly_up else "Belly-down" - notes = session_cfg["notes_template"].format( - pair=identity.pair, - usable_data=(session_row or {}).get("usable_data") or "not logged", - notes=(session_row or {}).get("notes") or "none", - ) + notes = session_notes(identity=identity, session_row=session_row, cfg=cfg) video_interfaces = _video_interfaces(placed_videos=placed_videos, cfg=cfg) converter = neuroconv.ConverterPipe(data_interfaces={"ecephys": interface, **video_interfaces}) @@ -499,7 +560,9 @@ def build_nwbfile(*, identity, headstage, cfg, placed_videos, nwb_path, belly_up def convert_session(*, rec, output_dir, cfg, session_log=None, overwrite=False): """Convert one recording into its two subject files; returns the written NWB paths.""" - identity = parse_rec_identity(rec) + identity = parse_rec_identity(rec, corrections=recording_corrections(cfg)) + if identity.correction_reason: + print(f"{identity.rec_path.name}: read through its correction. {identity.correction_reason}", flush=True) output_dir = Path(output_dir) # Checked before the videos are placed, which copies about 170 GB when it cannot hard-link. missing_ephys = [ @@ -509,7 +572,7 @@ def convert_session(*, rec, output_dir, cfg, session_log=None, overwrite=False): ] if missing_ephys: raise MissingInputError(f"no merged ephys file at {', '.join(str(path) for path in missing_ephys)}") - matched, unmatched = match_videos(identity.rec_path) + matched, unmatched = match_videos(identity) for path in unmatched: print(f"Ignoring unrecognized file in {VIDEOS_DIRNAME}/: {path.name}", flush=True) expected_cameras = set(cfg["video"]["cameras"]) diff --git a/labs/zhang/ferret-hyperscanning/code/batch_convert.py b/labs/zhang/ferret-hyperscanning/code/batch_convert.py index dcf8fba..38cd15b 100644 --- a/labs/zhang/ferret-hyperscanning/code/batch_convert.py +++ b/labs/zhang/ferret-hyperscanning/code/batch_convert.py @@ -66,6 +66,7 @@ load_session_log_for, output_paths, parse_rec_identity, + recording_corrections, ) RAW_SUBTREE = Path("sourcedata") / "raw" / "Ferret_Hyperscanning" @@ -99,8 +100,8 @@ def discover_recordings(incoming_dir, /): return recordings, ignored -def is_converted(*, rec_dir, standardized_dir): - identity = parse_rec_identity(rec_dir) +def is_converted(*, rec_dir, standardized_dir, corrections): + identity = parse_rec_identity(rec_dir, corrections=corrections) paths = output_paths(output_dir=standardized_dir, identity=identity) converted = all(path.is_file() for path in paths.values()) return converted @@ -148,11 +149,12 @@ def convert_batch(*, incoming_dir, standardized_dir, config_path, overwrite=Fals print(f"No .rec recordings found under {incoming_dir / RAW_SUBTREE}") return 0 + corrections = recording_corrections(load_cfg(config_path)) skipped = 0 pending = [] for rec_dir in recordings: try: - converted = is_converted(rec_dir=rec_dir, standardized_dir=standardized_dir) + converted = is_converted(rec_dir=rec_dir, standardized_dir=standardized_dir, corrections=corrections) except ValueError as error: print(f"Ignoring unrecognized recording: {error}", flush=True) skipped += 1 diff --git a/labs/zhang/ferret-hyperscanning/code/config.yaml b/labs/zhang/ferret-hyperscanning/code/config.yaml index 71fde21..01172af 100644 --- a/labs/zhang/ferret-hyperscanning/code/config.yaml +++ b/labs/zhang/ferret-hyperscanning/code/config.yaml @@ -91,3 +91,40 @@ session_log: # 24-hour local wall clock; rows whose sheet cell had no AM/PM were read as # afternoon when the hour was below 8. path: session_log.csv + +corrections: + # Recordings whose uploaded names contradict their own files, keyed by the recording name as + # uploaded (https://github.com/brain-bbqs/data-ingest-task-force/issues/62). Each was checked + # against the merged files' headers, which carry the recording's start time and the serial of + # the headstage that recorded them (03103 00106 is HS3, 00105 HS4, 00127 HS5, 00130 HS6). + # "name" replaces the name the animals, headstages and session are read from. "merged" gives an + # animal's merged file, relative to the recording directory and keyed by HS. + # It is used only while that file exists, so a rename on the lab's side takes over. + 0235HS3_0237HS4_20260325_124806: + merged: + 0235HS3: 0235HS3/0235HS3_0237HS4_20260325_124806_0236HS3_merged.rec + reason: >- + The merged file in 0235HS3/ is named for 0236. It carries HS3's serial, and 0236 was not + recorded that day. + 0235HS6_0237HS6_20260425_132118: + name: 0235HS5_0237HS6_20260425_132118 + reason: >- + The recording name puts both animals on HS6. The 0235HS5/ folder, both merged file names and + 0235's serial say 0235 wore HS5. + 0236HS5_0237HS6_20260408_155939: + merged: + 0237HS6: 0237HS8/0236HS5_0237HS6_20260408_155939_0237HS6_merged.rec + reason: >- + 0237's merged file sits in a folder named 0237HS8. Its own name and its serial say HS6. + 02355HS6_0236HS5_20260408_134217: + name: 0235HS6_0236HS5_20260408_134217 + reason: >- + The recording name has an extra 5 in 0235. The pair folder, the 0235HS6/ folder and its merged + file name say 0235, and its serial matches HS6. + 0236HS3_0237HS4_Session1_20260308_130347: + merged: + 0236HS3: 0236HS3/0236HS3_0237HS4_Session1_20260308_130347_HS30236_merged.rec + 0237HS4: 0237HS4/0236HS3_0237HS4_Session1_20260308_130347_merged.rec + reason: >- + The merged files are named HS30236 and with no headstage at all. Each sits in its animal's + folder and carries that headstage's serial. diff --git a/labs/zhang/ferret-hyperscanning/prompts/initial.md b/labs/zhang/ferret-hyperscanning/prompts/initial.md index 88bf7af..9c1e739 100644 --- a/labs/zhang/ferret-hyperscanning/prompts/initial.md +++ b/labs/zhang/ferret-hyperscanning/prompts/initial.md @@ -550,3 +550,14 @@ Opened https://github.com/brain-bbqs/data-ingest-task-force/issues/62, listing t > ok adapt to the missing underscore for the .rec; explain the animal 0236 and 0235 and other naming mismatch to me in more detail The converter now reads `CamNN-Calibration-` videos, session tokens (`Session1`, `sess1`, `Sess1`) and a `Merge` prefix in recording names, merged files without the underscore before `merged`, and the `Session_1/` subfolders that hold a day's two recordings, which answers Q13. A test recording (`RecordingTest1`) and a five-digit animal id (`02355`) stay unrecognized, and the animal and headstage mismatches stay pending until the lab confirms them. + +## Request 14 — Correct the mislabeled recordings + +> yeah do all three + +The three were the follow-ups offered after the mismatches were explained. +1. Post the renames and questions for the lab on the issue. +2. Correct the mislabeled recordings in the converter. +3. Delete local copies of files removed from EMBER. + +`config.yaml` now has a `corrections` section for five recordings, each checked against the headstage serials in its merged files' headers. They are 03.25 (a merged file named for 0236), 04.25 (both animals on HS6), 04.08 15:59 (an HS8 folder), `02355` and 03.08 13:03 (two malformed merged names). Merged files loose in the recording folder, as on 03.30, are found too. On the current archive 107 of 111 recordings now convert. The other four are the three recordings with no merged files and `RecordingTest1`. Dispatch's session pattern now also covers the `Session_/` folders, whose 22 recordings it had not been discovering. diff --git a/labs/zhang/ferret-hyperscanning/tests/test_batch_convert.py b/labs/zhang/ferret-hyperscanning/tests/test_batch_convert.py index 5c38b18..91a9ad2 100644 --- a/labs/zhang/ferret-hyperscanning/tests/test_batch_convert.py +++ b/labs/zhang/ferret-hyperscanning/tests/test_batch_convert.py @@ -30,6 +30,12 @@ pytestmark = pytest.mark.ai_generated REC_STEM = "0236HS3_0237HS4_20260401_130419" +CONFIG_PATH = PROJECT / "code" / "config.yaml" +CORRECTIONS = { + "02355HS6_0236HS5_20260408_134217": {"name": "0235HS6_0236HS5_20260408_134217", "reason": "An extra 5."}, + "0235HS6_0237HS6_20260425_132118": {"name": "0235HS5_0237HS6_20260425_132118", "reason": "Both on HS6."}, + REC_STEM: {"merged": {"0236HS3": "0236HS3/misnamed_merged.rec"}, "reason": "Named for another animal."}, +} def make_rec_dir(root, /, *, pair="0236-0237", date_dirname="04.01.2026", stem=REC_STEM): @@ -83,6 +89,38 @@ def test_parse_rec_identity(tmp_path): assert identity.name_start == datetime.datetime(2026, 4, 1, 13, 4, 19) +def test_parse_rec_identity_applies_a_name_correction(tmp_path): + stem = "02355HS6_0236HS5_20260408_134217" + rec_dir = make_rec_dir(tmp_path, pair="0235-0236", date_dirname="04.08.2026", stem=stem) + with pytest.raises(ValueError, match="not a recognized recording name"): + core.parse_rec_identity(rec_dir) + identity = core.parse_rec_identity(rec_dir, corrections=CORRECTIONS) + assert [headstage.tag for headstage in identity.headstages] == ["0235HS6", "0236HS5"] + assert identity.session_label == "20260408T134217" + assert identity.rec_path == rec_dir / f"{stem}.rec" + assert identity.correction_reason == "An extra 5." + + +@pytest.mark.parametrize( + ("uploaded", "correction", "message"), + [ + (REC_STEM, {"reason": "Checked.", "note": "Typo."}, "unknown keys"), + (REC_STEM, {"merged": {"0236HS3": "0236HS3/file_merged.rec"}}, "gives no reason"), + ("02355HS6_0236HS5_20260408_134217", {"reason": "Checked."}, "unrecognized recording"), + (REC_STEM, {"merged": {"0235HS6": "0235HS6/file_merged.rec"}, "reason": "Checked."}, "not headstages"), + ], +) +def test_recording_corrections_rejects_malformed_entries(uploaded, correction, message): + with pytest.raises(ValueError, match=message): + core.recording_corrections({"corrections": {uploaded: correction}}) + + +def test_config_corrections_are_well_formed(): + corrections = core.recording_corrections(core.load_cfg(CONFIG_PATH)) + assert corrections + assert all(correction["reason"] for correction in corrections.values()) + + @pytest.mark.parametrize("pair", ["0236-0237", "0235-0237"]) @pytest.mark.parametrize("date_dirname", ["04.01.2026", "04.01.2026/Session_1"]) def test_parse_rec_identity_checks_pair_through_session_folders(tmp_path, pair, date_dirname): @@ -121,7 +159,7 @@ def test_match_videos_picks_nearest_stamp_per_camera(tmp_path, calibration, date "README.txt", ]: (videos / name).write_bytes(b"") - matched, unmatched = core.match_videos(rec_dir) + matched, unmatched = core.match_videos(core.parse_rec_identity(rec_dir)) assert matched[14]["video"].name == "cam14-04012026122439-0000.avi" assert matched[14]["calibration"].name == calibration assert matched[46] == {"video": videos / "cam46-04012026122433-0000.avi"} @@ -129,7 +167,7 @@ def test_match_videos_picks_nearest_stamp_per_camera(tmp_path, calibration, date def test_match_videos_without_videos_dir(tmp_path): - matched, unmatched = core.match_videos(make_rec_dir(tmp_path)) + matched, unmatched = core.match_videos(core.parse_rec_identity(make_rec_dir(tmp_path))) assert (matched, unmatched) == ({}, []) @@ -200,6 +238,66 @@ def test_merged_rec_path_accepts_a_missing_underscore(tmp_path): assert paths == [rec_dir / "0236HS3" / f"{REC_STEM}_0236HS3_merged.rec", written] +@pytest.mark.parametrize("in_folder", [False, True]) +def test_merged_rec_path_finds_merged_files_loose_in_the_recording_folder(tmp_path, in_folder): + rec_dir = make_rec_dir(tmp_path) + identity = core.parse_rec_identity(rec_dir) + loose = [rec_dir / f"{REC_STEM}_{headstage.tag}_merged.rec" for headstage in identity.headstages] + for path in loose: + path.write_bytes(b"") + expected = loose + if in_folder: + expected = [rec_dir / headstage.tag / path.name for headstage, path in zip(identity.headstages, loose)] + for path in expected: + path.parent.mkdir() + path.write_bytes(b"") + paths = [core.merged_rec_path(identity=identity, headstage=headstage) for headstage in identity.headstages] + assert paths == expected + + +@pytest.mark.parametrize( + ("pair", "uploaded", "merged_stem"), + [ + ("0235-0236", "02355HS6_0236HS5_20260408_134217", "02355HS6_0236HS5_20260408_134217"), + ("0235-0237", "0235HS6_0237HS6_20260425_132118", "0235HS5_0237HS6_20260425_132118"), + ], +) +def test_merged_rec_path_reads_the_uploaded_or_corrected_name(tmp_path, pair, uploaded, merged_stem): + rec_dir = make_rec_dir(tmp_path, pair=pair, stem=uploaded) + identity = core.parse_rec_identity(rec_dir, corrections=CORRECTIONS) + written = [ + rec_dir / headstage.tag / f"{merged_stem}_{headstage.tag}_merged.rec" for headstage in identity.headstages + ] + for path in written: + path.parent.mkdir() + path.write_bytes(b"") + paths = [core.merged_rec_path(identity=identity, headstage=headstage) for headstage in identity.headstages] + assert paths == written + + +@pytest.mark.parametrize("renamed_by_lab", [False, True]) +def test_merged_rec_path_takes_a_corrected_file_while_it_exists(tmp_path, renamed_by_lab): + rec_dir = make_rec_dir(tmp_path) + identity = core.parse_rec_identity(rec_dir, corrections=CORRECTIONS) + misnamed = rec_dir / "0236HS3" / "misnamed_merged.rec" + standard = rec_dir / "0236HS3" / f"{REC_STEM}_0236HS3_merged.rec" + present = standard if renamed_by_lab else misnamed + present.parent.mkdir() + present.write_bytes(b"") + assert core.merged_rec_path(identity=identity, headstage=identity.headstages[0]) == present + + +def test_session_notes_record_a_correction(tmp_path): + cfg = core.load_cfg(CONFIG_PATH) + rec_dir = make_rec_dir(tmp_path) + plain = core.session_notes(identity=core.parse_rec_identity(rec_dir), session_row=None, cfg=cfg) + corrected = core.session_notes( + identity=core.parse_rec_identity(rec_dir, corrections=CORRECTIONS), session_row=None, cfg=cfg + ) + assert plain == "Pair 0236-0237. Session log usable data: not logged. Session log notes: none." + assert corrected == f"{plain} Source names corrected for {rec_dir.name}: Named for another animal." + + def test_convert_session_fails_before_placing_videos_without_merged_ephys(tmp_path, monkeypatch): rec_dir = make_rec_dir(tmp_path) placed = [] @@ -231,6 +329,20 @@ def test_discover_recordings_reads_session_folders(tmp_path): assert ignored == [second.parent / "notes.txt"] +def test_convert_batch_applies_config_corrections(tmp_path, monkeypatch, capsys): + rec_dir = make_rec_dir( + tmp_path, pair="0235-0236", date_dirname="04.08.2026", stem="02355HS6_0236HS5_20260408_134217" + ) + calls = [] + monkeypatch.setattr(batch_convert, "convert_session", lambda **kwargs: calls.append(kwargs["rec"]) or []) + exit_code = batch_convert.convert_batch( + incoming_dir=tmp_path, standardized_dir=tmp_path / "out", config_path=CONFIG_PATH, max_workers=1 + ) + assert exit_code == 0 + assert calls == [rec_dir] + assert "unrecognized" not in capsys.readouterr().out + + def test_empty_incoming_exits_zero(tmp_path, capsys): exit_code = batch_convert.convert_batch( incoming_dir=tmp_path, standardized_dir=tmp_path / "out", config_path=PROJECT / "code" / "config.yaml"